From 6dd0ade20a9b5f3a779265b59f3f63ce1e671df7 Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Wed, 9 Sep 2026 07:12:26 +1000 Subject: [PATCH 01/35] feat(reconciliation): bind governed data products to runtime providers (#205) * feat(observation): add runtime provider binding contracts * feat(reconciliation): derive runtime asset specs from governed product * feat(reconciliation): support explicit logical runtime bindings * feat(databricks): add runtime product provider * feat(reconciliation): export runtime asset spec projection * feat(observation): export runtime provider contracts * feat(databricks): export runtime provider * test(reconciliation): cover runtime product binding and provider seam * test(reconciliation): isolate missing bound asset scenario --- semapact/observation/__init__.py | 10 + semapact/observation/providers.py | 77 ++++++ semapact/platforms/databricks/__init__.py | 2 + semapact/platforms/databricks/runtime.py | 102 ++++++++ semapact/reconciliation/__init__.py | 2 + semapact/reconciliation/binding.py | 43 +++ semapact/reconciliation/engine.py | 83 +++++- tests/test_runtime_product_binding.py | 304 ++++++++++++++++++++++ 8 files changed, 618 insertions(+), 5 deletions(-) create mode 100644 semapact/observation/providers.py create mode 100644 semapact/platforms/databricks/runtime.py create mode 100644 semapact/reconciliation/binding.py create mode 100644 tests/test_runtime_product_binding.py diff --git a/semapact/observation/__init__.py b/semapact/observation/__init__.py index 69c40796..19934ad5 100644 --- a/semapact/observation/__init__.py +++ b/semapact/observation/__init__.py @@ -15,6 +15,12 @@ ObservedPropertyIdentity, serialize_observed_state, ) +from semapact.observation.providers import ( + RuntimeAssetBinding, + RuntimeAssetSpec, + RuntimeProvider, + RuntimeProviderRegistry, +) __all__ = [ "OBSERVED_STATE_FINGERPRINT_ALGORITHM", @@ -24,6 +30,10 @@ "ObservedPlatformState", "ObservedProperty", "ObservedPropertyIdentity", + "RuntimeAssetBinding", + "RuntimeAssetSpec", + "RuntimeProvider", + "RuntimeProviderRegistry", "canonical_observed_state_payload", "fingerprint_observed_state", "serialize_observed_state", diff --git a/semapact/observation/providers.py b/semapact/observation/providers.py new file mode 100644 index 00000000..f555d3b0 --- /dev/null +++ b/semapact/observation/providers.py @@ -0,0 +1,77 @@ +"""Platform-neutral runtime product provider contracts.""" + +from __future__ import annotations + +from typing import Protocol, Sequence + +from pydantic import BaseModel, ConfigDict + +from semapact.observation.models import ObservedAssetIdentity, ObservedPlatformState + + +class RuntimeProviderModel(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + +class RuntimeAssetSpec(RuntimeProviderModel): + """Governed logical asset plus its physical-name hint.""" + + governed_asset: str + physical_name: str + + +class RuntimeAssetBinding(RuntimeProviderModel): + """Explicit logical governed asset to provider-local runtime identity binding.""" + + governed_asset: str + observed_asset: ObservedAssetIdentity + + +class RuntimeProvider(Protocol): + """Minimal provider seam for runtime product binding and observation.""" + + key: str + + def resolve_bindings( + self, + *, + runtime_target: str, + assets: Sequence[RuntimeAssetSpec], + ) -> tuple[RuntimeAssetBinding, ...]: ... + + def observe( + self, + *, + bindings: Sequence[RuntimeAssetBinding], + ) -> ObservedPlatformState: ... + + +class RuntimeProviderRegistry: + """Small deterministic provider registry used by application interfaces.""" + + def __init__(self, providers: Sequence[RuntimeProvider] = ()) -> None: + self._providers: dict[str, RuntimeProvider] = {} + for provider in providers: + self.register(provider) + + def register(self, provider: RuntimeProvider) -> None: + key = provider.key.strip().casefold() + if not key: + raise ValueError("Runtime provider key is required") + if key in self._providers: + raise ValueError(f"Runtime provider already registered: {provider.key}") + self._providers[key] = provider + + def get(self, key: str) -> RuntimeProvider: + normalized = key.strip().casefold() + try: + return self._providers[normalized] + except KeyError: + supported = ", ".join(sorted(self._providers)) or "none" + raise ValueError( + f"Unsupported runtime provider '{key}'. Registered providers: {supported}" + ) from None + + @property + def keys(self) -> tuple[str, ...]: + return tuple(sorted(self._providers)) diff --git a/semapact/platforms/databricks/__init__.py b/semapact/platforms/databricks/__init__.py index 9a44ddb6..64d75278 100644 --- a/semapact/platforms/databricks/__init__.py +++ b/semapact/platforms/databricks/__init__.py @@ -2,8 +2,10 @@ from semapact.platforms.databricks.client import create_databricks_workspace_client from semapact.platforms.databricks.discovery import discover_databricks_tables +from semapact.platforms.databricks.runtime import DatabricksRuntimeProvider __all__ = [ + "DatabricksRuntimeProvider", "create_databricks_workspace_client", "discover_databricks_tables", ] diff --git a/semapact/platforms/databricks/runtime.py b/semapact/platforms/databricks/runtime.py new file mode 100644 index 00000000..2ebb5888 --- /dev/null +++ b/semapact/platforms/databricks/runtime.py @@ -0,0 +1,102 @@ +"""Databricks implementation of the platform-neutral runtime provider seam.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Sequence + +from semapact.observation.databricks import observe_databricks_table +from semapact.observation.fingerprint import with_observed_state_fingerprint +from semapact.observation.models import ObservedAssetIdentity, ObservedPlatformState +from semapact.observation.providers import RuntimeAssetBinding, RuntimeAssetSpec + + +class DatabricksRuntimeProvider: + """Bind and observe a governed data product in one Unity Catalog namespace.""" + + key = "databricks" + + def __init__(self, *, client: Any, source_identifier: str) -> None: + if not source_identifier.strip(): + raise ValueError("source_identifier is required") + self._client = client + self._source_identifier = source_identifier.strip().rstrip("/") + + def resolve_bindings( + self, + *, + runtime_target: str, + assets: Sequence[RuntimeAssetSpec], + ) -> tuple[RuntimeAssetBinding, ...]: + """Resolve ``catalog.schema`` plus asset physical names into UC identities.""" + namespace = _parse_runtime_target(runtime_target) + bindings = tuple( + RuntimeAssetBinding( + governed_asset=asset.governed_asset, + observed_asset=ObservedAssetIdentity( + platform=self.key, + namespace=namespace, + asset=asset.physical_name, + ), + ) + for asset in assets + ) + return tuple(sorted(bindings, key=lambda item: item.governed_asset)) + + def observe( + self, + *, + bindings: Sequence[RuntimeAssetBinding], + ) -> ObservedPlatformState: + """Observe every bound UC asset; missing tables remain absent evidence.""" + captured_at = datetime.now(timezone.utc) + assets = [] + not_found_error = _load_databricks_not_found_error() + + for binding in sorted(bindings, key=lambda item: item.governed_asset): + identity = binding.observed_asset + if identity.platform.casefold() != self.key: + raise ValueError("Databricks provider received a non-Databricks binding") + table_fqn = ".".join((*identity.namespace, identity.asset)) + if len(identity.namespace) != 2: + raise ValueError( + "Databricks runtime asset identity requires catalog and schema namespace" + ) + try: + observed = observe_databricks_table( + client=self._client, + table_fqn=table_fqn, + source_identifier=self._source_identifier, + captured_at=captured_at, + ) + except not_found_error: + continue + assets.extend(observed.assets) + + state = ObservedPlatformState( + platform=self.key, + source_identifier=self._source_identifier, + assets=tuple(assets), + captured_at=captured_at, + fingerprint=None, + ) + return with_observed_state_fingerprint(state) + + +def _parse_runtime_target(value: str) -> tuple[str, str]: + parts = tuple(part.strip() for part in value.split(".")) + if len(parts) != 2 or not all(parts): + raise ValueError( + "Databricks runtime target must use catalog.schema format for a data product" + ) + return parts[0], parts[1] + + +def _load_databricks_not_found_error() -> type[BaseException]: + try: + from databricks.sdk.errors import NotFound + except ImportError as exc: + raise RuntimeError( + 'Databricks support requires the optional extra: pip install "semapact[databricks]"' + ) from exc + return NotFound diff --git a/semapact/reconciliation/__init__.py b/semapact/reconciliation/__init__.py index 6a910522..87fa8296 100644 --- a/semapact/reconciliation/__init__.py +++ b/semapact/reconciliation/__init__.py @@ -1,5 +1,6 @@ """Platform-neutral governed-desired-vs-observed reconciliation.""" +from semapact.reconciliation.binding import runtime_asset_specs_from_contract from semapact.reconciliation.engine import reconcile_governed_contract from semapact.reconciliation.models import ( ReconciliationDifference, @@ -23,5 +24,6 @@ "RuntimeReasonCode", "classify_reconciliation_status", "reconcile_governed_contract", + "runtime_asset_specs_from_contract", "serialize_reconciliation_result", ] diff --git a/semapact/reconciliation/binding.py b/semapact/reconciliation/binding.py new file mode 100644 index 00000000..d3a42812 --- /dev/null +++ b/semapact/reconciliation/binding.py @@ -0,0 +1,43 @@ +"""Governed data-product asset specs for runtime binding.""" + +from __future__ import annotations + +from open_data_contract_standard.model import OpenDataContractStandard + +from semapact.exceptions import ValidationError +from semapact.lifecycle.identity import normalize_identity_name +from semapact.observation.providers import RuntimeAssetSpec + + +def runtime_asset_specs_from_contract( + contract: OpenDataContractStandard, +) -> tuple[RuntimeAssetSpec, ...]: + """Project governed schemas into logical asset specs without changing identity. + + ``schema.name`` remains the governed logical identity. ``physicalName`` is used + only as a runtime binding hint and falls back to the governed name when absent. + """ + specs: list[RuntimeAssetSpec] = [] + seen: set[str] = set() + + for schema in contract.schema_ or []: + raw_name = getattr(schema, "name", None) + if raw_name is None: + raise ValidationError("Governed schema name is required for runtime binding") + governed_asset = normalize_identity_name(str(raw_name), "Schema") + if governed_asset in seen: + raise ValidationError( + f"Duplicate canonical governed asset identity found: '{governed_asset}'" + ) + seen.add(governed_asset) + + physical_name_value = getattr(schema, "physicalName", None) + physical_name = str(physical_name_value).strip() if physical_name_value else "" + specs.append( + RuntimeAssetSpec( + governed_asset=governed_asset, + physical_name=physical_name or str(raw_name).strip(), + ) + ) + + return tuple(sorted(specs, key=lambda item: item.governed_asset)) diff --git a/semapact/reconciliation/engine.py b/semapact/reconciliation/engine.py index 1b224ecc..eb586bcd 100644 --- a/semapact/reconciliation/engine.py +++ b/semapact/reconciliation/engine.py @@ -2,6 +2,8 @@ from __future__ import annotations +from collections.abc import Sequence + from open_data_contract_standard.model import OpenDataContractStandard, SchemaProperty from semapact.exceptions import ValidationError @@ -13,6 +15,7 @@ ) from semapact.observation.fingerprint import fingerprint_observed_state from semapact.observation.models import ObservedAsset, ObservedPlatformState, ObservedProperty +from semapact.observation.providers import RuntimeAssetBinding from semapact.reconciliation.models import ( ReconciliationDifference, ReconciliationDifferenceType, @@ -61,16 +64,26 @@ def reconcile_governed_contract( contract: OpenDataContractStandard, observation: ObservedPlatformState, + *, + asset_bindings: Sequence[RuntimeAssetBinding] | None = None, ) -> ReconciliationResult: """Compare governed ODCS desired state with platform-neutral observed state. - The caller is responsible for selecting the authoritative governed contract - revision. Reconciliation reports deterministic runtime differences and raw - evidence gaps only; it does not determine approval/authorization, drift cause, - operational status, or mutate either input. + ``asset_bindings`` explicitly maps governed logical schema identity to + provider-local observed asset identity. Operational runtime-product flows + should provide bindings so physical names never redefine governed identity. + The legacy name-matching path remains available for existing library callers. """ governed_assets = build_schema_index(contract) - observed_assets = _build_observed_asset_index(observation) + if asset_bindings is None: + observed_assets = _build_observed_asset_index(observation) + else: + observed_assets = _build_bound_observed_asset_index( + observation=observation, + governed_keys=set(governed_assets), + bindings=asset_bindings, + ) + differences: list[ReconciliationDifference] = [] unverified_paths: list[str] = [] @@ -247,6 +260,66 @@ def _build_observed_asset_index( return index +def _build_bound_observed_asset_index( + *, + observation: ObservedPlatformState, + governed_keys: set[str], + bindings: Sequence[RuntimeAssetBinding], +) -> dict[str, ObservedAsset]: + binding_by_governed: dict[str, RuntimeAssetBinding] = {} + bound_runtime_keys: set[tuple[str, ...]] = set() + + for binding in bindings: + governed_key = normalize_identity_name( + binding.governed_asset, "Runtime binding governed asset" + ) + if governed_key in binding_by_governed: + raise ValidationError( + f"Duplicate runtime binding for governed asset: '{governed_key}'" + ) + if binding.observed_asset.platform.casefold() != observation.platform.casefold(): + raise ValidationError( + "Runtime binding platform must match observed platform state" + ) + runtime_key = binding.observed_asset.canonical_key + if runtime_key in bound_runtime_keys: + raise ValidationError("Multiple governed assets cannot bind to one runtime asset") + bound_runtime_keys.add(runtime_key) + binding_by_governed[governed_key] = binding + + binding_keys = set(binding_by_governed) + if binding_keys != governed_keys: + missing = sorted(governed_keys - binding_keys) + unexpected = sorted(binding_keys - governed_keys) + raise ValidationError( + "Runtime bindings must cover exactly the governed data-product assets; " + f"missing={missing}, unexpected={unexpected}" + ) + + observed_by_runtime: dict[tuple[str, ...], ObservedAsset] = {} + for asset in observation.assets: + runtime_key = asset.identity.canonical_key + if runtime_key in observed_by_runtime: + raise ValidationError( + "Duplicate canonical observed runtime asset identity found: " + f"{runtime_key}" + ) + observed_by_runtime[runtime_key] = asset + + unbound_runtime = sorted(set(observed_by_runtime) - bound_runtime_keys) + if unbound_runtime: + raise ValidationError( + "Observed state contains assets outside the explicit runtime product bindings" + ) + + index: dict[str, ObservedAsset] = {} + for governed_key, binding in binding_by_governed.items(): + observed_asset = observed_by_runtime.get(binding.observed_asset.canonical_key) + if observed_asset is not None: + index[governed_key] = observed_asset + return index + + def _build_observed_property_index( asset_key: str, asset: ObservedAsset, diff --git a/tests/test_runtime_product_binding.py b/tests/test_runtime_product_binding.py new file mode 100644 index 00000000..82bb8bfd --- /dev/null +++ b/tests/test_runtime_product_binding.py @@ -0,0 +1,304 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from types import SimpleNamespace + +import pytest +from open_data_contract_standard.model import ( + OpenDataContractStandard, + SchemaObject, + SchemaProperty, +) + +from semapact.exceptions import ValidationError +from semapact.observation import ( + ObservedAsset, + ObservedAssetIdentity, + ObservedPlatformState, + ObservedProperty, + ObservedPropertyIdentity, + RuntimeAssetBinding, + RuntimeAssetSpec, + RuntimeProviderRegistry, + with_observed_state_fingerprint, +) +from semapact.platforms.databricks import runtime as databricks_runtime +from semapact.platforms.databricks.runtime import DatabricksRuntimeProvider +from semapact.reconciliation import ( + RuntimeReasonCode, + reconcile_governed_contract, + runtime_asset_specs_from_contract, +) + +CAPTURED_AT = datetime(2026, 9, 8, 20, 0, tzinfo=timezone.utc) + + +def _contract(*schemas: SchemaObject) -> OpenDataContractStandard: + return OpenDataContractStandard.model_construct( + id="sales-product", + version="1.0.0", + schema_=list(schemas), + ) + + +def _observed_asset( + *, + namespace: tuple[str, ...], + name: str, + property_name: str = "id", +) -> ObservedAsset: + identity = ObservedAssetIdentity( + platform="databricks", + namespace=namespace, + asset=name, + ) + return ObservedAsset( + identity=identity, + properties=( + ObservedProperty( + identity=ObservedPropertyIdentity( + asset=identity, + property=property_name, + ), + physical_type="BIGINT", + nullable=False, + ), + ), + ) + + +def _observation(*assets: ObservedAsset) -> ObservedPlatformState: + return with_observed_state_fingerprint( + ObservedPlatformState( + platform="databricks", + source_identifier="https://adb.example", + assets=tuple(assets), + captured_at=CAPTURED_AT, + fingerprint=None, + ) + ) + + +def test_runtime_asset_specs_keep_logical_identity_separate_from_physical_name() -> None: + contract = _contract( + SchemaObject.model_construct( + name="orders", + physicalName="fact_orders_v2", + properties=[], + ), + SchemaObject.model_construct( + name="customers", + physicalName=None, + properties=[], + ), + ) + + assert runtime_asset_specs_from_contract(contract) == ( + RuntimeAssetSpec(governed_asset="customers", physical_name="customers"), + RuntimeAssetSpec(governed_asset="orders", physical_name="fact_orders_v2"), + ) + + +def test_explicit_bindings_reconcile_multi_asset_product_without_name_equality() -> None: + contract = _contract( + SchemaObject.model_construct( + name="orders", + physicalName="fact_orders_v2", + properties=[ + SchemaProperty( + name="id", + type="integer", + physicalType="BIGINT", + required=True, + ) + ], + ), + SchemaObject.model_construct( + name="customers", + physicalName="dim_customer", + properties=[ + SchemaProperty( + name="id", + type="integer", + physicalType="BIGINT", + required=True, + ) + ], + ), + ) + orders_identity = ObservedAssetIdentity( + platform="databricks", namespace=("main", "sales"), asset="fact_orders_v2" + ) + customers_identity = ObservedAssetIdentity( + platform="databricks", namespace=("main", "sales"), asset="dim_customer" + ) + observation = _observation( + _observed_asset(namespace=("main", "sales"), name="dim_customer"), + _observed_asset(namespace=("main", "sales"), name="fact_orders_v2"), + ) + bindings = ( + RuntimeAssetBinding(governed_asset="orders", observed_asset=orders_identity), + RuntimeAssetBinding(governed_asset="customers", observed_asset=customers_identity), + ) + + result = reconcile_governed_contract( + contract, + observation, + asset_bindings=bindings, + ) + + assert result.differences == () + assert result.unverified_paths == () + + +def test_missing_bound_runtime_asset_reports_logical_governed_asset() -> None: + contract = _contract( + SchemaObject.model_construct(name="orders", properties=[]), + SchemaObject.model_construct(name="customers", properties=[]), + ) + orders_identity = ObservedAssetIdentity( + platform="databricks", + namespace=("main", "sales"), + asset="orders", + ) + observation = _observation(ObservedAsset(identity=orders_identity, properties=())) + bindings = ( + RuntimeAssetBinding( + governed_asset="orders", + observed_asset=orders_identity, + ), + RuntimeAssetBinding( + governed_asset="customers", + observed_asset=ObservedAssetIdentity( + platform="databricks", + namespace=("main", "sales"), + asset="dim_customer", + ), + ), + ) + + result = reconcile_governed_contract( + contract, + observation, + asset_bindings=bindings, + ) + + assert len(result.differences) == 1 + difference = result.differences[0] + assert difference.reason_code is RuntimeReasonCode.RUNTIME_SCHEMA_REMOVED + assert difference.asset_identity == "customers" + assert difference.path == "schema[customers]" + + +def test_runtime_bindings_must_cover_exact_governed_product() -> None: + contract = _contract( + SchemaObject.model_construct(name="orders", properties=[]), + SchemaObject.model_construct(name="customers", properties=[]), + ) + observation = _observation() + bindings = ( + RuntimeAssetBinding( + governed_asset="orders", + observed_asset=ObservedAssetIdentity( + platform="databricks", + namespace=("main", "sales"), + asset="orders", + ), + ), + ) + + with pytest.raises( + ValidationError, + match="Runtime bindings must cover exactly the governed data-product assets", + ): + reconcile_governed_contract( + contract, + observation, + asset_bindings=bindings, + ) + + +def test_provider_registry_is_vendor_neutral() -> None: + class _Provider: + def __init__(self, key: str) -> None: + self.key = key + + def resolve_bindings(self, **kwargs: object) -> tuple[RuntimeAssetBinding, ...]: + return () + + def observe(self, **kwargs: object) -> ObservedPlatformState: + return _observation() + + registry = RuntimeProviderRegistry( + (_Provider("databricks"), _Provider("snowflake")) + ) + + assert registry.keys == ("databricks", "snowflake") + assert registry.get("SNOWFLAKE").key == "snowflake" + + +class _FakeTableInfo: + def __init__(self, *, catalog: str, schema: str, name: str) -> None: + self._payload = { + "catalog_name": catalog, + "schema_name": schema, + "name": name, + "full_name": f"{catalog}.{schema}.{name}", + "columns": [ + { + "name": "id", + "type_text": "BIGINT", + "nullable": False, + "position": 0, + } + ], + } + + def as_dict(self) -> dict[str, object]: + return self._payload + + +class _FakeNotFound(Exception): + pass + + +def test_databricks_provider_resolves_and_observes_complete_bound_product( + monkeypatch: pytest.MonkeyPatch, +) -> None: + tables = { + "main.sales.fact_orders_v2": _FakeTableInfo( + catalog="main", schema="sales", name="fact_orders_v2" + ) + } + + class _Tables: + def get(self, full_name: str) -> _FakeTableInfo: + try: + return tables[full_name] + except KeyError: + raise _FakeNotFound(full_name) from None + + provider = DatabricksRuntimeProvider( + client=SimpleNamespace(tables=_Tables()), + source_identifier="https://adb.example/", + ) + monkeypatch.setattr( + databricks_runtime, + "_load_databricks_not_found_error", + lambda: _FakeNotFound, + ) + specs = ( + RuntimeAssetSpec(governed_asset="customers", physical_name="dim_customer"), + RuntimeAssetSpec(governed_asset="orders", physical_name="fact_orders_v2"), + ) + + bindings = provider.resolve_bindings(runtime_target="main.sales", assets=specs) + state = provider.observe(bindings=bindings) + + assert [binding.governed_asset for binding in bindings] == ["customers", "orders"] + assert bindings[0].observed_asset.namespace == ("main", "sales") + assert bindings[1].observed_asset.asset == "fact_orders_v2" + assert [asset.identity.asset for asset in state.assets] == ["fact_orders_v2"] + assert state.platform == "databricks" + assert state.source_identifier == "https://adb.example" + assert state.fingerprint is not None From ea0a86665f8dccb4304a1deace73d3568e930acb Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Wed, 9 Sep 2026 08:06:22 +1000 Subject: [PATCH 02/35] feat(reconciliation): expose contract-first provider-neutral runtime assurance (#206) * feat(reconciliation): add provider-neutral reconciliation service * feat(platforms): compose selected runtime provider * feat(cli): add provider-neutral reconcile command adapter * feat(cli): add runtime assurance outcomes * feat(services): export reconciliation service * feat(cli): expose runtime product reconciliation * test(reconciliation): cover provider-neutral service boundary * test(cli): freeze runtime assurance exit semantics * test(cli): cover provider-neutral reconcile command * refactor(cli): make reconcile text output product-scoped * docs(reconciliation): document provider-neutral CLI * refactor(platforms): classify provider selection as validation * refactor(databricks): classify runtime target syntax as validation * refactor(test): keep reconciliation service doubles protocol-compatible * test(platforms): freeze runtime provider validation boundaries * refactor(reconciliation): resolve contract runtime before CLI fallback * refactor(reconciliation): prefer contract runtime metadata * refactor(cli): treat runtime arguments as contract fallback * refactor(cli): make runtime location a contract fallback * test(reconciliation): cover contract-first runtime resolution * test(cli): make runtime fallback optional * test(reconciliation): freeze runtime source precedence * fix(reconciliation): read ODCS server aliases deterministically * test(reconciliation): exercise real ODCS server aliases * docs(reconciliation): document contract-first runtime fallback * refactor(reconciliation): type runtime servers explicitly * test(reconciliation): use typed ODCS server fixtures * refactor(reconciliation): simplify typed runtime resolution --- docs/runtime_reconciliation.md | 115 ++++++++++++ semapact/interfaces/cli.py | 34 ++++ semapact/interfaces/commands/reconcile_cmd.py | 101 +++++++++++ semapact/interfaces/outcomes.py | 20 +++ semapact/platforms/databricks/runtime.py | 3 +- semapact/platforms/runtime_registry.py | 164 ++++++++++++++++++ semapact/services/__init__.py | 11 +- semapact/services/reconciliation_service.py | 88 ++++++++++ tests/interfaces/test_cli_outcomes.py | 25 ++- tests/interfaces/test_reconcile_cmd.py | 146 ++++++++++++++++ tests/test_reconciliation_service.py | 146 ++++++++++++++++ tests/test_runtime_location_resolution.py | 101 +++++++++++ tests/test_runtime_provider_validation.py | 27 +++ 13 files changed, 977 insertions(+), 4 deletions(-) create mode 100644 docs/runtime_reconciliation.md create mode 100644 semapact/interfaces/commands/reconcile_cmd.py create mode 100644 semapact/platforms/runtime_registry.py create mode 100644 semapact/services/reconciliation_service.py create mode 100644 tests/interfaces/test_reconcile_cmd.py create mode 100644 tests/test_reconciliation_service.py create mode 100644 tests/test_runtime_location_resolution.py create mode 100644 tests/test_runtime_provider_validation.py diff --git a/docs/runtime_reconciliation.md b/docs/runtime_reconciliation.md new file mode 100644 index 00000000..cbabb736 --- /dev/null +++ b/docs/runtime_reconciliation.md @@ -0,0 +1,115 @@ +# Runtime Product Reconciliation + +SemaPact reconciles a governed ODCS data product against an observed runtime product without treating either a single table or Databricks as the core abstraction. + +```text +Governed ODCS data product + ↓ +Runtime location resolution +(contract server first, CLI fallback only when absent) + ↓ +Runtime asset specs from schema.name / physicalName + ↓ +Selected runtime provider + ↓ +Logical ↔ physical bindings + ↓ +ObservedPlatformState + ↓ +ReconciliationResult + ↓ +IN_SYNC / DRIFT / INDETERMINATE +``` + +## Runtime location precedence + +Runtime metadata is resolved deterministically and fail-closed. + +1. If the contract defines one `server`, SemaPact uses it automatically. +2. If the contract defines multiple servers, select one with `--server`. +3. If the contract defines no servers, both `--platform` and `--runtime` are required as CLI fallback values. +4. CLI fallback values never override a server already defined by the contract. + +The contract remains authoritative for asset identity and physical naming: + +```text +schema.name -> governed logical asset identity +schema.physicalName -> physical runtime asset name +``` + +CLI fallback supplies only the missing runtime location. It never supplies or rewrites logical-to-physical table mappings. + +## CLI + +When the contract contains one complete runtime server: + +```bash +semapact reconcile --contract ./contracts/sales.yaml +``` + +When it contains multiple servers: + +```bash +semapact reconcile \ + --contract ./contracts/sales.yaml \ + --server production +``` + +When it contains no server metadata, provide a complete runtime-location fallback: + +```bash +semapact reconcile \ + --contract ./contracts/sales.yaml \ + --platform databricks \ + --runtime main.sales +``` + +For the Databricks provider, the runtime target currently uses `catalog.schema`. Each governed schema is bound to its physical Unity Catalog asset using contract `physicalName` when present, otherwise the governed schema name. + +`--runtime` is provider-local. Core SemaPact does not define it as a table FQN. A future provider can interpret the fallback differently, for example: + +```bash +semapact reconcile \ + --contract sales.yaml \ + --platform snowflake \ + --runtime ANALYTICS.SALES +``` + +## Authentication + +The generic `reconcile` command does not expose Databricks-specific credential flags. The selected contract server may provide non-secret connection metadata such as its host. Credentials remain outside the contract and are resolved by the provider authentication mechanism. + +The Databricks provider uses the Databricks SDK unified authentication chain. Install provider support with: + +```bash +pip install "semapact[databricks]" +``` + +This keeps the base SemaPact installation and reconciliation core independent from the Databricks SDK. + +## Machine-readable output + +```bash +semapact reconcile \ + --contract ./contracts/sales.yaml \ + --server production \ + --output json +``` + +JSON output includes the resolved provider and runtime target, whether runtime metadata came from the contract or CLI fallback, the selected server identifier when applicable, logical-to-physical bindings, runtime status, deterministic reconciliation differences, stable runtime reason codes, expected/observed values, unverified paths, and the observation fingerprint. + +## CI exit codes + +Runtime assurance uses additive process outcomes and does not reuse governance blocking semantics: + +| Status | Exit code | +| --- | ---: | +| `IN_SYNC` | `0` | +| `DRIFT` | `6` | +| `INDETERMINATE` | `7` | + +Existing M0 exit codes remain unchanged: validation `2`, governance blocked `3`, review required `4`, and runtime/infrastructure error `5`. + +Missing or ambiguous runtime location is a validation failure. Examples include multiple contract servers without `--server`, or a contract with no servers and an incomplete CLI fallback. + +The reconciliation workflow is read-only. It does not mutate contracts, deploy runtime assets, infer deployment causality, or perform remediation. diff --git a/semapact/interfaces/cli.py b/semapact/interfaces/cli.py index ce8eaf1d..f1f293b5 100644 --- a/semapact/interfaces/cli.py +++ b/semapact/interfaces/cli.py @@ -316,6 +316,32 @@ def _build_parser() -> argparse.ArgumentParser: release_prs_parser.add_argument("--pat-token") release_prs_parser.add_argument("--push", action="store_true") + reconcile_parser = subparsers.add_parser( + "reconcile", + help="Compare a governed data product with its runtime implementation", + ) + reconcile_parser.add_argument( + "--contract", required=True, help="Path or URL to the governed ODCS contract" + ) + reconcile_parser.add_argument( + "--server", + help="Contract server identifier when the contract defines multiple servers", + ) + reconcile_parser.add_argument( + "--platform", + help="Fallback runtime provider when the contract defines no servers", + ) + reconcile_parser.add_argument( + "--runtime", + help="Fallback provider-local runtime product target when the contract defines no servers", + ) + reconcile_parser.add_argument( + "--output", + choices=["text", "json"], + default="text", + help="Output format (default: text)", + ) + return parser @@ -406,6 +432,14 @@ def main() -> int: print(json.dumps(payload, indent=2, sort_keys=True)) return 0 + if args.command == "reconcile": + from semapact.interfaces.commands.reconcile_cmd import run_reconcile + from semapact.interfaces.outcomes import exit_code_from_outcome + + result = run_reconcile(args) + print(result.output) + return int(exit_code_from_outcome(result.outcome)) + if args.command == "release": from semapact.interfaces.commands.release_cmd import ( run_release_classify, run_release_classify_repo, run_release_build_manifest, diff --git a/semapact/interfaces/commands/reconcile_cmd.py b/semapact/interfaces/commands/reconcile_cmd.py new file mode 100644 index 00000000..83e0e091 --- /dev/null +++ b/semapact/interfaces/commands/reconcile_cmd.py @@ -0,0 +1,101 @@ +"""CLI adapter for read-only runtime product reconciliation.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +import json +from typing import Literal + +from semapact.interfaces.outcomes import ProcessOutcome, outcome_from_reconciliation_status +from semapact.services.reconciliation_service import ReconciliationService, RuntimeReconciliation + + +@dataclass(frozen=True) +class ReconcileCommandResult: + """Rendered command output plus its semantic process outcome.""" + + output: str + outcome: ProcessOutcome + + +def run_reconcile(args: argparse.Namespace) -> ReconcileCommandResult: + """Execute the provider-neutral reconciliation workflow.""" + analysis = ReconciliationService().reconcile( + contract_path=args.contract, + server_name=args.server, + fallback_platform=args.platform, + fallback_runtime_target=args.runtime, + ) + output_format: Literal["text", "json"] = args.output + rendered = ( + _format_json(analysis) + if output_format == "json" + else _format_text(analysis) + ) + return ReconcileCommandResult( + output=rendered, + outcome=outcome_from_reconciliation_status(analysis.status), + ) + + +def _format_json(analysis: RuntimeReconciliation) -> str: + payload = { + "platform": analysis.platform, + "runtimeTarget": analysis.runtime_target, + "runtimeSource": analysis.runtime_source, + "server": analysis.server_name, + "status": analysis.status.value, + "bindings": [ + { + "governedAsset": binding.governed_asset, + "observedAsset": binding.observed_asset.model_dump(mode="json"), + } + for binding in analysis.bindings + ], + "reconciliation": analysis.result.model_dump(mode="json"), + } + return json.dumps(payload, indent=2, ensure_ascii=False, sort_keys=True) + + +def _format_text(analysis: RuntimeReconciliation) -> str: + result = analysis.result + runtime_source = analysis.runtime_source + if analysis.server_name: + runtime_source += f" ({analysis.server_name})" + lines = [ + f"Status: {analysis.status.value}", + f"Contract: {result.contract_id}@{result.contract_version}", + f"Platform: {analysis.platform}", + f"Runtime: {analysis.runtime_target}", + f"Runtime source: {runtime_source}", + f"Observation fingerprint: {result.observation_fingerprint}", + "Bindings:", + ] + for binding in analysis.bindings: + identity = binding.observed_asset + physical_name = ".".join((*identity.namespace, identity.asset)) + lines.append( + f" - {binding.governed_asset} -> {identity.platform}:{physical_name}" + ) + + if result.differences: + lines.append("Differences:") + for difference in result.differences: + detail = f" - {difference.reason_code.value} {difference.path}" + if difference.expected is not None or difference.observed is not None: + detail += ( + f" expected={difference.expected!r}" + f" observed={difference.observed!r}" + ) + lines.append(detail) + else: + lines.append("Differences: none") + + if result.unverified_paths: + lines.append("Unverified paths:") + lines.extend(f" - {path}" for path in result.unverified_paths) + else: + lines.append("Unverified paths: none") + + return "\n".join(lines) diff --git a/semapact/interfaces/outcomes.py b/semapact/interfaces/outcomes.py index 0959fcaf..8e001aae 100644 --- a/semapact/interfaces/outcomes.py +++ b/semapact/interfaces/outcomes.py @@ -12,6 +12,7 @@ if TYPE_CHECKING: from semapact.governance.gate import GovernanceGateResult + from semapact.reconciliation.status import RuntimeDriftStatus logger = logging.getLogger("semapact") @@ -25,6 +26,8 @@ class ProcessOutcome(str, Enum): GOVERNANCE_BLOCKED = "GOVERNANCE_BLOCKED" REVIEW_REQUIRED = "REVIEW_REQUIRED" RUNTIME_ERROR = "RUNTIME_ERROR" + RUNTIME_DRIFT = "RUNTIME_DRIFT" + RUNTIME_INDETERMINATE = "RUNTIME_INDETERMINATE" class CliExitCode(IntEnum): @@ -35,6 +38,8 @@ class CliExitCode(IntEnum): GOVERNANCE_BLOCKED = 3 REVIEW_REQUIRED = 4 RUNTIME_ERROR = 5 + RUNTIME_DRIFT = 6 + RUNTIME_INDETERMINATE = 7 _OUTCOME_TO_EXIT_CODE: dict[ProcessOutcome, CliExitCode] = { @@ -43,6 +48,8 @@ class CliExitCode(IntEnum): ProcessOutcome.GOVERNANCE_BLOCKED: CliExitCode.GOVERNANCE_BLOCKED, ProcessOutcome.REVIEW_REQUIRED: CliExitCode.REVIEW_REQUIRED, ProcessOutcome.RUNTIME_ERROR: CliExitCode.RUNTIME_ERROR, + ProcessOutcome.RUNTIME_DRIFT: CliExitCode.RUNTIME_DRIFT, + ProcessOutcome.RUNTIME_INDETERMINATE: CliExitCode.RUNTIME_INDETERMINATE, } @@ -65,6 +72,19 @@ def outcome_from_gate_result(gate_result: GovernanceGateResult) -> ProcessOutcom raise ValueError(f"Unsupported GovernanceGateResult reason: {gate_result.reason!r}") +def outcome_from_reconciliation_status(status: RuntimeDriftStatus) -> ProcessOutcome: + """Map M1 runtime assurance status without conflating it with governance.""" + from semapact.reconciliation.status import RuntimeDriftStatus + + if status is RuntimeDriftStatus.IN_SYNC: + return ProcessOutcome.SUCCESS + if status is RuntimeDriftStatus.DRIFT: + return ProcessOutcome.RUNTIME_DRIFT + if status is RuntimeDriftStatus.INDETERMINATE: + return ProcessOutcome.RUNTIME_INDETERMINATE + raise ValueError(f"Unsupported RuntimeDriftStatus: {status!r}") + + def outcome_from_exception(exc: BaseException) -> ProcessOutcome: """Determine the semantic ProcessOutcome for a given exception.""" from semapact.exceptions import ( diff --git a/semapact/platforms/databricks/runtime.py b/semapact/platforms/databricks/runtime.py index 2ebb5888..8888a8d6 100644 --- a/semapact/platforms/databricks/runtime.py +++ b/semapact/platforms/databricks/runtime.py @@ -5,6 +5,7 @@ from datetime import datetime, timezone from typing import Any, Sequence +from semapact.exceptions import ValidationError from semapact.observation.databricks import observe_databricks_table from semapact.observation.fingerprint import with_observed_state_fingerprint from semapact.observation.models import ObservedAssetIdentity, ObservedPlatformState @@ -86,7 +87,7 @@ def observe( def _parse_runtime_target(value: str) -> tuple[str, str]: parts = tuple(part.strip() for part in value.split(".")) if len(parts) != 2 or not all(parts): - raise ValueError( + raise ValidationError( "Databricks runtime target must use catalog.schema format for a data product" ) return parts[0], parts[1] diff --git a/semapact/platforms/runtime_registry.py b/semapact/platforms/runtime_registry.py new file mode 100644 index 00000000..87b86dd4 --- /dev/null +++ b/semapact/platforms/runtime_registry.py @@ -0,0 +1,164 @@ +"""Runtime-location resolution and provider composition boundary.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +from open_data_contract_standard.model import OpenDataContractStandard, Server + +from semapact.exceptions import ValidationError +from semapact.observation import RuntimeProvider, RuntimeProviderRegistry + + +@dataclass(frozen=True) +class ResolvedRuntimeLocation: + """Runtime location resolved from contract metadata or explicit CLI fallback.""" + + platform: str + runtime_target: str + source: Literal["contract", "cli"] + server_name: str | None = None + contract_server: Server | None = None + + +def resolve_runtime_location( + contract: OpenDataContractStandard, + *, + server_name: str | None = None, + fallback_platform: str | None = None, + fallback_runtime_target: str | None = None, +) -> ResolvedRuntimeLocation: + """Resolve runtime location with contract-first, fail-closed precedence.""" + servers = tuple(contract.servers or ()) + if servers: + selected = _select_contract_server(servers, server_name) + platform = _required(selected.type, "Selected contract server must define a runtime type") + return ResolvedRuntimeLocation( + platform=platform, + runtime_target=_runtime_target_from_server(platform, selected), + source="contract", + server_name=_required( + selected.server, + "Selected contract server must define a server identifier", + ), + contract_server=selected, + ) + + if _clean(server_name): + raise ValidationError( + "--server cannot be used because the contract defines no servers" + ) + + platform = _clean(fallback_platform) + runtime_target = _clean(fallback_runtime_target) + if not platform or not runtime_target: + raise ValidationError( + "Contract defines no servers; provide both --platform and --runtime" + ) + + return ResolvedRuntimeLocation( + platform=platform, + runtime_target=runtime_target, + source="cli", + ) + + +def create_runtime_provider_registry( + platform: str, + *, + contract_server: Server | None = None, +) -> RuntimeProviderRegistry: + """Create only the selected provider, keeping optional dependencies lazy.""" + normalized = platform.strip().casefold() + if normalized == "databricks": + return RuntimeProviderRegistry( + (_create_databricks_provider(contract_server=contract_server),) + ) + raise ValidationError( + f"Unsupported runtime provider '{platform}'. Supported providers: databricks" + ) + + +def _select_contract_server( + servers: tuple[Server, ...], + requested_name: str | None, +) -> Server: + requested = _clean(requested_name) + if requested: + matches = [ + server + for server in servers + if _clean(server.server, casefold=True) == requested.casefold() + ] + if len(matches) == 1: + return matches[0] + if len(matches) > 1: + raise ValidationError( + f"Contract contains duplicate server identifier '{requested}'" + ) + raise ValidationError( + f"Contract server '{requested}' was not found. " + f"Available servers: {_available_server_names(servers)}" + ) + + if len(servers) == 1: + return servers[0] + + raise ValidationError( + "Multiple contract servers are defined; select one with --server. " + f"Available servers: {_available_server_names(servers)}" + ) + + +def _runtime_target_from_server(platform: str, server: Server) -> str: + """Project one ODCS server into the selected provider's runtime target.""" + if platform.strip().casefold() == "databricks": + catalog = _required(server.catalog, "Databricks contract server must define catalog") + schema = _required(server.schema_, "Databricks contract server must define schema") + return f"{catalog}.{schema}" + raise ValidationError( + f"Unsupported runtime provider '{platform}'. Supported providers: databricks" + ) + + +def _create_databricks_provider( + *, + contract_server: Server | None = None, +) -> RuntimeProvider: + from semapact.platforms.databricks import ( + DatabricksRuntimeProvider, + create_databricks_workspace_client, + ) + + client = create_databricks_workspace_client( + workspace_url=_clean(contract_server.host) if contract_server else None + ) + source_identifier = getattr(getattr(client, "config", None), "host", None) + if not isinstance(source_identifier, str) or not source_identifier.strip(): + raise RuntimeError("Databricks SDK did not resolve a workspace host") + return DatabricksRuntimeProvider( + client=client, + source_identifier=source_identifier, + ) + + +def _available_server_names(servers: tuple[Server, ...]) -> str: + names = sorted(name for server in servers if (name := _clean(server.server))) + return ", ".join(names) or "none" + + +def _required(value: object, message: str) -> str: + cleaned = _clean(value) + if not cleaned: + raise ValidationError(message) + return cleaned + + +def _clean(value: object, *, casefold: bool = False) -> str | None: + if value is None: + return None + cleaned = str(value).strip() + if not cleaned: + return None + return cleaned.casefold() if casefold else cleaned diff --git a/semapact/services/__init__.py b/semapact/services/__init__.py index f9f34caa..232d9b93 100644 --- a/semapact/services/__init__.py +++ b/semapact/services/__init__.py @@ -1,5 +1,14 @@ """UI-independent application services for SemaPact workflows.""" from semapact.services.governance_service import GovernanceAnalysis, GovernanceService +from semapact.services.reconciliation_service import ( + ReconciliationService, + RuntimeReconciliation, +) -__all__ = ["GovernanceAnalysis", "GovernanceService"] +__all__ = [ + "GovernanceAnalysis", + "GovernanceService", + "ReconciliationService", + "RuntimeReconciliation", +] diff --git a/semapact/services/reconciliation_service.py b/semapact/services/reconciliation_service.py new file mode 100644 index 00000000..70b81619 --- /dev/null +++ b/semapact/services/reconciliation_service.py @@ -0,0 +1,88 @@ +"""Application service for provider-neutral runtime product reconciliation.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +from semapact.core.loader import ContractLoader +from semapact.observation import RuntimeAssetBinding, RuntimeProviderRegistry +from semapact.platforms.runtime_registry import ( + create_runtime_provider_registry, + resolve_runtime_location, +) +from semapact.reconciliation import ( + ReconciliationResult, + RuntimeDriftStatus, + classify_reconciliation_status, + reconcile_governed_contract, + runtime_asset_specs_from_contract, +) + + +@dataclass(frozen=True) +class RuntimeReconciliation: + """One complete read-only reconciliation of a governed data product.""" + + platform: str + runtime_target: str + runtime_source: Literal["contract", "cli"] + server_name: str | None + bindings: tuple[RuntimeAssetBinding, ...] + result: ReconciliationResult + status: RuntimeDriftStatus + + +class ReconciliationService: + """Orchestrate load, runtime resolution, bind, observe, reconcile, and classify.""" + + def __init__( + self, + provider_registry: RuntimeProviderRegistry | None = None, + *, + contract_loader: ContractLoader | None = None, + ) -> None: + self._provider_registry = provider_registry + self._contract_loader = contract_loader or ContractLoader() + + def reconcile( + self, + *, + contract_path: str, + server_name: str | None = None, + fallback_platform: str | None = None, + fallback_runtime_target: str | None = None, + ) -> RuntimeReconciliation: + """Reconcile one governed data product against its resolved runtime location.""" + contract = self._contract_loader.load(contract_path) + location = resolve_runtime_location( + contract, + server_name=server_name, + fallback_platform=fallback_platform, + fallback_runtime_target=fallback_runtime_target, + ) + registry = self._provider_registry or create_runtime_provider_registry( + location.platform, + contract_server=location.contract_server, + ) + provider = registry.get(location.platform) + asset_specs = runtime_asset_specs_from_contract(contract) + bindings = provider.resolve_bindings( + runtime_target=location.runtime_target, + assets=asset_specs, + ) + observation = provider.observe(bindings=bindings) + result = reconcile_governed_contract( + contract, + observation, + asset_bindings=bindings, + ) + return RuntimeReconciliation( + platform=provider.key, + runtime_target=location.runtime_target, + runtime_source=location.source, + server_name=location.server_name, + bindings=bindings, + result=result, + status=classify_reconciliation_status(result), + ) diff --git a/tests/interfaces/test_cli_outcomes.py b/tests/interfaces/test_cli_outcomes.py index 2c0eb5a8..bad45dcf 100644 --- a/tests/interfaces/test_cli_outcomes.py +++ b/tests/interfaces/test_cli_outcomes.py @@ -30,7 +30,9 @@ exit_code_from_outcome, outcome_from_exception, outcome_from_gate_result, + outcome_from_reconciliation_status, ) +from semapact.reconciliation import RuntimeDriftStatus def _make_dummy_decision( @@ -59,17 +61,21 @@ def test_process_outcome_and_exit_code_mappings(): ProcessOutcome.GOVERNANCE_BLOCKED: CliExitCode.GOVERNANCE_BLOCKED, ProcessOutcome.REVIEW_REQUIRED: CliExitCode.REVIEW_REQUIRED, ProcessOutcome.RUNTIME_ERROR: CliExitCode.RUNTIME_ERROR, + ProcessOutcome.RUNTIME_DRIFT: CliExitCode.RUNTIME_DRIFT, + ProcessOutcome.RUNTIME_INDETERMINATE: CliExitCode.RUNTIME_INDETERMINATE, } for outcome, expected_code in expected_mappings.items(): assert exit_code_from_outcome(outcome) == expected_code - assert int(expected_code) in (0, 2, 3, 4, 5) + assert int(expected_code) in (0, 2, 3, 4, 5, 6, 7) assert int(CliExitCode.SUCCESS) == 0 assert int(CliExitCode.VALIDATION_FAILED) == 2 assert int(CliExitCode.GOVERNANCE_BLOCKED) == 3 assert int(CliExitCode.REVIEW_REQUIRED) == 4 assert int(CliExitCode.RUNTIME_ERROR) == 5 + assert int(CliExitCode.RUNTIME_DRIFT) == 6 + assert int(CliExitCode.RUNTIME_INDETERMINATE) == 7 def test_outcome_from_gate_result(): @@ -90,6 +96,22 @@ def test_outcome_from_gate_result(): assert outcome_from_gate_result(review_res) == ProcessOutcome.REVIEW_REQUIRED +def test_outcome_from_reconciliation_status(): + """Runtime assurance outcomes remain distinct from governance outcomes.""" + assert ( + outcome_from_reconciliation_status(RuntimeDriftStatus.IN_SYNC) + == ProcessOutcome.SUCCESS + ) + assert ( + outcome_from_reconciliation_status(RuntimeDriftStatus.DRIFT) + == ProcessOutcome.RUNTIME_DRIFT + ) + assert ( + outcome_from_reconciliation_status(RuntimeDriftStatus.INDETERMINATE) + == ProcessOutcome.RUNTIME_INDETERMINATE + ) + + def test_outcome_from_exception_and_exit_code(): """Verify domain, validation, and runtime exceptions map to standardized outcomes and exit codes.""" blocked_exc = GovernanceBlockedError("Change is blocked") @@ -110,7 +132,6 @@ def test_outcome_from_exception_and_exit_code(): assert outcome_from_exception(release_val_exc) == ProcessOutcome.VALIDATION_FAILED assert exit_code_from_exception(release_val_exc) == 2 - # Runtime and infrastructure exceptions runtime_exc = RuntimeError("Database unreachable") assert outcome_from_exception(runtime_exc) == ProcessOutcome.RUNTIME_ERROR diff --git a/tests/interfaces/test_reconcile_cmd.py b/tests/interfaces/test_reconcile_cmd.py new file mode 100644 index 00000000..0ef576d0 --- /dev/null +++ b/tests/interfaces/test_reconcile_cmd.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import json +import sys + +import pytest + +from semapact.interfaces import cli +from semapact.interfaces.commands import reconcile_cmd +from semapact.interfaces.commands.reconcile_cmd import ReconcileCommandResult +from semapact.interfaces.outcomes import ProcessOutcome +from semapact.observation import ObservedAssetIdentity, RuntimeAssetBinding +from semapact.reconciliation import ( + ReconciliationDifference, + ReconciliationDifferenceType, + ReconciliationResult, + ReconciliationSubject, + RuntimeDriftStatus, + RuntimeReasonCode, +) +from semapact.services.reconciliation_service import RuntimeReconciliation + + +def _analysis() -> RuntimeReconciliation: + binding = RuntimeAssetBinding( + governed_asset="orders", + observed_asset=ObservedAssetIdentity( + platform="warehouse", + namespace=("analytics",), + asset="fact_orders", + ), + ) + result = ReconciliationResult( + contract_id="sales-product", + contract_version="1.0.0", + observation_source_identifier="warehouse://test", + observation_fingerprint="sha256:test", + differences=( + ReconciliationDifference( + difference_type=ReconciliationDifferenceType.MISMATCH, + subject=ReconciliationSubject.PHYSICAL_TYPE, + reason_code=RuntimeReasonCode.RUNTIME_PHYSICAL_TYPE_CHANGED, + path="schema[orders].properties[id].physicalType", + asset_identity="orders", + property_identity="id", + expected="BIGINT", + observed="STRING", + ), + ), + unverified_paths=(), + ) + return RuntimeReconciliation( + platform="warehouse", + runtime_target="analytics", + runtime_source="cli", + server_name=None, + bindings=(binding,), + result=result, + status=RuntimeDriftStatus.DRIFT, + ) + + +def test_reconcile_parser_allows_contract_owned_runtime_metadata() -> None: + args = cli._build_parser().parse_args( + [ + "reconcile", + "--contract", + "contracts/sales.yaml", + "--output", + "json", + ] + ) + + assert args.command == "reconcile" + assert args.contract == "contracts/sales.yaml" + assert args.server is None + assert args.platform is None + assert args.runtime is None + assert args.output == "json" + assert not hasattr(args, "workspace_url") + assert not hasattr(args, "token") + + +def test_reconcile_parser_accepts_explicit_runtime_fallback() -> None: + args = cli._build_parser().parse_args( + [ + "reconcile", + "--contract", + "contracts/sales.yaml", + "--platform", + "warehouse", + "--runtime", + "analytics", + ] + ) + + assert args.platform == "warehouse" + assert args.runtime == "analytics" + + +def test_json_output_is_deterministic_and_product_scoped() -> None: + payload = json.loads(reconcile_cmd._format_json(_analysis())) + + assert payload["platform"] == "warehouse" + assert payload["runtimeTarget"] == "analytics" + assert payload["runtimeSource"] == "cli" + assert payload["server"] is None + assert payload["status"] == "DRIFT" + assert payload["bindings"][0]["governedAsset"] == "orders" + difference = payload["reconciliation"]["differences"][0] + assert difference["reason_code"] == "RUNTIME_PHYSICAL_TYPE_CHANGED" + assert difference["path"] == "schema[orders].properties[id].physicalType" + + +@pytest.mark.parametrize( + "outcome,expected_exit", + [ + (ProcessOutcome.SUCCESS, 0), + (ProcessOutcome.RUNTIME_DRIFT, 6), + (ProcessOutcome.RUNTIME_INDETERMINATE, 7), + ], +) +def test_main_preserves_runtime_assurance_exit_semantics( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + outcome: ProcessOutcome, + expected_exit: int, +) -> None: + monkeypatch.setattr( + reconcile_cmd, + "run_reconcile", + lambda args: ReconcileCommandResult(output="result", outcome=outcome), + ) + monkeypatch.setattr( + sys, + "argv", + [ + "semapact", + "reconcile", + "--contract", + "contracts/sales.yaml", + ], + ) + + assert cli.main() == expected_exit + assert capsys.readouterr().out.strip() == "result" diff --git a/tests/test_reconciliation_service.py b/tests/test_reconciliation_service.py new file mode 100644 index 00000000..6d5f937f --- /dev/null +++ b/tests/test_reconciliation_service.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Sequence, cast + +from open_data_contract_standard.model import OpenDataContractStandard, SchemaObject, Server + +from semapact.core.loader import ContractLoader +from semapact.observation import ( + ObservedAsset, + ObservedAssetIdentity, + ObservedPlatformState, + RuntimeAssetBinding, + RuntimeAssetSpec, + RuntimeProviderRegistry, + with_observed_state_fingerprint, +) +from semapact.reconciliation import RuntimeDriftStatus +from semapact.services.reconciliation_service import ReconciliationService + + +class _Loader: + def __init__(self, contract: OpenDataContractStandard) -> None: + self.contract = contract + self.loaded_path: str | None = None + + def load(self, contract_path: str) -> OpenDataContractStandard: + self.loaded_path = contract_path + return self.contract + + +class _Provider: + def __init__(self, key: str) -> None: + self.key = key + self.runtime_target: str | None = None + self.specs: tuple[RuntimeAssetSpec, ...] = () + self.observed_bindings: tuple[RuntimeAssetBinding, ...] = () + + def resolve_bindings( + self, + *, + runtime_target: str, + assets: Sequence[RuntimeAssetSpec], + ) -> tuple[RuntimeAssetBinding, ...]: + self.runtime_target = runtime_target + self.specs = tuple(assets) + return ( + RuntimeAssetBinding( + governed_asset="orders", + observed_asset=ObservedAssetIdentity( + platform=self.key, + namespace=("analytics",), + asset="fact_orders_v2", + ), + ), + ) + + def observe( + self, + *, + bindings: Sequence[RuntimeAssetBinding], + ) -> ObservedPlatformState: + self.observed_bindings = tuple(bindings) + identity = bindings[0].observed_asset + state = ObservedPlatformState( + platform=self.key, + source_identifier=f"{self.key}://test", + assets=(ObservedAsset(identity=identity, properties=()),), + captured_at=datetime(2026, 9, 8, 21, 0, tzinfo=timezone.utc), + fingerprint=None, + ) + return with_observed_state_fingerprint(state) + + +def _contract(*, servers: list[Server] | None = None) -> OpenDataContractStandard: + return OpenDataContractStandard.model_construct( + id="sales-product", + version="1.0.0", + servers=servers, + schema_=[ + SchemaObject.model_construct( + name="orders", + physicalName="fact_orders_v2", + properties=[], + ) + ], + ) + + +def test_reconciliation_service_uses_cli_runtime_only_when_contract_has_no_server() -> None: + loader = _Loader(_contract()) + provider = _Provider("warehouse") + service = ReconciliationService( + RuntimeProviderRegistry((provider,)), + contract_loader=cast(ContractLoader, loader), + ) + + analysis = service.reconcile( + contract_path="contracts/sales.yaml", + fallback_platform="WAREHOUSE", + fallback_runtime_target="analytics", + ) + + assert loader.loaded_path == "contracts/sales.yaml" + assert provider.runtime_target == "analytics" + assert provider.specs == ( + RuntimeAssetSpec(governed_asset="orders", physical_name="fact_orders_v2"), + ) + assert provider.observed_bindings == analysis.bindings + assert analysis.platform == "warehouse" + assert analysis.runtime_target == "analytics" + assert analysis.runtime_source == "cli" + assert analysis.server_name is None + assert analysis.status is RuntimeDriftStatus.IN_SYNC + assert analysis.result.differences == () + + +def test_reconciliation_service_prefers_contract_server_over_cli_fallback() -> None: + server = Server.model_validate( + { + "server": "production", + "type": "databricks", + "host": "https://workspace.example", + "catalog": "main", + "schema": "sales", + } + ) + loader = _Loader(_contract(servers=[server])) + provider = _Provider("databricks") + service = ReconciliationService( + RuntimeProviderRegistry((provider,)), + contract_loader=cast(ContractLoader, loader), + ) + + analysis = service.reconcile( + contract_path="contracts/sales.yaml", + fallback_platform="warehouse", + fallback_runtime_target="ignored", + ) + + assert provider.runtime_target == "main.sales" + assert analysis.platform == "databricks" + assert analysis.runtime_target == "main.sales" + assert analysis.runtime_source == "contract" + assert analysis.server_name == "production" + assert analysis.status is RuntimeDriftStatus.IN_SYNC diff --git a/tests/test_runtime_location_resolution.py b/tests/test_runtime_location_resolution.py new file mode 100644 index 00000000..79e69d3e --- /dev/null +++ b/tests/test_runtime_location_resolution.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import pytest +from open_data_contract_standard.model import OpenDataContractStandard, Server + +from semapact.exceptions import ValidationError +from semapact.platforms.runtime_registry import resolve_runtime_location + + +def _contract(*servers: Server) -> OpenDataContractStandard: + return OpenDataContractStandard.model_construct( + id="sales-product", + version="1.0.0", + servers=list(servers) or None, + schema_=[], + ) + + +def _databricks_server(name: str, *, catalog: str, schema: str) -> Server: + return Server.model_validate( + { + "server": name, + "type": "databricks", + "host": "https://workspace.example", + "catalog": catalog, + "schema": schema, + } + ) + + +def test_single_contract_server_is_authoritative_over_cli_fallback() -> None: + server = _databricks_server("production", catalog="main", schema="sales") + assert server.schema_ == "sales" + + location = resolve_runtime_location( + _contract(server), + fallback_platform="snowflake", + fallback_runtime_target="ignored.target", + ) + + assert location.platform == "databricks" + assert location.runtime_target == "main.sales" + assert location.source == "contract" + assert location.server_name == "production" + assert location.contract_server is server + + +def test_multiple_contract_servers_require_explicit_server_selection() -> None: + contract = _contract( + _databricks_server("staging", catalog="staging", schema="sales"), + _databricks_server("production", catalog="main", schema="sales"), + ) + + with pytest.raises(ValidationError, match="select one with --server"): + resolve_runtime_location(contract) + + selected = resolve_runtime_location(contract, server_name="PRODUCTION") + assert selected.runtime_target == "main.sales" + assert selected.server_name == "production" + + +def test_contract_without_servers_uses_complete_cli_fallback() -> None: + location = resolve_runtime_location( + _contract(), + fallback_platform="warehouse", + fallback_runtime_target="analytics.sales", + ) + + assert location.platform == "warehouse" + assert location.runtime_target == "analytics.sales" + assert location.source == "cli" + assert location.server_name is None + assert location.contract_server is None + + +@pytest.mark.parametrize( + "platform,runtime", + [ + (None, None), + ("databricks", None), + (None, "main.sales"), + ], +) +def test_contract_without_servers_requires_both_fallback_values( + platform: str | None, + runtime: str | None, +) -> None: + with pytest.raises( + ValidationError, + match="provide both --platform and --runtime", + ): + resolve_runtime_location( + _contract(), + fallback_platform=platform, + fallback_runtime_target=runtime, + ) + + +def test_server_selector_is_invalid_when_contract_has_no_servers() -> None: + with pytest.raises(ValidationError, match="contract defines no servers"): + resolve_runtime_location(_contract(), server_name="production") diff --git a/tests/test_runtime_provider_validation.py b/tests/test_runtime_provider_validation.py new file mode 100644 index 00000000..db99ec4e --- /dev/null +++ b/tests/test_runtime_provider_validation.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from semapact.exceptions import ValidationError +from semapact.platforms.databricks.runtime import DatabricksRuntimeProvider +from semapact.platforms.runtime_registry import create_runtime_provider_registry + + +def test_unsupported_runtime_provider_is_user_validation() -> None: + with pytest.raises(ValidationError, match="Unsupported runtime provider 'snowflake'"): + create_runtime_provider_registry("snowflake") + + +def test_databricks_runtime_target_syntax_is_user_validation() -> None: + provider = DatabricksRuntimeProvider( + client=SimpleNamespace(), + source_identifier="https://adb.example", + ) + + with pytest.raises( + ValidationError, + match="Databricks runtime target must use catalog.schema format", + ): + provider.resolve_bindings(runtime_target="main.sales.orders", assets=()) From 6a98c1e34377e64c7e12dac7e87cacec2bca18b7 Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Wed, 9 Sep 2026 13:22:35 +1000 Subject: [PATCH 03/35] feat(contractops): represent governed proposals as ChangeSets (#207) * feat(contractops): add immutable ChangeSet model * feat(contractops): build deterministic ChangeSets * feat(contractops): expose ChangeSet API * feat(services): expose governed ChangeSet proposals * feat(services): export GovernanceProposal * test(contractops): freeze ChangeSet determinism * test(services): freeze governed proposal boundary * refactor(contractops): make ChangeSet identity cover full record * test(contractops): align ChangeSet record identity * refactor(test): type proposal evaluator double --- semapact/contractops/__init__.py | 13 ++ semapact/contractops/changeset.py | 112 +++++++++++++++ semapact/contractops/models.py | 52 +++++++ semapact/services/__init__.py | 7 +- semapact/services/governance_service.py | 41 ++++++ tests/test_change_set.py | 173 ++++++++++++++++++++++++ tests/test_governance_proposal.py | 88 ++++++++++++ 7 files changed, 485 insertions(+), 1 deletion(-) create mode 100644 semapact/contractops/__init__.py create mode 100644 semapact/contractops/changeset.py create mode 100644 semapact/contractops/models.py create mode 100644 tests/test_change_set.py create mode 100644 tests/test_governance_proposal.py diff --git a/semapact/contractops/__init__.py b/semapact/contractops/__init__.py new file mode 100644 index 00000000..fda4e10e --- /dev/null +++ b/semapact/contractops/__init__.py @@ -0,0 +1,13 @@ +"""M2 ContractOps proposal domain.""" + +from semapact.contractops.changeset import ( + build_change_set, + build_change_set_from_decision, +) +from semapact.contractops.models import ChangeSet + +__all__ = [ + "ChangeSet", + "build_change_set", + "build_change_set_from_decision", +] diff --git a/semapact/contractops/changeset.py b/semapact/contractops/changeset.py new file mode 100644 index 00000000..100e30e6 --- /dev/null +++ b/semapact/contractops/changeset.py @@ -0,0 +1,112 @@ +"""Deterministic ChangeSet construction from authoritative governance changes.""" + +from __future__ import annotations + +import json +import uuid +from collections.abc import Sequence + +from semapact.change_context import ChangeContext +from semapact.contractops.models import ChangeSet +from semapact.governance.models import GovernanceDecision +from semapact.lifecycle.changes import GovernanceChange, governance_change_sort_key + + +SEMAPACT_CHANGESET_NAMESPACE = uuid.UUID("3ea0f6d8-28ca-4bb4-94f5-ea1f0f48cb84") + + +def build_change_set( + *, + contract_id: str, + base_revision_ref: str, + candidate_revision_ref: str, + changes: Sequence[GovernanceChange], + context: ChangeContext, + source: str | None = None, + actor_reference: str | None = None, +) -> ChangeSet: + """Build one immutable proposal without re-diffing governed contracts. + + ``changes`` must already be authoritative M0 governance changes. Construction is + pure: it sorts those changes canonically, derives a UUID5 from the complete stable + proposal record, and does not inspect files, Git, clocks, or deployment state. + + Source and actor references are proposal provenance, not governance inputs. They + therefore do not affect the upstream GovernanceDecision, but they do participate + in ChangeSet identity so one ID always represents one immutable serialized record. + """ + canonical_changes = tuple(sorted(tuple(changes), key=governance_change_sort_key)) + cleaned_contract_id = _required_text(contract_id, "contract_id") + cleaned_base_ref = _required_text(base_revision_ref, "base_revision_ref") + cleaned_candidate_ref = _required_text( + candidate_revision_ref, + "candidate_revision_ref", + ) + cleaned_source = _optional_text(source) + cleaned_actor_reference = _optional_text(actor_reference) + + identity_payload = { + "contract_id": cleaned_contract_id, + "base_revision_ref": cleaned_base_ref, + "candidate_revision_ref": cleaned_candidate_ref, + "context": context.model_dump(mode="json"), + "changes": [change.model_dump(mode="json") for change in canonical_changes], + "source": cleaned_source, + "actor_reference": cleaned_actor_reference, + } + canonical_payload = json.dumps( + identity_payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) + change_set_id = str(uuid.uuid5(SEMAPACT_CHANGESET_NAMESPACE, canonical_payload)) + + return ChangeSet( + change_set_id=change_set_id, + contract_id=cleaned_contract_id, + base_revision_ref=cleaned_base_ref, + candidate_revision_ref=cleaned_candidate_ref, + changes=canonical_changes, + context=context, + source=cleaned_source, + actor_reference=cleaned_actor_reference, + ) + + +def build_change_set_from_decision( + decision: GovernanceDecision, + *, + base_revision_ref: str, + candidate_revision_ref: str, + source: str | None = None, + actor_reference: str | None = None, +) -> ChangeSet: + """Project one authoritative GovernanceDecision into its M2 proposal object.""" + return build_change_set( + contract_id=decision.contract_id, + base_revision_ref=base_revision_ref, + candidate_revision_ref=candidate_revision_ref, + changes=decision.changes, + context=decision.context, + source=source, + actor_reference=actor_reference, + ) + + +def _required_text(value: str, field_name: str) -> str: + if not isinstance(value, str): + raise TypeError(f"{field_name} must be a string") + cleaned = value.strip() + if not cleaned: + raise ValueError(f"{field_name} must not be empty") + return cleaned + + +def _optional_text(value: str | None) -> str | None: + if value is None: + return None + if not isinstance(value, str): + raise TypeError("optional provenance values must be strings") + cleaned = value.strip() + return cleaned or None diff --git a/semapact/contractops/models.py b/semapact/contractops/models.py new file mode 100644 index 00000000..b3ca3230 --- /dev/null +++ b/semapact/contractops/models.py @@ -0,0 +1,52 @@ +"""Immutable ContractOps domain models.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, field_validator + +from semapact.change_context import ChangeContext +from semapact.lifecycle.changes import GovernanceChange + + +class ContractOpsModel(BaseModel): + """Shared immutable base for M2 ContractOps domain models.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + +class ChangeSet(ContractOpsModel): + """Deterministic proposal over exact governed contract revisions. + + Revision references are opaque workflow-owned identifiers. They intentionally do + not assume Git SHAs, storage versions, or ODCS semantic versions. + """ + + change_set_id: str + contract_id: str + base_revision_ref: str + candidate_revision_ref: str + changes: tuple[GovernanceChange, ...] + context: ChangeContext + source: str | None = None + actor_reference: str | None = None + + @field_validator( + "change_set_id", + "contract_id", + "base_revision_ref", + "candidate_revision_ref", + ) + @classmethod + def _require_non_empty_text(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("value must not be empty") + return cleaned + + @field_validator("source", "actor_reference") + @classmethod + def _normalize_optional_text(cls, value: str | None) -> str | None: + if value is None: + return None + cleaned = value.strip() + return cleaned or None diff --git a/semapact/services/__init__.py b/semapact/services/__init__.py index 232d9b93..2d08fe54 100644 --- a/semapact/services/__init__.py +++ b/semapact/services/__init__.py @@ -1,6 +1,10 @@ """UI-independent application services for SemaPact workflows.""" -from semapact.services.governance_service import GovernanceAnalysis, GovernanceService +from semapact.services.governance_service import ( + GovernanceAnalysis, + GovernanceProposal, + GovernanceService, +) from semapact.services.reconciliation_service import ( ReconciliationService, RuntimeReconciliation, @@ -8,6 +12,7 @@ __all__ = [ "GovernanceAnalysis", + "GovernanceProposal", "GovernanceService", "ReconciliationService", "RuntimeReconciliation", diff --git a/semapact/services/governance_service.py b/semapact/services/governance_service.py index 9ce6c5a1..653238e1 100644 --- a/semapact/services/governance_service.py +++ b/semapact/services/governance_service.py @@ -9,6 +9,7 @@ from open_data_contract_standard.model import OpenDataContractStandard from semapact.change_context import ChangeContext +from semapact.contractops import ChangeSet, build_change_set_from_decision from semapact.governance.evaluator import evaluate_governance_decision from semapact.governance.models import GovernanceDecision from semapact.lifecycle.merge_engine import ContractMergeEngine, MergeConflict, MergeResult @@ -23,6 +24,14 @@ class GovernanceAnalysis: decision: GovernanceDecision +@dataclass(frozen=True) +class GovernanceProposal: + """One evaluated proposal represented by a ChangeSet and its decision.""" + + change_set: ChangeSet + decision: GovernanceDecision + + class GovernanceService: """Own ChangeContext construction and delegate governance domain work.""" @@ -61,6 +70,38 @@ def evaluate( merge_conflicts=merge_conflicts, ) + def evaluate_proposal( + self, + base_contract: OpenDataContractStandard, + candidate_contract: OpenDataContractStandard, + *, + effective_date: date | str, + base_revision_ref: str, + candidate_revision_ref: str, + merge_conflicts: Sequence[MergeConflict] = (), + source: str | None = None, + actor_reference: str | None = None, + ) -> GovernanceProposal: + """Evaluate once and project the authoritative decision changes into a ChangeSet.""" + context = self.create_context(effective_date) + decision = evaluate_governance_decision( + base_contract, + candidate_contract, + context=context, + merge_conflicts=merge_conflicts, + ) + change_set = build_change_set_from_decision( + decision, + base_revision_ref=base_revision_ref, + candidate_revision_ref=candidate_revision_ref, + source=source, + actor_reference=actor_reference, + ) + return GovernanceProposal( + change_set=change_set, + decision=decision, + ) + def merge_and_evaluate( self, source_contract: OpenDataContractStandard, diff --git a/tests/test_change_set.py b/tests/test_change_set.py new file mode 100644 index 00000000..87c1fd5a --- /dev/null +++ b/tests/test_change_set.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +from datetime import date + +import pytest +from open_data_contract_standard.model import ( + OpenDataContractStandard, + SchemaObject, + SchemaProperty, +) +from pydantic import ValidationError as PydanticValidationError + +from semapact.change_context import ChangeContext +from semapact.contractops import build_change_set, build_change_set_from_decision +from semapact.governance import evaluate_governance_decision +from semapact.lifecycle.changes import ( + GovernanceChange, + GovernanceChangeDomain, + GovernanceChangeType, + GovernanceEntityType, +) + + +def _change(name: str, *, before: str, after: str) -> GovernanceChange: + return GovernanceChange( + change_type=GovernanceChangeType.MODIFY, + entity_type=GovernanceEntityType.PROPERTY, + identity=("orders", name), + path=f"schema[orders].properties[{name}].physicalType", + field="physicalType", + before=before, + after=after, + domain=GovernanceChangeDomain.STRUCTURE, + ) + + +def _contract(physical_type: str) -> OpenDataContractStandard: + return OpenDataContractStandard( + apiVersion="v3.1.0", + kind="DataContract", + id="orders-product", + name="orders-product", + version="1.0.0", + status="active", + schema=[ + SchemaObject( + name="orders", + properties=[ + SchemaProperty( + name="id", + physicalType=physical_type, + logicalType="string", + ) + ], + ) + ], + ) + + +def test_changeset_identity_is_deterministic_and_change_order_independent() -> None: + context = ChangeContext(effective_date=date(2026, 9, 9)) + id_change = _change("id", before="STRING", after="BIGINT") + amount_change = _change("amount", before="DECIMAL(10,2)", after="DECIMAL(18,2)") + + first = build_change_set( + contract_id="orders-product", + base_revision_ref="git:abc123", + candidate_revision_ref="git:def456", + changes=(id_change, amount_change), + context=context, + ) + second = build_change_set( + contract_id="orders-product", + base_revision_ref="git:abc123", + candidate_revision_ref="git:def456", + changes=(amount_change, id_change), + context=context, + ) + + assert first == second + assert first.change_set_id == second.change_set_id + assert first.changes == (amount_change, id_change) + assert first.model_dump(mode="json") == second.model_dump(mode="json") + + +def test_changeset_identity_changes_with_revision_or_governance_context() -> None: + change = _change("id", before="STRING", after="BIGINT") + base_args = { + "contract_id": "orders-product", + "base_revision_ref": "git:abc123", + "candidate_revision_ref": "git:def456", + "changes": (change,), + } + + first = build_change_set( + **base_args, + context=ChangeContext(effective_date=date(2026, 9, 9)), + ) + different_candidate = build_change_set( + **{**base_args, "candidate_revision_ref": "git:ghi789"}, + context=ChangeContext(effective_date=date(2026, 9, 9)), + ) + different_context = build_change_set( + **base_args, + context=ChangeContext(effective_date=date(2026, 9, 10)), + ) + + assert first.change_set_id != different_candidate.change_set_id + assert first.change_set_id != different_context.change_set_id + + +def test_changeset_identity_covers_provenance_without_affecting_governance() -> None: + change = _change("id", before="STRING", after="BIGINT") + context = ChangeContext(effective_date=date(2026, 9, 9)) + + first = build_change_set( + contract_id="orders-product", + base_revision_ref="git:abc123", + candidate_revision_ref="git:def456", + changes=(change,), + context=context, + source="cli", + actor_reference="user:alice", + ) + second = build_change_set( + contract_id="orders-product", + base_revision_ref="git:abc123", + candidate_revision_ref="git:def456", + changes=(change,), + context=context, + source="api", + actor_reference="service:ci", + ) + + assert first.change_set_id != second.change_set_id + assert first.changes == second.changes + assert first.context == second.context + assert first.source == "cli" + assert second.source == "api" + assert first.actor_reference == "user:alice" + assert second.actor_reference == "service:ci" + + +def test_changeset_is_immutable() -> None: + change_set = build_change_set( + contract_id="orders-product", + base_revision_ref="git:abc123", + candidate_revision_ref="git:def456", + changes=(), + context=ChangeContext(effective_date=date(2026, 9, 9)), + ) + + with pytest.raises(PydanticValidationError): + change_set.contract_id = "other" # type: ignore[misc] + + +def test_changeset_from_decision_reuses_authoritative_changes_and_context() -> None: + base = _contract("STRING") + candidate = _contract("BIGINT") + context = ChangeContext(effective_date=date(2026, 9, 9)) + decision = evaluate_governance_decision(base, candidate, context=context) + + change_set = build_change_set_from_decision( + decision, + base_revision_ref="git:abc123", + candidate_revision_ref="git:def456", + ) + + assert change_set.contract_id == decision.contract_id + assert change_set.context == decision.context + assert change_set.changes == decision.changes + assert change_set.base_revision_ref == "git:abc123" + assert change_set.candidate_revision_ref == "git:def456" diff --git a/tests/test_governance_proposal.py b/tests/test_governance_proposal.py new file mode 100644 index 00000000..108c7ade --- /dev/null +++ b/tests/test_governance_proposal.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +from collections.abc import Sequence + +import pytest +from open_data_contract_standard.model import ( + OpenDataContractStandard, + SchemaObject, + SchemaProperty, +) + +import semapact.services.governance_service as governance_service_module +from semapact.change_context import ChangeContext +from semapact.governance.models import GovernanceDecision +from semapact.lifecycle.merge_engine import MergeConflict +from semapact.services import GovernanceService + + +def _contract(physical_type: str) -> OpenDataContractStandard: + return OpenDataContractStandard( + apiVersion="v3.1.0", + kind="DataContract", + id="orders-product", + name="orders-product", + version="1.0.0", + status="active", + schema=[ + SchemaObject( + name="orders", + properties=[ + SchemaProperty( + name="id", + physicalType=physical_type, + logicalType="string", + ) + ], + ) + ], + ) + + +def test_evaluate_proposal_evaluates_once_and_reuses_authoritative_changes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + base = _contract("STRING") + candidate = _contract("BIGINT") + original = governance_service_module.evaluate_governance_decision + calls = 0 + + def counted_evaluator( + base_contract: OpenDataContractStandard, + candidate_contract: OpenDataContractStandard, + *, + context: ChangeContext, + merge_conflicts: Sequence[MergeConflict] = (), + ) -> GovernanceDecision: + nonlocal calls + calls += 1 + return original( + base_contract, + candidate_contract, + context=context, + merge_conflicts=merge_conflicts, + ) + + monkeypatch.setattr( + governance_service_module, + "evaluate_governance_decision", + counted_evaluator, + ) + + proposal = GovernanceService().evaluate_proposal( + base, + candidate, + effective_date="2026-09-09", + base_revision_ref="git:abc123", + candidate_revision_ref="git:def456", + source="api", + actor_reference="service:ci", + ) + + assert calls == 1 + assert proposal.change_set.changes == proposal.decision.changes + assert proposal.change_set.context == proposal.decision.context + assert proposal.change_set.base_revision_ref == "git:abc123" + assert proposal.change_set.candidate_revision_ref == "git:def456" + assert proposal.change_set.source == "api" + assert proposal.change_set.actor_reference == "service:ci" From 57e8e6df7c18b7eff6b9341b3251658665b99d89 Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Wed, 9 Sep 2026 16:17:33 +1000 Subject: [PATCH 04/35] feat(contractops): plan governed releases deterministically (#208) * feat(contractops): add immutable release plan model * feat(contractops): build deterministic release plans * feat(contractops): export release planning API * test(contractops): freeze release plan semantics * test(contractops): use ODCS-native metadata fixture --- semapact/contractops/__init__.py | 8 +- semapact/contractops/models.py | 35 +++++ semapact/contractops/release_plan.py | 105 +++++++++++++++ tests/test_release_plan.py | 185 +++++++++++++++++++++++++++ 4 files changed, 331 insertions(+), 2 deletions(-) create mode 100644 semapact/contractops/release_plan.py create mode 100644 tests/test_release_plan.py diff --git a/semapact/contractops/__init__.py b/semapact/contractops/__init__.py index fda4e10e..094f2faa 100644 --- a/semapact/contractops/__init__.py +++ b/semapact/contractops/__init__.py @@ -1,13 +1,17 @@ -"""M2 ContractOps proposal domain.""" +"""M2 ContractOps domain.""" from semapact.contractops.changeset import ( build_change_set, build_change_set_from_decision, ) -from semapact.contractops.models import ChangeSet +from semapact.contractops.models import ChangeSet, ReleasePlan, ReleasePrecondition +from semapact.contractops.release_plan import build_release_plan __all__ = [ "ChangeSet", + "ReleasePlan", + "ReleasePrecondition", "build_change_set", "build_change_set_from_decision", + "build_release_plan", ] diff --git a/semapact/contractops/models.py b/semapact/contractops/models.py index b3ca3230..98940bef 100644 --- a/semapact/contractops/models.py +++ b/semapact/contractops/models.py @@ -2,9 +2,12 @@ from __future__ import annotations +from enum import Enum + from pydantic import BaseModel, ConfigDict, field_validator from semapact.change_context import ChangeContext +from semapact.core.release import RequiredBump from semapact.lifecycle.changes import GovernanceChange @@ -50,3 +53,35 @@ def _normalize_optional_text(cls, value: str | None) -> str | None: return None cleaned = value.strip() return cleaned or None + + +class ReleasePrecondition(str, Enum): + """Explicit prerequisite that must be satisfied before release publication.""" + + REVIEW_AUTHORIZATION_REQUIRED = "REVIEW_AUTHORIZATION_REQUIRED" + + +class ReleasePlan(ContractOpsModel): + """Pure plan for releasing the exact candidate revision from a ChangeSet.""" + + release_plan_id: str + contract_id: str + change_set_id: str + decision_id: str + release_revision_ref: str + required_version_bump: RequiredBump + preconditions: tuple[ReleasePrecondition, ...] = () + + @field_validator( + "release_plan_id", + "contract_id", + "change_set_id", + "decision_id", + "release_revision_ref", + ) + @classmethod + def _require_release_text(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("value must not be empty") + return cleaned diff --git a/semapact/contractops/release_plan.py b/semapact/contractops/release_plan.py new file mode 100644 index 00000000..12c3b457 --- /dev/null +++ b/semapact/contractops/release_plan.py @@ -0,0 +1,105 @@ +"""Deterministic release planning from authoritative ContractOps artifacts.""" + +from __future__ import annotations + +import json +import uuid + +from semapact.contractops.models import ChangeSet, ReleasePlan, ReleasePrecondition +from semapact.exceptions import ReleaseValidationError +from semapact.governance.gate import ( + GovernanceOperation, + enforce_governance_gate, + evaluate_governance_gate, +) +from semapact.governance.models import GovernanceDecision + + +SEMAPACT_RELEASE_PLAN_NAMESPACE = uuid.UUID("7d2ad1de-c196-4f12-b1af-fdf79105eb04") + + +def build_release_plan( + change_set: ChangeSet, + decision: GovernanceDecision, +) -> ReleasePlan: + """Build a pure ReleasePlan from one exact ChangeSet and governance decision. + + The function never re-evaluates contract changes, policy, or version + classification. BLOCK decisions cannot be planned. REVIEW decisions remain + REVIEW and carry an explicit authorization precondition for later ContractOps + authorization (#120). + """ + if not isinstance(change_set, ChangeSet): + raise TypeError( + f"change_set must be ChangeSet, got {type(change_set).__name__}" + ) + if not isinstance(decision, GovernanceDecision): + raise TypeError( + f"decision must be GovernanceDecision, got {type(decision).__name__}" + ) + + _validate_proposal_consistency(change_set, decision) + + # Release planning is a pure PROPOSE operation. Reuse the authoritative M0 gate + # rather than duplicating ALLOW/REVIEW/BLOCK mapping in ContractOps. + enforce_governance_gate(decision, GovernanceOperation.PROPOSE) + + if not decision.evidence.has_changes: + raise ReleaseValidationError("Cannot plan a release for a proposal with no changes") + + publish_gate = evaluate_governance_gate(decision, GovernanceOperation.PUBLISH) + if publish_gate.allowed: + preconditions: tuple[ReleasePrecondition, ...] = () + elif publish_gate.reason == "review_required": + preconditions = (ReleasePrecondition.REVIEW_AUTHORIZATION_REQUIRED,) + else: + # PROPOSE already rejects BLOCK. Reaching this state would mean the M0 gate + # returned internally inconsistent results for the same immutable decision. + raise RuntimeError("Governance gate returned inconsistent release eligibility") + + stable_record = { + "contract_id": change_set.contract_id, + "change_set_id": change_set.change_set_id, + "decision_id": decision.decision_id, + "release_revision_ref": change_set.candidate_revision_ref, + "required_version_bump": decision.required_version_bump, + "preconditions": [item.value for item in preconditions], + } + canonical_payload = json.dumps( + stable_record, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) + release_plan_id = str( + uuid.uuid5(SEMAPACT_RELEASE_PLAN_NAMESPACE, canonical_payload) + ) + + return ReleasePlan( + release_plan_id=release_plan_id, + contract_id=change_set.contract_id, + change_set_id=change_set.change_set_id, + decision_id=decision.decision_id, + release_revision_ref=change_set.candidate_revision_ref, + required_version_bump=decision.required_version_bump, + preconditions=preconditions, + ) + + +def _validate_proposal_consistency( + change_set: ChangeSet, + decision: GovernanceDecision, +) -> None: + """Fail closed when artifacts do not describe the same evaluated proposal.""" + if change_set.contract_id != decision.contract_id: + raise ReleaseValidationError( + "ChangeSet and GovernanceDecision contract IDs do not match" + ) + if change_set.context != decision.context: + raise ReleaseValidationError( + "ChangeSet and GovernanceDecision governance contexts do not match" + ) + if change_set.changes != decision.changes: + raise ReleaseValidationError( + "ChangeSet changes do not match authoritative GovernanceDecision changes" + ) diff --git a/tests/test_release_plan.py b/tests/test_release_plan.py new file mode 100644 index 00000000..4c0b943d --- /dev/null +++ b/tests/test_release_plan.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +from datetime import date + +import pytest +from open_data_contract_standard.model import ( + OpenDataContractStandard, + SchemaObject, + SchemaProperty, +) +from pydantic import ValidationError as PydanticValidationError + +from semapact.change_context import ChangeContext +from semapact.contractops import ( + ReleasePrecondition, + build_change_set_from_decision, + build_release_plan, +) +from semapact.exceptions import GovernanceBlockedError, ReleaseValidationError +from semapact.governance import DecisionResult, evaluate_governance_decision + + +CONTEXT = ChangeContext(effective_date=date(2026, 9, 9)) + + +def _contract( + *, + contract_id: str = "orders-product", + contract_name: str | None = None, + include_created_at: bool = False, +) -> OpenDataContractStandard: + properties = [ + SchemaProperty( + name="id", + logicalType="string", + physicalType="varchar(255)", + required=True, + ) + ] + if include_created_at: + properties.append( + SchemaProperty( + name="created_at", + logicalType="timestamp", + physicalType="timestamp", + required=False, + ) + ) + return OpenDataContractStandard( + apiVersion="v3.1.0", + kind="DataContract", + id=contract_id, + name=contract_name or contract_id, + version="1.0.0", + status="active", + schema=[SchemaObject(name="orders", properties=properties)], + ) + + +def _proposal( + base: OpenDataContractStandard, + candidate: OpenDataContractStandard, + *, + base_revision_ref: str = "rev:base", + candidate_revision_ref: str = "rev:candidate", +): + decision = evaluate_governance_decision(base, candidate, context=CONTEXT) + change_set = build_change_set_from_decision( + decision, + base_revision_ref=base_revision_ref, + candidate_revision_ref=candidate_revision_ref, + source="test", + actor_reference="actor:test", + ) + return change_set, decision + + +def test_allow_release_plan_is_deterministic_and_has_no_review_precondition() -> None: + base = _contract(contract_name="orders-old") + candidate = _contract(contract_name="orders-new") + change_set, decision = _proposal(base, candidate) + + assert decision.decision is DecisionResult.ALLOW + assert decision.required_version_bump == "none" + assert decision.evidence.has_changes is True + + first = build_release_plan(change_set, decision) + second = build_release_plan(change_set, decision) + + assert first == second + assert first.release_plan_id == second.release_plan_id + assert first.contract_id == change_set.contract_id + assert first.change_set_id == change_set.change_set_id + assert first.decision_id == decision.decision_id + assert first.release_revision_ref == change_set.candidate_revision_ref + assert first.required_version_bump == decision.required_version_bump + assert first.preconditions == () + assert first.model_dump(mode="json") == second.model_dump(mode="json") + + +def test_review_release_plan_preserves_decision_and_requires_authorization() -> None: + base = _contract() + candidate = _contract(include_created_at=True) + change_set, decision = _proposal(base, candidate) + + assert decision.decision is DecisionResult.REVIEW + assert decision.required_version_bump == "minor" + + plan = build_release_plan(change_set, decision) + + assert decision.decision is DecisionResult.REVIEW + assert plan.required_version_bump == "minor" + assert plan.preconditions == ( + ReleasePrecondition.REVIEW_AUTHORIZATION_REQUIRED, + ) + + +def test_block_decision_cannot_produce_release_plan() -> None: + base = _contract(contract_id="orders-product") + candidate = _contract(contract_id="other-product") + change_set, decision = _proposal(base, candidate) + + assert decision.decision is DecisionResult.BLOCK + + with pytest.raises(GovernanceBlockedError): + build_release_plan(change_set, decision) + + +def test_release_plan_rejects_no_change_proposal() -> None: + base = _contract() + candidate = _contract() + change_set, decision = _proposal(base, candidate) + + assert decision.decision is DecisionResult.ALLOW + assert decision.evidence.has_changes is False + + with pytest.raises(ReleaseValidationError, match="no changes"): + build_release_plan(change_set, decision) + + +def test_release_plan_fails_closed_for_mismatched_proposal_artifacts() -> None: + base = _contract() + review_candidate = _contract(include_created_at=True) + metadata_candidate = _contract(contract_name="orders-changed") + + review_change_set, review_decision = _proposal(base, review_candidate) + metadata_change_set, _ = _proposal(base, metadata_candidate) + + assert review_change_set.context == metadata_change_set.context + assert review_change_set.contract_id == metadata_change_set.contract_id + assert review_change_set.changes != metadata_change_set.changes + + with pytest.raises(ReleaseValidationError, match="changes do not match"): + build_release_plan(metadata_change_set, review_decision) + + +def test_release_plan_identity_changes_with_exact_release_revision() -> None: + base = _contract(contract_name="orders-old") + candidate = _contract(contract_name="orders-new") + first_change_set, first_decision = _proposal( + base, + candidate, + candidate_revision_ref="rev:candidate-1", + ) + second_change_set, second_decision = _proposal( + base, + candidate, + candidate_revision_ref="rev:candidate-2", + ) + + first = build_release_plan(first_change_set, first_decision) + second = build_release_plan(second_change_set, second_decision) + + assert first.release_revision_ref != second.release_revision_ref + assert first.release_plan_id != second.release_plan_id + + +def test_release_plan_is_immutable() -> None: + base = _contract(contract_name="orders-old") + candidate = _contract(contract_name="orders-new") + change_set, decision = _proposal(base, candidate) + plan = build_release_plan(change_set, decision) + + with pytest.raises(PydanticValidationError): + plan.contract_id = "other" # type: ignore[misc] From 8f1f78f3b89fe48dad1b1468744f2bbcaf5abcd9 Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Wed, 9 Sep 2026 20:52:07 +1000 Subject: [PATCH 05/35] feat(contractops): resolve configurable release version authority * refactor(release): expose semantic version increment helper * feat(contractops): add version authority domain models * feat(contractops): resolve release versions by configured authority * refactor(contractops): fail closed on version config invariants * feat(contractops): export version authority API * feat(services): resolve configured contract version authority * feat(services): export version authority service * test(contractops): cover deterministic version authority resolution * test(services): cover configured version authority boundary * test(services): isolate version authority configuration * test(contractops): tighten version authority test types * docs(contractops): document release version authority * refactor(release): expose semantic version normalization * refactor(contractops): use explicit semantic version normalization * refactor(contractops): enforce version resolution provenance invariant * fix(contractops): keep git authority repository-scoped * test(contractops): lock repository-scoped git authority * docs(contractops): clarify repository-scoped git authority * test(contractops): use repository git release convention * test(version-authority): lock intended authority use cases * docs(version-authority): clarify managed version use cases * test(version-authority): align git-managed service scenario * docs(version-authority): clarify repository topologies * test(version-authority): remove shared git-release implication * docs(version-authority): clarify git co-versioning intent * test(version-authority): preserve service contract after semantic clarification * docs(version-authority): finalize topology distinction * docs(versioning): clarify recommended authority topology --- docs/version_authority.md | 175 +++++++++++ semapact/contractops/__init__.py | 18 +- semapact/contractops/models.py | 85 +++++- semapact/contractops/version_authority.py | 214 ++++++++++++++ semapact/core/release.py | 29 +- semapact/services/__init__.py | 2 + .../services/version_authority_service.py | 84 ++++++ tests/test_version_authority.py | 279 ++++++++++++++++++ tests/test_version_authority_service.py | 128 ++++++++ 9 files changed, 1006 insertions(+), 8 deletions(-) create mode 100644 docs/version_authority.md create mode 100644 semapact/contractops/version_authority.py create mode 100644 semapact/services/version_authority_service.py create mode 100644 tests/test_version_authority.py create mode 100644 tests/test_version_authority_service.py diff --git a/docs/version_authority.md b/docs/version_authority.md new file mode 100644 index 00000000..a67c9176 --- /dev/null +++ b/docs/version_authority.md @@ -0,0 +1,175 @@ +# Contract version authority + +SemaPact separates governance classification from release version selection. + +```text +GovernanceDecision.requiredVersionBump + ↓ +ReleasePlan + + +current released ODCS version + + +version-authority configuration + ↓ +VersionResolution + ↓ +explicit ContractOps authorization / APPLY / PUBLISH +``` + +`VersionResolution` is read-only planning output. Resolving a version does not mutate the ODCS contract, write Git state, or publish anything. + +## Recommended usage + +Choose version authority from the **release topology**, not simply from whether the contract files are stored in Git. + +| Repository / release topology | Recommended authority | Version owner | +| --- | --- | --- | +| A contracts repository contains many independently governed contracts | `semapact` | Each contract owns its own ODCS version | +| A contract lives with the data product/code it describes and they are released together | `git` | The product/repository Git release tag owns the version | + +### Recommended default: `semapact` + +Use SemaPact-managed authority when contracts are centrally governed and their lifecycles are independent, even when all contract files are stored in one Git repository. + +```text +contracts/ +├── orders.yaml version: 1.3.0 +├── customers.yaml version: 4.7.3 +└── payments.yaml version: 3.0.0 +``` + +The Git repository is the storage and collaboration boundary; it is **not** the version boundary. Each contract can receive a different semantic-version bump from its own governed changes. + +Do not choose Git-managed authority merely because the contracts repository uses Git or CI/CD. + +### Recommended co-versioning case: `git` + +Use Git-managed authority when one repository represents a releasable data product and contains both the product implementation and its contract. + +```text +orders-data-product/ +├── src/... +├── pipelines/... +├── contract.yaml +└── deployment/... + +Git release: v1.4.0 +``` + +In this topology the product and contract are intentionally co-versioned: + +```text +Git tag v1.4.0 + ↓ +data product release = 1.4.0 + ↓ +contract.version = 1.4.0 +``` + +SemaPact does not calculate a competing contract version. It validates that the Git-selected version satisfies the contract's governance-required minimum bump. + +### Decision rule + +```text +Does the contract have an independent lifecycle/version from the code or product it describes? + +YES +→ use semapact +→ version each contract independently + +NO, the contract and data product are deliberately released as one versioned unit +→ use git +→ let the Git/data-product release tag select the contract version +``` + +If the repository topology is ambiguous, prefer `semapact` until the product and contract have an explicit co-versioning policy. This avoids accidentally coupling otherwise independent contract lifecycles to a repository-wide release number. + +## SemaPact-managed versions + +This is the default mode for a contract repository that can contain many governed contracts while each contract retains its own independent ODCS lifecycle and version. + +```yaml +release: + versionAuthority: semapact +``` + +SemaPact selects the smallest valid next release version separately for each contract from that contract's current released ODCS version: + +| Governance minimum | Current | Selected | +| --- | --- | --- | +| `none` | `1.2.3` | `1.2.4` | +| `minor` | `1.2.3` | `1.3.0` | +| `major` | `1.2.3` | `2.0.0` | + +For example, two contracts in the same repository can evolve independently: + +```text +orders 1.2.3 + minor → 1.3.0 +customers 4.7.2 + none → 4.7.3 +``` + +Their versions are not coupled merely because they are stored in the same Git repository. + +`none` means governance does not require a minor or major bump. When an already-planned governed revision is explicitly released, the smallest distinct release version is therefore a patch bump. + +## Git-managed versions + +Git-managed mode is intended for a different repository topology: the data product implementation and its contract live and release together in the same repository. The product/repository Git tag owns the release version, and the contract follows that version. + +```yaml +release: + versionAuthority: git + tagPattern: "v{version}" +``` + +For example: + +```text +contract + data product repo tag = v1.4.0 + ↓ +Git-selected release version = 1.4.0 + ↓ +SemaPact validates 1.4.0 against the contract's requiredVersionBump + ↓ +VersionResolution.selectedVersion = 1.4.0 +``` + +SemaPact does not calculate a competing contract version in Git-managed mode. The later APPLY boundary can synchronize the Git-selected value into canonical ODCS `version`. + +The release reference is repository/workflow provenance, not a per-contract tag convention. SemaPact therefore does not derive tag names from `contractId`. + +A repository can use a literal product-specific prefix when that is its own release convention: + +```yaml +release: + versionAuthority: git + tagPattern: "orders-data-product/v{version}" +``` + +The only supported placeholder is: + +- `{version}` — required exactly once + +All other pattern text is literal. Unknown placeholders fail closed. + +## Configuration precedence + +`VersionAuthorityService` uses the existing SemaPact configuration hierarchy. Environment variables can explicitly override file configuration: + +```text +SEMAPACT_RELEASE_VERSION_AUTHORITY +SEMAPACT_RELEASE_TAG_PATTERN +``` + +The corresponding YAML keys are: + +```text +release.versionAuthority +release.tagPattern +``` + +A Git authority without `tagPattern`, or a SemaPact authority with a Git-only `tagPattern`, is rejected rather than silently ignored. + +## Write boundary + +ODCS `version` remains the canonical released contract version, but version resolution itself does not change it. The later explicit ContractOps APPLY boundary owns synchronization of the selected version into the candidate contract. Git tags, SHAs, authority metadata, and deployment provenance remain outside canonical ODCS. diff --git a/semapact/contractops/__init__.py b/semapact/contractops/__init__.py index 094f2faa..a3dd6958 100644 --- a/semapact/contractops/__init__.py +++ b/semapact/contractops/__init__.py @@ -4,14 +4,30 @@ build_change_set, build_change_set_from_decision, ) -from semapact.contractops.models import ChangeSet, ReleasePlan, ReleasePrecondition +from semapact.contractops.models import ( + ChangeSet, + ReleasePlan, + ReleasePrecondition, + VersionAuthority, + VersionAuthorityConfig, + VersionResolution, +) from semapact.contractops.release_plan import build_release_plan +from semapact.contractops.version_authority import ( + extract_version_from_release_reference, + resolve_release_version, +) __all__ = [ "ChangeSet", "ReleasePlan", "ReleasePrecondition", + "VersionAuthority", + "VersionAuthorityConfig", + "VersionResolution", "build_change_set", "build_change_set_from_decision", "build_release_plan", + "extract_version_from_release_reference", + "resolve_release_version", ] diff --git a/semapact/contractops/models.py b/semapact/contractops/models.py index 98940bef..bd0c6ee5 100644 --- a/semapact/contractops/models.py +++ b/semapact/contractops/models.py @@ -4,10 +4,10 @@ from enum import Enum -from pydantic import BaseModel, ConfigDict, field_validator +from pydantic import BaseModel, ConfigDict, field_validator, model_validator from semapact.change_context import ChangeContext -from semapact.core.release import RequiredBump +from semapact.core.release import ActualVersionBump, RequiredBump from semapact.lifecycle.changes import GovernanceChange @@ -85,3 +85,84 @@ def _require_release_text(cls, value: str) -> str: if not cleaned: raise ValueError("value must not be empty") return cleaned + + +class VersionAuthority(str, Enum): + """Authority that selects the actual released ODCS semantic version.""" + + SEMAPACT = "semapact" + GIT = "git" + + +class VersionAuthorityConfig(ContractOpsModel): + """Typed configuration for resolving one ReleasePlan version.""" + + authority: VersionAuthority = VersionAuthority.SEMAPACT + tag_pattern: str | None = None + + @field_validator("tag_pattern") + @classmethod + def _normalize_tag_pattern(cls, value: str | None) -> str | None: + if value is None: + return None + cleaned = value.strip() + return cleaned or None + + @model_validator(mode="after") + def _validate_authority_configuration(self) -> VersionAuthorityConfig: + if self.authority is VersionAuthority.GIT and self.tag_pattern is None: + raise ValueError("git version authority requires tag_pattern") + if self.authority is VersionAuthority.SEMAPACT and self.tag_pattern is not None: + raise ValueError("tag_pattern is only valid for git version authority") + return self + + +class VersionResolution(ContractOpsModel): + """Pure resolution of the actual release version for one ReleasePlan.""" + + version_resolution_id: str + release_plan_id: str + contract_id: str + release_revision_ref: str + authority: VersionAuthority + current_version: str + required_version_bump: RequiredBump + selected_version: str + actual_bump: ActualVersionBump + authority_reference: str | None = None + + @field_validator( + "version_resolution_id", + "release_plan_id", + "contract_id", + "release_revision_ref", + "current_version", + "selected_version", + ) + @classmethod + def _require_version_resolution_text(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("value must not be empty") + return cleaned + + @field_validator("authority_reference") + @classmethod + def _normalize_authority_reference(cls, value: str | None) -> str | None: + if value is None: + return None + cleaned = value.strip() + return cleaned or None + + @model_validator(mode="after") + def _validate_authority_reference(self) -> VersionResolution: + if self.authority is VersionAuthority.GIT and self.authority_reference is None: + raise ValueError("git version resolution requires authority_reference") + if ( + self.authority is VersionAuthority.SEMAPACT + and self.authority_reference is not None + ): + raise ValueError( + "SemaPact version resolution must not contain authority_reference" + ) + return self diff --git a/semapact/contractops/version_authority.py b/semapact/contractops/version_authority.py new file mode 100644 index 00000000..3f6d00d8 --- /dev/null +++ b/semapact/contractops/version_authority.py @@ -0,0 +1,214 @@ +"""Pure contract version resolution for deterministic governed releases.""" + +from __future__ import annotations + +import json +import re +import uuid + +from semapact.contractops.models import ( + ReleasePlan, + VersionAuthority, + VersionAuthorityConfig, + VersionResolution, +) +from semapact.core.release import ( + ActualVersionBump, + RequiredBump, + classify_version_bump, + increment_version, + normalize_semver, +) +from semapact.exceptions import ReleaseValidationError + + +SEMAPACT_VERSION_RESOLUTION_NAMESPACE = uuid.UUID( + "4a2bd29d-2e44-44a0-9fc9-a1f4c81a2108" +) +_VERSION_TOKEN = "{version}" + + +def resolve_release_version( + release_plan: ReleasePlan, + *, + current_version: str, + config: VersionAuthorityConfig, + authority_reference: str | None = None, +) -> VersionResolution: + """Resolve one contract's release version without mutating governed state. + + SemaPact-managed authority independently versions the contract. Git-managed + authority is for a contract co-versioned with its data product/repository and + consumes that repository's explicit release reference. Governance + classification is never recalculated here. + """ + if not isinstance(release_plan, ReleasePlan): + raise TypeError( + f"release_plan must be ReleasePlan, got {type(release_plan).__name__}" + ) + if not isinstance(config, VersionAuthorityConfig): + raise TypeError( + f"config must be VersionAuthorityConfig, got {type(config).__name__}" + ) + + canonical_current = _canonical_current_version(current_version) + + if config.authority is VersionAuthority.SEMAPACT: + if authority_reference is not None: + raise ReleaseValidationError( + "SemaPact version authority does not accept an external authority reference" + ) + selected_version, actual_bump = _resolve_semapact_version( + canonical_current, + release_plan.required_version_bump, + ) + normalized_reference = None + elif config.authority is VersionAuthority.GIT: + if authority_reference is None or not authority_reference.strip(): + raise ReleaseValidationError( + "Git version authority requires an explicit release reference" + ) + tag_pattern = config.tag_pattern + if tag_pattern is None: + raise RuntimeError( + "VersionAuthorityConfig invariant violation: git authority requires tag_pattern" + ) + normalized_reference = authority_reference.strip() + selected_version = extract_version_from_release_reference( + normalized_reference, + tag_pattern=tag_pattern, + ) + try: + actual_bump = classify_version_bump( + canonical_current, + selected_version, + ) + except ValueError as exc: + raise ReleaseValidationError(str(exc)) from exc + _validate_minimum_bump( + actual_bump, + release_plan.required_version_bump, + selected_version=selected_version, + ) + else: # pragma: no cover - enum exhaustiveness guard + raise RuntimeError(f"Unsupported version authority: {config.authority}") + + stable_record = { + "release_plan_id": release_plan.release_plan_id, + "contract_id": release_plan.contract_id, + "release_revision_ref": release_plan.release_revision_ref, + "authority": config.authority.value, + "current_version": canonical_current, + "required_version_bump": release_plan.required_version_bump, + "selected_version": selected_version, + "actual_bump": actual_bump, + "authority_reference": normalized_reference, + } + canonical_payload = json.dumps( + stable_record, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) + version_resolution_id = str( + uuid.uuid5(SEMAPACT_VERSION_RESOLUTION_NAMESPACE, canonical_payload) + ) + + return VersionResolution( + version_resolution_id=version_resolution_id, + release_plan_id=release_plan.release_plan_id, + contract_id=release_plan.contract_id, + release_revision_ref=release_plan.release_revision_ref, + authority=config.authority, + current_version=canonical_current, + required_version_bump=release_plan.required_version_bump, + selected_version=selected_version, + actual_bump=actual_bump, + authority_reference=normalized_reference, + ) + + +def extract_version_from_release_reference( + reference: str, + *, + tag_pattern: str, +) -> str: + """Extract a semantic version from a configured literal Git tag pattern. + + The only supported placeholder is ``{version}``. All other pattern text is + literal repository/workflow convention rather than ContractOps domain data. + """ + cleaned_reference = str(reference or "").strip() + cleaned_pattern = str(tag_pattern or "").strip() + if not cleaned_reference: + raise ReleaseValidationError("Release reference must not be empty") + if not cleaned_pattern: + raise ReleaseValidationError("Tag pattern must not be empty") + + _validate_tag_pattern(cleaned_pattern) + + regex = re.escape(cleaned_pattern) + regex = regex.replace( + re.escape(_VERSION_TOKEN), + r"(?P\d+\.\d+\.\d+)", + ) + match = re.fullmatch(regex, cleaned_reference) + if match is None: + raise ReleaseValidationError( + f"Release reference '{cleaned_reference}' does not match tag pattern " + f"'{cleaned_pattern}'" + ) + return match.group("version") + + +def _canonical_current_version(current_version: str) -> str: + try: + return normalize_semver(current_version) + except ValueError as exc: + raise ReleaseValidationError(str(exc)) from exc + + +def _resolve_semapact_version( + current_version: str, + required_bump: RequiredBump, +) -> tuple[str, ActualVersionBump]: + if required_bump == "major": + actual_bump: ActualVersionBump = "major" + elif required_bump == "minor": + actual_bump = "minor" + else: + # A metadata-only governed revision still needs a distinct released ODCS + # version when it is explicitly published. ``none`` means no minimum + # minor/major requirement; patch is the smallest actual release bump. + actual_bump = "patch" + return increment_version(current_version, actual_bump), actual_bump + + +def _validate_minimum_bump( + actual_bump: ActualVersionBump, + required_bump: RequiredBump, + *, + selected_version: str, +) -> None: + insufficient = ( + (required_bump == "major" and actual_bump != "major") + or (required_bump == "minor" and actual_bump == "patch") + ) + if insufficient: + raise ReleaseValidationError( + f"Resolved version '{selected_version}' applies a {actual_bump} bump, " + f"but release requires at least a {required_bump} bump" + ) + + +def _validate_tag_pattern(tag_pattern: str) -> None: + if tag_pattern.count(_VERSION_TOKEN) != 1: + raise ReleaseValidationError( + "Tag pattern must contain exactly one {version} placeholder" + ) + + remaining = tag_pattern.replace(_VERSION_TOKEN, "") + if "{" in remaining or "}" in remaining: + raise ReleaseValidationError( + "Tag pattern supports only the {version} placeholder" + ) diff --git a/semapact/core/release.py b/semapact/core/release.py index 5bf5ccf3..1cd8c77c 100644 --- a/semapact/core/release.py +++ b/semapact/core/release.py @@ -196,7 +196,6 @@ def apply_release_candidate( f"{required_bump} bump" ) - promoted = candidate_model.model_copy(deep=True) promoted.version = target_version return PromotionResult( @@ -258,6 +257,12 @@ def parse_release_tag_version(release_tag: str) -> str: return match.group("version") +def normalize_semver(version: str) -> str: + """Validate and return the canonical ``major.minor.patch`` representation.""" + major, minor, patch = _parse_semver(version) + return f"{major}.{minor}.{patch}" + + def classify_version_bump( current_version: str, target_version: str ) -> ActualVersionBump: @@ -275,6 +280,21 @@ def classify_version_bump( return "patch" +def increment_version( + current_version: str, + bump: ActualVersionBump, +) -> str: + """Return the next semantic version for an explicit actual release bump.""" + major, minor, patch = _parse_semver(current_version) + if bump == "major": + return f"{major + 1}.0.0" + if bump == "minor": + return f"{major}.{minor + 1}.0" + if bump == "patch": + return f"{major}.{minor}.{patch + 1}" + raise ValueError(f"Unsupported version bump: {bump}") + + def suggest_release_version( current_version: str, required_bump: RequiredBump, @@ -290,12 +310,11 @@ def suggest_release_version( - required bump stays `major` - suggested release version stays `2.0.0`, not `2.1.0` """ - major, minor, patch = _parse_semver(current_version) if required_bump == "major": - return f"{major + 1}.0.0" + return increment_version(current_version, "major") if required_bump == "minor": - return f"{major}.{minor + 1}.0" - return f"{major}.{minor}.{patch}" + return increment_version(current_version, "minor") + return normalize_semver(current_version) def _parse_semver(version: str) -> tuple[int, int, int]: diff --git a/semapact/services/__init__.py b/semapact/services/__init__.py index 2d08fe54..161e41e3 100644 --- a/semapact/services/__init__.py +++ b/semapact/services/__init__.py @@ -9,6 +9,7 @@ ReconciliationService, RuntimeReconciliation, ) +from semapact.services.version_authority_service import VersionAuthorityService __all__ = [ "GovernanceAnalysis", @@ -16,4 +17,5 @@ "GovernanceService", "ReconciliationService", "RuntimeReconciliation", + "VersionAuthorityService", ] diff --git a/semapact/services/version_authority_service.py b/semapact/services/version_authority_service.py new file mode 100644 index 00000000..89682003 --- /dev/null +++ b/semapact/services/version_authority_service.py @@ -0,0 +1,84 @@ +"""Application boundary for configured contract version authority.""" + +from __future__ import annotations + +from open_data_contract_standard.model import OpenDataContractStandard +from pydantic import ValidationError as PydanticValidationError + +from semapact.contractops import ( + ReleasePlan, + VersionAuthority, + VersionAuthorityConfig, + VersionResolution, + resolve_release_version, +) +from semapact.core.config import ConfigManager +from semapact.exceptions import ReleaseValidationError, ValidationError + + +class VersionAuthorityService: + """Resolve a ReleasePlan using application configuration and released ODCS state.""" + + def __init__(self, config_manager: ConfigManager | None = None) -> None: + self._config = config_manager or ConfigManager() + + def resolve( + self, + release_plan: ReleasePlan, + released_contract: OpenDataContractStandard, + *, + authority_reference: str | None = None, + ) -> VersionResolution: + """Resolve the actual release version without mutating the contract.""" + if not isinstance(release_plan, ReleasePlan): + raise TypeError( + f"release_plan must be ReleasePlan, got {type(release_plan).__name__}" + ) + if not isinstance(released_contract, OpenDataContractStandard): + raise TypeError( + "released_contract must be OpenDataContractStandard, " + f"got {type(released_contract).__name__}" + ) + + contract_id = str(released_contract.id or "").strip() + if contract_id != release_plan.contract_id: + raise ReleaseValidationError( + "Released contract ID does not match ReleasePlan contract ID" + ) + + current_version = str(released_contract.version or "").strip() + if not current_version: + raise ReleaseValidationError( + "Released contract must define the current ODCS version" + ) + + return resolve_release_version( + release_plan, + current_version=current_version, + config=self.load_config(), + authority_reference=authority_reference, + ) + + def load_config(self) -> VersionAuthorityConfig: + """Resolve typed version-authority config from standard SemaPact config sources.""" + authority = self._config.get( + "release.versionAuthority", + env_var="SEMAPACT_RELEASE_VERSION_AUTHORITY", + default=VersionAuthority.SEMAPACT.value, + ) + tag_pattern = self._config.get( + "release.tagPattern", + env_var="SEMAPACT_RELEASE_TAG_PATTERN", + default=None, + ) + + try: + return VersionAuthorityConfig( + authority=authority, + tag_pattern=tag_pattern, + ) + except PydanticValidationError as exc: + message = exc.errors()[0].get("msg", "invalid version authority configuration") + raise ValidationError( + f"Invalid release version authority configuration: {message}" + ) from exc diff --git a/tests/test_version_authority.py b/tests/test_version_authority.py new file mode 100644 index 00000000..5e327138 --- /dev/null +++ b/tests/test_version_authority.py @@ -0,0 +1,279 @@ +from __future__ import annotations + +import pytest +from pydantic import ValidationError as PydanticValidationError + +from semapact.contractops import ( + ReleasePlan, + VersionAuthority, + VersionAuthorityConfig, + extract_version_from_release_reference, + resolve_release_version, +) +from semapact.core.release import ActualVersionBump, RequiredBump +from semapact.exceptions import ReleaseValidationError + + +def _plan( + required_bump: RequiredBump, + *, + contract_id: str = "orders-product", + release_plan_id: str = "release-plan-1", +) -> ReleasePlan: + return ReleasePlan( + release_plan_id=release_plan_id, + contract_id=contract_id, + change_set_id="change-set-1", + decision_id="decision-1", + release_revision_ref="rev:candidate", + required_version_bump=required_bump, + ) + + +@pytest.mark.parametrize( + ("required_bump", "expected_version", "expected_actual_bump"), + [ + ("none", "1.2.4", "patch"), + ("minor", "1.3.0", "minor"), + ("major", "2.0.0", "major"), + ], +) +def test_semapact_authority_selects_smallest_valid_release_version( + required_bump: RequiredBump, + expected_version: str, + expected_actual_bump: ActualVersionBump, +) -> None: + resolution = resolve_release_version( + _plan(required_bump), + current_version="1.2.3", + config=VersionAuthorityConfig(), + ) + + assert resolution.authority is VersionAuthority.SEMAPACT + assert resolution.current_version == "1.2.3" + assert resolution.required_version_bump == required_bump + assert resolution.selected_version == expected_version + assert resolution.actual_bump == expected_actual_bump + assert resolution.authority_reference is None + + +def test_semapact_authority_versions_contracts_independently() -> None: + orders = resolve_release_version( + _plan("minor", contract_id="orders", release_plan_id="release-orders"), + current_version="1.2.3", + config=VersionAuthorityConfig(), + ) + customers = resolve_release_version( + _plan("none", contract_id="customers", release_plan_id="release-customers"), + current_version="4.7.2", + config=VersionAuthorityConfig(), + ) + + assert orders.selected_version == "1.3.0" + assert customers.selected_version == "4.7.3" + assert orders.version_resolution_id != customers.version_resolution_id + + +def test_semapact_version_resolution_is_deterministic() -> None: + plan = _plan("minor") + config = VersionAuthorityConfig(authority=VersionAuthority.SEMAPACT) + + first = resolve_release_version( + plan, + current_version="1.2.3", + config=config, + ) + second = resolve_release_version( + plan, + current_version="1.2.3", + config=config, + ) + + assert first == second + assert first.version_resolution_id == second.version_resolution_id + assert first.model_dump(mode="json") == second.model_dump(mode="json") + + +def test_semapact_authority_rejects_external_reference() -> None: + with pytest.raises( + ReleaseValidationError, + match="does not accept an external authority reference", + ): + resolve_release_version( + _plan("minor"), + current_version="1.2.3", + config=VersionAuthorityConfig(), + authority_reference="v1.3.0", + ) + + +def test_git_authority_uses_exact_product_repository_release_version() -> None: + resolution = resolve_release_version( + _plan("minor"), + current_version="1.2.3", + config=VersionAuthorityConfig( + authority=VersionAuthority.GIT, + tag_pattern="v{version}", + ), + authority_reference="v1.4.0", + ) + + assert resolution.authority is VersionAuthority.GIT + assert resolution.selected_version == "1.4.0" + assert resolution.actual_bump == "minor" + assert resolution.authority_reference == "v1.4.0" + + +def test_git_authority_allows_repository_specific_literal_tag_prefix() -> None: + resolution = resolve_release_version( + _plan("major"), + current_version="1.2.3", + config=VersionAuthorityConfig( + authority=VersionAuthority.GIT, + tag_pattern="orders-data-product/v{version}", + ), + authority_reference="orders-data-product/v2.1.0", + ) + + assert resolution.selected_version == "2.1.0" + assert resolution.actual_bump == "major" + + +def test_git_authority_rejects_insufficient_bump() -> None: + with pytest.raises(ReleaseValidationError, match="requires at least a major bump"): + resolve_release_version( + _plan("major"), + current_version="1.2.3", + config=VersionAuthorityConfig( + authority=VersionAuthority.GIT, + tag_pattern="v{version}", + ), + authority_reference="v1.3.0", + ) + + +def test_git_authority_accepts_any_positive_bump_when_governance_requires_none() -> None: + resolution = resolve_release_version( + _plan("none"), + current_version="1.2.3", + config=VersionAuthorityConfig( + authority=VersionAuthority.GIT, + tag_pattern="v{version}", + ), + authority_reference="v2.0.0", + ) + + assert resolution.selected_version == "2.0.0" + assert resolution.actual_bump == "major" + + +def test_git_authority_requires_version_greater_than_current() -> None: + with pytest.raises(ReleaseValidationError, match="must be greater than current"): + resolve_release_version( + _plan("none"), + current_version="1.2.3", + config=VersionAuthorityConfig( + authority=VersionAuthority.GIT, + tag_pattern="v{version}", + ), + authority_reference="v1.2.3", + ) + + +def test_git_authority_requires_explicit_release_reference() -> None: + with pytest.raises( + ReleaseValidationError, + match="requires an explicit release reference", + ): + resolve_release_version( + _plan("minor"), + current_version="1.2.3", + config=VersionAuthorityConfig( + authority=VersionAuthority.GIT, + tag_pattern="v{version}", + ), + ) + + +def test_release_reference_pattern_treats_non_version_text_as_literal() -> None: + assert ( + extract_version_from_release_reference( + "orders.product/v1.2.3", + tag_pattern="orders.product/v{version}", + ) + == "1.2.3" + ) + + with pytest.raises(ReleaseValidationError, match="does not match tag pattern"): + extract_version_from_release_reference( + "ordersXproduct/v1.2.3", + tag_pattern="orders.product/v{version}", + ) + + +def test_release_reference_rejects_contract_or_unknown_pattern_placeholders() -> None: + with pytest.raises(ReleaseValidationError, match="supports only"): + extract_version_from_release_reference( + "orders-product/v1.2.3", + tag_pattern="{contractId}/v{version}", + ) + + with pytest.raises(ReleaseValidationError, match="supports only"): + extract_version_from_release_reference( + "main/v1.2.3", + tag_pattern="{repo}/v{version}", + ) + + +def test_version_authority_config_fails_closed_on_incompatible_fields() -> None: + with pytest.raises(PydanticValidationError, match="requires tag_pattern"): + VersionAuthorityConfig(authority=VersionAuthority.GIT) + + with pytest.raises(PydanticValidationError, match="only valid for git"): + VersionAuthorityConfig( + authority=VersionAuthority.SEMAPACT, + tag_pattern="v{version}", + ) + + +def test_version_resolution_rejects_invalid_current_version() -> None: + with pytest.raises(ReleaseValidationError, match="must be a semantic version"): + resolve_release_version( + _plan("minor"), + current_version="latest", + config=VersionAuthorityConfig(), + ) + + +def test_version_resolution_identity_changes_with_authoritative_inputs() -> None: + plan = _plan("minor") + config = VersionAuthorityConfig( + authority=VersionAuthority.GIT, + tag_pattern="v{version}", + ) + + first = resolve_release_version( + plan, + current_version="1.2.3", + config=config, + authority_reference="v1.3.0", + ) + second = resolve_release_version( + plan, + current_version="1.2.3", + config=config, + authority_reference="v1.4.0", + ) + + assert first.version_resolution_id != second.version_resolution_id + + +def test_version_resolution_is_immutable() -> None: + resolution = resolve_release_version( + _plan("minor"), + current_version="1.2.3", + config=VersionAuthorityConfig(), + ) + + with pytest.raises(PydanticValidationError): + resolution.selected_version = "9.9.9" diff --git a/tests/test_version_authority_service.py b/tests/test_version_authority_service.py new file mode 100644 index 00000000..7b5bb904 --- /dev/null +++ b/tests/test_version_authority_service.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +import pytest + +from semapact.contractops import ReleasePlan, VersionAuthority +from semapact.core.config import ConfigManager +from semapact.core.release import RequiredBump +from semapact.exceptions import ReleaseValidationError, ValidationError +from semapact.services import VersionAuthorityService + + +@pytest.fixture(autouse=True) +def _clear_release_environment(monkeypatch: pytest.MonkeyPatch) -> None: + for name in ( + "SEMAPACT_RELEASE_VERSION_AUTHORITY", + "SEMAPACT_RELEASE_TAG_PATTERN", + "CONTRACTHUB_RELEASE_VERSION_AUTHORITY", + "CONTRACTHUB_RELEASE_TAG_PATTERN", + ): + monkeypatch.delenv(name, raising=False) + + +def _config(data: dict[str, object] | None = None) -> ConfigManager: + config = ConfigManager() + config.config_data = {} + if data is not None: + config.update_config(data) + return config + + +def _plan(required_bump: RequiredBump = "minor") -> ReleasePlan: + return ReleasePlan( + release_plan_id="release-plan-1", + contract_id="orders-product", + change_set_id="change-set-1", + decision_id="decision-1", + release_revision_ref="rev:candidate", + required_version_bump=required_bump, + ) + + +def _released_contract(sample_odcs_model): + contract = sample_odcs_model.model_copy(deep=True) + contract.id = "orders-product" + contract.version = "1.2.3" + return contract + + +def test_service_defaults_to_semapact_authority(sample_odcs_model) -> None: + resolution = VersionAuthorityService(_config()).resolve( + _plan("minor"), + _released_contract(sample_odcs_model), + ) + + assert resolution.authority is VersionAuthority.SEMAPACT + assert resolution.selected_version == "1.3.0" + + +def test_service_reads_git_authority_from_config(sample_odcs_model) -> None: + config = _config( + { + "release": { + "versionAuthority": "git", + "tagPattern": "v{version}", + } + } + ) + + resolution = VersionAuthorityService(config).resolve( + _plan("minor"), + _released_contract(sample_odcs_model), + authority_reference="v1.4.0", + ) + + assert resolution.authority is VersionAuthority.GIT + assert resolution.selected_version == "1.4.0" + assert resolution.authority_reference == "v1.4.0" + + +def test_service_environment_overrides_file_authority( + sample_odcs_model, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config = _config({"release": {"versionAuthority": "semapact"}}) + monkeypatch.setenv("SEMAPACT_RELEASE_VERSION_AUTHORITY", "git") + monkeypatch.setenv("SEMAPACT_RELEASE_TAG_PATTERN", "v{version}") + + resolution = VersionAuthorityService(config).resolve( + _plan("major"), + _released_contract(sample_odcs_model), + authority_reference="v2.0.0", + ) + + assert resolution.authority is VersionAuthority.GIT + assert resolution.selected_version == "2.0.0" + + +def test_service_rejects_mismatched_released_contract(sample_odcs_model) -> None: + contract = _released_contract(sample_odcs_model) + contract.id = "other-product" + + with pytest.raises(ReleaseValidationError, match="contract ID does not match"): + VersionAuthorityService(_config()).resolve(_plan(), contract) + + +def test_service_converts_invalid_configuration_to_validation_error( + sample_odcs_model, +) -> None: + config = _config({"release": {"versionAuthority": "git"}}) + + with pytest.raises( + ValidationError, + match="Invalid release version authority configuration", + ): + VersionAuthorityService(config).resolve( + _plan(), + _released_contract(sample_odcs_model), + authority_reference="v1.3.0", + ) + + +def test_service_does_not_mutate_released_contract(sample_odcs_model) -> None: + contract = _released_contract(sample_odcs_model) + before = contract.model_dump(mode="json") + + VersionAuthorityService(_config()).resolve(_plan(), contract) + + assert contract.model_dump(mode="json") == before From 989159d92de59505d47f530d1695e07f187af628 Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Wed, 9 Sep 2026 21:40:47 +1000 Subject: [PATCH 06/35] feat(contractops): authorize review-required release actions * feat(contractops): add review authorization domain models * feat(contractops): authorize exact review-required operations * feat(contractops): export review authorization boundary * test(contractops): lock review authorization matrix * docs(contractops): explain review authorization boundary --- docs/contractops_authorization.md | 97 +++++++ semapact/contractops/__init__.py | 10 + semapact/contractops/authorization.py | 279 ++++++++++++++++++++ semapact/contractops/models.py | 121 ++++++++- tests/test_contractops_authorization.py | 328 ++++++++++++++++++++++++ 5 files changed, 834 insertions(+), 1 deletion(-) create mode 100644 docs/contractops_authorization.md create mode 100644 semapact/contractops/authorization.py create mode 100644 tests/test_contractops_authorization.py diff --git a/docs/contractops_authorization.md b/docs/contractops_authorization.md new file mode 100644 index 00000000..aa670018 --- /dev/null +++ b/docs/contractops_authorization.md @@ -0,0 +1,97 @@ +# ContractOps review authorization + +SemaPact keeps governance classification and review authorization as separate immutable facts. + +```text +GovernanceDecision(REVIEW) + + +exact version-resolved release context + + +matching explicit approval evidence + ↓ +ContractOpsAuthorization(allowed=true) +``` + +The original `GovernanceDecision` remains `REVIEW`. Approval never rewrites it to `ALLOW`. + +## Why authorization happens after version resolution + +A review must authorize the release that will actually be executed, not an earlier approximation of it. + +```text +GovernanceDecision +→ ChangeSet +→ ReleasePlan +→ VersionResolution +→ ContractOpsAuthorization +→ APPLY / PUBLISH +``` + +Review evidence is therefore scoped to all of: + +```text +decisionId +changeSetId +releasePlanId +versionResolutionId +operation +``` + +Changing the proposal, release plan, selected version, or requested operation invalidates the evidence for that new action. + +## Decision behavior + +| Governance gate result | Review evidence | Authorization | +| --- | --- | --- | +| `allowed` | not required | allowed by governance | +| `review_required` | missing | denied: authorization required | +| `review_required` | exact `APPROVE` | allowed by review | +| `review_required` | `REJECT` / `REQUEST_CHANGES` | denied | +| `review_required` | stale or mismatched | denied | +| `blocked` | any | denied; review cannot override BLOCK | + +M0 `GovernanceGateResult` remains authoritative for whether the operation is already allowed, requires review, or is blocked. ContractOps authorization only satisfies `review_required`; it does not re-run policy. + +## Explicit evidence, not comment parsing + +M2 consumes structured evidence: + +```text +ReviewAuthorizationEvidence +├── evidenceReference +├── decisionId +├── changeSetId +├── releasePlanId +├── versionResolutionId +├── operation +└── action +``` + +`evidenceReference` is opaque. This layer does not store approvals or infer approval from human comments, PR text, Slack messages, or similar free-form content. + +Future durable review history belongs to M3. A persisted `ApprovalRecord` can later be resolved/projected into `ReviewAuthorizationEvidence` without changing the ContractOps authorization semantics. + +## Operation scope + +Approval is not a generic bypass token. + +```text +approval for APPLY +≠ approval for PUBLISH +``` + +Likewise, approval for one `VersionResolution` does not authorize a different selected version. + +## Invalid context vs denied authorization + +SemaPact distinguishes malformed artifact composition from a valid release that simply lacks approval. + +If `GovernanceDecision`, `ChangeSet`, `ReleasePlan`, and `VersionResolution` do not refer to the same immutable release context, authorization fails closed with a validation error. + +If the release context is valid but evidence is missing, rejected, stale, or mismatched, SemaPact returns a deterministic `ContractOpsAuthorization` with `allowed=false` and a machine-readable reason. + +This lets later APPLY/PUBLISH boundaries consume one authoritative authorization result without implementing their own approval rules. + +## Non-goals + +This boundary does not implement approval persistence, reviewer routing, quorum, IAM/SSO, approval UI, GitHub review parsing, or BLOCK overrides. diff --git a/semapact/contractops/__init__.py b/semapact/contractops/__init__.py index a3dd6958..d21454b3 100644 --- a/semapact/contractops/__init__.py +++ b/semapact/contractops/__init__.py @@ -1,13 +1,18 @@ """M2 ContractOps domain.""" +from semapact.contractops.authorization import authorize_contract_operation from semapact.contractops.changeset import ( build_change_set, build_change_set_from_decision, ) from semapact.contractops.models import ( + AuthorizationReason, ChangeSet, + ContractOpsAuthorization, ReleasePlan, ReleasePrecondition, + ReviewAuthorizationEvidence, + ReviewEvidenceAction, VersionAuthority, VersionAuthorityConfig, VersionResolution, @@ -19,12 +24,17 @@ ) __all__ = [ + "AuthorizationReason", "ChangeSet", + "ContractOpsAuthorization", "ReleasePlan", "ReleasePrecondition", + "ReviewAuthorizationEvidence", + "ReviewEvidenceAction", "VersionAuthority", "VersionAuthorityConfig", "VersionResolution", + "authorize_contract_operation", "build_change_set", "build_change_set_from_decision", "build_release_plan", diff --git a/semapact/contractops/authorization.py b/semapact/contractops/authorization.py new file mode 100644 index 00000000..9c498238 --- /dev/null +++ b/semapact/contractops/authorization.py @@ -0,0 +1,279 @@ +"""Pure authorization composition for review-required ContractOps actions.""" + +from __future__ import annotations + +import json +import uuid + +from semapact.contractops.models import ( + AuthorizationReason, + ChangeSet, + ContractOpsAuthorization, + ReleasePlan, + ReviewAuthorizationEvidence, + ReviewEvidenceAction, + VersionResolution, +) +from semapact.exceptions import ReleaseValidationError +from semapact.governance.gate import GovernanceOperation, evaluate_governance_gate +from semapact.governance.models import GovernanceDecision + + +SEMAPACT_CONTRACTOPS_AUTHORIZATION_NAMESPACE = uuid.UUID( + "b6218d0c-3f9d-44a2-8d68-e3b0ee170948" +) + + +def authorize_contract_operation( + decision: GovernanceDecision, + change_set: ChangeSet, + release_plan: ReleasePlan, + version_resolution: VersionResolution, + operation: GovernanceOperation, + *, + evidence: ReviewAuthorizationEvidence | None = None, +) -> ContractOpsAuthorization: + """Authorize one exact version-resolved ContractOps operation. + + M0 governance remains authoritative for decision-level semantics. This function + only satisfies a ``review_required`` gate result with explicit, exact approval + evidence. It never mutates or reinterprets the GovernanceDecision. + """ + _validate_types( + decision, + change_set, + release_plan, + version_resolution, + operation, + evidence, + ) + _validate_release_context(decision, change_set, release_plan, version_resolution) + + gate = evaluate_governance_gate(decision, operation) + + if gate.reason == "blocked": + return _build_authorization( + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + operation=operation, + allowed=False, + reason=AuthorizationReason.BLOCKED_BY_GOVERNANCE, + ) + + if gate.allowed: + return _build_authorization( + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + operation=operation, + allowed=True, + reason=AuthorizationReason.ALLOWED_BY_GOVERNANCE, + ) + + if gate.reason != "review_required": # pragma: no cover - gate exhaustiveness guard + raise RuntimeError(f"Unsupported governance gate reason: {gate.reason}") + + if evidence is None: + return _build_authorization( + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + operation=operation, + allowed=False, + reason=AuthorizationReason.REVIEW_AUTHORIZATION_REQUIRED, + ) + + if not _evidence_matches_release_context( + evidence, + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + operation=operation, + ): + return _build_authorization( + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + operation=operation, + allowed=False, + reason=AuthorizationReason.REVIEW_AUTHORIZATION_MISMATCH, + evidence=evidence, + ) + + if evidence.action is not ReviewEvidenceAction.APPROVE: + return _build_authorization( + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + operation=operation, + allowed=False, + reason=AuthorizationReason.REVIEW_AUTHORIZATION_REJECTED, + evidence=evidence, + ) + + return _build_authorization( + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + operation=operation, + allowed=True, + reason=AuthorizationReason.ALLOWED_BY_REVIEW, + evidence=evidence, + ) + + +def _validate_types( + decision: GovernanceDecision, + change_set: ChangeSet, + release_plan: ReleasePlan, + version_resolution: VersionResolution, + operation: GovernanceOperation, + evidence: ReviewAuthorizationEvidence | None, +) -> None: + expected = ( + (decision, GovernanceDecision, "decision"), + (change_set, ChangeSet, "change_set"), + (release_plan, ReleasePlan, "release_plan"), + (version_resolution, VersionResolution, "version_resolution"), + (operation, GovernanceOperation, "operation"), + ) + for value, expected_type, name in expected: + if not isinstance(value, expected_type): + raise TypeError( + f"{name} must be {expected_type.__name__}, got {type(value).__name__}" + ) + if evidence is not None and not isinstance(evidence, ReviewAuthorizationEvidence): + raise TypeError( + "evidence must be ReviewAuthorizationEvidence or None, " + f"got {type(evidence).__name__}" + ) + + +def _validate_release_context( + decision: GovernanceDecision, + change_set: ChangeSet, + release_plan: ReleasePlan, + version_resolution: VersionResolution, +) -> None: + """Fail closed when immutable artifacts do not describe one release context.""" + if change_set.contract_id != decision.contract_id: + raise ReleaseValidationError( + "ChangeSet and GovernanceDecision contract IDs do not match" + ) + if change_set.context != decision.context: + raise ReleaseValidationError( + "ChangeSet and GovernanceDecision governance contexts do not match" + ) + if change_set.changes != decision.changes: + raise ReleaseValidationError( + "ChangeSet changes do not match authoritative GovernanceDecision changes" + ) + + if release_plan.contract_id != change_set.contract_id: + raise ReleaseValidationError("ReleasePlan and ChangeSet contract IDs do not match") + if release_plan.change_set_id != change_set.change_set_id: + raise ReleaseValidationError("ReleasePlan does not reference the supplied ChangeSet") + if release_plan.decision_id != decision.decision_id: + raise ReleaseValidationError( + "ReleasePlan does not reference the supplied GovernanceDecision" + ) + if release_plan.release_revision_ref != change_set.candidate_revision_ref: + raise ReleaseValidationError( + "ReleasePlan release revision does not match ChangeSet candidate revision" + ) + if release_plan.required_version_bump != decision.required_version_bump: + raise ReleaseValidationError( + "ReleasePlan required version bump does not match GovernanceDecision" + ) + + if version_resolution.release_plan_id != release_plan.release_plan_id: + raise ReleaseValidationError( + "VersionResolution does not reference the supplied ReleasePlan" + ) + if version_resolution.contract_id != release_plan.contract_id: + raise ReleaseValidationError( + "VersionResolution and ReleasePlan contract IDs do not match" + ) + if version_resolution.release_revision_ref != release_plan.release_revision_ref: + raise ReleaseValidationError( + "VersionResolution release revision does not match ReleasePlan" + ) + if version_resolution.required_version_bump != release_plan.required_version_bump: + raise ReleaseValidationError( + "VersionResolution required version bump does not match ReleasePlan" + ) + + +def _evidence_matches_release_context( + evidence: ReviewAuthorizationEvidence, + *, + decision: GovernanceDecision, + change_set: ChangeSet, + release_plan: ReleasePlan, + version_resolution: VersionResolution, + operation: GovernanceOperation, +) -> bool: + return ( + evidence.decision_id == decision.decision_id + and evidence.change_set_id == change_set.change_set_id + and evidence.release_plan_id == release_plan.release_plan_id + and evidence.version_resolution_id == version_resolution.version_resolution_id + and evidence.operation is operation + ) + + +def _build_authorization( + *, + decision: GovernanceDecision, + change_set: ChangeSet, + release_plan: ReleasePlan, + version_resolution: VersionResolution, + operation: GovernanceOperation, + allowed: bool, + reason: AuthorizationReason, + evidence: ReviewAuthorizationEvidence | None = None, +) -> ContractOpsAuthorization: + evidence_reference = evidence.evidence_reference if evidence is not None else None + evidence_action = evidence.action if evidence is not None else None + + stable_record = { + "decision_id": decision.decision_id, + "change_set_id": change_set.change_set_id, + "release_plan_id": release_plan.release_plan_id, + "version_resolution_id": version_resolution.version_resolution_id, + "operation": operation.value, + "allowed": allowed, + "reason": reason.value, + "evidence_reference": evidence_reference, + "evidence_action": evidence_action.value if evidence_action is not None else None, + } + canonical_payload = json.dumps( + stable_record, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) + authorization_id = str( + uuid.uuid5(SEMAPACT_CONTRACTOPS_AUTHORIZATION_NAMESPACE, canonical_payload) + ) + + return ContractOpsAuthorization( + authorization_id=authorization_id, + decision_id=decision.decision_id, + change_set_id=change_set.change_set_id, + release_plan_id=release_plan.release_plan_id, + version_resolution_id=version_resolution.version_resolution_id, + operation=operation, + allowed=allowed, + reason=reason, + evidence_reference=evidence_reference, + evidence_action=evidence_action, + ) diff --git a/semapact/contractops/models.py b/semapact/contractops/models.py index bd0c6ee5..37fc429f 100644 --- a/semapact/contractops/models.py +++ b/semapact/contractops/models.py @@ -4,10 +4,11 @@ from enum import Enum -from pydantic import BaseModel, ConfigDict, field_validator, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from semapact.change_context import ChangeContext from semapact.core.release import ActualVersionBump, RequiredBump +from semapact.governance.gate import GovernanceOperation from semapact.lifecycle.changes import GovernanceChange @@ -166,3 +167,121 @@ def _validate_authority_reference(self) -> VersionResolution: "SemaPact version resolution must not contain authority_reference" ) return self + + +class ReviewEvidenceAction(str, Enum): + """Explicit review outcome consumed by ContractOps authorization.""" + + APPROVE = "APPROVE" + REJECT = "REJECT" + REQUEST_CHANGES = "REQUEST_CHANGES" + + +class ReviewAuthorizationEvidence(ContractOpsModel): + """Opaque review evidence projected onto one exact version-resolved action.""" + + evidence_reference: str + decision_id: str + change_set_id: str + release_plan_id: str + version_resolution_id: str + operation: GovernanceOperation + action: ReviewEvidenceAction + + @field_validator( + "evidence_reference", + "decision_id", + "change_set_id", + "release_plan_id", + "version_resolution_id", + ) + @classmethod + def _require_review_evidence_text(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("value must not be empty") + return cleaned + + +class AuthorizationReason(str, Enum): + """Machine-readable outcome of ContractOps authorization composition.""" + + ALLOWED_BY_GOVERNANCE = "allowed_by_governance" + ALLOWED_BY_REVIEW = "allowed_by_review" + BLOCKED_BY_GOVERNANCE = "blocked_by_governance" + REVIEW_AUTHORIZATION_REQUIRED = "review_authorization_required" + REVIEW_AUTHORIZATION_REJECTED = "review_authorization_rejected" + REVIEW_AUTHORIZATION_MISMATCH = "review_authorization_mismatch" + + +class ContractOpsAuthorization(ContractOpsModel): + """Authoritative authorization result for one exact ContractOps operation.""" + + authorization_id: str + decision_id: str + change_set_id: str + release_plan_id: str + version_resolution_id: str + operation: GovernanceOperation + allowed: bool = Field(strict=True) + reason: AuthorizationReason + evidence_reference: str | None = None + evidence_action: ReviewEvidenceAction | None = None + + @field_validator( + "authorization_id", + "decision_id", + "change_set_id", + "release_plan_id", + "version_resolution_id", + ) + @classmethod + def _require_authorization_text(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("value must not be empty") + return cleaned + + @field_validator("evidence_reference") + @classmethod + def _normalize_evidence_reference(cls, value: str | None) -> str | None: + if value is None: + return None + cleaned = value.strip() + return cleaned or None + + @model_validator(mode="after") + def _validate_authorization_invariants(self) -> ContractOpsAuthorization: + allowed_reasons = { + AuthorizationReason.ALLOWED_BY_GOVERNANCE, + AuthorizationReason.ALLOWED_BY_REVIEW, + } + if self.allowed != (self.reason in allowed_reasons): + raise ValueError("ContractOpsAuthorization allowed/reason invariant violation") + + evidence_reasons = { + AuthorizationReason.ALLOWED_BY_REVIEW, + AuthorizationReason.REVIEW_AUTHORIZATION_REJECTED, + AuthorizationReason.REVIEW_AUTHORIZATION_MISMATCH, + } + if self.reason in evidence_reasons: + if self.evidence_reference is None or self.evidence_action is None: + raise ValueError( + "ContractOpsAuthorization evidence reason requires evidence provenance" + ) + elif self.evidence_reference is not None or self.evidence_action is not None: + raise ValueError( + "ContractOpsAuthorization non-evidence reason must not contain evidence provenance" + ) + + if ( + self.reason is AuthorizationReason.ALLOWED_BY_REVIEW + and self.evidence_action is not ReviewEvidenceAction.APPROVE + ): + raise ValueError("Allowed review authorization requires APPROVE evidence") + if ( + self.reason is AuthorizationReason.REVIEW_AUTHORIZATION_REJECTED + and self.evidence_action is ReviewEvidenceAction.APPROVE + ): + raise ValueError("Rejected review authorization cannot contain APPROVE evidence") + return self diff --git a/tests/test_contractops_authorization.py b/tests/test_contractops_authorization.py new file mode 100644 index 00000000..04e4ddb9 --- /dev/null +++ b/tests/test_contractops_authorization.py @@ -0,0 +1,328 @@ +from __future__ import annotations + +from datetime import date + +import pytest +from open_data_contract_standard.model import ( + OpenDataContractStandard, + SchemaObject, + SchemaProperty, +) + +from semapact.change_context import ChangeContext +from semapact.contractops import ( + AuthorizationReason, + ReleasePlan, + ReviewAuthorizationEvidence, + ReviewEvidenceAction, + VersionAuthorityConfig, + authorize_contract_operation, + build_change_set_from_decision, + build_release_plan, + resolve_release_version, +) +from semapact.exceptions import ReleaseValidationError +from semapact.governance import DecisionResult, evaluate_governance_decision +from semapact.governance.gate import GovernanceOperation + + +CONTEXT = ChangeContext(effective_date=date(2026, 9, 9)) + + +def _contract( + *, + contract_id: str = "orders-product", + contract_name: str | None = None, + include_created_at: bool = False, +) -> OpenDataContractStandard: + properties = [ + SchemaProperty( + name="id", + logicalType="string", + physicalType="varchar(255)", + required=True, + ) + ] + if include_created_at: + properties.append( + SchemaProperty( + name="created_at", + logicalType="timestamp", + physicalType="timestamp", + required=False, + ) + ) + return OpenDataContractStandard( + apiVersion="v3.1.0", + kind="DataContract", + id=contract_id, + name=contract_name or contract_id, + version="1.0.0", + status="active", + schema=[SchemaObject(name="orders", properties=properties)], + ) + + +def _release_context(kind: str): + base = _contract(contract_name="orders-old") + if kind == "allow": + candidate = _contract(contract_name="orders-new") + elif kind == "review": + candidate = _contract(contract_name="orders-old", include_created_at=True) + elif kind == "block": + candidate = _contract(contract_id="other-product", contract_name="orders-old") + else: # pragma: no cover - test helper guard + raise ValueError(kind) + + decision = evaluate_governance_decision(base, candidate, context=CONTEXT) + change_set = build_change_set_from_decision( + decision, + base_revision_ref="rev:base", + candidate_revision_ref="rev:candidate", + source="test", + actor_reference="actor:test", + ) + + if decision.decision is DecisionResult.BLOCK: + # BLOCK cannot normally produce a ReleasePlan. Constructing an internally + # associated plan here proves that explicit review evidence still cannot + # override the authoritative M0 BLOCK result. + release_plan = ReleasePlan( + release_plan_id="release-plan-block-test", + contract_id=change_set.contract_id, + change_set_id=change_set.change_set_id, + decision_id=decision.decision_id, + release_revision_ref=change_set.candidate_revision_ref, + required_version_bump=decision.required_version_bump, + ) + else: + release_plan = build_release_plan(change_set, decision) + + version_resolution = resolve_release_version( + release_plan, + current_version="1.0.0", + config=VersionAuthorityConfig(), + ) + return decision, change_set, release_plan, version_resolution + + +def _evidence( + decision, + change_set, + release_plan, + version_resolution, + *, + operation: GovernanceOperation = GovernanceOperation.PUBLISH, + action: ReviewEvidenceAction = ReviewEvidenceAction.APPROVE, + evidence_reference: str = "approval:test-1", + version_resolution_id: str | None = None, +) -> ReviewAuthorizationEvidence: + return ReviewAuthorizationEvidence( + evidence_reference=evidence_reference, + decision_id=decision.decision_id, + change_set_id=change_set.change_set_id, + release_plan_id=release_plan.release_plan_id, + version_resolution_id=( + version_resolution_id or version_resolution.version_resolution_id + ), + operation=operation, + action=action, + ) + + +def test_allow_requires_no_synthetic_review_evidence() -> None: + decision, change_set, release_plan, version_resolution = _release_context("allow") + + result = authorize_contract_operation( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.PUBLISH, + ) + + assert decision.decision is DecisionResult.ALLOW + assert result.allowed is True + assert result.reason is AuthorizationReason.ALLOWED_BY_GOVERNANCE + assert result.evidence_reference is None + assert result.evidence_action is None + + +def test_review_without_evidence_remains_denied() -> None: + decision, change_set, release_plan, version_resolution = _release_context("review") + + result = authorize_contract_operation( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.PUBLISH, + ) + + assert decision.decision is DecisionResult.REVIEW + assert result.allowed is False + assert result.reason is AuthorizationReason.REVIEW_AUTHORIZATION_REQUIRED + + +def test_matching_approval_authorizes_review_without_rewriting_decision() -> None: + decision, change_set, release_plan, version_resolution = _release_context("review") + evidence = _evidence(decision, change_set, release_plan, version_resolution) + + result = authorize_contract_operation( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.PUBLISH, + evidence=evidence, + ) + + assert decision.decision is DecisionResult.REVIEW + assert result.allowed is True + assert result.reason is AuthorizationReason.ALLOWED_BY_REVIEW + assert result.evidence_reference == evidence.evidence_reference + assert result.evidence_action is ReviewEvidenceAction.APPROVE + + +@pytest.mark.parametrize( + "action", + [ReviewEvidenceAction.REJECT, ReviewEvidenceAction.REQUEST_CHANGES], +) +def test_rejected_review_evidence_cannot_authorize( + action: ReviewEvidenceAction, +) -> None: + decision, change_set, release_plan, version_resolution = _release_context("review") + evidence = _evidence( + decision, + change_set, + release_plan, + version_resolution, + action=action, + ) + + result = authorize_contract_operation( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.PUBLISH, + evidence=evidence, + ) + + assert result.allowed is False + assert result.reason is AuthorizationReason.REVIEW_AUTHORIZATION_REJECTED + assert result.evidence_action is action + + +def test_stale_version_resolution_evidence_cannot_authorize() -> None: + decision, change_set, release_plan, version_resolution = _release_context("review") + evidence = _evidence( + decision, + change_set, + release_plan, + version_resolution, + version_resolution_id="version-resolution:stale", + ) + + result = authorize_contract_operation( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.PUBLISH, + evidence=evidence, + ) + + assert result.allowed is False + assert result.reason is AuthorizationReason.REVIEW_AUTHORIZATION_MISMATCH + + +def test_approval_is_operation_scoped() -> None: + decision, change_set, release_plan, version_resolution = _release_context("review") + evidence = _evidence( + decision, + change_set, + release_plan, + version_resolution, + operation=GovernanceOperation.APPLY, + ) + + result = authorize_contract_operation( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.PUBLISH, + evidence=evidence, + ) + + assert result.allowed is False + assert result.reason is AuthorizationReason.REVIEW_AUTHORIZATION_MISMATCH + + +def test_block_cannot_be_overridden_by_approval_evidence() -> None: + decision, change_set, release_plan, version_resolution = _release_context("block") + evidence = _evidence(decision, change_set, release_plan, version_resolution) + + result = authorize_contract_operation( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.PUBLISH, + evidence=evidence, + ) + + assert decision.decision is DecisionResult.BLOCK + assert result.allowed is False + assert result.reason is AuthorizationReason.BLOCKED_BY_GOVERNANCE + assert result.evidence_reference is None + assert result.evidence_action is None + + +def test_invalid_release_context_fails_closed_before_authorization() -> None: + decision, change_set, release_plan, version_resolution = _release_context("review") + invalid_plan = ReleasePlan( + release_plan_id=release_plan.release_plan_id, + contract_id=release_plan.contract_id, + change_set_id="change-set:other", + decision_id=release_plan.decision_id, + release_revision_ref=release_plan.release_revision_ref, + required_version_bump=release_plan.required_version_bump, + preconditions=release_plan.preconditions, + ) + + with pytest.raises(ReleaseValidationError, match="supplied ChangeSet"): + authorize_contract_operation( + decision, + change_set, + invalid_plan, + version_resolution, + GovernanceOperation.PUBLISH, + ) + + +def test_authorization_is_deterministic_for_equivalent_inputs() -> None: + decision, change_set, release_plan, version_resolution = _release_context("review") + evidence = _evidence(decision, change_set, release_plan, version_resolution) + + first = authorize_contract_operation( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.PUBLISH, + evidence=evidence, + ) + second = authorize_contract_operation( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.PUBLISH, + evidence=evidence, + ) + + assert first == second + assert first.authorization_id == second.authorization_id + assert first.model_dump(mode="json") == second.model_dump(mode="json") From 30ab39767354f1cbc963e831ecf6bc08de993500 Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Wed, 9 Sep 2026 22:17:21 +1000 Subject: [PATCH 07/35] feat(contractops): enforce explicit apply and publish phases * refactor(contractops): share release context validation * refactor(contractops): reuse release context validator * feat(contractops): add explicit authorization error * feat(contractops): model applied releases and publication results * feat(contractops): add explicit apply and publish boundaries * feat(contractops): export apply and publish boundaries * test(contractops): cover explicit apply and publish boundaries * docs(contractops): define explicit release execution phases * chore: noop * chore: remove accidental noop file --- docs/contractops_phases.md | 117 +++++++ semapact/contractops/__init__.py | 14 + semapact/contractops/authorization.py | 59 +--- semapact/contractops/context.py | 62 ++++ semapact/contractops/execution.py | 278 +++++++++++++++ semapact/contractops/execution_models.py | 77 +++++ semapact/exceptions.py | 6 +- tests/test_contractops_execution.py | 415 +++++++++++++++++++++++ 8 files changed, 970 insertions(+), 58 deletions(-) create mode 100644 docs/contractops_phases.md create mode 100644 semapact/contractops/context.py create mode 100644 semapact/contractops/execution.py create mode 100644 semapact/contractops/execution_models.py create mode 100644 tests/test_contractops_execution.py diff --git a/docs/contractops_phases.md b/docs/contractops_phases.md new file mode 100644 index 00000000..c36b7118 --- /dev/null +++ b/docs/contractops_phases.md @@ -0,0 +1,117 @@ +# ContractOps execution phases + +ContractOps separates reasoning from side effects so CI, agents, APIs, and user interfaces can inspect a governed release before anything external changes. + +## Canonical contract release flow + +```text +ANALYZE +→ GovernanceDecision + +PLAN +→ ChangeSet +→ ReleasePlan +→ VersionResolution + +AUTHORIZE +→ ContractOpsAuthorization + +APPLY +→ AppliedContractRelease + +PUBLISH +→ PublicationResult +``` + +Each phase consumes artifacts from the previous phases. Later phases do not recalculate earlier decisions. + +## ANALYZE + +ANALYZE evaluates the candidate against the governed base contract and produces the authoritative `GovernanceDecision`. + +It is pure. It does not write files, create Git branches or tags, update ODCS, or mutate a runtime platform. + +## PLAN + +PLAN converts the authoritative decision into deterministic release artifacts: + +- `ChangeSet` identifies the exact base and candidate revisions and carries the authoritative governance changes. +- `ReleasePlan` describes the exact candidate revision intended for release and its minimum required version bump. +- `VersionResolution` selects the actual release version according to the configured version authority. + +PLAN remains pure. + +## AUTHORIZE + +AUTHORIZE evaluates whether one exact operation can cross a side-effect boundary. + +An authorization is scoped to: + +```text +decisionId +changeSetId +releasePlanId +versionResolutionId +operation +``` + +`GovernanceDecision(REVIEW)` remains `REVIEW` after approval. Matching explicit review evidence produces an allowed `ContractOpsAuthorization`; it does not rewrite governance history. + +APPLY and PUBLISH require separate operation-scoped authorizations. + +## APPLY + +APPLY materializes the exact released ODCS state from the planned candidate and `VersionResolution.selectedVersion`. + +The canonical APPLY path: + +- requires an allowed `ContractOpsAuthorization(operation=APPLY)`; +- requires the supplied candidate revision reference to match the planned revision; +- validates contract identity and the expected current version; +- copies the candidate and synchronizes only the selected release version; +- does not mutate the input candidate; +- does not rerun diffing, lifecycle policy, breaking-change classification, or version authority. + +The output is an immutable `AppliedContractRelease` containing provenance IDs and a canonical JSON snapshot of the released ODCS state. Consumers can materialize a fresh ODCS model from that snapshot. + +This also covers metadata-only governed releases: `requiredVersionBump=none` may have been resolved by SemaPact version authority to an actual patch release, and APPLY uses that already-selected version directly. + +## PUBLISH + +PUBLISH is the first explicit external publication boundary. + +It requires an allowed `ContractOpsAuthorization(operation=PUBLISH)` matching the exact applied release context before the publisher adapter is invoked. An APPLY authorization cannot authorize PUBLISH. + +ContractOps defines only a narrow publisher port: + +```text +AppliedContractRelease + ↓ +ContractReleasePublisher.publish(...) + ↓ +opaque publication reference + ↓ +PublicationResult +``` + +Git, storage, deployment, and platform-specific publication behavior belongs in adapters rather than the ContractOps domain. + +## Failure semantics + +ContractOps distinguishes invalid context from denied authorization: + +- mismatched revision/artifact/operation context → `ReleaseValidationError`; +- a valid context whose explicit authorization is denied → `ContractOpsAuthorizationError`; +- a publisher is never invoked when authorization validation fails. + +Unexpected publisher/runtime failures are not converted into governance decisions; they propagate as execution failures. + +## Legacy release helpers + +`semapact.core.release.prepare_release_candidate()` remains a backward-compatible helper and is not the canonical ContractOps APPLY path because it may classify changes itself. + +New ContractOps flows consume the existing authoritative `GovernanceDecision`, `ReleasePlan`, and `VersionResolution` instead of recomputing them. + +## Deployment boundary + +`AppliedContractRelease` is the released contract state that downstream deployment planning can consume. `DeploymentPlan` and platform deployment adapters are intentionally handled by later M2 work and are not part of the contract-release phase implementation. diff --git a/semapact/contractops/__init__.py b/semapact/contractops/__init__.py index d21454b3..4123e916 100644 --- a/semapact/contractops/__init__.py +++ b/semapact/contractops/__init__.py @@ -5,6 +5,15 @@ build_change_set, build_change_set_from_decision, ) +from semapact.contractops.execution import ( + ContractReleasePublisher, + apply_contract_release, + publish_contract_release, +) +from semapact.contractops.execution_models import ( + AppliedContractRelease, + PublicationResult, +) from semapact.contractops.models import ( AuthorizationReason, ChangeSet, @@ -24,9 +33,12 @@ ) __all__ = [ + "AppliedContractRelease", "AuthorizationReason", "ChangeSet", "ContractOpsAuthorization", + "ContractReleasePublisher", + "PublicationResult", "ReleasePlan", "ReleasePrecondition", "ReviewAuthorizationEvidence", @@ -34,10 +46,12 @@ "VersionAuthority", "VersionAuthorityConfig", "VersionResolution", + "apply_contract_release", "authorize_contract_operation", "build_change_set", "build_change_set_from_decision", "build_release_plan", "extract_version_from_release_reference", + "publish_contract_release", "resolve_release_version", ] diff --git a/semapact/contractops/authorization.py b/semapact/contractops/authorization.py index 9c498238..56819f43 100644 --- a/semapact/contractops/authorization.py +++ b/semapact/contractops/authorization.py @@ -5,6 +5,7 @@ import json import uuid +from semapact.contractops.context import validate_release_context from semapact.contractops.models import ( AuthorizationReason, ChangeSet, @@ -14,7 +15,6 @@ ReviewEvidenceAction, VersionResolution, ) -from semapact.exceptions import ReleaseValidationError from semapact.governance.gate import GovernanceOperation, evaluate_governance_gate from semapact.governance.models import GovernanceDecision @@ -47,7 +47,7 @@ def authorize_contract_operation( operation, evidence, ) - _validate_release_context(decision, change_set, release_plan, version_resolution) + validate_release_context(decision, change_set, release_plan, version_resolution) gate = evaluate_governance_gate(decision, operation) @@ -157,61 +157,6 @@ def _validate_types( ) -def _validate_release_context( - decision: GovernanceDecision, - change_set: ChangeSet, - release_plan: ReleasePlan, - version_resolution: VersionResolution, -) -> None: - """Fail closed when immutable artifacts do not describe one release context.""" - if change_set.contract_id != decision.contract_id: - raise ReleaseValidationError( - "ChangeSet and GovernanceDecision contract IDs do not match" - ) - if change_set.context != decision.context: - raise ReleaseValidationError( - "ChangeSet and GovernanceDecision governance contexts do not match" - ) - if change_set.changes != decision.changes: - raise ReleaseValidationError( - "ChangeSet changes do not match authoritative GovernanceDecision changes" - ) - - if release_plan.contract_id != change_set.contract_id: - raise ReleaseValidationError("ReleasePlan and ChangeSet contract IDs do not match") - if release_plan.change_set_id != change_set.change_set_id: - raise ReleaseValidationError("ReleasePlan does not reference the supplied ChangeSet") - if release_plan.decision_id != decision.decision_id: - raise ReleaseValidationError( - "ReleasePlan does not reference the supplied GovernanceDecision" - ) - if release_plan.release_revision_ref != change_set.candidate_revision_ref: - raise ReleaseValidationError( - "ReleasePlan release revision does not match ChangeSet candidate revision" - ) - if release_plan.required_version_bump != decision.required_version_bump: - raise ReleaseValidationError( - "ReleasePlan required version bump does not match GovernanceDecision" - ) - - if version_resolution.release_plan_id != release_plan.release_plan_id: - raise ReleaseValidationError( - "VersionResolution does not reference the supplied ReleasePlan" - ) - if version_resolution.contract_id != release_plan.contract_id: - raise ReleaseValidationError( - "VersionResolution and ReleasePlan contract IDs do not match" - ) - if version_resolution.release_revision_ref != release_plan.release_revision_ref: - raise ReleaseValidationError( - "VersionResolution release revision does not match ReleasePlan" - ) - if version_resolution.required_version_bump != release_plan.required_version_bump: - raise ReleaseValidationError( - "VersionResolution required version bump does not match ReleasePlan" - ) - - def _evidence_matches_release_context( evidence: ReviewAuthorizationEvidence, *, diff --git a/semapact/contractops/context.py b/semapact/contractops/context.py new file mode 100644 index 00000000..92921075 --- /dev/null +++ b/semapact/contractops/context.py @@ -0,0 +1,62 @@ +"""Shared fail-closed validation for one ContractOps release context.""" + +from __future__ import annotations + +from semapact.contractops.models import ChangeSet, ReleasePlan, VersionResolution +from semapact.exceptions import ReleaseValidationError +from semapact.governance.models import GovernanceDecision + + +def validate_release_context( + decision: GovernanceDecision, + change_set: ChangeSet, + release_plan: ReleasePlan, + version_resolution: VersionResolution, +) -> None: + """Fail closed unless immutable artifacts describe one exact release context.""" + if change_set.contract_id != decision.contract_id: + raise ReleaseValidationError( + "ChangeSet and GovernanceDecision contract IDs do not match" + ) + if change_set.context != decision.context: + raise ReleaseValidationError( + "ChangeSet and GovernanceDecision governance contexts do not match" + ) + if change_set.changes != decision.changes: + raise ReleaseValidationError( + "ChangeSet changes do not match authoritative GovernanceDecision changes" + ) + + if release_plan.contract_id != change_set.contract_id: + raise ReleaseValidationError("ReleasePlan and ChangeSet contract IDs do not match") + if release_plan.change_set_id != change_set.change_set_id: + raise ReleaseValidationError("ReleasePlan does not reference the supplied ChangeSet") + if release_plan.decision_id != decision.decision_id: + raise ReleaseValidationError( + "ReleasePlan does not reference the supplied GovernanceDecision" + ) + if release_plan.release_revision_ref != change_set.candidate_revision_ref: + raise ReleaseValidationError( + "ReleasePlan release revision does not match ChangeSet candidate revision" + ) + if release_plan.required_version_bump != decision.required_version_bump: + raise ReleaseValidationError( + "ReleasePlan required version bump does not match GovernanceDecision" + ) + + if version_resolution.release_plan_id != release_plan.release_plan_id: + raise ReleaseValidationError( + "VersionResolution does not reference the supplied ReleasePlan" + ) + if version_resolution.contract_id != release_plan.contract_id: + raise ReleaseValidationError( + "VersionResolution and ReleasePlan contract IDs do not match" + ) + if version_resolution.release_revision_ref != release_plan.release_revision_ref: + raise ReleaseValidationError( + "VersionResolution release revision does not match ReleasePlan" + ) + if version_resolution.required_version_bump != release_plan.required_version_bump: + raise ReleaseValidationError( + "VersionResolution required version bump does not match ReleasePlan" + ) diff --git a/semapact/contractops/execution.py b/semapact/contractops/execution.py new file mode 100644 index 00000000..a59cdb34 --- /dev/null +++ b/semapact/contractops/execution.py @@ -0,0 +1,278 @@ +"""Explicit ContractOps APPLY and PUBLISH execution boundaries.""" + +from __future__ import annotations + +import json +import uuid +from typing import Protocol + +from open_data_contract_standard.model import OpenDataContractStandard + +from semapact.contractops.context import validate_release_context +from semapact.contractops.execution_models import AppliedContractRelease, PublicationResult +from semapact.contractops.models import ( + ChangeSet, + ContractOpsAuthorization, + ReleasePlan, + VersionResolution, +) +from semapact.core.release import normalize_semver +from semapact.exceptions import ContractOpsAuthorizationError, ReleaseValidationError +from semapact.governance.gate import GovernanceOperation +from semapact.governance.models import GovernanceDecision + + +SEMAPACT_APPLIED_RELEASE_NAMESPACE = uuid.UUID( + "a5de2e65-aee7-48ac-9cb6-6a785f4cdf33" +) +SEMAPACT_PUBLICATION_NAMESPACE = uuid.UUID( + "f776fc77-b37f-43ef-bf6d-d8dcf38d264f" +) + + +class ContractReleasePublisher(Protocol): + """Narrow external publication port for one applied contract release.""" + + def publish(self, release: AppliedContractRelease) -> str: + """Publish the exact applied release and return an opaque reference.""" + ... + + +def apply_contract_release( + candidate_contract: OpenDataContractStandard, + *, + candidate_revision_ref: str, + decision: GovernanceDecision, + change_set: ChangeSet, + release_plan: ReleasePlan, + version_resolution: VersionResolution, + authorization: ContractOpsAuthorization, +) -> AppliedContractRelease: + """Materialize the exact released ODCS state after explicit APPLY authorization. + + This is the canonical M2 apply path. It never re-runs governance, change + classification, or version authority. The input candidate is not mutated. + """ + if not isinstance(candidate_contract, OpenDataContractStandard): + raise TypeError( + "candidate_contract must be OpenDataContractStandard, " + f"got {type(candidate_contract).__name__}" + ) + if not isinstance(candidate_revision_ref, str): + raise TypeError("candidate_revision_ref must be str") + + validate_release_context(decision, change_set, release_plan, version_resolution) + _validate_authorization( + authorization, + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + operation=GovernanceOperation.APPLY, + ) + + supplied_revision_ref = candidate_revision_ref.strip() + if not supplied_revision_ref: + raise ReleaseValidationError("candidate_revision_ref must not be empty") + if supplied_revision_ref != release_plan.release_revision_ref: + raise ReleaseValidationError( + "Supplied candidate revision does not match the planned release revision" + ) + + if str(candidate_contract.id or "") != release_plan.contract_id: + raise ReleaseValidationError( + "Candidate contract ID does not match the planned release contract" + ) + + current_version = _canonical_version( + version_resolution.current_version, + field_name="VersionResolution current_version", + ) + selected_version = _canonical_version( + version_resolution.selected_version, + field_name="VersionResolution selected_version", + ) + candidate_version = _canonical_version( + str(candidate_contract.version or ""), + field_name="candidate contract version", + ) + if candidate_version != current_version: + raise ReleaseValidationError( + "Candidate contract version does not match VersionResolution current_version" + ) + + released_contract = candidate_contract.model_copy(deep=True) + released_contract.version = selected_version + released_contract_json = _canonical_contract_json(released_contract) + + stable_record = { + "contract_id": release_plan.contract_id, + "decision_id": decision.decision_id, + "change_set_id": change_set.change_set_id, + "release_plan_id": release_plan.release_plan_id, + "version_resolution_id": version_resolution.version_resolution_id, + "release_revision_ref": release_plan.release_revision_ref, + "selected_version": selected_version, + "authorization_id": authorization.authorization_id, + "released_contract_json": released_contract_json, + } + applied_release_id = str( + uuid.uuid5( + SEMAPACT_APPLIED_RELEASE_NAMESPACE, + json.dumps( + stable_record, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ), + ) + ) + + return AppliedContractRelease( + applied_release_id=applied_release_id, + contract_id=release_plan.contract_id, + decision_id=decision.decision_id, + change_set_id=change_set.change_set_id, + release_plan_id=release_plan.release_plan_id, + version_resolution_id=version_resolution.version_resolution_id, + release_revision_ref=release_plan.release_revision_ref, + selected_version=selected_version, + authorization_id=authorization.authorization_id, + released_contract_json=released_contract_json, + ) + + +def publish_contract_release( + release: AppliedContractRelease, + *, + authorization: ContractOpsAuthorization, + publisher: ContractReleasePublisher, +) -> PublicationResult: + """Invoke one external publisher only after exact PUBLISH authorization.""" + if not isinstance(release, AppliedContractRelease): + raise TypeError( + f"release must be AppliedContractRelease, got {type(release).__name__}" + ) + if not isinstance(authorization, ContractOpsAuthorization): + raise TypeError( + "authorization must be ContractOpsAuthorization, " + f"got {type(authorization).__name__}" + ) + if not hasattr(publisher, "publish") or not callable(publisher.publish): + raise TypeError("publisher must provide a callable publish(release) method") + + _validate_publication_authorization(release, authorization) + + publication_reference = publisher.publish(release) + if not isinstance(publication_reference, str): + raise TypeError("publisher.publish() must return str") + publication_reference = publication_reference.strip() + if not publication_reference: + raise ReleaseValidationError( + "publisher.publish() returned an empty publication reference" + ) + + stable_record = { + "applied_release_id": release.applied_release_id, + "authorization_id": authorization.authorization_id, + "publication_reference": publication_reference, + } + publication_id = str( + uuid.uuid5( + SEMAPACT_PUBLICATION_NAMESPACE, + json.dumps( + stable_record, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ), + ) + ) + return PublicationResult( + publication_id=publication_id, + applied_release_id=release.applied_release_id, + authorization_id=authorization.authorization_id, + publication_reference=publication_reference, + ) + + +def _validate_authorization( + authorization: ContractOpsAuthorization, + *, + decision: GovernanceDecision, + change_set: ChangeSet, + release_plan: ReleasePlan, + version_resolution: VersionResolution, + operation: GovernanceOperation, +) -> None: + if not isinstance(authorization, ContractOpsAuthorization): + raise TypeError( + "authorization must be ContractOpsAuthorization, " + f"got {type(authorization).__name__}" + ) + if authorization.operation is not operation: + raise ReleaseValidationError( + f"Authorization operation must be {operation.value}, " + f"got {authorization.operation.value}" + ) + if ( + authorization.decision_id != decision.decision_id + or authorization.change_set_id != change_set.change_set_id + or authorization.release_plan_id != release_plan.release_plan_id + or authorization.version_resolution_id + != version_resolution.version_resolution_id + ): + raise ReleaseValidationError( + "Authorization does not match the exact version-resolved release context" + ) + if not authorization.allowed: + raise ContractOpsAuthorizationError( + f"ContractOps {operation.value} is not authorized: " + f"{authorization.reason.value}" + ) + + +def _validate_publication_authorization( + release: AppliedContractRelease, + authorization: ContractOpsAuthorization, +) -> None: + if authorization.operation is not GovernanceOperation.PUBLISH: + raise ReleaseValidationError( + "Publication requires operation-scoped PUBLISH authorization" + ) + if ( + authorization.decision_id != release.decision_id + or authorization.change_set_id != release.change_set_id + or authorization.release_plan_id != release.release_plan_id + or authorization.version_resolution_id != release.version_resolution_id + ): + raise ReleaseValidationError( + "PUBLISH authorization does not match the applied release context" + ) + if not authorization.allowed: + raise ContractOpsAuthorizationError( + "ContractOps PUBLISH is not authorized: " + f"{authorization.reason.value}" + ) + + +def _canonical_version(version: str, *, field_name: str) -> str: + try: + canonical = normalize_semver(version) + except ValueError as exc: + raise ReleaseValidationError(f"{field_name} is not valid semantic version") from exc + if canonical != str(version).strip(): + raise ReleaseValidationError( + f"{field_name} must use canonical major.minor.patch form" + ) + return canonical + + +def _canonical_contract_json(contract: OpenDataContractStandard) -> str: + payload = contract.model_dump(mode="json", by_alias=True, exclude_none=True) + return json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) diff --git a/semapact/contractops/execution_models.py b/semapact/contractops/execution_models.py new file mode 100644 index 00000000..0604dfb1 --- /dev/null +++ b/semapact/contractops/execution_models.py @@ -0,0 +1,77 @@ +"""Immutable artifacts for explicit ContractOps APPLY and PUBLISH phases.""" + +from __future__ import annotations + +from open_data_contract_standard.model import OpenDataContractStandard +from pydantic import field_validator, model_validator + +from semapact.contractops.models import ContractOpsModel + + +class AppliedContractRelease(ContractOpsModel): + """Exact released ODCS state produced by an authorized APPLY operation.""" + + applied_release_id: str + contract_id: str + decision_id: str + change_set_id: str + release_plan_id: str + version_resolution_id: str + release_revision_ref: str + selected_version: str + authorization_id: str + released_contract_json: str + + @field_validator( + "applied_release_id", + "contract_id", + "decision_id", + "change_set_id", + "release_plan_id", + "version_resolution_id", + "release_revision_ref", + "selected_version", + "authorization_id", + "released_contract_json", + ) + @classmethod + def _require_non_empty_text(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("value must not be empty") + return cleaned + + @model_validator(mode="after") + def _validate_released_contract_snapshot(self) -> AppliedContractRelease: + contract = OpenDataContractStandard.model_validate_json(self.released_contract_json) + if str(contract.id or "") != self.contract_id: + raise ValueError("released contract ID does not match AppliedContractRelease") + if str(contract.version or "") != self.selected_version: + raise ValueError("released contract version does not match selected_version") + return self + + def to_contract(self) -> OpenDataContractStandard: + """Materialize a fresh mutable ODCS model from the immutable JSON snapshot.""" + return OpenDataContractStandard.model_validate_json(self.released_contract_json) + + +class PublicationResult(ContractOpsModel): + """Successful result of one explicitly authorized external publication.""" + + publication_id: str + applied_release_id: str + authorization_id: str + publication_reference: str + + @field_validator( + "publication_id", + "applied_release_id", + "authorization_id", + "publication_reference", + ) + @classmethod + def _require_publication_text(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("value must not be empty") + return cleaned diff --git a/semapact/exceptions.py b/semapact/exceptions.py index 5b80f188..2bf2aaee 100644 --- a/semapact/exceptions.py +++ b/semapact/exceptions.py @@ -26,13 +26,17 @@ class ValidationError(SemaPactError, ValueError): pass - class ReleaseValidationError(ValidationError): """Raised when release candidate validation fails (e.g. insufficient version bump or no changes to release).""" pass +class ContractOpsAuthorizationError(SemaPactError): + """Raised when an explicit ContractOps side effect is not authorized.""" + + pass + class MergeConflictError(SemaPactError): """Raised by the merge engine when business and technical metadata fatally conflict.""" diff --git a/tests/test_contractops_execution.py b/tests/test_contractops_execution.py new file mode 100644 index 00000000..164bf309 --- /dev/null +++ b/tests/test_contractops_execution.py @@ -0,0 +1,415 @@ +from __future__ import annotations + +from datetime import date + +import pytest +from open_data_contract_standard.model import ( + OpenDataContractStandard, + SchemaObject, + SchemaProperty, +) + +from semapact.change_context import ChangeContext +from semapact.contractops import ( + ReviewAuthorizationEvidence, + ReviewEvidenceAction, + VersionAuthorityConfig, + apply_contract_release, + authorize_contract_operation, + build_change_set_from_decision, + build_release_plan, + publish_contract_release, + resolve_release_version, +) +from semapact.exceptions import ContractOpsAuthorizationError, ReleaseValidationError +from semapact.governance import DecisionResult, evaluate_governance_decision +from semapact.governance.gate import GovernanceOperation + + +CONTEXT = ChangeContext(effective_date=date(2026, 9, 9)) + + +def _contract( + *, + contract_id: str = "orders-product", + contract_name: str = "orders", + version: str = "1.0.0", + include_created_at: bool = False, +) -> OpenDataContractStandard: + properties = [ + SchemaProperty( + name="id", + logicalType="string", + physicalType="varchar(255)", + required=True, + ) + ] + if include_created_at: + properties.append( + SchemaProperty( + name="created_at", + logicalType="timestamp", + physicalType="timestamp", + required=False, + ) + ) + return OpenDataContractStandard( + apiVersion="v3.1.0", + kind="DataContract", + id=contract_id, + name=contract_name, + version=version, + status="active", + schema=[SchemaObject(name="orders", properties=properties)], + ) + + +def _release_context(kind: str): + base = _contract(contract_name="orders-old") + if kind == "allow": + candidate = _contract(contract_name="orders-new") + elif kind == "review": + candidate = _contract(contract_name="orders-old", include_created_at=True) + else: # pragma: no cover - test helper guard + raise ValueError(kind) + + decision = evaluate_governance_decision(base, candidate, context=CONTEXT) + change_set = build_change_set_from_decision( + decision, + base_revision_ref="rev:base", + candidate_revision_ref="rev:candidate", + source="test", + actor_reference="actor:test", + ) + release_plan = build_release_plan(change_set, decision) + version_resolution = resolve_release_version( + release_plan, + current_version="1.0.0", + config=VersionAuthorityConfig(), + ) + return candidate, decision, change_set, release_plan, version_resolution + + +def _review_evidence( + decision, + change_set, + release_plan, + version_resolution, + operation: GovernanceOperation, +) -> ReviewAuthorizationEvidence: + return ReviewAuthorizationEvidence( + evidence_reference=f"approval:{operation.value.lower()}", + decision_id=decision.decision_id, + change_set_id=change_set.change_set_id, + release_plan_id=release_plan.release_plan_id, + version_resolution_id=version_resolution.version_resolution_id, + operation=operation, + action=ReviewEvidenceAction.APPROVE, + ) + + +def _authorization( + decision, + change_set, + release_plan, + version_resolution, + operation: GovernanceOperation, + *, + approve_review: bool = True, +): + evidence = None + if decision.decision is DecisionResult.REVIEW and approve_review: + evidence = _review_evidence( + decision, + change_set, + release_plan, + version_resolution, + operation, + ) + return authorize_contract_operation( + decision, + change_set, + release_plan, + version_resolution, + operation, + evidence=evidence, + ) + + +class RecordingPublisher: + def __init__(self) -> None: + self.calls = [] + + def publish(self, release) -> str: + self.calls.append(release) + return f"published:{release.applied_release_id}" + + +def test_apply_metadata_only_release_uses_selected_patch_without_mutating_candidate() -> None: + candidate, decision, change_set, release_plan, version_resolution = _release_context( + "allow" + ) + authorization = _authorization( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.APPLY, + ) + + assert decision.decision is DecisionResult.ALLOW + assert release_plan.required_version_bump == "none" + assert version_resolution.selected_version == "1.0.1" + + first = apply_contract_release( + candidate, + candidate_revision_ref="rev:candidate", + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + authorization=authorization, + ) + second = apply_contract_release( + candidate, + candidate_revision_ref="rev:candidate", + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + authorization=authorization, + ) + + assert first == second + assert first.applied_release_id == second.applied_release_id + assert first.selected_version == "1.0.1" + assert first.release_revision_ref == "rev:candidate" + assert first.to_contract().version == "1.0.1" + assert first.to_contract().name == "orders-new" + assert candidate.version == "1.0.0" + + +def test_authorized_review_can_apply_without_rewriting_decision() -> None: + candidate, decision, change_set, release_plan, version_resolution = _release_context( + "review" + ) + authorization = _authorization( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.APPLY, + ) + + release = apply_contract_release( + candidate, + candidate_revision_ref="rev:candidate", + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + authorization=authorization, + ) + + assert decision.decision is DecisionResult.REVIEW + assert authorization.allowed is True + assert release.selected_version == "1.1.0" + assert release.to_contract().version == "1.1.0" + assert len(release.to_contract().schema_[0].properties) == 2 + + +def test_review_without_approval_cannot_apply() -> None: + candidate, decision, change_set, release_plan, version_resolution = _release_context( + "review" + ) + authorization = _authorization( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.APPLY, + approve_review=False, + ) + + assert authorization.allowed is False + with pytest.raises(ContractOpsAuthorizationError, match="not authorized"): + apply_contract_release( + candidate, + candidate_revision_ref="rev:candidate", + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + authorization=authorization, + ) + + +def test_apply_fails_closed_for_stale_candidate_revision() -> None: + candidate, decision, change_set, release_plan, version_resolution = _release_context( + "allow" + ) + authorization = _authorization( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.APPLY, + ) + + with pytest.raises(ReleaseValidationError, match="planned release revision"): + apply_contract_release( + candidate, + candidate_revision_ref="rev:stale", + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + authorization=authorization, + ) + + +def test_apply_fails_closed_when_candidate_version_drifted() -> None: + candidate, decision, change_set, release_plan, version_resolution = _release_context( + "allow" + ) + authorization = _authorization( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.APPLY, + ) + drifted_candidate = candidate.model_copy(deep=True) + drifted_candidate.version = "9.0.0" + + with pytest.raises(ReleaseValidationError, match="current_version"): + apply_contract_release( + drifted_candidate, + candidate_revision_ref="rev:candidate", + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + authorization=authorization, + ) + + +def test_apply_authorization_cannot_authorize_publish() -> None: + candidate, decision, change_set, release_plan, version_resolution = _release_context( + "allow" + ) + apply_authorization = _authorization( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.APPLY, + ) + release = apply_contract_release( + candidate, + candidate_revision_ref="rev:candidate", + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + authorization=apply_authorization, + ) + publisher = RecordingPublisher() + + with pytest.raises(ReleaseValidationError, match="PUBLISH authorization"): + publish_contract_release( + release, + authorization=apply_authorization, + publisher=publisher, + ) + + assert publisher.calls == [] + + +def test_publish_invokes_adapter_only_after_exact_publish_authorization() -> None: + candidate, decision, change_set, release_plan, version_resolution = _release_context( + "allow" + ) + apply_authorization = _authorization( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.APPLY, + ) + publish_authorization = _authorization( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.PUBLISH, + ) + release = apply_contract_release( + candidate, + candidate_revision_ref="rev:candidate", + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + authorization=apply_authorization, + ) + publisher = RecordingPublisher() + + first = publish_contract_release( + release, + authorization=publish_authorization, + publisher=publisher, + ) + second = publish_contract_release( + release, + authorization=publish_authorization, + publisher=RecordingPublisher(), + ) + + assert len(publisher.calls) == 1 + assert publisher.calls[0] == release + assert first == second + assert first.publication_id == second.publication_id + assert first.applied_release_id == release.applied_release_id + assert first.authorization_id == publish_authorization.authorization_id + assert first.publication_reference == f"published:{release.applied_release_id}" + + +def test_review_without_publish_approval_never_invokes_publisher() -> None: + candidate, decision, change_set, release_plan, version_resolution = _release_context( + "review" + ) + apply_authorization = _authorization( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.APPLY, + ) + denied_publish_authorization = _authorization( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.PUBLISH, + approve_review=False, + ) + release = apply_contract_release( + candidate, + candidate_revision_ref="rev:candidate", + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + authorization=apply_authorization, + ) + publisher = RecordingPublisher() + + with pytest.raises(ContractOpsAuthorizationError, match="not authorized"): + publish_contract_release( + release, + authorization=denied_publish_authorization, + publisher=publisher, + ) + + assert publisher.calls == [] From 79854284a4a8d5d37af6414df611aeb099d6b151 Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Thu, 10 Sep 2026 08:29:08 +1000 Subject: [PATCH 08/35] feat(deployment): compile applied releases into deployment plans * feat(deployment): define deterministic deployment plan models * feat(deployment): build deployment plans from applied releases * feat(deployment): expose deployment planning API * test(deployment): cover deterministic deployment planning * docs(deployment): document deployment plan boundary * refactor(deployment): type released schema serialization explicitly * test(deployment): avoid type-ignore in immutability check --- docs/deployment_plans.md | 109 +++++++++++++++++ semapact/deployment/__init__.py | 17 +++ semapact/deployment/models.py | 134 +++++++++++++++++++++ semapact/deployment/planner.py | 108 +++++++++++++++++ tests/test_deployment_plan.py | 207 ++++++++++++++++++++++++++++++++ 5 files changed, 575 insertions(+) create mode 100644 docs/deployment_plans.md create mode 100644 semapact/deployment/__init__.py create mode 100644 semapact/deployment/models.py create mode 100644 semapact/deployment/planner.py create mode 100644 tests/test_deployment_plan.py diff --git a/docs/deployment_plans.md b/docs/deployment_plans.md new file mode 100644 index 00000000..33db0bc5 --- /dev/null +++ b/docs/deployment_plans.md @@ -0,0 +1,109 @@ +# Deployment plans + +SemaPact treats a released ODCS contract as governed desired state, not as an executable SQL, Terraform, or platform program. + +The deployment planning boundary is therefore: + +```text +AppliedContractRelease ++ DeploymentTarget + ↓ +DeploymentPlan + ↓ +platform adapter validate / preview / publish + ↓ +runtime + ↓ +reconciliation verifies convergence +``` + +## What a DeploymentPlan means + +A `DeploymentPlan` is a deterministic, provider-neutral statement of the runtime state that an exact applied contract release intends to converge toward. + +It is built only from `AppliedContractRelease`; drafts and raw candidate contracts are not deployment authority. + +The initial action vocabulary deliberately contains only: + +```text +ENSURE_ASSET_STATE +``` + +One action is emitted for each governed ODCS schema. The action carries the governed logical asset identity, its physical-name binding hint, and the canonical released schema snapshot. + +## Why plans do not say CREATE / ALTER / DROP + +Planning sees released desired state only. Without observed runtime state SemaPact cannot know whether a provider must create an asset, alter an existing asset, or do nothing. + +Likewise, an object that exists in runtime but is absent from one contract must not be interpreted as safe to drop. The contract may not own that object. + +Concrete provider-native operations therefore begin at the platform adapter boundary, where `validate` and `preview` can combine the DeploymentPlan with actual provider semantics and, where required, runtime evidence. + +## Identity and physical binding + +The same rule used by M1 reconciliation applies on the write side: + +```text +schema.name += governed logical identity + +schema.physicalName += deployment/runtime binding hint only +``` + +For example: + +```yaml +schema: + - name: Orders + physicalName: prod_orders_v2 +``` + +produces an action whose governed identity is `orders` while the physical binding hint remains `prod_orders_v2`. + +Changing a physical name does not redefine the governed contract identity. + +## Targeting + +A plan requires an explicit target: + +```text +DeploymentTarget +├── platform +├── runtimeTarget +└── serverName? # optional provenance +``` + +`platform` is the downstream adapter dispatch key. `runtimeTarget` is an opaque provider-local target descriptor at this layer. + +The plan does not contain credentials, workspace clients, SQL connections, or provider sessions. + +## Determinism + +`deploymentPlanId` is UUID5-derived from the full stable plan record: + +- exact `AppliedContractRelease` identity and provenance; +- exact deployment target; +- canonical actions ordered by governed asset identity; +- plan schema version. + +The same exact applied release and target therefore produce the same DeploymentPlan. + +Action ordering is canonical even when schemas appear in a different order in source ODCS. However, DeploymentPlan does not redefine release identity: two distinct `AppliedContractRelease` artifacts remain distinct authorities even if their projected actions happen to be equivalent. + +## Provider support belongs to the adapter + +DeploymentPlan intentionally does not contain generic `preconditions`, `adapterKey`, or guessed platform-specific operations. + +The next boundary is responsible for explicit support: + +```text +DeploymentAdapter +├── validate(plan) +├── preview(plan) +└── publish(plan, authorization) +``` + +A provider adapter must explicitly report unsupported ODCS-to-platform mappings. It must never silently ignore unsupported governed state. + +A successful plan or publish call is also not proof of convergence. Runtime convergence is verified separately through SemaPact reconciliation. diff --git a/semapact/deployment/__init__.py b/semapact/deployment/__init__.py new file mode 100644 index 00000000..719527d1 --- /dev/null +++ b/semapact/deployment/__init__.py @@ -0,0 +1,17 @@ +"""Provider-neutral deployment planning boundary.""" + +from semapact.deployment.models import ( + DeploymentAction, + DeploymentActionKind, + DeploymentPlan, + DeploymentTarget, +) +from semapact.deployment.planner import build_deployment_plan + +__all__ = [ + "DeploymentAction", + "DeploymentActionKind", + "DeploymentPlan", + "DeploymentTarget", + "build_deployment_plan", +] diff --git a/semapact/deployment/models.py b/semapact/deployment/models.py new file mode 100644 index 00000000..b7651dcf --- /dev/null +++ b/semapact/deployment/models.py @@ -0,0 +1,134 @@ +"""Provider-neutral immutable deployment planning artifacts.""" + +from __future__ import annotations + +from enum import Enum +from typing import Literal + +from open_data_contract_standard.model import SchemaObject +from pydantic import BaseModel, ConfigDict, field_validator, model_validator + +from semapact.lifecycle.identity import normalize_identity_name + + +class DeploymentModel(BaseModel): + """Shared immutable base for deployment planning artifacts.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + +class DeploymentActionKind(str, Enum): + """Provider-neutral convergence intents emitted by deployment planning.""" + + ENSURE_ASSET_STATE = "ENSURE_ASSET_STATE" + + +class DeploymentTarget(DeploymentModel): + """Explicit runtime target for one DeploymentPlan.""" + + platform: str + runtime_target: str + server_name: str | None = None + + @field_validator("platform") + @classmethod + def _normalize_platform(cls, value: str) -> str: + cleaned = value.strip().casefold() + if not cleaned: + raise ValueError("platform must not be empty") + return cleaned + + @field_validator("runtime_target") + @classmethod + def _normalize_runtime_target(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("runtime_target must not be empty") + return cleaned + + @field_validator("server_name") + @classmethod + def _normalize_server_name(cls, value: str | None) -> str | None: + if value is None: + return None + cleaned = value.strip() + return cleaned or None + + +class DeploymentAction(DeploymentModel): + """One machine-readable desired-state convergence intent.""" + + kind: DeploymentActionKind + governed_asset: str + physical_name: str + desired_state_json: str + + @field_validator("governed_asset", "physical_name", "desired_state_json") + @classmethod + def _require_non_empty_text(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("value must not be empty") + return cleaned + + @model_validator(mode="after") + def _validate_desired_state_identity(self) -> DeploymentAction: + schema = SchemaObject.model_validate_json(self.desired_state_json) + schema_name = getattr(schema, "name", None) + if schema_name is None: + raise ValueError("desired schema state must define name") + governed_asset = normalize_identity_name(str(schema_name), "Schema") + if governed_asset != self.governed_asset: + raise ValueError( + "desired schema state identity does not match governed_asset" + ) + + physical_value = getattr(schema, "physicalName", None) + expected_physical = ( + str(physical_value).strip() + if physical_value is not None and str(physical_value).strip() + else str(schema_name).strip() + ) + if expected_physical != self.physical_name: + raise ValueError( + "desired schema physicalName does not match deployment physical_name" + ) + return self + + +class DeploymentPlan(DeploymentModel): + """Pure deterministic runtime convergence plan for one applied release.""" + + deployment_plan_id: str + applied_release_id: str + contract_id: str + release_plan_id: str + released_revision_ref: str + selected_version: str + target: DeploymentTarget + actions: tuple[DeploymentAction, ...] + plan_version: Literal["1"] = "1" + + @field_validator( + "deployment_plan_id", + "applied_release_id", + "contract_id", + "release_plan_id", + "released_revision_ref", + "selected_version", + ) + @classmethod + def _require_plan_text(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("value must not be empty") + return cleaned + + @model_validator(mode="after") + def _validate_action_order_and_identity(self) -> DeploymentPlan: + governed_assets = [action.governed_asset for action in self.actions] + if governed_assets != sorted(governed_assets): + raise ValueError("DeploymentPlan actions must be ordered by governed_asset") + if len(governed_assets) != len(set(governed_assets)): + raise ValueError("DeploymentPlan cannot contain duplicate governed assets") + return self diff --git a/semapact/deployment/planner.py b/semapact/deployment/planner.py new file mode 100644 index 00000000..6c2a23da --- /dev/null +++ b/semapact/deployment/planner.py @@ -0,0 +1,108 @@ +"""Pure compilation of applied contract releases into deployment convergence plans.""" + +from __future__ import annotations + +import json +import uuid + +from open_data_contract_standard.model import SchemaObject + +from semapact.contractops.execution_models import AppliedContractRelease +from semapact.deployment.models import ( + DeploymentAction, + DeploymentActionKind, + DeploymentPlan, + DeploymentTarget, +) +from semapact.lifecycle.identity import normalize_identity_name +from semapact.reconciliation.binding import runtime_asset_specs_from_contract + + +SEMAPACT_DEPLOYMENT_PLAN_NAMESPACE = uuid.UUID( + "d59eaa31-997a-478d-9978-4659beee673d" +) + + +def build_deployment_plan( + release: AppliedContractRelease, + target: DeploymentTarget, +) -> DeploymentPlan: + """Build one deterministic provider-neutral convergence plan. + + Planning consumes exact released desired state only. It does not observe runtime, + choose CREATE/ALTER/DROP operations, contact a provider, or recompute governance. + """ + if not isinstance(release, AppliedContractRelease): + raise TypeError( + f"release must be AppliedContractRelease, got {type(release).__name__}" + ) + if not isinstance(target, DeploymentTarget): + raise TypeError( + f"target must be DeploymentTarget, got {type(target).__name__}" + ) + + contract = release.to_contract() + asset_specs = runtime_asset_specs_from_contract(contract) + specs_by_asset = {spec.governed_asset: spec for spec in asset_specs} + + actions: list[DeploymentAction] = [] + for schema in contract.schema_ or []: + raw_name = getattr(schema, "name", None) + if raw_name is None: + # runtime_asset_specs_from_contract() already validates this path; keep + # this guard explicit for planner exhaustiveness. + raise RuntimeError("Governed schema identity unexpectedly missing") + governed_asset = normalize_identity_name(str(raw_name), "Schema") + spec = specs_by_asset[governed_asset] + actions.append( + DeploymentAction( + kind=DeploymentActionKind.ENSURE_ASSET_STATE, + governed_asset=governed_asset, + physical_name=spec.physical_name, + desired_state_json=_canonical_schema_json(schema), + ) + ) + + ordered_actions = tuple(sorted(actions, key=lambda action: action.governed_asset)) + stable_record = { + "applied_release_id": release.applied_release_id, + "contract_id": release.contract_id, + "release_plan_id": release.release_plan_id, + "released_revision_ref": release.release_revision_ref, + "selected_version": release.selected_version, + "target": target.model_dump(mode="json"), + "actions": [action.model_dump(mode="json") for action in ordered_actions], + "plan_version": "1", + } + deployment_plan_id = str( + uuid.uuid5( + SEMAPACT_DEPLOYMENT_PLAN_NAMESPACE, + json.dumps( + stable_record, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ), + ) + ) + + return DeploymentPlan( + deployment_plan_id=deployment_plan_id, + applied_release_id=release.applied_release_id, + contract_id=release.contract_id, + release_plan_id=release.release_plan_id, + released_revision_ref=release.release_revision_ref, + selected_version=release.selected_version, + target=target, + actions=ordered_actions, + ) + + +def _canonical_schema_json(schema: SchemaObject) -> str: + payload = schema.model_dump(mode="json", by_alias=True, exclude_none=True) + return json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) diff --git a/tests/test_deployment_plan.py b/tests/test_deployment_plan.py new file mode 100644 index 00000000..05e705c4 --- /dev/null +++ b/tests/test_deployment_plan.py @@ -0,0 +1,207 @@ +from __future__ import annotations + +import json + +import pytest +from open_data_contract_standard.model import ( + OpenDataContractStandard, + SchemaObject, + SchemaProperty, +) +from pydantic import ValidationError as PydanticValidationError + +from semapact.contractops import AppliedContractRelease +from semapact.deployment import ( + DeploymentAction, + DeploymentActionKind, + DeploymentTarget, + build_deployment_plan, +) + + +def _schema( + name: str, + *, + physical_name: str | None = None, +) -> SchemaObject: + return SchemaObject( + name=name, + physicalName=physical_name, + properties=[ + SchemaProperty( + name="id", + logicalType="string", + physicalType="varchar(255)", + required=True, + ) + ], + ) + + +def _release( + *, + schemas: list[SchemaObject] | None = None, + applied_release_id: str = "applied-release:test", +) -> AppliedContractRelease: + contract = OpenDataContractStandard( + apiVersion="v3.1.0", + kind="DataContract", + id="orders-product", + name="Orders", + version="1.2.0", + status="active", + schema=schemas or [_schema("orders")], + ) + released_contract_json = json.dumps( + contract.model_dump(mode="json", by_alias=True, exclude_none=True), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) + return AppliedContractRelease( + applied_release_id=applied_release_id, + contract_id="orders-product", + decision_id="decision:test", + change_set_id="change-set:test", + release_plan_id="release-plan:test", + version_resolution_id="version-resolution:test", + release_revision_ref="rev:released", + selected_version="1.2.0", + authorization_id="authorization:test", + released_contract_json=released_contract_json, + ) + + +def _target() -> DeploymentTarget: + return DeploymentTarget( + platform="Databricks", + runtime_target="main.analytics", + server_name="production", + ) + + +def test_same_exact_release_and_target_produce_same_plan() -> None: + release = _release(schemas=[_schema("zeta"), _schema("alpha")]) + + first = build_deployment_plan(release, _target()) + second = build_deployment_plan(release, _target()) + + assert first == second + assert first.deployment_plan_id == second.deployment_plan_id + assert first.model_dump(mode="json") == second.model_dump(mode="json") + assert [action.governed_asset for action in first.actions] == ["alpha", "zeta"] + + +def test_plan_preserves_exact_applied_release_provenance() -> None: + release = _release() + + plan = build_deployment_plan(release, _target()) + + assert plan.applied_release_id == release.applied_release_id + assert plan.contract_id == release.contract_id + assert plan.release_plan_id == release.release_plan_id + assert plan.released_revision_ref == release.release_revision_ref + assert plan.selected_version == release.selected_version + assert plan.plan_version == "1" + + +def test_actions_are_provider_neutral_ensure_state_intents() -> None: + plan = build_deployment_plan( + _release(schemas=[_schema("orders"), _schema("customers")]), + _target(), + ) + + assert plan.actions + assert all( + action.kind is DeploymentActionKind.ENSURE_ASSET_STATE + for action in plan.actions + ) + assert {action.kind.value for action in plan.actions} == {"ENSURE_ASSET_STATE"} + + +def test_physical_name_is_binding_hint_not_governed_identity() -> None: + plan = build_deployment_plan( + _release(schemas=[_schema("Orders", physical_name="prod_orders_v2")]), + _target(), + ) + + action = plan.actions[0] + assert action.governed_asset == "orders" + assert action.physical_name == "prod_orders_v2" + + desired = SchemaObject.model_validate_json(action.desired_state_json) + assert desired.name == "Orders" + assert desired.physicalName == "prod_orders_v2" + + +def test_missing_physical_name_falls_back_to_governed_schema_name() -> None: + plan = build_deployment_plan( + _release(schemas=[_schema("Orders")]), + _target(), + ) + + action = plan.actions[0] + assert action.governed_asset == "orders" + assert action.physical_name == "Orders" + + +def test_target_is_explicit_and_changes_plan_identity() -> None: + release = _release() + + production = build_deployment_plan(release, _target()) + staging = build_deployment_plan( + release, + DeploymentTarget( + platform="databricks", + runtime_target="main.staging", + server_name="staging", + ), + ) + + assert production.deployment_plan_id != staging.deployment_plan_id + assert production.target.platform == "databricks" + assert staging.target.runtime_target == "main.staging" + + +def test_schema_order_is_canonicalized_within_each_exact_release() -> None: + first_release = _release( + schemas=[_schema("zeta"), _schema("alpha")], + applied_release_id="applied-release:first", + ) + second_release = _release( + schemas=[_schema("alpha"), _schema("zeta")], + applied_release_id="applied-release:second", + ) + + first = build_deployment_plan(first_release, _target()) + second = build_deployment_plan(second_release, _target()) + + assert [action.governed_asset for action in first.actions] == ["alpha", "zeta"] + assert [action.governed_asset for action in second.actions] == ["alpha", "zeta"] + # Exact release identity remains authoritative; #115 does not collapse two + # distinct AppliedContractRelease artifacts into one plan identity. + assert first.deployment_plan_id != second.deployment_plan_id + + +def test_desired_state_identity_mismatch_fails_closed() -> None: + schema = _schema("orders") + desired_state_json = json.dumps( + schema.model_dump(mode="json", by_alias=True, exclude_none=True), + sort_keys=True, + separators=(",", ":"), + ) + + with pytest.raises(PydanticValidationError, match="governed_asset"): + DeploymentAction( + kind=DeploymentActionKind.ENSURE_ASSET_STATE, + governed_asset="customers", + physical_name="orders", + desired_state_json=desired_state_json, + ) + + +def test_deployment_models_are_immutable() -> None: + plan = build_deployment_plan(_release(), _target()) + + with pytest.raises(PydanticValidationError): + setattr(plan, "selected_version", "9.9.9") From dbda3bc8db753a68c0e9f8c5a8041de700b2c6cc Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Thu, 10 Sep 2026 11:28:58 +1000 Subject: [PATCH 09/35] refactor(architecture): consolidate contractops boundaries (#213) * refactor(runtime): centralize governed asset projection * refactor(runtime): centralize governed asset projection * refactor(runtime): decouple providers from asset projection ownership * refactor(runtime): preserve reconciliation binding compatibility * refactor(deployment): depend on neutral runtime assets * refactor(observation): re-export neutral runtime asset spec * refactor(versioning): extract canonical semver primitives * refactor(governance): own release change classification * refactor(release): depend on canonical governance and versioning * refactor(governance): depend on canonical change classification * refactor(contractops): depend on canonical versioning * refactor(contractops): use canonical version primitives * refactor(contractops): remove legacy release dependency * refactor(contractops): centralize proposal context validation * refactor(contractops): reuse proposal context validation * refactor(utils): centralize deterministic artifact identity * refactor(contractops): reuse deterministic identity helper * refactor(contractops): reuse deterministic identity helper * refactor(contractops): centralize version resolution identity * refactor(contractops): centralize authorization identity * refactor(contractops): centralize execution artifact identity * refactor(deployment): centralize deployment plan identity * fix(governance): distinguish runtime deployment operation * fix(deployment): model exact plan authorization * fix(deployment): bind deploy authorization to exact plan * fix(deployment): expose deployment authorization boundary * test(governance): cover explicit deploy gate semantics * fix(contractops): preserve downstream review scope * fix(contractops): carry opaque downstream review scope * fix(deployment): enforce exact review scope for deploy * test(architecture): lock consolidation and deploy scope invariants * docs(architecture): record contractops composition boundaries * refactor(lifecycle): own release change classification * refactor(lifecycle): remove governance-owned classification copy * fix(architecture): remove governance package import cycle * refactor(governance): preserve classification import compatibility * docs(architecture): align classification ownership * docs(architecture): keep internal consolidation notes private * docs(contractops): remove internal milestone language * docs(contractops): keep execution phases public-safe * docs(deployment): clarify public deploy boundary * docs(governance): remove internal milestone references * docs(lifecycle): remove roadmap-only entrypoint * docs(runtime): keep reconciliation docs public-safe --- docs/contractops_authorization.md | 21 +- docs/contractops_phases.md | 29 ++- docs/deployment_plans.md | 27 +- docs/governance_analysis_boundary.md | 10 +- docs/lifecycle_semantics.md | 5 +- docs/runtime_reconciliation.md | 11 +- semapact/contractops/authorization.py | 22 +- semapact/contractops/changeset.py | 10 +- semapact/contractops/context.py | 18 +- semapact/contractops/execution.py | 37 +-- semapact/contractops/models.py | 32 ++- semapact/contractops/release_plan.py | 34 +-- semapact/contractops/version_authority.py | 22 +- semapact/core/release.py | 257 ++++--------------- semapact/deployment/__init__.py | 6 +- semapact/deployment/authorization.py | 124 +++++++++ semapact/deployment/models.py | 29 ++- semapact/deployment/planner.py | 24 +- semapact/governance/change_classification.py | 8 + semapact/governance/evaluator.py | 7 +- semapact/governance/gate.py | 15 +- semapact/lifecycle/change_classification.py | 137 ++++++++++ semapact/observation/__init__.py | 2 +- semapact/observation/providers.py | 8 +- semapact/reconciliation/binding.py | 46 +--- semapact/runtime/__init__.py | 8 + semapact/runtime/assets.py | 52 ++++ semapact/utils/deterministic.py | 22 ++ semapact/versioning.py | 87 +++++++ tests/test_architecture_consolidation.py | 219 ++++++++++++++++ tests/test_governance_gate.py | 22 +- 31 files changed, 901 insertions(+), 450 deletions(-) create mode 100644 semapact/deployment/authorization.py create mode 100644 semapact/governance/change_classification.py create mode 100644 semapact/lifecycle/change_classification.py create mode 100644 semapact/runtime/__init__.py create mode 100644 semapact/runtime/assets.py create mode 100644 semapact/utils/deterministic.py create mode 100644 semapact/versioning.py create mode 100644 tests/test_architecture_consolidation.py diff --git a/docs/contractops_authorization.md b/docs/contractops_authorization.md index aa670018..e47f4b73 100644 --- a/docs/contractops_authorization.md +++ b/docs/contractops_authorization.md @@ -24,10 +24,10 @@ GovernanceDecision → ReleasePlan → VersionResolution → ContractOpsAuthorization -→ APPLY / PUBLISH +→ APPLY / PUBLISH / DEPLOY ``` -Review evidence is therefore scoped to all of: +Review evidence is scoped to all of: ```text decisionId @@ -35,9 +35,10 @@ changeSetId releasePlanId versionResolutionId operation +scopeReference? # optional downstream scope ``` -Changing the proposal, release plan, selected version, or requested operation invalidates the evidence for that new action. +Changing the proposal, release plan, selected version, requested operation, or an explicitly bound downstream scope invalidates the evidence for that new action. ## Decision behavior @@ -50,11 +51,11 @@ Changing the proposal, release plan, selected version, or requested operation in | `review_required` | stale or mismatched | denied | | `blocked` | any | denied; review cannot override BLOCK | -M0 `GovernanceGateResult` remains authoritative for whether the operation is already allowed, requires review, or is blocked. ContractOps authorization only satisfies `review_required`; it does not re-run policy. +`GovernanceGateResult` remains authoritative for whether the operation is already allowed, requires review, or is blocked. ContractOps authorization only satisfies `review_required`; it does not re-run policy. ## Explicit evidence, not comment parsing -M2 consumes structured evidence: +ContractOps consumes structured evidence: ```text ReviewAuthorizationEvidence @@ -64,12 +65,13 @@ ReviewAuthorizationEvidence ├── releasePlanId ├── versionResolutionId ├── operation +├── scopeReference? └── action ``` -`evidenceReference` is opaque. This layer does not store approvals or infer approval from human comments, PR text, Slack messages, or similar free-form content. +`evidenceReference` is opaque. `scopeReference`, when present, is also opaque to ContractOps and may be used by a downstream boundary to bind approval to an exact target-specific artifact such as a deployment plan. -Future durable review history belongs to M3. A persisted `ApprovalRecord` can later be resolved/projected into `ReviewAuthorizationEvidence` without changing the ContractOps authorization semantics. +This layer does not store approvals or infer approval from human comments, PR text, Slack messages, or similar free-form content. Durable review history and reviewer workflow are separate persistence/application concerns and can project structured evidence into this boundary without changing its authorization semantics. ## Operation scope @@ -78,10 +80,13 @@ Approval is not a generic bypass token. ```text approval for APPLY ≠ approval for PUBLISH +≠ approval for DEPLOY ``` Likewise, approval for one `VersionResolution` does not authorize a different selected version. +Runtime deployment adds another scope boundary: a review-required deployment must bind approval to the exact `DeploymentPlan`, so approval for one runtime target cannot be rebound to another target. + ## Invalid context vs denied authorization SemaPact distinguishes malformed artifact composition from a valid release that simply lacks approval. @@ -90,7 +95,7 @@ If `GovernanceDecision`, `ChangeSet`, `ReleasePlan`, and `VersionResolution` do If the release context is valid but evidence is missing, rejected, stale, or mismatched, SemaPact returns a deterministic `ContractOpsAuthorization` with `allowed=false` and a machine-readable reason. -This lets later APPLY/PUBLISH boundaries consume one authoritative authorization result without implementing their own approval rules. +This lets APPLY, PUBLISH, and DEPLOY boundaries consume one authoritative release-context authorization result without implementing their own approval rules. ## Non-goals diff --git a/docs/contractops_phases.md b/docs/contractops_phases.md index c36b7118..1a127935 100644 --- a/docs/contractops_phases.md +++ b/docs/contractops_phases.md @@ -21,6 +21,10 @@ APPLY PUBLISH → PublicationResult + +DEPLOY +→ DeploymentPlan + DeploymentAuthorization +→ runtime mutation through a platform adapter ``` Each phase consumes artifacts from the previous phases. Later phases do not recalculate earlier decisions. @@ -45,7 +49,7 @@ PLAN remains pure. AUTHORIZE evaluates whether one exact operation can cross a side-effect boundary. -An authorization is scoped to: +A release-context authorization is scoped to: ```text decisionId @@ -53,11 +57,12 @@ changeSetId releasePlanId versionResolutionId operation +scopeReference? ``` `GovernanceDecision(REVIEW)` remains `REVIEW` after approval. Matching explicit review evidence produces an allowed `ContractOpsAuthorization`; it does not rewrite governance history. -APPLY and PUBLISH require separate operation-scoped authorizations. +APPLY, PUBLISH, and DEPLOY are distinct operation scopes. Runtime DEPLOY additionally binds authorization to the exact `DeploymentPlan` before mutation. ## APPLY @@ -78,7 +83,7 @@ This also covers metadata-only governed releases: `requiredVersionBump=none` may ## PUBLISH -PUBLISH is the first explicit external publication boundary. +PUBLISH publishes an applied contract release or release artifact. It is distinct from runtime deployment. It requires an allowed `ContractOpsAuthorization(operation=PUBLISH)` matching the exact applied release context before the publisher adapter is invoked. An APPLY authorization cannot authorize PUBLISH. @@ -94,7 +99,15 @@ opaque publication reference PublicationResult ``` -Git, storage, deployment, and platform-specific publication behavior belongs in adapters rather than the ContractOps domain. +Git, storage, and other release-artifact publication behavior belongs in adapters rather than the ContractOps domain. + +## DEPLOY + +DEPLOY mutates a runtime toward a provider-neutral `DeploymentPlan` and is separate from release publication. + +A release-context `ContractOpsAuthorization(operation=DEPLOY)` is not enough on its own. Runtime execution also requires a `DeploymentAuthorization` bound to the exact deployment plan, including its target. A review approval scoped to one deployment plan therefore cannot be rebound to another target. + +Platform-specific execution belongs behind a deployment adapter. The adapter must not recompute governance, version authority, release planning, or approval semantics. ## Failure semantics @@ -102,16 +115,12 @@ ContractOps distinguishes invalid context from denied authorization: - mismatched revision/artifact/operation context → `ReleaseValidationError`; - a valid context whose explicit authorization is denied → `ContractOpsAuthorizationError`; -- a publisher is never invoked when authorization validation fails. +- external publishers or runtime adapters are never invoked when authorization validation fails. Unexpected publisher/runtime failures are not converted into governance decisions; they propagate as execution failures. -## Legacy release helpers +## Compatibility helpers `semapact.core.release.prepare_release_candidate()` remains a backward-compatible helper and is not the canonical ContractOps APPLY path because it may classify changes itself. New ContractOps flows consume the existing authoritative `GovernanceDecision`, `ReleasePlan`, and `VersionResolution` instead of recomputing them. - -## Deployment boundary - -`AppliedContractRelease` is the released contract state that downstream deployment planning can consume. `DeploymentPlan` and platform deployment adapters are intentionally handled by later M2 work and are not part of the contract-release phase implementation. diff --git a/docs/deployment_plans.md b/docs/deployment_plans.md index 33db0bc5..f199eac9 100644 --- a/docs/deployment_plans.md +++ b/docs/deployment_plans.md @@ -10,7 +10,9 @@ AppliedContractRelease ↓ DeploymentPlan ↓ -platform adapter validate / preview / publish +DeploymentAuthorization + ↓ +platform adapter validate / preview / execute ↓ runtime ↓ @@ -37,11 +39,11 @@ Planning sees released desired state only. Without observed runtime state SemaPa Likewise, an object that exists in runtime but is absent from one contract must not be interpreted as safe to drop. The contract may not own that object. -Concrete provider-native operations therefore begin at the platform adapter boundary, where `validate` and `preview` can combine the DeploymentPlan with actual provider semantics and, where required, runtime evidence. +Concrete provider-native operations therefore begin at the platform adapter boundary, where validation and preview can combine the DeploymentPlan with provider semantics and, where required, runtime evidence. ## Identity and physical binding -The same rule used by M1 reconciliation applies on the write side: +The same governed identity rule applies on both deployment and reconciliation paths: ```text schema.name @@ -78,6 +80,14 @@ DeploymentTarget The plan does not contain credentials, workspace clients, SQL connections, or provider sessions. +## Authorization scope + +Runtime deployment is a separate protected operation from publishing a contract release artifact. + +A `ContractOpsAuthorization(operation=DEPLOY)` establishes release-context authorization. Before runtime mutation, it must be bound to the exact `DeploymentPlan` as a `DeploymentAuthorization`. + +For review-required changes, structured review evidence may carry an opaque `scopeReference`. Deployment requires that scope to match the exact `deploymentPlanId`, so an approval for one target cannot be reused for another target. + ## Determinism `deploymentPlanId` is UUID5-derived from the full stable plan record: @@ -95,15 +105,8 @@ Action ordering is canonical even when schemas appear in a different order in so DeploymentPlan intentionally does not contain generic `preconditions`, `adapterKey`, or guessed platform-specific operations. -The next boundary is responsible for explicit support: - -```text -DeploymentAdapter -├── validate(plan) -├── preview(plan) -└── publish(plan, authorization) -``` +A deployment adapter is responsible for explicit provider support and execution semantics. It receives an already-built DeploymentPlan and an allowed DeploymentAuthorization; it does not construct or reinterpret governance artifacts. A provider adapter must explicitly report unsupported ODCS-to-platform mappings. It must never silently ignore unsupported governed state. -A successful plan or publish call is also not proof of convergence. Runtime convergence is verified separately through SemaPact reconciliation. +A successful execution call is also not proof of convergence. Runtime convergence is verified separately through SemaPact reconciliation. diff --git a/docs/governance_analysis_boundary.md b/docs/governance_analysis_boundary.md index 79f52468..7bbb0a8b 100644 --- a/docs/governance_analysis_boundary.md +++ b/docs/governance_analysis_boundary.md @@ -13,11 +13,11 @@ Analyze Apply explicit local/candidate mutation -Publish - explicit external side effects +Publish / Deploy / CI + explicit protected side effects ``` -M0 establishes and protects the analysis boundary. Full ContractOps phase modeling belongs to later milestones. +Governance analysis remains upstream of all mutation-capable workflows. Later phases consume its result rather than recomputing governance semantics. ## Analysis Entry Points @@ -80,7 +80,7 @@ policy.valid = false decision = REVIEW ``` -This means the lifecycle policy found breaking evidence, not that every operation is prohibited. For example, analysis remains readable while a CI or publish operation may require review before side effects are allowed. +This means the lifecycle policy found breaking evidence, not that every operation is prohibited. For example, analysis remains readable while a CI, publish, or deploy operation may require review before side effects are allowed. External consumers must use `GovernanceDecision` and the operation-specific governance gate for authorization. They must not use `policy.valid`, `breaking`, or individual reason codes as independent permission checks. @@ -95,7 +95,7 @@ GovernanceDecision ↓ GovernanceOperation gate ↓ -explicit Apply / Publish / CI side effect +explicit APPLY / PUBLISH / DEPLOY / CI boundary ``` A client or adapter must not independently reinterpret breaking changes, validation, lifecycle policy, or version requirements to bypass the authoritative decision. diff --git a/docs/lifecycle_semantics.md b/docs/lifecycle_semantics.md index 4f961d8a..204a38c1 100644 --- a/docs/lifecycle_semantics.md +++ b/docs/lifecycle_semantics.md @@ -52,7 +52,7 @@ SemaPact strictly differentiates between **declared lifecycle** and **effective ### Declared Lifecycle - The status explicitly annotated on an individual entity (`customProperties.lifecycleStatus`). - Resolved via `resolve_declared_entity_lifecycle(entity)`. -- Used for release change classification (`_has_new_deprecations`) to determine if an entity was newly marked deprecated. +- Used for release change classification to determine if an entity was newly marked deprecated. ### Effective Governance Lifecycle - The status of an entity taking into account parent governance scope and hierarchy. @@ -149,11 +149,8 @@ Any semantic mutation against a retired base contract must be authoritatively bl | Release PR Creation | `semapact release create-pr` | `PROPOSE` | ❌ Blocked (`GovernanceBlockedError`) | | Batch Release Manifest | `semapact release build-manifest` | `PROPOSE` | ❌ Skipped from manifest tasks | | Automation Pipeline | `ContractPipeline.run()` | `CI` | ❌ Blocked (`GovernanceBlockedError`) | -| Future Draft Submission | `DraftService.submit()` | `PROPOSE` | ❌ Blocked (`GovernanceBlockedError`) | ### Lifecycle Transitions & Reactivations: - **Transition into Retired**: `ACTIVE / DEPRECATED / DRAFT -> RETIRED` produces `DecisionResult.REVIEW` with `GovernanceReasonCode.CONTRACT_RETIRED_TRANSITION`. This is a valid lifecycle transition requiring review, not a mutation of an already-retired contract. - **Reactivation**: `RETIRED -> ACTIVE / DRAFT / DEPRECATED` produces `DecisionResult.BLOCK` with `GovernanceReasonCode.RETIRED_CONTRACT_MODIFIED`. No unretire/reactivation semantics exist. - **Unchanged Retired Contracts**: `retired base == retired candidate` produces `DecisionResult.ALLOW` (when no other violation exists), preserving history, export, and verification workflows. - - diff --git a/docs/runtime_reconciliation.md b/docs/runtime_reconciliation.md index cbabb736..e3bae52b 100644 --- a/docs/runtime_reconciliation.md +++ b/docs/runtime_reconciliation.md @@ -66,14 +66,7 @@ semapact reconcile \ For the Databricks provider, the runtime target currently uses `catalog.schema`. Each governed schema is bound to its physical Unity Catalog asset using contract `physicalName` when present, otherwise the governed schema name. -`--runtime` is provider-local. Core SemaPact does not define it as a table FQN. A future provider can interpret the fallback differently, for example: - -```bash -semapact reconcile \ - --contract sales.yaml \ - --platform snowflake \ - --runtime ANALYTICS.SALES -``` +`--runtime` is provider-local. Core SemaPact does not define it as a table FQN; another provider may interpret the target using its own platform-local addressing convention. ## Authentication @@ -108,7 +101,7 @@ Runtime assurance uses additive process outcomes and does not reuse governance b | `DRIFT` | `6` | | `INDETERMINATE` | `7` | -Existing M0 exit codes remain unchanged: validation `2`, governance blocked `3`, review required `4`, and runtime/infrastructure error `5`. +Governance and validation exit codes remain unchanged: validation `2`, governance blocked `3`, review required `4`, and runtime/infrastructure error `5`. Missing or ambiguous runtime location is a validation failure. Examples include multiple contract servers without `--server`, or a contract with no servers and an incomplete CLI fallback. diff --git a/semapact/contractops/authorization.py b/semapact/contractops/authorization.py index 56819f43..97de58ee 100644 --- a/semapact/contractops/authorization.py +++ b/semapact/contractops/authorization.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json import uuid from semapact.contractops.context import validate_release_context @@ -17,6 +16,7 @@ ) from semapact.governance.gate import GovernanceOperation, evaluate_governance_gate from semapact.governance.models import GovernanceDecision +from semapact.utils.deterministic import deterministic_uuid5 SEMAPACT_CONTRACTOPS_AUTHORIZATION_NAMESPACE = uuid.UUID( @@ -37,7 +37,9 @@ def authorize_contract_operation( M0 governance remains authoritative for decision-level semantics. This function only satisfies a ``review_required`` gate result with explicit, exact approval - evidence. It never mutates or reinterprets the GovernanceDecision. + evidence. It never mutates or reinterprets the GovernanceDecision. Optional + downstream scope provenance is preserved opaquely for the owning domain to + validate later. """ _validate_types( decision, @@ -188,6 +190,7 @@ def _build_authorization( ) -> ContractOpsAuthorization: evidence_reference = evidence.evidence_reference if evidence is not None else None evidence_action = evidence.action if evidence is not None else None + scope_reference = evidence.scope_reference if evidence is not None else None stable_record = { "decision_id": decision.decision_id, @@ -200,14 +203,14 @@ def _build_authorization( "evidence_reference": evidence_reference, "evidence_action": evidence_action.value if evidence_action is not None else None, } - canonical_payload = json.dumps( + # Preserve all pre-#122 authorization IDs byte-for-byte when no downstream + # scope was supplied. Scoped authorizations intentionally gain a new identity. + if scope_reference is not None: + stable_record["scope_reference"] = scope_reference + + authorization_id = deterministic_uuid5( + SEMAPACT_CONTRACTOPS_AUTHORIZATION_NAMESPACE, stable_record, - sort_keys=True, - separators=(",", ":"), - ensure_ascii=False, - ) - authorization_id = str( - uuid.uuid5(SEMAPACT_CONTRACTOPS_AUTHORIZATION_NAMESPACE, canonical_payload) ) return ContractOpsAuthorization( @@ -221,4 +224,5 @@ def _build_authorization( reason=reason, evidence_reference=evidence_reference, evidence_action=evidence_action, + scope_reference=scope_reference, ) diff --git a/semapact/contractops/changeset.py b/semapact/contractops/changeset.py index 100e30e6..24e134d0 100644 --- a/semapact/contractops/changeset.py +++ b/semapact/contractops/changeset.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json import uuid from collections.abc import Sequence @@ -10,6 +9,7 @@ from semapact.contractops.models import ChangeSet from semapact.governance.models import GovernanceDecision from semapact.lifecycle.changes import GovernanceChange, governance_change_sort_key +from semapact.utils.deterministic import deterministic_uuid5 SEMAPACT_CHANGESET_NAMESPACE = uuid.UUID("3ea0f6d8-28ca-4bb4-94f5-ea1f0f48cb84") @@ -54,13 +54,7 @@ def build_change_set( "source": cleaned_source, "actor_reference": cleaned_actor_reference, } - canonical_payload = json.dumps( - identity_payload, - sort_keys=True, - separators=(",", ":"), - ensure_ascii=False, - ) - change_set_id = str(uuid.uuid5(SEMAPACT_CHANGESET_NAMESPACE, canonical_payload)) + change_set_id = deterministic_uuid5(SEMAPACT_CHANGESET_NAMESPACE, identity_payload) return ChangeSet( change_set_id=change_set_id, diff --git a/semapact/contractops/context.py b/semapact/contractops/context.py index 92921075..833dc711 100644 --- a/semapact/contractops/context.py +++ b/semapact/contractops/context.py @@ -1,4 +1,4 @@ -"""Shared fail-closed validation for one ContractOps release context.""" +"""Shared fail-closed validation for ContractOps artifact contexts.""" from __future__ import annotations @@ -7,13 +7,11 @@ from semapact.governance.models import GovernanceDecision -def validate_release_context( +def validate_proposal_context( decision: GovernanceDecision, change_set: ChangeSet, - release_plan: ReleasePlan, - version_resolution: VersionResolution, ) -> None: - """Fail closed unless immutable artifacts describe one exact release context.""" + """Fail closed unless ChangeSet exactly projects one GovernanceDecision.""" if change_set.contract_id != decision.contract_id: raise ReleaseValidationError( "ChangeSet and GovernanceDecision contract IDs do not match" @@ -27,6 +25,16 @@ def validate_release_context( "ChangeSet changes do not match authoritative GovernanceDecision changes" ) + +def validate_release_context( + decision: GovernanceDecision, + change_set: ChangeSet, + release_plan: ReleasePlan, + version_resolution: VersionResolution, +) -> None: + """Fail closed unless immutable artifacts describe one exact release context.""" + validate_proposal_context(decision, change_set) + if release_plan.contract_id != change_set.contract_id: raise ReleaseValidationError("ReleasePlan and ChangeSet contract IDs do not match") if release_plan.change_set_id != change_set.change_set_id: diff --git a/semapact/contractops/execution.py b/semapact/contractops/execution.py index a59cdb34..a6bda1c6 100644 --- a/semapact/contractops/execution.py +++ b/semapact/contractops/execution.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json import uuid from typing import Protocol @@ -16,10 +15,11 @@ ReleasePlan, VersionResolution, ) -from semapact.core.release import normalize_semver from semapact.exceptions import ContractOpsAuthorizationError, ReleaseValidationError from semapact.governance.gate import GovernanceOperation from semapact.governance.models import GovernanceDecision +from semapact.utils.deterministic import canonical_compact_json, deterministic_uuid5 +from semapact.versioning import normalize_semver SEMAPACT_APPLIED_RELEASE_NAMESPACE = uuid.UUID( @@ -116,16 +116,9 @@ def apply_contract_release( "authorization_id": authorization.authorization_id, "released_contract_json": released_contract_json, } - applied_release_id = str( - uuid.uuid5( - SEMAPACT_APPLIED_RELEASE_NAMESPACE, - json.dumps( - stable_record, - sort_keys=True, - separators=(",", ":"), - ensure_ascii=False, - ), - ) + applied_release_id = deterministic_uuid5( + SEMAPACT_APPLIED_RELEASE_NAMESPACE, + stable_record, ) return AppliedContractRelease( @@ -177,16 +170,9 @@ def publish_contract_release( "authorization_id": authorization.authorization_id, "publication_reference": publication_reference, } - publication_id = str( - uuid.uuid5( - SEMAPACT_PUBLICATION_NAMESPACE, - json.dumps( - stable_record, - sort_keys=True, - separators=(",", ":"), - ensure_ascii=False, - ), - ) + publication_id = deterministic_uuid5( + SEMAPACT_PUBLICATION_NAMESPACE, + stable_record, ) return PublicationResult( publication_id=publication_id, @@ -270,9 +256,4 @@ def _canonical_version(version: str, *, field_name: str) -> str: def _canonical_contract_json(contract: OpenDataContractStandard) -> str: payload = contract.model_dump(mode="json", by_alias=True, exclude_none=True) - return json.dumps( - payload, - sort_keys=True, - separators=(",", ":"), - ensure_ascii=False, - ) + return canonical_compact_json(payload) diff --git a/semapact/contractops/models.py b/semapact/contractops/models.py index 37fc429f..a2fa6d91 100644 --- a/semapact/contractops/models.py +++ b/semapact/contractops/models.py @@ -7,9 +7,9 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from semapact.change_context import ChangeContext -from semapact.core.release import ActualVersionBump, RequiredBump from semapact.governance.gate import GovernanceOperation from semapact.lifecycle.changes import GovernanceChange +from semapact.versioning import ActualVersionBump, RequiredBump class ContractOpsModel(BaseModel): @@ -178,7 +178,12 @@ class ReviewEvidenceAction(str, Enum): class ReviewAuthorizationEvidence(ContractOpsModel): - """Opaque review evidence projected onto one exact version-resolved action.""" + """Opaque review evidence projected onto one exact version-resolved action. + + ``scope_reference`` is optional downstream scope provenance. ContractOps preserves + but does not interpret it. For example, deployment review can bind the evidence + to an exact ``DeploymentPlan`` without making ContractOps depend on deployment. + """ evidence_reference: str decision_id: str @@ -187,6 +192,7 @@ class ReviewAuthorizationEvidence(ContractOpsModel): version_resolution_id: str operation: GovernanceOperation action: ReviewEvidenceAction + scope_reference: str | None = None @field_validator( "evidence_reference", @@ -202,6 +208,14 @@ def _require_review_evidence_text(cls, value: str) -> str: raise ValueError("value must not be empty") return cleaned + @field_validator("scope_reference") + @classmethod + def _normalize_scope_reference(cls, value: str | None) -> str | None: + if value is None: + return None + cleaned = value.strip() + return cleaned or None + class AuthorizationReason(str, Enum): """Machine-readable outcome of ContractOps authorization composition.""" @@ -227,6 +241,7 @@ class ContractOpsAuthorization(ContractOpsModel): reason: AuthorizationReason evidence_reference: str | None = None evidence_action: ReviewEvidenceAction | None = None + scope_reference: str | None = None @field_validator( "authorization_id", @@ -242,9 +257,12 @@ def _require_authorization_text(cls, value: str) -> str: raise ValueError("value must not be empty") return cleaned - @field_validator("evidence_reference") + @field_validator("evidence_reference", "scope_reference") @classmethod - def _normalize_evidence_reference(cls, value: str | None) -> str | None: + def _normalize_optional_authorization_reference( + cls, + value: str | None, + ) -> str | None: if value is None: return None cleaned = value.strip() @@ -269,7 +287,11 @@ def _validate_authorization_invariants(self) -> ContractOpsAuthorization: raise ValueError( "ContractOpsAuthorization evidence reason requires evidence provenance" ) - elif self.evidence_reference is not None or self.evidence_action is not None: + elif ( + self.evidence_reference is not None + or self.evidence_action is not None + or self.scope_reference is not None + ): raise ValueError( "ContractOpsAuthorization non-evidence reason must not contain evidence provenance" ) diff --git a/semapact/contractops/release_plan.py b/semapact/contractops/release_plan.py index 12c3b457..c4ba0abc 100644 --- a/semapact/contractops/release_plan.py +++ b/semapact/contractops/release_plan.py @@ -2,9 +2,9 @@ from __future__ import annotations -import json import uuid +from semapact.contractops.context import validate_proposal_context from semapact.contractops.models import ChangeSet, ReleasePlan, ReleasePrecondition from semapact.exceptions import ReleaseValidationError from semapact.governance.gate import ( @@ -13,6 +13,7 @@ evaluate_governance_gate, ) from semapact.governance.models import GovernanceDecision +from semapact.utils.deterministic import deterministic_uuid5 SEMAPACT_RELEASE_PLAN_NAMESPACE = uuid.UUID("7d2ad1de-c196-4f12-b1af-fdf79105eb04") @@ -38,7 +39,7 @@ def build_release_plan( f"decision must be GovernanceDecision, got {type(decision).__name__}" ) - _validate_proposal_consistency(change_set, decision) + validate_proposal_context(decision, change_set) # Release planning is a pure PROPOSE operation. Reuse the authoritative M0 gate # rather than duplicating ALLOW/REVIEW/BLOCK mapping in ContractOps. @@ -65,15 +66,7 @@ def build_release_plan( "required_version_bump": decision.required_version_bump, "preconditions": [item.value for item in preconditions], } - canonical_payload = json.dumps( - stable_record, - sort_keys=True, - separators=(",", ":"), - ensure_ascii=False, - ) - release_plan_id = str( - uuid.uuid5(SEMAPACT_RELEASE_PLAN_NAMESPACE, canonical_payload) - ) + release_plan_id = deterministic_uuid5(SEMAPACT_RELEASE_PLAN_NAMESPACE, stable_record) return ReleasePlan( release_plan_id=release_plan_id, @@ -84,22 +77,3 @@ def build_release_plan( required_version_bump=decision.required_version_bump, preconditions=preconditions, ) - - -def _validate_proposal_consistency( - change_set: ChangeSet, - decision: GovernanceDecision, -) -> None: - """Fail closed when artifacts do not describe the same evaluated proposal.""" - if change_set.contract_id != decision.contract_id: - raise ReleaseValidationError( - "ChangeSet and GovernanceDecision contract IDs do not match" - ) - if change_set.context != decision.context: - raise ReleaseValidationError( - "ChangeSet and GovernanceDecision governance contexts do not match" - ) - if change_set.changes != decision.changes: - raise ReleaseValidationError( - "ChangeSet changes do not match authoritative GovernanceDecision changes" - ) diff --git a/semapact/contractops/version_authority.py b/semapact/contractops/version_authority.py index 3f6d00d8..dddc2c5a 100644 --- a/semapact/contractops/version_authority.py +++ b/semapact/contractops/version_authority.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json import re import uuid @@ -12,14 +11,16 @@ VersionAuthorityConfig, VersionResolution, ) -from semapact.core.release import ( +from semapact.exceptions import ReleaseValidationError +from semapact.utils.deterministic import deterministic_uuid5 +from semapact.versioning import ( ActualVersionBump, RequiredBump, classify_version_bump, increment_version, normalize_semver, + version_bump_satisfies, ) -from semapact.exceptions import ReleaseValidationError SEMAPACT_VERSION_RESOLUTION_NAMESPACE = uuid.UUID( @@ -104,14 +105,9 @@ def resolve_release_version( "actual_bump": actual_bump, "authority_reference": normalized_reference, } - canonical_payload = json.dumps( + version_resolution_id = deterministic_uuid5( + SEMAPACT_VERSION_RESOLUTION_NAMESPACE, stable_record, - sort_keys=True, - separators=(",", ":"), - ensure_ascii=False, - ) - version_resolution_id = str( - uuid.uuid5(SEMAPACT_VERSION_RESOLUTION_NAMESPACE, canonical_payload) ) return VersionResolution( @@ -190,11 +186,7 @@ def _validate_minimum_bump( *, selected_version: str, ) -> None: - insufficient = ( - (required_bump == "major" and actual_bump != "major") - or (required_bump == "minor" and actual_bump == "patch") - ) - if insufficient: + if not version_bump_satisfies(actual_bump, required_bump): raise ReleaseValidationError( f"Resolved version '{selected_version}' applies a {actual_bump} bump, " f"but release requires at least a {required_bump} bump" diff --git a/semapact/core/release.py b/semapact/core/release.py index 1cd8c77c..17212933 100644 --- a/semapact/core/release.py +++ b/semapact/core/release.py @@ -1,40 +1,32 @@ from __future__ import annotations -import re import logging +import re from dataclasses import dataclass, field -from typing import Any, Literal, Sequence +from typing import Any from open_data_contract_standard.model import OpenDataContractStandard -from semapact.lifecycle.changes import ( - GovernanceChange, - GovernanceChangeDomain, - GovernanceChangeType, - GovernanceEntityType, - analyze_governance_changes, +from semapact.lifecycle.change_classification import ( + ContractChangeAssessment, + classify_contract_change, ) -from semapact.lifecycle.policy import BreakingChange, PolicyEvaluation, evaluate_merge_policy +from semapact.lifecycle.policy import BreakingChange from semapact.utils.schema_utils import contract_to_model +from semapact.versioning import ( + ActualVersionBump, + RequiredBump, + classify_version_bump, + increment_version, + normalize_semver, + suggest_release_version, + version_bump_satisfies, +) LOGGER = logging.getLogger(__name__) -RequiredBump = Literal["none", "minor", "major"] -ActualVersionBump = Literal["patch", "minor", "major"] - SEMVER_TAG_RE = re.compile(r"(?:^|[/-])v?(?P\d+\.\d+\.\d+)$") -VERSION_RANK = {"none": 0, "patch": 1, "minor": 2, "major": 3} - - -@dataclass(slots=True) -class ContractChangeAssessment: - """Per-contract change classification used by release workflows.""" - - has_changes: bool - required_bump: RequiredBump - reasons: list[str] = field(default_factory=list) - breaking_changes: list[BreakingChange] = field(default_factory=list) @dataclass(slots=True) @@ -51,102 +43,6 @@ class PromotionResult: breaking_changes: list[BreakingChange] = field(default_factory=list) -def classify_contract_change( - base_contract: OpenDataContractStandard | dict[str, Any], - candidate_contract: OpenDataContractStandard | dict[str, Any], - *, - changes: Sequence[GovernanceChange] | None = None, - policy_evaluation: PolicyEvaluation | None = None, -) -> ContractChangeAssessment: - """Classify required version bump for one contract change set. - - Rules: - - `major`: any lifecycle policy breaking change - - `minor`: additive, deprecation, quality, or other non-breaking structural changes - - `none`: only descriptive metadata changes - """ - base_model = contract_to_model(base_contract) - candidate_model = contract_to_model(candidate_contract) - - canonical_changes = ( - analyze_governance_changes(base_model, candidate_model) - if changes is None - else tuple(changes) - ) - - if not canonical_changes: - return ContractChangeAssessment( - has_changes=False, - required_bump="none", - reasons=["No contract changes detected"], - ) - - LOGGER.debug( - "Classifying changes for contract %s (base version: %s)", - base_model.id, - base_model.version, - ) - - policy = ( - policy_evaluation - if policy_evaluation is not None - else evaluate_merge_policy(base_model, candidate_model, changes=canonical_changes) - ) - if policy.breaking_changes: - LOGGER.info( - "Breaking changes detected in contract %s requiring major bump: %s", - base_model.id, - policy.breaking_changes, - ) - return ContractChangeAssessment( - has_changes=True, - required_bump="major", - reasons=["Breaking lifecycle changes require a major version bump"], - breaking_changes=policy.breaking_changes, - ) - - reasons: list[str] = [] - has_additions = any( - c.change_type == GovernanceChangeType.ADD - and c.entity_type in (GovernanceEntityType.SCHEMA, GovernanceEntityType.PROPERTY) - for c in canonical_changes - ) - has_deprecations = any( - c.change_type == GovernanceChangeType.DEPRECATE - and c.entity_type in (GovernanceEntityType.SCHEMA, GovernanceEntityType.PROPERTY) - for c in canonical_changes - ) - has_structural = any( - c.domain in ( - GovernanceChangeDomain.STRUCTURE, - GovernanceChangeDomain.RELATIONSHIP, - GovernanceChangeDomain.QUALITY, - ) - and c.change_type != GovernanceChangeType.DEPRECATE - for c in canonical_changes - ) - - if has_additions: - reasons.append("Schema or property additions require a minor version bump") - if has_deprecations: - reasons.append("New schema/property deprecations require a minor version bump") - if has_structural: - reasons.append( - "Non-breaking structural or quality changes require a minor version bump" - ) - - if reasons: - return ContractChangeAssessment( - has_changes=True, required_bump="minor", reasons=_dedupe(reasons) - ) - - return ContractChangeAssessment( - has_changes=True, - required_bump="none", - reasons=["Only descriptive metadata changed; no required version bump"], - ) - - def apply_release_candidate( base_contract: OpenDataContractStandard, candidate_contract: OpenDataContractStandard, @@ -154,10 +50,10 @@ def apply_release_candidate( *, required_bump: RequiredBump, ) -> PromotionResult: - """Apply a release candidate transformation using a pre-calculated required_bump. + """Apply a legacy release candidate using a pre-calculated required bump. - Avoids duplicate policy evaluation and classification when an authoritative - GovernanceDecision is already available. + This compatibility workflow preserves its historical semantics. Canonical M2 + ContractOps release execution lives outside this module. """ if not isinstance(base_contract, OpenDataContractStandard): raise TypeError( @@ -165,7 +61,8 @@ def apply_release_candidate( ) if not isinstance(candidate_contract, OpenDataContractStandard): raise TypeError( - f"candidate_contract must be OpenDataContractStandard, got {type(candidate_contract).__name__}" + "candidate_contract must be OpenDataContractStandard, " + f"got {type(candidate_contract).__name__}" ) base_model = base_contract @@ -188,7 +85,7 @@ def apply_release_candidate( target_version = parse_release_tag_version(release_tag) actual_bump = classify_version_bump(str(base_model.version or ""), target_version) - if VERSION_RANK[actual_bump] < VERSION_RANK[required_bump]: + if not version_bump_satisfies(actual_bump, required_bump): from semapact.exceptions import ReleaseValidationError raise ReleaseValidationError( @@ -215,11 +112,7 @@ def prepare_release_candidate( candidate_contract: OpenDataContractStandard | dict[str, Any], release_tag: str, ) -> PromotionResult: - """Prepare a promoted contract candidate from one governed contract. - - Maintained for backward compatibility. Runs classify_contract_change then - delegates to apply_release_candidate. - """ + """Prepare a promoted contract candidate through the legacy compatibility path.""" base_model = contract_to_model(base_contract) candidate_model = contract_to_model(candidate_contract) @@ -228,26 +121,26 @@ def prepare_release_candidate( LOGGER.error("Preparation failed: contract %s has no changes", base_model.id) raise ValueError("Cannot promote a contract with no changes") - res = apply_release_candidate( + result = apply_release_candidate( base_model, candidate_model, release_tag, required_bump=assessment.required_bump, ) return PromotionResult( - contract=res.contract, - required_bump=res.required_bump, - current_version=res.current_version, - target_version=res.target_version, - actual_bump=res.actual_bump, - release_tag=res.release_tag, + contract=result.contract, + required_bump=result.required_bump, + current_version=result.current_version, + target_version=result.target_version, + actual_bump=result.actual_bump, + release_tag=result.release_tag, reasons=assessment.reasons, breaking_changes=assessment.breaking_changes, ) def parse_release_tag_version(release_tag: str) -> str: - """Extract semantic version from an explicit per-contract release tag.""" + """Extract semantic version from an explicit legacy release tag.""" text = str(release_tag or "").strip() match = SEMVER_TAG_RE.search(text) if not match: @@ -257,79 +150,17 @@ def parse_release_tag_version(release_tag: str) -> str: return match.group("version") -def normalize_semver(version: str) -> str: - """Validate and return the canonical ``major.minor.patch`` representation.""" - major, minor, patch = _parse_semver(version) - return f"{major}.{minor}.{patch}" - - -def classify_version_bump( - current_version: str, target_version: str -) -> ActualVersionBump: - """Classify actual semantic version bump between current and target versions.""" - current = _parse_semver(current_version) - target = _parse_semver(target_version) - if target <= current: - raise ValueError( - f"Target version '{target_version}' must be greater than current version '{current_version}'" - ) - if target[0] > current[0]: - return "major" - if target[1] > current[1]: - return "minor" - return "patch" - - -def increment_version( - current_version: str, - bump: ActualVersionBump, -) -> str: - """Return the next semantic version for an explicit actual release bump.""" - major, minor, patch = _parse_semver(current_version) - if bump == "major": - return f"{major + 1}.0.0" - if bump == "minor": - return f"{major}.{minor + 1}.0" - if bump == "patch": - return f"{major}.{minor}.{patch + 1}" - raise ValueError(f"Unsupported version bump: {bump}") - - -def suggest_release_version( - current_version: str, - required_bump: RequiredBump, -) -> str: - """Suggest the next release version from the last released version. - - This helper always computes from the last released contract version. - It does not chain intermediate unreleased bumps together. - - Example: - - last released: 1.2.0 - - current unreleased delta includes both a breaking removal and an additive field - - required bump stays `major` - - suggested release version stays `2.0.0`, not `2.1.0` - """ - if required_bump == "major": - return increment_version(current_version, "major") - if required_bump == "minor": - return increment_version(current_version, "minor") - return normalize_semver(current_version) - - -def _parse_semver(version: str) -> tuple[int, int, int]: - match = re.fullmatch(r"v?(\d+)\.(\d+)\.(\d+)", str(version or "").strip()) - if not match: - raise ValueError(f"Version '{version}' must be a semantic version like 1.2.3") - return (int(match.group(1)), int(match.group(2)), int(match.group(3))) - - -def _dedupe(values: list[str]) -> list[str]: - seen: set[str] = set() - result: list[str] = [] - for item in values: - if item in seen: - continue - seen.add(item) - result.append(item) - return result +__all__ = [ + "ActualVersionBump", + "ContractChangeAssessment", + "PromotionResult", + "RequiredBump", + "apply_release_candidate", + "classify_contract_change", + "classify_version_bump", + "increment_version", + "normalize_semver", + "parse_release_tag_version", + "prepare_release_candidate", + "suggest_release_version", +] diff --git a/semapact/deployment/__init__.py b/semapact/deployment/__init__.py index 719527d1..576c2f2e 100644 --- a/semapact/deployment/__init__.py +++ b/semapact/deployment/__init__.py @@ -1,8 +1,10 @@ -"""Provider-neutral deployment planning boundary.""" +"""Provider-neutral deployment planning and authorization boundary.""" +from semapact.deployment.authorization import authorize_deployment from semapact.deployment.models import ( DeploymentAction, DeploymentActionKind, + DeploymentAuthorization, DeploymentPlan, DeploymentTarget, ) @@ -11,7 +13,9 @@ __all__ = [ "DeploymentAction", "DeploymentActionKind", + "DeploymentAuthorization", "DeploymentPlan", "DeploymentTarget", + "authorize_deployment", "build_deployment_plan", ] diff --git a/semapact/deployment/authorization.py b/semapact/deployment/authorization.py new file mode 100644 index 00000000..1c8de48a --- /dev/null +++ b/semapact/deployment/authorization.py @@ -0,0 +1,124 @@ +"""Bind release-level DEPLOY authorization to one exact DeploymentPlan.""" + +from __future__ import annotations + +import uuid + +from semapact.contractops.execution_models import AppliedContractRelease +from semapact.contractops.models import AuthorizationReason, ContractOpsAuthorization +from semapact.deployment.models import DeploymentAuthorization, DeploymentPlan +from semapact.exceptions import ReleaseValidationError +from semapact.governance.gate import GovernanceOperation +from semapact.utils.deterministic import deterministic_uuid5 + + +SEMAPACT_DEPLOYMENT_AUTHORIZATION_NAMESPACE = uuid.UUID( + "c1dd6b40-cb67-44c1-b0f2-5f133ba6a3f5" +) + + +def authorize_deployment( + plan: DeploymentPlan, + release: AppliedContractRelease, + authorization: ContractOpsAuthorization, +) -> DeploymentAuthorization: + """Bind one DEPLOY authorization to the exact deployment target/plan. + + Upstream ContractOps remains authoritative for governance/review semantics. + Review-based DEPLOY authorization must carry ``scope_reference`` equal to this + exact ``deployment_plan_id``; governance-ALLOW paths need no synthetic review + scope but are still bound here before an adapter may execute the plan. + """ + if not isinstance(plan, DeploymentPlan): + raise TypeError(f"plan must be DeploymentPlan, got {type(plan).__name__}") + if not isinstance(release, AppliedContractRelease): + raise TypeError( + f"release must be AppliedContractRelease, got {type(release).__name__}" + ) + if not isinstance(authorization, ContractOpsAuthorization): + raise TypeError( + "authorization must be ContractOpsAuthorization, " + f"got {type(authorization).__name__}" + ) + + if authorization.operation is not GovernanceOperation.DEPLOY: + raise ReleaseValidationError( + "Deployment requires operation-scoped DEPLOY authorization" + ) + + _validate_plan_release_context(plan, release) + _validate_authorization_release_context(authorization, release) + + if ( + authorization.allowed + and authorization.reason is AuthorizationReason.ALLOWED_BY_REVIEW + and authorization.scope_reference != plan.deployment_plan_id + ): + raise ReleaseValidationError( + "Review-based DEPLOY authorization is not scoped to this DeploymentPlan" + ) + + stable_record = { + "contract_ops_authorization_id": authorization.authorization_id, + "deployment_plan_id": plan.deployment_plan_id, + "applied_release_id": release.applied_release_id, + "allowed": authorization.allowed, + } + return DeploymentAuthorization( + deployment_authorization_id=deterministic_uuid5( + SEMAPACT_DEPLOYMENT_AUTHORIZATION_NAMESPACE, + stable_record, + ), + contract_ops_authorization_id=authorization.authorization_id, + deployment_plan_id=plan.deployment_plan_id, + applied_release_id=release.applied_release_id, + allowed=authorization.allowed, + ) + + +def _validate_plan_release_context( + plan: DeploymentPlan, + release: AppliedContractRelease, +) -> None: + if plan.applied_release_id != release.applied_release_id: + raise ReleaseValidationError( + "DeploymentPlan does not reference the supplied AppliedContractRelease" + ) + if plan.contract_id != release.contract_id: + raise ReleaseValidationError( + "DeploymentPlan and AppliedContractRelease contract IDs do not match" + ) + if plan.release_plan_id != release.release_plan_id: + raise ReleaseValidationError( + "DeploymentPlan and AppliedContractRelease release plan IDs do not match" + ) + if plan.released_revision_ref != release.release_revision_ref: + raise ReleaseValidationError( + "DeploymentPlan released revision does not match AppliedContractRelease" + ) + if plan.selected_version != release.selected_version: + raise ReleaseValidationError( + "DeploymentPlan selected version does not match AppliedContractRelease" + ) + + +def _validate_authorization_release_context( + authorization: ContractOpsAuthorization, + release: AppliedContractRelease, +) -> None: + if authorization.decision_id != release.decision_id: + raise ReleaseValidationError( + "DEPLOY authorization decision does not match AppliedContractRelease" + ) + if authorization.change_set_id != release.change_set_id: + raise ReleaseValidationError( + "DEPLOY authorization ChangeSet does not match AppliedContractRelease" + ) + if authorization.release_plan_id != release.release_plan_id: + raise ReleaseValidationError( + "DEPLOY authorization ReleasePlan does not match AppliedContractRelease" + ) + if authorization.version_resolution_id != release.version_resolution_id: + raise ReleaseValidationError( + "DEPLOY authorization version resolution does not match AppliedContractRelease" + ) diff --git a/semapact/deployment/models.py b/semapact/deployment/models.py index b7651dcf..0f3ec980 100644 --- a/semapact/deployment/models.py +++ b/semapact/deployment/models.py @@ -1,4 +1,4 @@ -"""Provider-neutral immutable deployment planning artifacts.""" +"""Provider-neutral immutable deployment planning and authorization artifacts.""" from __future__ import annotations @@ -6,13 +6,13 @@ from typing import Literal from open_data_contract_standard.model import SchemaObject -from pydantic import BaseModel, ConfigDict, field_validator, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from semapact.lifecycle.identity import normalize_identity_name class DeploymentModel(BaseModel): - """Shared immutable base for deployment planning artifacts.""" + """Shared immutable base for deployment domain artifacts.""" model_config = ConfigDict(frozen=True, extra="forbid") @@ -132,3 +132,26 @@ def _validate_action_order_and_identity(self) -> DeploymentPlan: if len(governed_assets) != len(set(governed_assets)): raise ValueError("DeploymentPlan cannot contain duplicate governed assets") return self + + +class DeploymentAuthorization(DeploymentModel): + """Authorization bound to one exact DeploymentPlan and applied release.""" + + deployment_authorization_id: str + contract_ops_authorization_id: str + deployment_plan_id: str + applied_release_id: str + allowed: bool = Field(strict=True) + + @field_validator( + "deployment_authorization_id", + "contract_ops_authorization_id", + "deployment_plan_id", + "applied_release_id", + ) + @classmethod + def _require_authorization_text(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("value must not be empty") + return cleaned diff --git a/semapact/deployment/planner.py b/semapact/deployment/planner.py index 6c2a23da..e5d3677e 100644 --- a/semapact/deployment/planner.py +++ b/semapact/deployment/planner.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json import uuid from open_data_contract_standard.model import SchemaObject @@ -15,7 +14,8 @@ DeploymentTarget, ) from semapact.lifecycle.identity import normalize_identity_name -from semapact.reconciliation.binding import runtime_asset_specs_from_contract +from semapact.runtime import runtime_asset_specs_from_contract +from semapact.utils.deterministic import canonical_compact_json, deterministic_uuid5 SEMAPACT_DEPLOYMENT_PLAN_NAMESPACE = uuid.UUID( @@ -74,16 +74,9 @@ def build_deployment_plan( "actions": [action.model_dump(mode="json") for action in ordered_actions], "plan_version": "1", } - deployment_plan_id = str( - uuid.uuid5( - SEMAPACT_DEPLOYMENT_PLAN_NAMESPACE, - json.dumps( - stable_record, - sort_keys=True, - separators=(",", ":"), - ensure_ascii=False, - ), - ) + deployment_plan_id = deterministic_uuid5( + SEMAPACT_DEPLOYMENT_PLAN_NAMESPACE, + stable_record, ) return DeploymentPlan( @@ -100,9 +93,4 @@ def build_deployment_plan( def _canonical_schema_json(schema: SchemaObject) -> str: payload = schema.model_dump(mode="json", by_alias=True, exclude_none=True) - return json.dumps( - payload, - sort_keys=True, - separators=(",", ":"), - ensure_ascii=False, - ) + return canonical_compact_json(payload) diff --git a/semapact/governance/change_classification.py b/semapact/governance/change_classification.py new file mode 100644 index 00000000..a143a919 --- /dev/null +++ b/semapact/governance/change_classification.py @@ -0,0 +1,8 @@ +"""Compatibility import for lifecycle-owned governed change classification.""" + +from semapact.lifecycle.change_classification import ( + ContractChangeAssessment, + classify_contract_change, +) + +__all__ = ["ContractChangeAssessment", "classify_contract_change"] diff --git a/semapact/governance/evaluator.py b/semapact/governance/evaluator.py index 7fa95fb7..a4263af1 100644 --- a/semapact/governance/evaluator.py +++ b/semapact/governance/evaluator.py @@ -10,9 +10,12 @@ from open_data_contract_standard.model import OpenDataContractStandard from semapact.change_context import ChangeContext -from semapact.core.release import ContractChangeAssessment, classify_contract_change from semapact.core.validator import ContractValidator from semapact.exceptions import ValidationError +from semapact.governance.change_classification import ( + ContractChangeAssessment, + classify_contract_change, +) from semapact.governance.models import ( ChangeEvidence, DecisionResult, @@ -380,8 +383,6 @@ def _determine_decision( return DecisionResult.ALLOW - - def _generate_decision_id( base_contract: OpenDataContractStandard, candidate_contract: OpenDataContractStandard, diff --git a/semapact/governance/gate.py b/semapact/governance/gate.py index ffcb3d27..d5242046 100644 --- a/semapact/governance/gate.py +++ b/semapact/governance/gate.py @@ -23,6 +23,7 @@ class GovernanceOperation(str, Enum): PROPOSE = "PROPOSE" APPLY = "APPLY" PUBLISH = "PUBLISH" + DEPLOY = "DEPLOY" CI = "CI" @@ -54,18 +55,24 @@ def evaluate_governance_gate( decision_id = decision.decision_id - # 1. ANALYZE operation is always allowed (displays decision and reasons without mutation/side-effects) + # ANALYZE is observational and always allowed. if operation == GovernanceOperation.ANALYZE: return GovernanceGateResult(allowed=True, reason="allowed", decision_id=decision_id) - # 2. PROPOSE operation allows ALLOW and REVIEW (e.g. producing candidate YAML/PR), but blocks BLOCK + # PROPOSE may create candidate/planning artifacts for ALLOW or REVIEW, never BLOCK. if operation == GovernanceOperation.PROPOSE: if decision.decision in (DecisionResult.ALLOW, DecisionResult.REVIEW): return GovernanceGateResult(allowed=True, reason="allowed", decision_id=decision_id) return GovernanceGateResult(allowed=False, reason="blocked", decision_id=decision_id) - # 3. APPLY, PUBLISH, and CI operations allow only ALLOW; REVIEW is gated, and BLOCK is forbidden - if operation in (GovernanceOperation.APPLY, GovernanceOperation.PUBLISH, GovernanceOperation.CI): + # Side-effect-capable operations require ALLOW or explicit review satisfaction + # at the downstream authorization layer. BLOCK is never overrideable. + if operation in ( + GovernanceOperation.APPLY, + GovernanceOperation.PUBLISH, + GovernanceOperation.DEPLOY, + GovernanceOperation.CI, + ): if decision.decision == DecisionResult.ALLOW: return GovernanceGateResult(allowed=True, reason="allowed", decision_id=decision_id) if decision.decision == DecisionResult.REVIEW: diff --git a/semapact/lifecycle/change_classification.py b/semapact/lifecycle/change_classification.py new file mode 100644 index 00000000..c13075b0 --- /dev/null +++ b/semapact/lifecycle/change_classification.py @@ -0,0 +1,137 @@ +"""Canonical lifecycle change classification for release-version requirements.""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from typing import Any, Sequence + +from open_data_contract_standard.model import OpenDataContractStandard + +from semapact.lifecycle.changes import ( + GovernanceChange, + GovernanceChangeDomain, + GovernanceChangeType, + GovernanceEntityType, + analyze_governance_changes, +) +from semapact.lifecycle.policy import BreakingChange, PolicyEvaluation, evaluate_merge_policy +from semapact.utils.schema_utils import contract_to_model +from semapact.versioning import RequiredBump + + +LOGGER = logging.getLogger(__name__) + + +@dataclass(slots=True) +class ContractChangeAssessment: + """Per-contract lifecycle classification used by governed release workflows.""" + + has_changes: bool + required_bump: RequiredBump + reasons: list[str] = field(default_factory=list) + breaking_changes: list[BreakingChange] = field(default_factory=list) + + +def classify_contract_change( + base_contract: OpenDataContractStandard | dict[str, Any], + candidate_contract: OpenDataContractStandard | dict[str, Any], + *, + changes: Sequence[GovernanceChange] | None = None, + policy_evaluation: PolicyEvaluation | None = None, +) -> ContractChangeAssessment: + """Classify the minimum governed release-version bump for one contract delta.""" + base_model = contract_to_model(base_contract) + candidate_model = contract_to_model(candidate_contract) + + canonical_changes = ( + analyze_governance_changes(base_model, candidate_model) + if changes is None + else tuple(changes) + ) + + if not canonical_changes: + return ContractChangeAssessment( + has_changes=False, + required_bump="none", + reasons=["No contract changes detected"], + ) + + LOGGER.debug( + "Classifying changes for contract %s (base version: %s)", + base_model.id, + base_model.version, + ) + + policy = ( + policy_evaluation + if policy_evaluation is not None + else evaluate_merge_policy(base_model, candidate_model, changes=canonical_changes) + ) + if policy.breaking_changes: + LOGGER.info( + "Breaking changes detected in contract %s requiring major bump: %s", + base_model.id, + policy.breaking_changes, + ) + return ContractChangeAssessment( + has_changes=True, + required_bump="major", + reasons=["Breaking lifecycle changes require a major version bump"], + breaking_changes=policy.breaking_changes, + ) + + reasons: list[str] = [] + has_additions = any( + change.change_type == GovernanceChangeType.ADD + and change.entity_type in (GovernanceEntityType.SCHEMA, GovernanceEntityType.PROPERTY) + for change in canonical_changes + ) + has_deprecations = any( + change.change_type == GovernanceChangeType.DEPRECATE + and change.entity_type in (GovernanceEntityType.SCHEMA, GovernanceEntityType.PROPERTY) + for change in canonical_changes + ) + has_structural = any( + change.domain + in ( + GovernanceChangeDomain.STRUCTURE, + GovernanceChangeDomain.RELATIONSHIP, + GovernanceChangeDomain.QUALITY, + ) + and change.change_type != GovernanceChangeType.DEPRECATE + for change in canonical_changes + ) + + if has_additions: + reasons.append("Schema or property additions require a minor version bump") + if has_deprecations: + reasons.append("New schema/property deprecations require a minor version bump") + if has_structural: + reasons.append( + "Non-breaking structural or quality changes require a minor version bump" + ) + + if reasons: + return ContractChangeAssessment( + has_changes=True, + required_bump="minor", + reasons=_dedupe(reasons), + ) + + return ContractChangeAssessment( + has_changes=True, + required_bump="none", + reasons=["Only descriptive metadata changed; no required version bump"], + ) + + +def _dedupe(values: list[str]) -> list[str]: + seen: set[str] = set() + result: list[str] = [] + for item in values: + if item in seen: + continue + seen.add(item) + result.append(item) + return result diff --git a/semapact/observation/__init__.py b/semapact/observation/__init__.py index 19934ad5..1ac37915 100644 --- a/semapact/observation/__init__.py +++ b/semapact/observation/__init__.py @@ -17,10 +17,10 @@ ) from semapact.observation.providers import ( RuntimeAssetBinding, - RuntimeAssetSpec, RuntimeProvider, RuntimeProviderRegistry, ) +from semapact.runtime import RuntimeAssetSpec __all__ = [ "OBSERVED_STATE_FINGERPRINT_ALGORITHM", diff --git a/semapact/observation/providers.py b/semapact/observation/providers.py index f555d3b0..1a46fbce 100644 --- a/semapact/observation/providers.py +++ b/semapact/observation/providers.py @@ -7,19 +7,13 @@ from pydantic import BaseModel, ConfigDict from semapact.observation.models import ObservedAssetIdentity, ObservedPlatformState +from semapact.runtime import RuntimeAssetSpec class RuntimeProviderModel(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") -class RuntimeAssetSpec(RuntimeProviderModel): - """Governed logical asset plus its physical-name hint.""" - - governed_asset: str - physical_name: str - - class RuntimeAssetBinding(RuntimeProviderModel): """Explicit logical governed asset to provider-local runtime identity binding.""" diff --git a/semapact/reconciliation/binding.py b/semapact/reconciliation/binding.py index d3a42812..1637e918 100644 --- a/semapact/reconciliation/binding.py +++ b/semapact/reconciliation/binding.py @@ -1,43 +1,9 @@ -"""Governed data-product asset specs for runtime binding.""" +"""Compatibility exports for governed runtime asset projection. -from __future__ import annotations +The canonical provider-neutral projection lives in :mod:`semapact.runtime`. This +module remains as a compatibility import path for existing reconciliation callers. +""" -from open_data_contract_standard.model import OpenDataContractStandard +from semapact.runtime import RuntimeAssetSpec, runtime_asset_specs_from_contract -from semapact.exceptions import ValidationError -from semapact.lifecycle.identity import normalize_identity_name -from semapact.observation.providers import RuntimeAssetSpec - - -def runtime_asset_specs_from_contract( - contract: OpenDataContractStandard, -) -> tuple[RuntimeAssetSpec, ...]: - """Project governed schemas into logical asset specs without changing identity. - - ``schema.name`` remains the governed logical identity. ``physicalName`` is used - only as a runtime binding hint and falls back to the governed name when absent. - """ - specs: list[RuntimeAssetSpec] = [] - seen: set[str] = set() - - for schema in contract.schema_ or []: - raw_name = getattr(schema, "name", None) - if raw_name is None: - raise ValidationError("Governed schema name is required for runtime binding") - governed_asset = normalize_identity_name(str(raw_name), "Schema") - if governed_asset in seen: - raise ValidationError( - f"Duplicate canonical governed asset identity found: '{governed_asset}'" - ) - seen.add(governed_asset) - - physical_name_value = getattr(schema, "physicalName", None) - physical_name = str(physical_name_value).strip() if physical_name_value else "" - specs.append( - RuntimeAssetSpec( - governed_asset=governed_asset, - physical_name=physical_name or str(raw_name).strip(), - ) - ) - - return tuple(sorted(specs, key=lambda item: item.governed_asset)) +__all__ = ["RuntimeAssetSpec", "runtime_asset_specs_from_contract"] diff --git a/semapact/runtime/__init__.py b/semapact/runtime/__init__.py new file mode 100644 index 00000000..58da644b --- /dev/null +++ b/semapact/runtime/__init__.py @@ -0,0 +1,8 @@ +"""Provider-neutral governed runtime asset projections.""" + +from semapact.runtime.assets import RuntimeAssetSpec, runtime_asset_specs_from_contract + +__all__ = [ + "RuntimeAssetSpec", + "runtime_asset_specs_from_contract", +] diff --git a/semapact/runtime/assets.py b/semapact/runtime/assets.py new file mode 100644 index 00000000..b0288291 --- /dev/null +++ b/semapact/runtime/assets.py @@ -0,0 +1,52 @@ +"""Neutral governed asset projection shared by runtime read/write paths.""" + +from __future__ import annotations + +from open_data_contract_standard.model import OpenDataContractStandard +from pydantic import BaseModel, ConfigDict + +from semapact.exceptions import ValidationError +from semapact.lifecycle.identity import normalize_identity_name + + +class RuntimeAssetSpec(BaseModel): + """Governed logical asset plus its physical runtime-name hint.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + governed_asset: str + physical_name: str + + +def runtime_asset_specs_from_contract( + contract: OpenDataContractStandard, +) -> tuple[RuntimeAssetSpec, ...]: + """Project governed schemas into provider-neutral runtime asset specs. + + ``schema.name`` remains canonical governed identity. ``physicalName`` is only + a runtime/deployment binding hint and falls back to the governed name. + """ + specs: list[RuntimeAssetSpec] = [] + seen: set[str] = set() + + for schema in contract.schema_ or []: + raw_name = getattr(schema, "name", None) + if raw_name is None: + raise ValidationError("Governed schema name is required for runtime binding") + governed_asset = normalize_identity_name(str(raw_name), "Schema") + if governed_asset in seen: + raise ValidationError( + f"Duplicate canonical governed asset identity found: '{governed_asset}'" + ) + seen.add(governed_asset) + + physical_name_value = getattr(schema, "physicalName", None) + physical_name = str(physical_name_value).strip() if physical_name_value else "" + specs.append( + RuntimeAssetSpec( + governed_asset=governed_asset, + physical_name=physical_name or str(raw_name).strip(), + ) + ) + + return tuple(sorted(specs, key=lambda item: item.governed_asset)) diff --git a/semapact/utils/deterministic.py b/semapact/utils/deterministic.py new file mode 100644 index 00000000..a8e67806 --- /dev/null +++ b/semapact/utils/deterministic.py @@ -0,0 +1,22 @@ +"""Stable serialization helpers for deterministic domain artifact identities.""" + +from __future__ import annotations + +import json +import uuid +from typing import Any + + +def canonical_compact_json(payload: Any) -> str: + """Serialize a payload using the compact canonical form used by M2 artifacts.""" + return json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) + + +def deterministic_uuid5(namespace: uuid.UUID, payload: Any) -> str: + """Return the stable UUID5 for one compact-canonical payload.""" + return str(uuid.uuid5(namespace, canonical_compact_json(payload))) diff --git a/semapact/versioning.py b/semapact/versioning.py new file mode 100644 index 00000000..2af636b1 --- /dev/null +++ b/semapact/versioning.py @@ -0,0 +1,87 @@ +"""Canonical semantic-version primitives shared by governance and ContractOps.""" + +from __future__ import annotations + +import re +from typing import Literal + +RequiredBump = Literal["none", "minor", "major"] +ActualVersionBump = Literal["patch", "minor", "major"] + +_VERSION_RANK: dict[str, int] = { + "none": 0, + "patch": 1, + "minor": 2, + "major": 3, +} + + +def normalize_semver(version: str) -> str: + """Validate and return canonical ``major.minor.patch`` form.""" + major, minor, patch = _parse_semver(version) + return f"{major}.{minor}.{patch}" + + +def classify_version_bump( + current_version: str, + target_version: str, +) -> ActualVersionBump: + """Classify the actual positive semantic-version bump.""" + current = _parse_semver(current_version) + target = _parse_semver(target_version) + if target <= current: + raise ValueError( + f"Target version '{target_version}' must be greater than current version '{current_version}'" + ) + if target[0] > current[0]: + return "major" + if target[1] > current[1]: + return "minor" + return "patch" + + +def increment_version( + current_version: str, + bump: ActualVersionBump, +) -> str: + """Return the next semantic version for an explicit actual release bump.""" + major, minor, patch = _parse_semver(current_version) + if bump == "major": + return f"{major + 1}.0.0" + if bump == "minor": + return f"{major}.{minor + 1}.0" + if bump == "patch": + return f"{major}.{minor}.{patch + 1}" + raise ValueError(f"Unsupported version bump: {bump}") + + +def version_bump_satisfies( + actual_bump: ActualVersionBump, + required_bump: RequiredBump, +) -> bool: + """Return whether an actual release bump satisfies a governance minimum.""" + return _VERSION_RANK[actual_bump] >= _VERSION_RANK[required_bump] + + +def suggest_release_version( + current_version: str, + required_bump: RequiredBump, +) -> str: + """Suggest the next version from one last released contract version. + + ``none`` preserves legacy suggestion semantics and returns the normalized current + version. ContractOps may independently choose a patch when an explicit governed + release still requires a distinct version. + """ + if required_bump == "major": + return increment_version(current_version, "major") + if required_bump == "minor": + return increment_version(current_version, "minor") + return normalize_semver(current_version) + + +def _parse_semver(version: str) -> tuple[int, int, int]: + match = re.fullmatch(r"v?(\d+)\.(\d+)\.(\d+)", str(version or "").strip()) + if not match: + raise ValueError(f"Version '{version}' must be a semantic version like 1.2.3") + return (int(match.group(1)), int(match.group(2)), int(match.group(3))) diff --git a/tests/test_architecture_consolidation.py b/tests/test_architecture_consolidation.py new file mode 100644 index 00000000..e298b120 --- /dev/null +++ b/tests/test_architecture_consolidation.py @@ -0,0 +1,219 @@ +from __future__ import annotations + +import json +import uuid +from datetime import date + +import pytest +from open_data_contract_standard.model import ( + OpenDataContractStandard, + SchemaObject, + SchemaProperty, +) + +from semapact.change_context import ChangeContext +from semapact.contractops import ( + AppliedContractRelease, + AuthorizationReason, + ReviewAuthorizationEvidence, + ReviewEvidenceAction, + VersionAuthorityConfig, + authorize_contract_operation, + build_change_set_from_decision, + build_release_plan, + resolve_release_version, +) +from semapact.core.release import ( + classify_contract_change as legacy_classify_contract_change, + normalize_semver as legacy_normalize_semver, +) +from semapact.deployment import ( + DeploymentTarget, + authorize_deployment, + build_deployment_plan, +) +from semapact.exceptions import ReleaseValidationError +from semapact.governance import DecisionResult, evaluate_governance_decision +from semapact.governance.change_classification import classify_contract_change +from semapact.governance.gate import GovernanceOperation +from semapact.observation import RuntimeAssetSpec as ObservationRuntimeAssetSpec +from semapact.reconciliation.binding import ( + runtime_asset_specs_from_contract as legacy_runtime_asset_specs_from_contract, +) +from semapact.runtime import RuntimeAssetSpec, runtime_asset_specs_from_contract +from semapact.utils.deterministic import canonical_compact_json, deterministic_uuid5 +from semapact.versioning import normalize_semver + + +CONTEXT = ChangeContext(effective_date=date(2026, 9, 10)) + + +def _contract(*, include_created_at: bool = False) -> OpenDataContractStandard: + properties = [ + SchemaProperty( + name="id", + logicalType="string", + physicalType="varchar(255)", + required=True, + ) + ] + if include_created_at: + properties.append( + SchemaProperty( + name="created_at", + logicalType="timestamp", + physicalType="timestamp", + required=False, + ) + ) + return OpenDataContractStandard( + apiVersion="v3.1.0", + kind="DataContract", + id="orders-product", + name="Orders", + version="1.0.0", + status="active", + schema=[SchemaObject(name="orders", properties=properties)], + ) + + +def _review_release_chain(): + base = _contract() + candidate = _contract(include_created_at=True) + decision = evaluate_governance_decision(base, candidate, context=CONTEXT) + assert decision.decision is DecisionResult.REVIEW + + change_set = build_change_set_from_decision( + decision, + base_revision_ref="rev:base", + candidate_revision_ref="rev:candidate", + ) + release_plan = build_release_plan(change_set, decision) + version_resolution = resolve_release_version( + release_plan, + current_version="1.0.0", + config=VersionAuthorityConfig(), + ) + + released_contract = candidate.model_copy(deep=True) + released_contract.version = version_resolution.selected_version + released_json = canonical_compact_json( + released_contract.model_dump(mode="json", by_alias=True, exclude_none=True) + ) + release = AppliedContractRelease( + applied_release_id="applied:review-test", + contract_id=release_plan.contract_id, + decision_id=decision.decision_id, + change_set_id=change_set.change_set_id, + release_plan_id=release_plan.release_plan_id, + version_resolution_id=version_resolution.version_resolution_id, + release_revision_ref=release_plan.release_revision_ref, + selected_version=version_resolution.selected_version, + authorization_id="authorization:apply-test", + released_contract_json=released_json, + ) + return decision, change_set, release_plan, version_resolution, release + + +def test_deterministic_helper_preserves_existing_compact_uuid_formula() -> None: + namespace = uuid.UUID("3ea0f6d8-28ca-4bb4-94f5-ea1f0f48cb84") + payload = {"z": "é", "a": [2, 1], "nested": {"b": True}} + + legacy_json = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) + legacy_id = str(uuid.uuid5(namespace, legacy_json)) + + assert canonical_compact_json(payload) == legacy_json + assert deterministic_uuid5(namespace, payload) == legacy_id + + +def test_legacy_release_imports_delegate_to_canonical_owners() -> None: + assert legacy_normalize_semver is normalize_semver + assert legacy_classify_contract_change is classify_contract_change + assert legacy_normalize_semver("v1.2.3") == "1.2.3" + + +def test_reconciliation_asset_projection_remains_compatible_but_neutral() -> None: + assert legacy_runtime_asset_specs_from_contract is runtime_asset_specs_from_contract + assert ObservationRuntimeAssetSpec is RuntimeAssetSpec + + specs = runtime_asset_specs_from_contract(_contract()) + assert specs == (RuntimeAssetSpec(governed_asset="orders", physical_name="orders"),) + + +def test_review_deploy_authorization_is_bound_to_exact_deployment_plan() -> None: + decision, change_set, release_plan, version_resolution, release = _review_release_chain() + production_plan = build_deployment_plan( + release, + DeploymentTarget(platform="databricks", runtime_target="main.production"), + ) + + evidence = ReviewAuthorizationEvidence( + evidence_reference="approval:deploy-production", + decision_id=decision.decision_id, + change_set_id=change_set.change_set_id, + release_plan_id=release_plan.release_plan_id, + version_resolution_id=version_resolution.version_resolution_id, + operation=GovernanceOperation.DEPLOY, + action=ReviewEvidenceAction.APPROVE, + scope_reference=production_plan.deployment_plan_id, + ) + contractops_authorization = authorize_contract_operation( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.DEPLOY, + evidence=evidence, + ) + + assert contractops_authorization.allowed is True + assert contractops_authorization.reason is AuthorizationReason.ALLOWED_BY_REVIEW + assert contractops_authorization.scope_reference == production_plan.deployment_plan_id + + deployment_authorization = authorize_deployment( + production_plan, + release, + contractops_authorization, + ) + assert deployment_authorization.allowed is True + assert deployment_authorization.deployment_plan_id == production_plan.deployment_plan_id + + staging_plan = build_deployment_plan( + release, + DeploymentTarget(platform="databricks", runtime_target="main.staging"), + ) + with pytest.raises(ReleaseValidationError, match="not scoped to this DeploymentPlan"): + authorize_deployment(staging_plan, release, contractops_authorization) + + +def test_publish_authorization_cannot_be_reused_for_runtime_deploy() -> None: + decision, change_set, release_plan, version_resolution, release = _review_release_chain() + plan = build_deployment_plan( + release, + DeploymentTarget(platform="databricks", runtime_target="main.production"), + ) + evidence = ReviewAuthorizationEvidence( + evidence_reference="approval:publish", + decision_id=decision.decision_id, + change_set_id=change_set.change_set_id, + release_plan_id=release_plan.release_plan_id, + version_resolution_id=version_resolution.version_resolution_id, + operation=GovernanceOperation.PUBLISH, + action=ReviewEvidenceAction.APPROVE, + ) + publish_authorization = authorize_contract_operation( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.PUBLISH, + evidence=evidence, + ) + + with pytest.raises(ReleaseValidationError, match="DEPLOY authorization"): + authorize_deployment(plan, release, publish_authorization) diff --git a/tests/test_governance_gate.py b/tests/test_governance_gate.py index a486c447..0dd17394 100644 --- a/tests/test_governance_gate.py +++ b/tests/test_governance_gate.py @@ -4,6 +4,7 @@ from datetime import date from pathlib import Path + import pytest from semapact.exceptions import GovernanceBlockedError, GovernanceReviewRequiredError @@ -78,15 +79,16 @@ def _make_decision(result: DecisionResult) -> GovernanceDecision: (DecisionResult.ALLOW, GovernanceOperation.PROPOSE, True, "allowed"), (DecisionResult.REVIEW, GovernanceOperation.PROPOSE, True, "allowed"), (DecisionResult.BLOCK, GovernanceOperation.PROPOSE, False, "blocked"), - # APPLY operation: ALLOW is allowed; REVIEW requires review; BLOCK is blocked + # Side-effect operations: ALLOW; REVIEW requires review; BLOCK is blocked (DecisionResult.ALLOW, GovernanceOperation.APPLY, True, "allowed"), (DecisionResult.REVIEW, GovernanceOperation.APPLY, False, "review_required"), (DecisionResult.BLOCK, GovernanceOperation.APPLY, False, "blocked"), - # PUBLISH operation: ALLOW is allowed; REVIEW requires review; BLOCK is blocked (DecisionResult.ALLOW, GovernanceOperation.PUBLISH, True, "allowed"), (DecisionResult.REVIEW, GovernanceOperation.PUBLISH, False, "review_required"), (DecisionResult.BLOCK, GovernanceOperation.PUBLISH, False, "blocked"), - # CI operation: ALLOW is allowed; REVIEW requires review; BLOCK is blocked + (DecisionResult.ALLOW, GovernanceOperation.DEPLOY, True, "allowed"), + (DecisionResult.REVIEW, GovernanceOperation.DEPLOY, False, "review_required"), + (DecisionResult.BLOCK, GovernanceOperation.DEPLOY, False, "blocked"), (DecisionResult.ALLOW, GovernanceOperation.CI, True, "allowed"), (DecisionResult.REVIEW, GovernanceOperation.CI, False, "review_required"), (DecisionResult.BLOCK, GovernanceOperation.CI, False, "blocked"), @@ -98,7 +100,7 @@ def test_evaluate_governance_gate_matrix( expected_allowed: bool, expected_reason: str, ): - """Verify all 15 combinations of DecisionResult x GovernanceOperation.""" + """Verify all DecisionResult x GovernanceOperation combinations.""" decision = _make_decision(decision_result) result = evaluate_governance_gate(decision, operation) @@ -111,22 +113,18 @@ def test_evaluate_governance_gate_type_checks(): """Verify evaluate_governance_gate rejects non-pydantic decision objects or raw string operations.""" decision = _make_decision(DecisionResult.ALLOW) - # Rejects dict input for decision with pytest.raises(TypeError, match="evaluate_governance_gate requires GovernanceDecision"): evaluate_governance_gate({"decision": "ALLOW"}, GovernanceOperation.CI) # type: ignore - # Rejects raw string input for operation with pytest.raises(TypeError, match="evaluate_governance_gate requires GovernanceOperation"): evaluate_governance_gate(decision, "CI") # type: ignore def test_governance_gate_result_invariants(): - """Verify GovernanceGateResult model invariants: allowed=True <-> reason='allowed'.""" - # allowed=True with reason='blocked' should fail + """Verify GovernanceGateResult model invariants.""" with pytest.raises(ValueError, match="GovernanceGateResult invariant violation"): GovernanceGateResult(allowed=True, reason="blocked", decision_id="dec-1") - # allowed=False with reason='allowed' should fail with pytest.raises(ValueError, match="GovernanceGateResult invariant violation"): GovernanceGateResult(allowed=False, reason="allowed", decision_id="dec-1") @@ -147,15 +145,15 @@ def test_enforce_governance_gate_raises_blocked_error(): def test_enforce_governance_gate_raises_review_required_error(): - """Verify enforce_governance_gate raises GovernanceReviewRequiredError on REVIEW decision for APPLY/CI.""" + """Verify REVIEW remains gated for side-effect operations.""" decision = _make_decision(DecisionResult.REVIEW) with pytest.raises(GovernanceReviewRequiredError) as exc_info: - enforce_governance_gate(decision, GovernanceOperation.APPLY) + enforce_governance_gate(decision, GovernanceOperation.DEPLOY) err = exc_info.value assert err.decision == decision - assert err.operation == GovernanceOperation.APPLY + assert err.operation == GovernanceOperation.DEPLOY assert "Governance decision REVIEW required" in str(err) From 6480f9070573ec660534471fba6490a992add262 Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Thu, 10 Sep 2026 17:11:22 +1000 Subject: [PATCH 10/35] feat(deployment): add guarded Databricks deployment adapter Adds a guarded write-side Databricks deployment adapter with deterministic preview artifacts, exact plan/authorization binding, runtime freshness checks, CREATE/add-nullable-column/NO_OP support, and fail-closed behavior for unsupported mutations. --- semapact/deployment/__init__.py | 10 +- semapact/deployment/adapters.py | 33 ++ semapact/deployment/authorization.py | 42 +- semapact/deployment/models.py | 202 ++++++++- semapact/deployment/planner.py | 33 +- semapact/platforms/databricks/__init__.py | 2 + semapact/platforms/databricks/deployment.py | 472 ++++++++++++++++++++ semapact/platforms/databricks/runtime.py | 15 +- semapact/platforms/databricks/target.py | 15 + semapact/platforms/runtime_registry.py | 37 ++ semapact/utils/__init__.py | 44 +- tests/test_deployment_databricks.py | 334 ++++++++++++++ 12 files changed, 1161 insertions(+), 78 deletions(-) create mode 100644 semapact/deployment/adapters.py create mode 100644 semapact/platforms/databricks/deployment.py create mode 100644 semapact/platforms/databricks/target.py create mode 100644 tests/test_deployment_databricks.py diff --git a/semapact/deployment/__init__.py b/semapact/deployment/__init__.py index 576c2f2e..70a0e9f3 100644 --- a/semapact/deployment/__init__.py +++ b/semapact/deployment/__init__.py @@ -1,21 +1,29 @@ -"""Provider-neutral deployment planning and authorization boundary.""" +"""Provider-neutral deployment planning, preview, and authorization boundary.""" +from semapact.deployment.adapters import DeploymentAdapter from semapact.deployment.authorization import authorize_deployment from semapact.deployment.models import ( DeploymentAction, DeploymentActionKind, DeploymentAuthorization, DeploymentPlan, + DeploymentPreview, DeploymentTarget, + NativeOperation, + NativeOperationKind, ) from semapact.deployment.planner import build_deployment_plan __all__ = [ "DeploymentAction", "DeploymentActionKind", + "DeploymentAdapter", "DeploymentAuthorization", "DeploymentPlan", + "DeploymentPreview", "DeploymentTarget", + "NativeOperation", + "NativeOperationKind", "authorize_deployment", "build_deployment_plan", ] diff --git a/semapact/deployment/adapters.py b/semapact/deployment/adapters.py new file mode 100644 index 00000000..08f69687 --- /dev/null +++ b/semapact/deployment/adapters.py @@ -0,0 +1,33 @@ +"""Replaceable write-side deployment adapter boundary.""" + +from __future__ import annotations + +from typing import Protocol + +from semapact.deployment.models import ( + DeploymentAuthorization, + DeploymentPlan, + DeploymentPreview, +) +from semapact.observation.models import ObservedPlatformState + + +class DeploymentAdapter(Protocol): + """Translate and execute one exact DeploymentPlan for a runtime provider.""" + + key: str + + def validate(self, plan: DeploymentPlan) -> None: ... + + def preview( + self, + plan: DeploymentPlan, + observed_state: ObservedPlatformState, + ) -> DeploymentPreview: ... + + def execute( + self, + plan: DeploymentPlan, + preview: DeploymentPreview, + authorization: DeploymentAuthorization, + ) -> None: ... diff --git a/semapact/deployment/authorization.py b/semapact/deployment/authorization.py index 1c8de48a..6f475fdb 100644 --- a/semapact/deployment/authorization.py +++ b/semapact/deployment/authorization.py @@ -2,19 +2,16 @@ from __future__ import annotations -import uuid - from semapact.contractops.execution_models import AppliedContractRelease from semapact.contractops.models import AuthorizationReason, ContractOpsAuthorization -from semapact.deployment.models import DeploymentAuthorization, DeploymentPlan +from semapact.deployment.models import ( + DeploymentAuthorization, + DeploymentPlan, + compute_deployment_authorization_id, + validate_deployment_plan_identity, +) from semapact.exceptions import ReleaseValidationError from semapact.governance.gate import GovernanceOperation -from semapact.utils.deterministic import deterministic_uuid5 - - -SEMAPACT_DEPLOYMENT_AUTHORIZATION_NAMESPACE = uuid.UUID( - "c1dd6b40-cb67-44c1-b0f2-5f133ba6a3f5" -) def authorize_deployment( @@ -22,13 +19,7 @@ def authorize_deployment( release: AppliedContractRelease, authorization: ContractOpsAuthorization, ) -> DeploymentAuthorization: - """Bind one DEPLOY authorization to the exact deployment target/plan. - - Upstream ContractOps remains authoritative for governance/review semantics. - Review-based DEPLOY authorization must carry ``scope_reference`` equal to this - exact ``deployment_plan_id``; governance-ALLOW paths need no synthetic review - scope but are still bound here before an adapter may execute the plan. - """ + """Bind one DEPLOY authorization to the exact deployment target/plan.""" if not isinstance(plan, DeploymentPlan): raise TypeError(f"plan must be DeploymentPlan, got {type(plan).__name__}") if not isinstance(release, AppliedContractRelease): @@ -41,6 +32,8 @@ def authorize_deployment( f"got {type(authorization).__name__}" ) + validate_deployment_plan_identity(plan) + if authorization.operation is not GovernanceOperation.DEPLOY: raise ReleaseValidationError( "Deployment requires operation-scoped DEPLOY authorization" @@ -58,17 +51,14 @@ def authorize_deployment( "Review-based DEPLOY authorization is not scoped to this DeploymentPlan" ) - stable_record = { - "contract_ops_authorization_id": authorization.authorization_id, - "deployment_plan_id": plan.deployment_plan_id, - "applied_release_id": release.applied_release_id, - "allowed": authorization.allowed, - } + deployment_authorization_id = compute_deployment_authorization_id( + contract_ops_authorization_id=authorization.authorization_id, + deployment_plan_id=plan.deployment_plan_id, + applied_release_id=release.applied_release_id, + allowed=authorization.allowed, + ) return DeploymentAuthorization( - deployment_authorization_id=deterministic_uuid5( - SEMAPACT_DEPLOYMENT_AUTHORIZATION_NAMESPACE, - stable_record, - ), + deployment_authorization_id=deployment_authorization_id, contract_ops_authorization_id=authorization.authorization_id, deployment_plan_id=plan.deployment_plan_id, applied_release_id=release.applied_release_id, diff --git a/semapact/deployment/models.py b/semapact/deployment/models.py index 0f3ec980..857468ec 100644 --- a/semapact/deployment/models.py +++ b/semapact/deployment/models.py @@ -1,14 +1,27 @@ -"""Provider-neutral immutable deployment planning and authorization artifacts.""" +"""Provider-neutral immutable deployment planning, preview, and authorization artifacts.""" from __future__ import annotations +import uuid from enum import Enum -from typing import Literal +from typing import Literal, Sequence from open_data_contract_standard.model import SchemaObject from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from semapact.lifecycle.identity import normalize_identity_name +from semapact.utils.deterministic import deterministic_uuid5 + + +SEMAPACT_DEPLOYMENT_PLAN_NAMESPACE = uuid.UUID( + "d59eaa31-997a-478d-9978-4659beee673d" +) +SEMAPACT_DEPLOYMENT_AUTHORIZATION_NAMESPACE = uuid.UUID( + "c1dd6b40-cb67-44c1-b0f2-5f133ba6a3f5" +) +SEMAPACT_DEPLOYMENT_PREVIEW_NAMESPACE = uuid.UUID( + "0ee613b9-a9ef-4a95-ac87-25d6c706b16c" +) class DeploymentModel(BaseModel): @@ -23,6 +36,14 @@ class DeploymentActionKind(str, Enum): ENSURE_ASSET_STATE = "ENSURE_ASSET_STATE" +class NativeOperationKind(str, Enum): + """Provider-native operation classes derived from runtime evidence.""" + + CREATE = "CREATE" + ALTER = "ALTER" + NO_OP = "NO_OP" + + class DeploymentTarget(DeploymentModel): """Explicit runtime target for one DeploymentPlan.""" @@ -131,6 +152,66 @@ def _validate_action_order_and_identity(self) -> DeploymentPlan: raise ValueError("DeploymentPlan actions must be ordered by governed_asset") if len(governed_assets) != len(set(governed_assets)): raise ValueError("DeploymentPlan cannot contain duplicate governed assets") + validate_deployment_plan_identity(self) + return self + + +class NativeOperation(DeploymentModel): + """One immutable provider-native operation selected by an adapter preview.""" + + kind: NativeOperationKind + governed_asset: str + statement: str | None = None + + @field_validator("governed_asset") + @classmethod + def _require_governed_asset(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("governed_asset must not be empty") + return cleaned + + @model_validator(mode="after") + def _validate_statement_shape(self) -> NativeOperation: + if self.kind is NativeOperationKind.NO_OP: + if self.statement is not None: + raise ValueError("NO_OP must not carry a provider statement") + return self + if self.statement is None or not self.statement.strip(): + raise ValueError(f"{self.kind.value} requires a provider statement") + return self + + +class DeploymentPreview(DeploymentModel): + """Deterministic provider-native preview bound to one observed runtime state.""" + + deployment_preview_id: str + deployment_plan_id: str + platform: str + runtime_target: str + source_identifier: str + observation_fingerprint: str + operations: tuple[NativeOperation, ...] + preview_version: Literal["1"] = "1" + + @field_validator( + "deployment_preview_id", + "deployment_plan_id", + "platform", + "runtime_target", + "source_identifier", + "observation_fingerprint", + ) + @classmethod + def _require_preview_text(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("preview fields must not be empty") + return cleaned + + @model_validator(mode="after") + def _validate_identity(self) -> DeploymentPreview: + validate_deployment_preview_identity(self) return self @@ -155,3 +236,120 @@ def _require_authorization_text(cls, value: str) -> str: if not cleaned: raise ValueError("value must not be empty") return cleaned + + @model_validator(mode="after") + def _validate_identity(self) -> DeploymentAuthorization: + validate_deployment_authorization_identity(self) + return self + + +def compute_deployment_plan_id( + *, + applied_release_id: str, + contract_id: str, + release_plan_id: str, + released_revision_ref: str, + selected_version: str, + target: DeploymentTarget, + actions: Sequence[DeploymentAction], + plan_version: str = "1", +) -> str: + return deterministic_uuid5( + SEMAPACT_DEPLOYMENT_PLAN_NAMESPACE, + { + "applied_release_id": applied_release_id, + "contract_id": contract_id, + "release_plan_id": release_plan_id, + "released_revision_ref": released_revision_ref, + "selected_version": selected_version, + "target": target.model_dump(mode="json"), + "actions": [action.model_dump(mode="json") for action in actions], + "plan_version": plan_version, + }, + ) + + +def compute_deployment_authorization_id( + *, + contract_ops_authorization_id: str, + deployment_plan_id: str, + applied_release_id: str, + allowed: bool, +) -> str: + return deterministic_uuid5( + SEMAPACT_DEPLOYMENT_AUTHORIZATION_NAMESPACE, + { + "contract_ops_authorization_id": contract_ops_authorization_id, + "deployment_plan_id": deployment_plan_id, + "applied_release_id": applied_release_id, + "allowed": allowed, + }, + ) + + +def compute_deployment_preview_id( + *, + deployment_plan_id: str, + platform: str, + runtime_target: str, + source_identifier: str, + observation_fingerprint: str, + operations: Sequence[NativeOperation], + preview_version: str = "1", +) -> str: + return deterministic_uuid5( + SEMAPACT_DEPLOYMENT_PREVIEW_NAMESPACE, + { + "deployment_plan_id": deployment_plan_id, + "platform": platform, + "runtime_target": runtime_target, + "source_identifier": source_identifier, + "observation_fingerprint": observation_fingerprint, + "operations": [operation.model_dump(mode="json") for operation in operations], + "preview_version": preview_version, + }, + ) + + +def validate_deployment_plan_identity(plan: DeploymentPlan) -> None: + expected = compute_deployment_plan_id( + applied_release_id=plan.applied_release_id, + contract_id=plan.contract_id, + release_plan_id=plan.release_plan_id, + released_revision_ref=plan.released_revision_ref, + selected_version=plan.selected_version, + target=plan.target, + actions=plan.actions, + plan_version=plan.plan_version, + ) + if expected != plan.deployment_plan_id: + raise ValueError("DeploymentPlan deterministic identity does not match content") + + +def validate_deployment_authorization_identity( + authorization: DeploymentAuthorization, +) -> None: + expected = compute_deployment_authorization_id( + contract_ops_authorization_id=authorization.contract_ops_authorization_id, + deployment_plan_id=authorization.deployment_plan_id, + applied_release_id=authorization.applied_release_id, + allowed=authorization.allowed, + ) + if expected != authorization.deployment_authorization_id: + raise ValueError( + "DeploymentAuthorization deterministic identity does not match content" + ) + + +def validate_deployment_preview_identity(preview: DeploymentPreview) -> None: + expected = compute_deployment_preview_id( + deployment_plan_id=preview.deployment_plan_id, + platform=preview.platform, + runtime_target=preview.runtime_target, + source_identifier=preview.source_identifier, + observation_fingerprint=preview.observation_fingerprint, + operations=preview.operations, + preview_version=preview.preview_version, + ) + if expected != preview.deployment_preview_id: + raise ValueError("DeploymentPreview deterministic identity does not match content") diff --git a/semapact/deployment/planner.py b/semapact/deployment/planner.py index e5d3677e..8aec8394 100644 --- a/semapact/deployment/planner.py +++ b/semapact/deployment/planner.py @@ -2,8 +2,6 @@ from __future__ import annotations -import uuid - from open_data_contract_standard.model import SchemaObject from semapact.contractops.execution_models import AppliedContractRelease @@ -12,15 +10,11 @@ DeploymentActionKind, DeploymentPlan, DeploymentTarget, + compute_deployment_plan_id, ) from semapact.lifecycle.identity import normalize_identity_name from semapact.runtime import runtime_asset_specs_from_contract -from semapact.utils.deterministic import canonical_compact_json, deterministic_uuid5 - - -SEMAPACT_DEPLOYMENT_PLAN_NAMESPACE = uuid.UUID( - "d59eaa31-997a-478d-9978-4659beee673d" -) +from semapact.utils.deterministic import canonical_compact_json def build_deployment_plan( @@ -49,8 +43,6 @@ def build_deployment_plan( for schema in contract.schema_ or []: raw_name = getattr(schema, "name", None) if raw_name is None: - # runtime_asset_specs_from_contract() already validates this path; keep - # this guard explicit for planner exhaustiveness. raise RuntimeError("Governed schema identity unexpectedly missing") governed_asset = normalize_identity_name(str(raw_name), "Schema") spec = specs_by_asset[governed_asset] @@ -64,19 +56,14 @@ def build_deployment_plan( ) ordered_actions = tuple(sorted(actions, key=lambda action: action.governed_asset)) - stable_record = { - "applied_release_id": release.applied_release_id, - "contract_id": release.contract_id, - "release_plan_id": release.release_plan_id, - "released_revision_ref": release.release_revision_ref, - "selected_version": release.selected_version, - "target": target.model_dump(mode="json"), - "actions": [action.model_dump(mode="json") for action in ordered_actions], - "plan_version": "1", - } - deployment_plan_id = deterministic_uuid5( - SEMAPACT_DEPLOYMENT_PLAN_NAMESPACE, - stable_record, + deployment_plan_id = compute_deployment_plan_id( + applied_release_id=release.applied_release_id, + contract_id=release.contract_id, + release_plan_id=release.release_plan_id, + released_revision_ref=release.release_revision_ref, + selected_version=release.selected_version, + target=target, + actions=ordered_actions, ) return DeploymentPlan( diff --git a/semapact/platforms/databricks/__init__.py b/semapact/platforms/databricks/__init__.py index 64d75278..6ae24d6b 100644 --- a/semapact/platforms/databricks/__init__.py +++ b/semapact/platforms/databricks/__init__.py @@ -1,10 +1,12 @@ """Databricks platform-access helpers.""" from semapact.platforms.databricks.client import create_databricks_workspace_client +from semapact.platforms.databricks.deployment import DatabricksDeploymentAdapter from semapact.platforms.databricks.discovery import discover_databricks_tables from semapact.platforms.databricks.runtime import DatabricksRuntimeProvider __all__ = [ + "DatabricksDeploymentAdapter", "DatabricksRuntimeProvider", "create_databricks_workspace_client", "discover_databricks_tables", diff --git a/semapact/platforms/databricks/deployment.py b/semapact/platforms/databricks/deployment.py new file mode 100644 index 00000000..fb8303a5 --- /dev/null +++ b/semapact/platforms/databricks/deployment.py @@ -0,0 +1,472 @@ +"""Databricks write-side deployment adapter.""" + +from __future__ import annotations + +import re +import time +from typing import Any + +from open_data_contract_standard.model import SchemaObject, SchemaProperty + +from semapact.deployment.models import ( + DeploymentActionKind, + DeploymentAuthorization, + DeploymentPlan, + DeploymentPreview, + NativeOperation, + NativeOperationKind, + compute_deployment_preview_id, + validate_deployment_authorization_identity, + validate_deployment_plan_identity, + validate_deployment_preview_identity, +) +from semapact.exceptions import ContractOpsAuthorizationError, ValidationError +from semapact.observation.fingerprint import fingerprint_observed_state +from semapact.observation.models import ObservedAsset, ObservedPlatformState +from semapact.observation.providers import RuntimeProvider +from semapact.platforms.databricks.target import parse_databricks_runtime_target +from semapact.runtime import RuntimeAssetSpec + +_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_$]*$") +_DECIMAL_RE = re.compile(r"^DECIMAL\((\d{1,2}),(\d{1,2})\)$") +_CHAR_RE = re.compile(r"^(CHAR|VARCHAR)\((\d+)\)$") +_PRIMITIVE_TYPES = { + "BIGINT", + "BINARY", + "BOOLEAN", + "BYTE", + "DATE", + "DOUBLE", + "FLOAT", + "INT", + "INTEGER", + "LONG", + "REAL", + "SHORT", + "SMALLINT", + "STRING", + "TIMESTAMP", + "TIMESTAMP_NTZ", + "TINYINT", +} +_TERMINAL_STATES = {"SUCCEEDED", "FAILED", "CANCELED", "CLOSED"} + + +class DatabricksDeploymentAdapter: + """Translate approved desired state into guarded Unity Catalog mutations.""" + + key = "databricks" + + def __init__( + self, + *, + client: Any, + runtime_provider: RuntimeProvider, + warehouse_id: str, + poll_interval_seconds: float = 1.0, + max_poll_attempts: int = 300, + ) -> None: + if not warehouse_id.strip(): + raise ValueError("warehouse_id is required for Databricks deployment") + if poll_interval_seconds < 0: + raise ValueError("poll_interval_seconds must be non-negative") + if max_poll_attempts < 1: + raise ValueError("max_poll_attempts must be positive") + self._client = client + self._runtime_provider = runtime_provider + self._warehouse_id = warehouse_id.strip() + self._poll_interval_seconds = poll_interval_seconds + self._max_poll_attempts = max_poll_attempts + + def validate(self, plan: DeploymentPlan) -> None: + """Fail closed when a released desired state cannot be safely translated.""" + validate_deployment_plan_identity(plan) + if plan.target.platform.casefold() != self.key: + raise ValidationError( + f"Databricks adapter cannot deploy platform '{plan.target.platform}'" + ) + catalog, schema_name = parse_databricks_runtime_target(plan.target.runtime_target) + _validate_identifier(catalog, "catalog") + _validate_identifier(schema_name, "schema") + + physical_assets: set[str] = set() + for action in plan.actions: + if action.kind is not DeploymentActionKind.ENSURE_ASSET_STATE: + raise ValidationError( + f"Unsupported deployment action kind: {action.kind.value}" + ) + _validate_identifier(action.physical_name, "asset") + asset_key = action.physical_name.casefold() + if asset_key in physical_assets: + raise ValidationError( + "Databricks deployment cannot bind multiple governed assets to " + f"the same physical asset '{action.physical_name}'" + ) + physical_assets.add(asset_key) + desired = SchemaObject.model_validate_json(action.desired_state_json) + _desired_columns(desired) + + def preview( + self, + plan: DeploymentPlan, + observed_state: ObservedPlatformState, + ) -> DeploymentPreview: + """Derive exact CREATE/ALTER/NO_OP operations from current runtime evidence.""" + self.validate(plan) + _validate_observation(plan, observed_state) + catalog, schema_name = parse_databricks_runtime_target(plan.target.runtime_target) + observed_by_asset = { + asset.identity.asset.casefold(): asset for asset in observed_state.assets + } + + operations: list[NativeOperation] = [] + for action in plan.actions: + desired = SchemaObject.model_validate_json(action.desired_state_json) + observed = observed_by_asset.get(action.physical_name.casefold()) + if observed is None: + statement = _create_table_statement( + catalog=catalog, + schema_name=schema_name, + table_name=action.physical_name, + desired=desired, + ) + operations.append( + NativeOperation( + kind=NativeOperationKind.CREATE, + governed_asset=action.governed_asset, + statement=statement, + ) + ) + continue + + additions = _required_additions(desired, observed) + if additions: + statement = _add_columns_statement( + catalog=catalog, + schema_name=schema_name, + table_name=action.physical_name, + additions=additions, + ) + operations.append( + NativeOperation( + kind=NativeOperationKind.ALTER, + governed_asset=action.governed_asset, + statement=statement, + ) + ) + else: + operations.append( + NativeOperation( + kind=NativeOperationKind.NO_OP, + governed_asset=action.governed_asset, + ) + ) + + ordered = tuple(operations) + assert observed_state.fingerprint is not None + preview_id = compute_deployment_preview_id( + deployment_plan_id=plan.deployment_plan_id, + platform=self.key, + runtime_target=plan.target.runtime_target, + source_identifier=observed_state.source_identifier, + observation_fingerprint=observed_state.fingerprint, + operations=ordered, + ) + return DeploymentPreview( + deployment_preview_id=preview_id, + deployment_plan_id=plan.deployment_plan_id, + platform=self.key, + runtime_target=plan.target.runtime_target, + source_identifier=observed_state.source_identifier, + observation_fingerprint=observed_state.fingerprint, + operations=ordered, + ) + + def execute( + self, + plan: DeploymentPlan, + preview: DeploymentPreview, + authorization: DeploymentAuthorization, + ) -> None: + """Execute only the exact preview while its runtime preconditions still hold.""" + validate_deployment_plan_identity(plan) + validate_deployment_preview_identity(preview) + validate_deployment_authorization_identity(authorization) + self.validate(plan) + + if not authorization.allowed: + raise ContractOpsAuthorizationError("DeploymentAuthorization is not allowed") + if authorization.deployment_plan_id != plan.deployment_plan_id: + raise ContractOpsAuthorizationError( + "DeploymentAuthorization is not bound to this DeploymentPlan" + ) + if authorization.applied_release_id != plan.applied_release_id: + raise ContractOpsAuthorizationError( + "DeploymentAuthorization release does not match DeploymentPlan" + ) + if preview.deployment_plan_id != plan.deployment_plan_id: + raise ValidationError("DeploymentPreview is not bound to this DeploymentPlan") + if preview.platform.casefold() != self.key: + raise ValidationError("DeploymentPreview platform does not match adapter") + if preview.runtime_target != plan.target.runtime_target: + raise ValidationError("DeploymentPreview target does not match DeploymentPlan") + + current = self._observe_plan_scope(plan) + if current.source_identifier != preview.source_identifier: + raise ValidationError( + "Runtime source changed since DeploymentPreview was produced" + ) + if current.fingerprint != preview.observation_fingerprint: + raise ValidationError( + "Runtime state changed since DeploymentPreview was produced" + ) + + expected = self.preview(plan, current) + if expected != preview: + raise ValidationError( + "DeploymentPreview no longer equals the deterministic preview for " + "the authorized plan and runtime evidence" + ) + + for operation in preview.operations: + if operation.kind is NativeOperationKind.NO_OP: + continue + assert operation.statement is not None + self._execute_statement(operation.statement) + + def _observe_plan_scope(self, plan: DeploymentPlan) -> ObservedPlatformState: + assets = tuple( + RuntimeAssetSpec( + governed_asset=action.governed_asset, + physical_name=action.physical_name, + ) + for action in plan.actions + ) + bindings = self._runtime_provider.resolve_bindings( + runtime_target=plan.target.runtime_target, + assets=assets, + ) + return self._runtime_provider.observe(bindings=bindings) + + def _execute_statement(self, statement: str) -> None: + response = self._client.statement_execution.execute_statement( + statement=statement, + warehouse_id=self._warehouse_id, + wait_timeout="10s", + ) + state = _statement_state(response) + attempts = 0 + while state not in _TERMINAL_STATES: + statement_id = getattr(response, "statement_id", None) + if not statement_id: + raise RuntimeError( + "Databricks statement is non-terminal but returned no statement_id" + ) + if attempts >= self._max_poll_attempts: + raise RuntimeError( + f"Databricks statement did not reach terminal state: {statement_id}" + ) + if self._poll_interval_seconds: + time.sleep(self._poll_interval_seconds) + response = self._client.statement_execution.get_statement(statement_id) + state = _statement_state(response) + attempts += 1 + + if state != "SUCCEEDED": + status = getattr(response, "status", None) + error = getattr(status, "error", None) + raise RuntimeError( + f"Databricks deployment statement finished with state {state}: {error}" + ) + + +def _validate_observation( + plan: DeploymentPlan, + observed_state: ObservedPlatformState, +) -> None: + if observed_state.platform.casefold() != "databricks": + raise ValidationError("Databricks preview requires Databricks runtime evidence") + if not observed_state.source_identifier.strip(): + raise ValidationError("Runtime observation source_identifier is required") + if observed_state.fingerprint is None: + raise ValidationError("Runtime observation fingerprint is required") + if observed_state.fingerprint != fingerprint_observed_state(observed_state): + raise ValidationError("Runtime observation fingerprint does not match its content") + + namespace = parse_databricks_runtime_target(plan.target.runtime_target) + expected_assets = {action.physical_name.casefold() for action in plan.actions} + seen: set[str] = set() + for asset in observed_state.assets: + identity = asset.identity + if identity.platform.casefold() != "databricks": + raise ValidationError("Observed asset platform does not match Databricks") + if tuple(part.casefold() for part in identity.namespace) != tuple( + part.casefold() for part in namespace + ): + raise ValidationError("Observed asset is outside DeploymentPlan runtime target") + asset_key = identity.asset.casefold() + if asset_key not in expected_assets: + raise ValidationError("Runtime evidence contains an asset outside plan scope") + if asset_key in seen: + raise ValidationError("Runtime evidence contains duplicate asset identities") + seen.add(asset_key) + + +def _desired_columns( + desired: SchemaObject, +) -> tuple[tuple[str, str, bool], ...]: + columns: list[tuple[str, str, bool]] = [] + seen: set[str] = set() + for prop in desired.properties or []: + physical_name = _property_physical_name(prop) + _validate_identifier(physical_name, "column") + key = physical_name.casefold() + if key in seen: + raise ValidationError( + f"Duplicate physical column binding in desired schema: '{physical_name}'" + ) + seen.add(key) + physical_type = getattr(prop, "physicalType", None) + if physical_type is None or not str(physical_type).strip(): + raise ValidationError( + f"Databricks deployment requires physicalType for column '{physical_name}'" + ) + rendered_type = _render_type(str(physical_type)) + columns.append((physical_name, rendered_type, bool(getattr(prop, "required", False)))) + if not columns: + raise ValidationError("Databricks deployment requires at least one schema property") + return tuple(columns) + + +def _required_additions( + desired: SchemaObject, + observed: ObservedAsset, +) -> tuple[tuple[str, str, bool], ...]: + if (observed.asset_type or "").strip().casefold() != "managed": + raise ValidationError( + "Existing Databricks asset must be a MANAGED table for deployment mutation" + ) + + observed_columns = { + prop.identity.property.casefold(): prop for prop in observed.properties + } + additions: list[tuple[str, str, bool]] = [] + for name, desired_type, required in _desired_columns(desired): + current = observed_columns.get(name.casefold()) + if current is None: + if required: + raise ValidationError( + f"Cannot add required column '{name}' without a safe default" + ) + additions.append((name, desired_type, required)) + continue + if current.physical_type is None: + raise ValidationError(f"Observed type is unknown for column '{name}'") + current_type = _render_type(current.physical_type) + if current_type != desired_type: + raise ValidationError( + f"Unsupported existing column type mutation for '{name}': " + f"{current_type} -> {desired_type}" + ) + if current.nullable is None: + raise ValidationError(f"Observed nullability is unknown for column '{name}'") + desired_nullable = not required + if current.nullable is not desired_nullable: + raise ValidationError( + f"Unsupported existing column nullability mutation for '{name}'" + ) + return tuple(additions) + + +def _property_physical_name(prop: SchemaProperty) -> str: + physical = getattr(prop, "physicalName", None) + if physical is not None and str(physical).strip(): + return str(physical).strip() + name = getattr(prop, "name", None) + if name is None or not str(name).strip(): + raise ValidationError("Schema property name is required for deployment binding") + return str(name).strip() + + +def _create_table_statement( + *, + catalog: str, + schema_name: str, + table_name: str, + desired: SchemaObject, +) -> str: + columns = [] + for name, physical_type, required in _desired_columns(desired): + suffix = " NOT NULL" if required else "" + columns.append(f"{_quote_identifier(name)} {physical_type}{suffix}") + column_sql = ", ".join(columns) + return ( + f"CREATE TABLE {_qualified_name(catalog, schema_name, table_name)} " + f"({column_sql}) USING DELTA" + ) + + +def _add_columns_statement( + *, + catalog: str, + schema_name: str, + table_name: str, + additions: tuple[tuple[str, str, bool], ...], +) -> str: + columns = ", ".join( + f"{_quote_identifier(name)} {physical_type}" + for name, physical_type, _required in additions + ) + return ( + f"ALTER TABLE {_qualified_name(catalog, schema_name, table_name)} " + f"ADD COLUMNS ({columns})" + ) + + +def _qualified_name(catalog: str, schema_name: str, table_name: str) -> str: + return ".".join( + _quote_identifier(part) for part in (catalog, schema_name, table_name) + ) + + +def _quote_identifier(value: str) -> str: + _validate_identifier(value, "identifier") + return f"`{value}`" + + +def _validate_identifier(value: str, role: str) -> None: + if not _IDENTIFIER_RE.fullmatch(value): + raise ValidationError( + f"Unsupported Databricks {role} identifier for M1 deployment: '{value}'" + ) + + +def _render_type(value: str) -> str: + normalized = re.sub(r"\s+", "", value.strip().upper()) + if normalized in _PRIMITIVE_TYPES: + return normalized + + decimal = _DECIMAL_RE.fullmatch(normalized) + if decimal: + precision = int(decimal.group(1)) + scale = int(decimal.group(2)) + if 1 <= precision <= 38 and 0 <= scale <= precision: + return f"DECIMAL({precision},{scale})" + raise ValidationError(f"Unsupported Databricks DECIMAL type: '{value}'") + + char_type = _CHAR_RE.fullmatch(normalized) + if char_type: + length = int(char_type.group(2)) + if length > 0: + return f"{char_type.group(1)}({length})" + + raise ValidationError(f"Unsupported Databricks physicalType: '{value}'") + + +def _statement_state(response: Any) -> str: + status = getattr(response, "status", None) + state = getattr(status, "state", None) + if state is None: + raise RuntimeError("Databricks statement response did not contain status.state") + value = getattr(state, "value", state) + return str(value).upper() diff --git a/semapact/platforms/databricks/runtime.py b/semapact/platforms/databricks/runtime.py index 8888a8d6..c92ff784 100644 --- a/semapact/platforms/databricks/runtime.py +++ b/semapact/platforms/databricks/runtime.py @@ -5,11 +5,11 @@ from datetime import datetime, timezone from typing import Any, Sequence -from semapact.exceptions import ValidationError from semapact.observation.databricks import observe_databricks_table from semapact.observation.fingerprint import with_observed_state_fingerprint from semapact.observation.models import ObservedAssetIdentity, ObservedPlatformState from semapact.observation.providers import RuntimeAssetBinding, RuntimeAssetSpec +from semapact.platforms.databricks.target import parse_databricks_runtime_target class DatabricksRuntimeProvider: @@ -30,7 +30,7 @@ def resolve_bindings( assets: Sequence[RuntimeAssetSpec], ) -> tuple[RuntimeAssetBinding, ...]: """Resolve ``catalog.schema`` plus asset physical names into UC identities.""" - namespace = _parse_runtime_target(runtime_target) + namespace = parse_databricks_runtime_target(runtime_target) bindings = tuple( RuntimeAssetBinding( governed_asset=asset.governed_asset, @@ -58,11 +58,11 @@ def observe( identity = binding.observed_asset if identity.platform.casefold() != self.key: raise ValueError("Databricks provider received a non-Databricks binding") - table_fqn = ".".join((*identity.namespace, identity.asset)) if len(identity.namespace) != 2: raise ValueError( "Databricks runtime asset identity requires catalog and schema namespace" ) + table_fqn = ".".join((*identity.namespace, identity.asset)) try: observed = observe_databricks_table( client=self._client, @@ -84,15 +84,6 @@ def observe( return with_observed_state_fingerprint(state) -def _parse_runtime_target(value: str) -> tuple[str, str]: - parts = tuple(part.strip() for part in value.split(".")) - if len(parts) != 2 or not all(parts): - raise ValidationError( - "Databricks runtime target must use catalog.schema format for a data product" - ) - return parts[0], parts[1] - - def _load_databricks_not_found_error() -> type[BaseException]: try: from databricks.sdk.errors import NotFound diff --git a/semapact/platforms/databricks/target.py b/semapact/platforms/databricks/target.py new file mode 100644 index 00000000..f0038d85 --- /dev/null +++ b/semapact/platforms/databricks/target.py @@ -0,0 +1,15 @@ +"""Shared Databricks runtime-target parsing.""" + +from __future__ import annotations + +from semapact.exceptions import ValidationError + + +def parse_databricks_runtime_target(value: str) -> tuple[str, str]: + """Parse the provider-local ``catalog.schema`` target descriptor.""" + parts = tuple(part.strip() for part in value.split(".")) + if len(parts) != 2 or not all(parts): + raise ValidationError( + "Databricks runtime target must use catalog.schema format for a data product" + ) + return parts[0], parts[1] diff --git a/semapact/platforms/runtime_registry.py b/semapact/platforms/runtime_registry.py index 87b86dd4..436d3b92 100644 --- a/semapact/platforms/runtime_registry.py +++ b/semapact/platforms/runtime_registry.py @@ -7,6 +7,7 @@ from open_data_contract_standard.model import OpenDataContractStandard, Server +from semapact.deployment.adapters import DeploymentAdapter from semapact.exceptions import ValidationError from semapact.observation import RuntimeProvider, RuntimeProviderRegistry @@ -80,6 +81,42 @@ def create_runtime_provider_registry( ) +def create_deployment_adapter( + platform: str, + *, + warehouse_id: str, + contract_server: Server | None = None, +) -> DeploymentAdapter: + """Compose the selected write adapter and provider clients lazily.""" + normalized = platform.strip().casefold() + if normalized != "databricks": + raise ValidationError( + f"Unsupported deployment adapter '{platform}'. Supported adapters: databricks" + ) + + from semapact.platforms.databricks import ( + DatabricksDeploymentAdapter, + DatabricksRuntimeProvider, + create_databricks_workspace_client, + ) + + client = create_databricks_workspace_client( + workspace_url=_clean(contract_server.host) if contract_server else None + ) + source_identifier = getattr(getattr(client, "config", None), "host", None) + if not isinstance(source_identifier, str) or not source_identifier.strip(): + raise RuntimeError("Databricks SDK did not resolve a workspace host") + runtime_provider = DatabricksRuntimeProvider( + client=client, + source_identifier=source_identifier, + ) + return DatabricksDeploymentAdapter( + client=client, + runtime_provider=runtime_provider, + warehouse_id=warehouse_id, + ) + + def _select_contract_server( servers: tuple[Server, ...], requested_name: str | None, diff --git a/semapact/utils/__init__.py b/semapact/utils/__init__.py index 49df098d..482e3856 100644 --- a/semapact/utils/__init__.py +++ b/semapact/utils/__init__.py @@ -1,14 +1,30 @@ -from semapact.utils.schema_utils import ( - contract_to_dict, - contract_to_model, - ensure_schema_key, -) -from semapact.utils.yaml_utils import dump_yaml, load_yaml - -__all__ = [ - "contract_to_model", - "contract_to_dict", - "ensure_schema_key", - "load_yaml", - "dump_yaml", -] +"""Utility exports kept lazy so focused helpers do not import unrelated core modules.""" + +from __future__ import annotations + +from importlib import import_module +from typing import Any + +_EXPORTS: dict[str, tuple[str, str]] = { + "contract_to_dict": ("semapact.utils.schema_utils", "contract_to_dict"), + "contract_to_model": ("semapact.utils.schema_utils", "contract_to_model"), + "ensure_schema_key": ("semapact.utils.schema_utils", "ensure_schema_key"), + "dump_yaml": ("semapact.utils.yaml_utils", "dump_yaml"), + "load_yaml": ("semapact.utils.yaml_utils", "load_yaml"), +} + +__all__ = list(_EXPORTS) + + +def __getattr__(name: str) -> Any: + target = _EXPORTS.get(name) + if target is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + module_name, attribute = target + value = getattr(import_module(module_name), attribute) + globals()[name] = value + return value + + +def __dir__() -> list[str]: + return sorted({*globals(), *__all__}) diff --git a/tests/test_deployment_databricks.py b/tests/test_deployment_databricks.py new file mode 100644 index 00000000..a70744b1 --- /dev/null +++ b/tests/test_deployment_databricks.py @@ -0,0 +1,334 @@ +from __future__ import annotations + +import json +from datetime import datetime, timezone +from types import SimpleNamespace + +import pytest +from open_data_contract_standard.model import SchemaObject, SchemaProperty + +from semapact.deployment.models import ( + DeploymentAction, + DeploymentActionKind, + DeploymentAuthorization, + DeploymentPlan, + DeploymentTarget, + NativeOperation, + NativeOperationKind, + compute_deployment_authorization_id, + compute_deployment_plan_id, + compute_deployment_preview_id, +) +from semapact.exceptions import ContractOpsAuthorizationError, ValidationError +from semapact.observation.fingerprint import with_observed_state_fingerprint +from semapact.observation.models import ( + ObservedAsset, + ObservedAssetIdentity, + ObservedPlatformState, + ObservedProperty, + ObservedPropertyIdentity, +) +from semapact.observation.providers import RuntimeAssetBinding +from semapact.platforms.databricks.deployment import DatabricksDeploymentAdapter + +CAPTURED_AT = datetime(2026, 9, 10, 5, 0, tzinfo=timezone.utc) + + +def _property(name: str, physical_type: str, *, required: bool = False) -> SchemaProperty: + return SchemaProperty( + name=name, + physicalName=name, + logicalType="string", + physicalType=physical_type, + required=required, + ) + + +def _plan(*properties: SchemaProperty) -> DeploymentPlan: + schema = SchemaObject( + name="orders", + physicalName="orders", + physicalType="table", + properties=list(properties), + ) + action = DeploymentAction( + kind=DeploymentActionKind.ENSURE_ASSET_STATE, + governed_asset="orders", + physical_name="orders", + desired_state_json=json.dumps( + schema.model_dump(mode="json", by_alias=True, exclude_none=True), + sort_keys=True, + separators=(",", ":"), + ), + ) + target = DeploymentTarget(platform="databricks", runtime_target="main.silver") + plan_id = compute_deployment_plan_id( + applied_release_id="applied:test", + contract_id="orders-product", + release_plan_id="release-plan:test", + released_revision_ref="rev:released", + selected_version="1.2.0", + target=target, + actions=(action,), + ) + return DeploymentPlan( + deployment_plan_id=plan_id, + applied_release_id="applied:test", + contract_id="orders-product", + release_plan_id="release-plan:test", + released_revision_ref="rev:released", + selected_version="1.2.0", + target=target, + actions=(action,), + ) + + +def _authorization(plan: DeploymentPlan, allowed: bool = True) -> DeploymentAuthorization: + authorization_id = compute_deployment_authorization_id( + contract_ops_authorization_id="contractops-auth:test", + deployment_plan_id=plan.deployment_plan_id, + applied_release_id=plan.applied_release_id, + allowed=allowed, + ) + return DeploymentAuthorization( + deployment_authorization_id=authorization_id, + contract_ops_authorization_id="contractops-auth:test", + deployment_plan_id=plan.deployment_plan_id, + applied_release_id=plan.applied_release_id, + allowed=allowed, + ) + + +def _state( + *columns: tuple[str, str, bool], + asset_type: str = "MANAGED", + source: str = "workspace-a", + present: bool = True, +) -> ObservedPlatformState: + identity = ObservedAssetIdentity( + platform="databricks", + namespace=("main", "silver"), + asset="orders", + ) + assets = () + if present: + assets = ( + ObservedAsset( + identity=identity, + asset_type=asset_type, + properties=tuple( + ObservedProperty( + identity=ObservedPropertyIdentity(asset=identity, property=name), + physical_type=physical_type, + nullable=nullable, + ) + for name, physical_type, nullable in columns + ), + ), + ) + raw = ObservedPlatformState( + platform="databricks", + source_identifier=source, + assets=assets, + captured_at=CAPTURED_AT, + fingerprint=None, + ) + return with_observed_state_fingerprint(raw) + + +class _Provider: + key = "databricks" + + def __init__(self, state: ObservedPlatformState) -> None: + self.state = state + + def resolve_bindings(self, *, runtime_target, assets): + return tuple( + RuntimeAssetBinding( + governed_asset=asset.governed_asset, + observed_asset=ObservedAssetIdentity( + platform="databricks", + namespace=("main", "silver"), + asset=asset.physical_name, + ), + ) + for asset in assets + ) + + def observe(self, *, bindings): + return self.state + + +class _Statements: + def __init__(self) -> None: + self.calls: list[str] = [] + + def execute_statement(self, *, statement, warehouse_id, wait_timeout): + self.calls.append(statement) + return SimpleNamespace( + statement_id="s-1", + status=SimpleNamespace(state="SUCCEEDED", error=None), + ) + + def get_statement(self, statement_id): + raise AssertionError(f"unexpected poll: {statement_id}") + + +class _Client: + def __init__(self) -> None: + self.statement_execution = _Statements() + + +def _adapter(state: ObservedPlatformState): + client = _Client() + provider = _Provider(state) + adapter = DatabricksDeploymentAdapter( + client=client, + runtime_provider=provider, + warehouse_id="warehouse-1", + poll_interval_seconds=0, + ) + return adapter, provider, client + + +def test_preview_create_alter_and_no_op() -> None: + create_plan = _plan(_property("id", "BIGINT", required=True)) + missing = _state(present=False) + adapter, _, _ = _adapter(missing) + create_preview = adapter.preview(create_plan, missing) + assert create_preview.operations[0].kind is NativeOperationKind.CREATE + + alter_plan = _plan( + _property("id", "BIGINT", required=True), + _property("note", "STRING"), + ) + current = _state(("id", "bigint", False)) + adapter, _, _ = _adapter(current) + alter_preview = adapter.preview(alter_plan, current) + assert alter_preview.operations[0].kind is NativeOperationKind.ALTER + assert "ADD COLUMNS (`note` STRING)" in alter_preview.operations[0].statement + + no_op = adapter.preview(create_plan, current) + assert no_op.operations == ( + NativeOperation(kind=NativeOperationKind.NO_OP, governed_asset="orders"), + ) + + +def test_preview_never_drops_extra_runtime_columns() -> None: + plan = _plan(_property("id", "BIGINT", required=True)) + current = _state(("id", "bigint", False), ("extra", "string", True)) + adapter, _, _ = _adapter(current) + assert adapter.preview(plan, current).operations[0].kind is NativeOperationKind.NO_OP + + +@pytest.mark.parametrize( + ("plan", "state", "message"), + [ + ( + _plan(_property("id", "STRING", required=True)), + _state(("id", "bigint", False)), + "type mutation", + ), + ( + _plan(_property("id", "BIGINT", required=True)), + _state(("id", "bigint", True)), + "nullability mutation", + ), + ( + _plan( + _property("id", "BIGINT", required=True), + _property("new_required", "STRING", required=True), + ), + _state(("id", "bigint", False)), + "safe default", + ), + ], +) +def test_unsafe_existing_mutations_fail_closed(plan, state, message) -> None: + adapter, _, _ = _adapter(state) + with pytest.raises(ValidationError, match=message): + adapter.preview(plan, state) + + +def test_non_managed_asset_and_unsafe_type_fail_closed() -> None: + plan = _plan(_property("id", "BIGINT", required=True)) + external = _state(("id", "bigint", False), asset_type="EXTERNAL") + adapter, _, _ = _adapter(external) + with pytest.raises(ValidationError, match="MANAGED"): + adapter.preview(plan, external) + + malicious_type = "STRING);DROP" + malicious = _plan(_property("id", malicious_type)) + with pytest.raises(ValidationError, match="physicalType"): + adapter.validate(malicious) + + +def test_execute_fails_closed_for_denied_stale_and_cross_source() -> None: + plan = _plan(_property("id", "BIGINT", required=True)) + before = _state(("id", "bigint", False), source="workspace-a") + adapter, provider, _ = _adapter(before) + preview = adapter.preview(plan, before) + + with pytest.raises(ContractOpsAuthorizationError, match="not allowed"): + adapter.execute(plan, preview, _authorization(plan, False)) + + provider.state = _state( + ("id", "bigint", False), + ("other", "string", True), + source="workspace-a", + ) + with pytest.raises(ValidationError, match="Runtime state changed"): + adapter.execute(plan, preview, _authorization(plan)) + + provider.state = before.model_copy(update={"source_identifier": "workspace-b"}) + with pytest.raises(ValidationError, match="Runtime source changed"): + adapter.execute(plan, preview, _authorization(plan)) + + +def test_execute_rejects_tampered_plan_and_forged_native_command() -> None: + plan = _plan(_property("id", "BIGINT", required=True)) + current = _state(("id", "bigint", False)) + adapter, _, client = _adapter(current) + preview = adapter.preview(plan, current) + + tampered = plan.model_copy(update={"selected_version": "9.9.9"}) + with pytest.raises(ValueError, match="DeploymentPlan deterministic identity"): + adapter.execute(tampered, preview, _authorization(plan)) + + forged_operations = ( + NativeOperation( + kind=NativeOperationKind.ALTER, + governed_asset="orders", + statement="DROP TABLE `main`.`silver`.`orders`", + ), + ) + forged_id = compute_deployment_preview_id( + deployment_plan_id=preview.deployment_plan_id, + platform=preview.platform, + runtime_target=preview.runtime_target, + source_identifier=preview.source_identifier, + observation_fingerprint=preview.observation_fingerprint, + operations=forged_operations, + ) + forged = preview.model_copy( + update={"deployment_preview_id": forged_id, "operations": forged_operations} + ) + with pytest.raises(ValidationError, match="no longer equals"): + adapter.execute(plan, forged, _authorization(plan)) + assert client.statement_execution.calls == [] + + +def test_execute_runs_exact_preview_statement() -> None: + plan = _plan( + _property("id", "BIGINT", required=True), + _property("note", "STRING"), + ) + current = _state(("id", "bigint", False)) + adapter, _, client = _adapter(current) + preview = adapter.preview(plan, current) + + adapter.execute(plan, preview, _authorization(plan)) + + assert client.statement_execution.calls == [ + "ALTER TABLE `main`.`silver`.`orders` ADD COLUMNS (`note` STRING)" + ] From 4e3598be5d28ba70f33474205d4582af805ed17f Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Thu, 10 Sep 2026 19:54:58 +1000 Subject: [PATCH 11/35] feat(deployment): verify convergence through reconciliation * feat(deployment): verify runtime convergence through reconciliation * fix(reconciliation): bind physical property names to governed identity * feat(deployment): export convergence verification * test(deployment): cover convergence verification --- semapact/deployment/__init__.py | 2 + semapact/deployment/verification.py | 76 +++++++++++ semapact/reconciliation/engine.py | 123 ++++++++--------- tests/test_deployment_verification.py | 185 ++++++++++++++++++++++++++ 4 files changed, 317 insertions(+), 69 deletions(-) create mode 100644 semapact/deployment/verification.py create mode 100644 tests/test_deployment_verification.py diff --git a/semapact/deployment/__init__.py b/semapact/deployment/__init__.py index 70a0e9f3..ed738096 100644 --- a/semapact/deployment/__init__.py +++ b/semapact/deployment/__init__.py @@ -13,6 +13,7 @@ NativeOperationKind, ) from semapact.deployment.planner import build_deployment_plan +from semapact.deployment.verification import verify_deployment_convergence __all__ = [ "DeploymentAction", @@ -26,4 +27,5 @@ "NativeOperationKind", "authorize_deployment", "build_deployment_plan", + "verify_deployment_convergence", ] diff --git a/semapact/deployment/verification.py b/semapact/deployment/verification.py new file mode 100644 index 00000000..050dab7e --- /dev/null +++ b/semapact/deployment/verification.py @@ -0,0 +1,76 @@ +"""Thin bridge from an exact DeploymentPlan to M1 runtime reconciliation.""" + +from __future__ import annotations + +from open_data_contract_standard.model import OpenDataContractStandard, SchemaObject + +from semapact.deployment.models import ( + DeploymentPlan, + validate_deployment_plan_identity, +) +from semapact.exceptions import ValidationError +from semapact.observation.providers import RuntimeProvider +from semapact.reconciliation import ReconciliationResult, reconcile_governed_contract +from semapact.runtime import RuntimeAssetSpec + + +def verify_deployment_convergence( + plan: DeploymentPlan, + runtime_provider: RuntimeProvider, +) -> ReconciliationResult: + """Observe and reconcile the exact desired state embedded in a DeploymentPlan. + + This function does not infer deployment causality or create a second convergence + status model. Callers classify the returned result with the existing + ``classify_reconciliation_status`` function. + """ + validate_deployment_plan_identity(plan) + + provider_key = runtime_provider.key.strip().casefold() + if provider_key != plan.target.platform: + raise ValidationError( + "Runtime provider does not match DeploymentPlan platform: " + f"{provider_key!r} != {plan.target.platform!r}" + ) + + desired_contract = _contract_projection_from_plan(plan) + assets = tuple( + RuntimeAssetSpec( + governed_asset=action.governed_asset, + physical_name=action.physical_name, + ) + for action in plan.actions + ) + bindings = runtime_provider.resolve_bindings( + runtime_target=plan.target.runtime_target, + assets=assets, + ) + observation = runtime_provider.observe(bindings=bindings) + + if observation.platform.strip().casefold() != plan.target.platform: + raise ValidationError( + "Observed runtime platform does not match DeploymentPlan platform" + ) + + return reconcile_governed_contract( + desired_contract, + observation, + asset_bindings=bindings, + ) + + +def _contract_projection_from_plan( + plan: DeploymentPlan, +) -> OpenDataContractStandard: + schemas = [ + SchemaObject.model_validate_json(action.desired_state_json) + for action in plan.actions + ] + # Reconciliation needs only exact contract identity/version plus governed schemas. + # The schemas were validated when the DeploymentPlan was constructed, and the + # deterministic plan identity was revalidated above. + return OpenDataContractStandard.model_construct( + id=plan.contract_id, + version=plan.selected_version, + schema_=schemas, + ) diff --git a/semapact/reconciliation/engine.py b/semapact/reconciliation/engine.py index eb586bcd..551eb079 100644 --- a/semapact/reconciliation/engine.py +++ b/semapact/reconciliation/engine.py @@ -34,30 +34,12 @@ _REASON_CODE_BY_RAW_DIFFERENCE: dict[ tuple[ReconciliationDifferenceType, ReconciliationSubject], RuntimeReasonCode ] = { - ( - ReconciliationDifferenceType.UNEXPECTED, - ReconciliationSubject.ASSET, - ): RuntimeReasonCode.RUNTIME_SCHEMA_ADDED, - ( - ReconciliationDifferenceType.MISSING, - ReconciliationSubject.ASSET, - ): RuntimeReasonCode.RUNTIME_SCHEMA_REMOVED, - ( - ReconciliationDifferenceType.UNEXPECTED, - ReconciliationSubject.PROPERTY, - ): RuntimeReasonCode.RUNTIME_PROPERTY_ADDED, - ( - ReconciliationDifferenceType.MISSING, - ReconciliationSubject.PROPERTY, - ): RuntimeReasonCode.RUNTIME_PROPERTY_REMOVED, - ( - ReconciliationDifferenceType.MISMATCH, - ReconciliationSubject.PHYSICAL_TYPE, - ): RuntimeReasonCode.RUNTIME_PHYSICAL_TYPE_CHANGED, - ( - ReconciliationDifferenceType.MISMATCH, - ReconciliationSubject.NULLABILITY, - ): RuntimeReasonCode.RUNTIME_REQUIRED_CHANGED, + (ReconciliationDifferenceType.UNEXPECTED, ReconciliationSubject.ASSET): RuntimeReasonCode.RUNTIME_SCHEMA_ADDED, + (ReconciliationDifferenceType.MISSING, ReconciliationSubject.ASSET): RuntimeReasonCode.RUNTIME_SCHEMA_REMOVED, + (ReconciliationDifferenceType.UNEXPECTED, ReconciliationSubject.PROPERTY): RuntimeReasonCode.RUNTIME_PROPERTY_ADDED, + (ReconciliationDifferenceType.MISSING, ReconciliationSubject.PROPERTY): RuntimeReasonCode.RUNTIME_PROPERTY_REMOVED, + (ReconciliationDifferenceType.MISMATCH, ReconciliationSubject.PHYSICAL_TYPE): RuntimeReasonCode.RUNTIME_PHYSICAL_TYPE_CHANGED, + (ReconciliationDifferenceType.MISMATCH, ReconciliationSubject.NULLABILITY): RuntimeReasonCode.RUNTIME_REQUIRED_CHANGED, } @@ -86,7 +68,6 @@ def reconcile_governed_contract( differences: list[ReconciliationDifference] = [] unverified_paths: list[str] = [] - governed_keys = set(governed_assets) observed_keys = set(observed_assets) @@ -122,13 +103,9 @@ def reconcile_governed_contract( ordered = tuple(sorted(differences, key=_difference_sort_key)) return ReconciliationResult( contract_id=_required_contract_text(getattr(contract, "id", None), field="id"), - contract_version=_required_contract_text( - getattr(contract, "version", None), field="version" - ), + contract_version=_required_contract_text(getattr(contract, "version", None), field="version"), observation_source_identifier=observation.source_identifier, - observation_fingerprint=( - observation.fingerprint or fingerprint_observed_state(observation) - ), + observation_fingerprint=observation.fingerprint or fingerprint_observed_state(observation), differences=ordered, unverified_paths=tuple(sorted(unverified_paths)), ) @@ -141,10 +118,13 @@ def _reconcile_properties( observed_asset: ObservedAsset, ) -> tuple[list[ReconciliationDifference], list[str]]: governed = build_property_index(asset_key, governed_properties) - observed = _build_observed_property_index(asset_key, observed_asset) + observed = _build_observed_property_index( + asset_key, + observed_asset, + governed_properties=governed_properties, + ) differences: list[ReconciliationDifference] = [] unverified_paths: list[str] = [] - governed_keys = set(governed) observed_keys = set(observed) @@ -203,10 +183,7 @@ def _reconcile_matching_property( property_identity=property_identity, ) ) - elif ( - _normalize_comparable_text(expected_physical) - != _normalize_comparable_text(observed_physical) - ): + elif _normalize_comparable_text(expected_physical) != _normalize_comparable_text(observed_physical): differences.append( _difference( difference_type=ReconciliationDifferenceType.MISMATCH, @@ -246,16 +223,12 @@ def _reconcile_matching_property( return differences, unverified_paths -def _build_observed_asset_index( - observation: ObservedPlatformState, -) -> dict[str, ObservedAsset]: +def _build_observed_asset_index(observation: ObservedPlatformState) -> dict[str, ObservedAsset]: index: dict[str, ObservedAsset] = {} for asset in observation.assets: key = normalize_identity_name(asset.identity.asset, "Observed asset") if key in index: - raise ValidationError( - f"Duplicate canonical observed asset identity found: '{key}'" - ) + raise ValidationError(f"Duplicate canonical observed asset identity found: '{key}'") index[key] = asset return index @@ -270,17 +243,11 @@ def _build_bound_observed_asset_index( bound_runtime_keys: set[tuple[str, ...]] = set() for binding in bindings: - governed_key = normalize_identity_name( - binding.governed_asset, "Runtime binding governed asset" - ) + governed_key = normalize_identity_name(binding.governed_asset, "Runtime binding governed asset") if governed_key in binding_by_governed: - raise ValidationError( - f"Duplicate runtime binding for governed asset: '{governed_key}'" - ) + raise ValidationError(f"Duplicate runtime binding for governed asset: '{governed_key}'") if binding.observed_asset.platform.casefold() != observation.platform.casefold(): - raise ValidationError( - "Runtime binding platform must match observed platform state" - ) + raise ValidationError("Runtime binding platform must match observed platform state") runtime_key = binding.observed_asset.canonical_key if runtime_key in bound_runtime_keys: raise ValidationError("Multiple governed assets cannot bind to one runtime asset") @@ -323,26 +290,51 @@ def _build_bound_observed_asset_index( def _build_observed_property_index( asset_key: str, asset: ObservedAsset, + *, + governed_properties: Sequence[SchemaProperty], ) -> dict[PropertyIdentity, ObservedProperty]: + physical_to_governed = _property_binding_index(governed_properties) index: dict[PropertyIdentity, ObservedProperty] = {} for prop in asset.properties: if prop.identity.asset != asset.identity: - raise ValidationError( - "Observed property asset identity must match its containing asset" - ) - prop_name = normalize_identity_name( - prop.identity.property, "Observed property" - ) - key: PropertyIdentity = (asset_key, prop_name) + raise ValidationError("Observed property asset identity must match its containing asset") + observed_name = normalize_identity_name(prop.identity.property, "Observed property") + governed_name = physical_to_governed.get(observed_name, observed_name) + key: PropertyIdentity = (asset_key, governed_name) if key in index: raise ValidationError( - f"Duplicate canonical observed property identity found: '{prop_name}'" + f"Duplicate canonical observed property identity found: '{governed_name}'" f" in asset '{asset_key}'" ) index[key] = prop return index +def _property_binding_index( + governed_properties: Sequence[SchemaProperty], +) -> dict[str, str]: + physical_to_governed: dict[str, str] = {} + for prop in governed_properties: + logical_raw = getattr(prop, "name", None) + if logical_raw is None: + raise ValidationError("Governed property name is required for runtime binding") + governed_name = normalize_identity_name(str(logical_raw), "Property") + physical_raw = getattr(prop, "physicalName", None) + physical_text = str(physical_raw).strip() if physical_raw is not None else "" + physical_name = normalize_identity_name( + physical_text or str(logical_raw), + "Property physical binding", + ) + existing = physical_to_governed.get(physical_name) + if existing is not None and existing != governed_name: + raise ValidationError( + "Multiple governed properties cannot bind to one physical runtime property: " + f"'{physical_name}'" + ) + physical_to_governed[physical_name] = governed_name + return physical_to_governed + + def _difference( *, difference_type: ReconciliationDifferenceType, @@ -355,10 +347,7 @@ def _difference( return ReconciliationDifference( difference_type=difference_type, subject=subject, - reason_code=_runtime_reason_code( - difference_type=difference_type, - subject=subject, - ), + reason_code=_runtime_reason_code(difference_type=difference_type, subject=subject), path=_difference_path( subject=subject, asset_identity=asset_identity, @@ -394,10 +383,8 @@ def _difference_path( asset_path = f"schema[{asset_identity}]" if subject is ReconciliationSubject.ASSET: return asset_path - if property_identity is None: raise ValueError(f"property_identity is required for {subject.value}") - property_path = f"{asset_path}.properties[{property_identity}]" if subject is ReconciliationSubject.PROPERTY: return property_path @@ -406,9 +393,7 @@ def _difference_path( return f"{property_path}.nullability" -def _difference_sort_key( - difference: ReconciliationDifference, -) -> tuple[str, str, int, str]: +def _difference_sort_key(difference: ReconciliationDifference) -> tuple[str, str, int, str]: return ( difference.asset_identity, difference.property_identity or "", diff --git a/tests/test_deployment_verification.py b/tests/test_deployment_verification.py new file mode 100644 index 00000000..78970ba3 --- /dev/null +++ b/tests/test_deployment_verification.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +from datetime import datetime, timezone + +import pytest +from open_data_contract_standard.model import SchemaObject, SchemaProperty + +from semapact.deployment import ( + DeploymentAction, + DeploymentActionKind, + DeploymentPlan, + DeploymentTarget, + verify_deployment_convergence, +) +from semapact.deployment.models import compute_deployment_plan_id +from semapact.exceptions import ValidationError +from semapact.observation import ( + ObservedAsset, + ObservedAssetIdentity, + ObservedPlatformState, + ObservedProperty, + ObservedPropertyIdentity, + RuntimeAssetBinding, + with_observed_state_fingerprint, +) +from semapact.reconciliation import RuntimeDriftStatus, classify_reconciliation_status + +CAPTURED_AT = datetime(2026, 9, 10, 7, 0, tzinfo=timezone.utc) + + +class FakeRuntimeProvider: + key = "databricks" + + def __init__(self, observation: ObservedPlatformState) -> None: + self.observation = observation + self.bindings: tuple[RuntimeAssetBinding, ...] = () + + def resolve_bindings(self, *, runtime_target: str, assets): + assert runtime_target == "main.silver" + self.bindings = tuple( + RuntimeAssetBinding( + governed_asset=asset.governed_asset, + observed_asset=ObservedAssetIdentity( + platform="databricks", + namespace=("main", "silver"), + asset=asset.physical_name, + ), + ) + for asset in assets + ) + return self.bindings + + def observe(self, *, bindings): + assert tuple(bindings) == self.bindings + return self.observation + + +def _plan() -> DeploymentPlan: + schema = SchemaObject( + name="orders", + physicalName="orders_v2", + properties=[ + SchemaProperty( + name="order_id", + physicalName="order_pk", + type="integer", + physicalType="BIGINT", + required=True, + ) + ], + ) + action = DeploymentAction( + kind=DeploymentActionKind.ENSURE_ASSET_STATE, + governed_asset="orders", + physical_name="orders_v2", + desired_state_json=schema.model_dump_json(by_alias=True, exclude_none=True), + ) + target = DeploymentTarget(platform="databricks", runtime_target="main.silver") + plan_id = compute_deployment_plan_id( + applied_release_id="release-1", + contract_id="orders-contract", + release_plan_id="release-plan-1", + released_revision_ref="abc123", + selected_version="1.2.3", + target=target, + actions=(action,), + ) + return DeploymentPlan( + deployment_plan_id=plan_id, + applied_release_id="release-1", + contract_id="orders-contract", + release_plan_id="release-plan-1", + released_revision_ref="abc123", + selected_version="1.2.3", + target=target, + actions=(action,), + ) + + +def _observation( + *, + physical_type: str | None = "BIGINT", + nullable: bool | None = False, +) -> ObservedPlatformState: + asset_identity = ObservedAssetIdentity( + platform="databricks", + namespace=("main", "silver"), + asset="orders_v2", + ) + state = ObservedPlatformState( + platform="databricks", + source_identifier="https://adb.example", + captured_at=CAPTURED_AT, + assets=( + ObservedAsset( + identity=asset_identity, + asset_type="MANAGED", + properties=( + ObservedProperty( + identity=ObservedPropertyIdentity( + asset=asset_identity, + property="order_pk", + ), + physical_type=physical_type, + nullable=nullable, + ), + ), + ), + ), + fingerprint=None, + ) + return with_observed_state_fingerprint(state) + + +def test_exact_plan_state_is_verified_in_sync_through_physical_bindings() -> None: + result = verify_deployment_convergence(_plan(), FakeRuntimeProvider(_observation())) + + assert result.contract_id == "orders-contract" + assert result.contract_version == "1.2.3" + assert result.differences == () + assert result.unverified_paths == () + assert classify_reconciliation_status(result) is RuntimeDriftStatus.IN_SYNC + + +def test_runtime_difference_is_reported_as_drift() -> None: + result = verify_deployment_convergence( + _plan(), + FakeRuntimeProvider(_observation(physical_type="STRING")), + ) + + assert len(result.differences) == 1 + assert result.differences[0].property_identity == "order_id" + assert classify_reconciliation_status(result) is RuntimeDriftStatus.DRIFT + + +def test_missing_runtime_evidence_is_indeterminate() -> None: + result = verify_deployment_convergence( + _plan(), + FakeRuntimeProvider(_observation(physical_type=None)), + ) + + assert result.differences == () + assert result.unverified_paths == ("schema[orders].properties[order_id].physicalType",) + assert classify_reconciliation_status(result) is RuntimeDriftStatus.INDETERMINATE + + +def test_verification_uses_exact_deployment_plan_version() -> None: + result = verify_deployment_convergence(_plan(), FakeRuntimeProvider(_observation())) + + assert result.contract_version == "1.2.3" + + +def test_provider_platform_mismatch_fails_closed() -> None: + provider = FakeRuntimeProvider(_observation()) + provider.key = "snowflake" + + with pytest.raises(ValidationError, match="does not match DeploymentPlan platform"): + verify_deployment_convergence(_plan(), provider) + + +def test_tampered_deployment_plan_identity_fails_closed() -> None: + plan = _plan().model_copy(update={"selected_version": "9.9.9"}) + + with pytest.raises(ValueError, match="deterministic identity"): + verify_deployment_convergence(plan, FakeRuntimeProvider(_observation())) From 3d43771132c4c926068181e5df42cc99c0e0273e Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Fri, 11 Sep 2026 14:46:49 +1000 Subject: [PATCH 12/35] feat(deployment): expose canonical deployment CLI (#216) * feat(deployment): add application service boundary * feat(deployment): export deployment service * refactor(deployment): decouple preview from warehouse execution config * refactor(deployment): require warehouse only for mutation * feat(deployment): add deployment CLI command adapter * feat(deployment): expose deployment CLI commands * fix(deployment): map artifact parse failures to validation outcomes * test(deployment): cover application deployment service * test(deployment): cover deployment CLI surface --- semapact/interfaces/cli.py | 66 +++++ .../interfaces/commands/deployment_cmd.py | 201 +++++++++++++ semapact/platforms/databricks/deployment.py | 10 +- semapact/platforms/runtime_registry.py | 9 +- semapact/services/__init__.py | 2 + semapact/services/deployment_service.py | 93 ++++++ tests/interfaces/test_deployment_cmd.py | 170 +++++++++++ tests/test_deployment_service.py | 271 ++++++++++++++++++ 8 files changed, 816 insertions(+), 6 deletions(-) create mode 100644 semapact/interfaces/commands/deployment_cmd.py create mode 100644 semapact/services/deployment_service.py create mode 100644 tests/interfaces/test_deployment_cmd.py create mode 100644 tests/test_deployment_service.py diff --git a/semapact/interfaces/cli.py b/semapact/interfaces/cli.py index f1f293b5..cd66c33b 100644 --- a/semapact/interfaces/cli.py +++ b/semapact/interfaces/cli.py @@ -342,6 +342,50 @@ def _build_parser() -> argparse.ArgumentParser: help="Output format (default: text)", ) + deployment_parser = subparsers.add_parser( + "deployment", + help="Plan, preview, execute, and verify governed runtime deployment", + ) + deployment_subparsers = deployment_parser.add_subparsers( + dest="deployment_command", required=True + ) + + deployment_plan_parser = deployment_subparsers.add_parser( + "plan", help="Build a DeploymentPlan from an exact AppliedContractRelease" + ) + deployment_plan_parser.add_argument("--release", required=True) + deployment_plan_parser.add_argument("--platform", required=True) + deployment_plan_parser.add_argument("--runtime", required=True) + deployment_plan_parser.add_argument("--server") + + deployment_preview_parser = deployment_subparsers.add_parser( + "preview", help="Observe runtime and derive provider-native deployment operations" + ) + deployment_preview_parser.add_argument("--plan", required=True) + + deployment_execute_parser = deployment_subparsers.add_parser( + "execute", help="Execute an exact authorized DeploymentPreview" + ) + deployment_execute_parser.add_argument("--plan", required=True) + deployment_execute_parser.add_argument("--preview", required=True) + deployment_execute_parser.add_argument("--authorization", required=True) + deployment_execute_parser.add_argument( + "--warehouse-id", + required=True, + help="Databricks SQL warehouse used only for runtime mutation", + ) + + deployment_verify_parser = deployment_subparsers.add_parser( + "verify", help="Verify DeploymentPlan convergence through M1 reconciliation" + ) + deployment_verify_parser.add_argument("--plan", required=True) + deployment_verify_parser.add_argument( + "--output", + choices=["text", "json"], + default="text", + help="Output format (default: text)", + ) + return parser @@ -440,6 +484,28 @@ def main() -> int: print(result.output) return int(exit_code_from_outcome(result.outcome)) + if args.command == "deployment": + from semapact.interfaces.commands.deployment_cmd import ( + run_deployment_execute, + run_deployment_plan, + run_deployment_preview, + run_deployment_verify, + ) + from semapact.interfaces.outcomes import exit_code_from_outcome + + if args.deployment_command == "plan": + result = run_deployment_plan(args) + elif args.deployment_command == "preview": + result = run_deployment_preview(args) + elif args.deployment_command == "execute": + result = run_deployment_execute(args) + elif args.deployment_command == "verify": + result = run_deployment_verify(args) + else: + parser.error(f"Unknown deployment command: {args.deployment_command}") + print(result.output) + return int(exit_code_from_outcome(result.outcome)) + if args.command == "release": from semapact.interfaces.commands.release_cmd import ( run_release_classify, run_release_classify_repo, run_release_build_manifest, diff --git a/semapact/interfaces/commands/deployment_cmd.py b/semapact/interfaces/commands/deployment_cmd.py new file mode 100644 index 00000000..b63e4ea1 --- /dev/null +++ b/semapact/interfaces/commands/deployment_cmd.py @@ -0,0 +1,201 @@ +"""CLI adapter for canonical deployment planning, execution, and verification.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +import json +from pathlib import Path +import sys +from typing import TypeVar + +from pydantic import BaseModel +from pydantic import ValidationError as PydanticValidationError + +from semapact.contractops import AppliedContractRelease +from semapact.deployment import ( + DeploymentAuthorization, + DeploymentPlan, + DeploymentPreview, + DeploymentTarget, +) +from semapact.exceptions import ValidationError +from semapact.interfaces.outcomes import ( + ProcessOutcome, + outcome_from_reconciliation_status, +) +from semapact.observation import RuntimeProvider +from semapact.reconciliation import classify_reconciliation_status +from semapact.services.deployment_service import DeploymentService + + +_ModelT = TypeVar("_ModelT", bound=BaseModel) + + +@dataclass(frozen=True) +class DeploymentCommandResult: + """Rendered CLI output plus its existing semantic process outcome.""" + + output: str + outcome: ProcessOutcome + + +def run_deployment_plan(args: argparse.Namespace) -> DeploymentCommandResult: + """Build one provider-neutral DeploymentPlan from an exact applied release.""" + release = _load_model(args.release, AppliedContractRelease) + target = DeploymentTarget( + platform=args.platform, + runtime_target=args.runtime, + server_name=args.server, + ) + plan = DeploymentService().plan(release, target) + return DeploymentCommandResult( + output=_model_json(plan), + outcome=ProcessOutcome.SUCCESS, + ) + + +def run_deployment_preview(args: argparse.Namespace) -> DeploymentCommandResult: + """Observe exact runtime scope and render the adapter's canonical preview.""" + plan = _load_model(args.plan, DeploymentPlan) + provider = _runtime_provider(plan) + + from semapact.platforms.runtime_registry import create_deployment_adapter + + adapter = create_deployment_adapter(plan.target.platform) + preview = DeploymentService().preview( + plan, + runtime_provider=provider, + adapter=adapter, + ) + return DeploymentCommandResult( + output=_model_json(preview), + outcome=ProcessOutcome.SUCCESS, + ) + + +def run_deployment_execute(args: argparse.Namespace) -> DeploymentCommandResult: + """Execute only an exact plan/preview/authorization tuple.""" + plan = _load_model(args.plan, DeploymentPlan) + preview = _load_model(args.preview, DeploymentPreview) + authorization = _load_model(args.authorization, DeploymentAuthorization) + + from semapact.platforms.runtime_registry import create_deployment_adapter + + adapter = create_deployment_adapter( + plan.target.platform, + warehouse_id=args.warehouse_id, + ) + DeploymentService().execute( + plan, + preview, + authorization, + adapter=adapter, + ) + return DeploymentCommandResult( + output=json.dumps( + { + "convergenceVerified": False, + "deploymentPlanId": plan.deployment_plan_id, + "deploymentPreviewId": preview.deployment_preview_id, + "providerExecution": "SUCCEEDED", + }, + indent=2, + sort_keys=True, + ), + outcome=ProcessOutcome.SUCCESS, + ) + + +def run_deployment_verify(args: argparse.Namespace) -> DeploymentCommandResult: + """Verify exact DeploymentPlan convergence through the existing M1 path.""" + plan = _load_model(args.plan, DeploymentPlan) + result = DeploymentService().verify( + plan, + runtime_provider=_runtime_provider(plan), + ) + status = classify_reconciliation_status(result) + rendered = ( + _verification_json(status.value, result) + if args.output == "json" + else _verification_text(status.value, result) + ) + return DeploymentCommandResult( + output=rendered, + outcome=outcome_from_reconciliation_status(status), + ) + + +def _runtime_provider(plan: DeploymentPlan) -> RuntimeProvider: + from semapact.platforms.runtime_registry import create_runtime_provider_registry + + registry = create_runtime_provider_registry(plan.target.platform) + return registry.get(plan.target.platform) + + +def _load_model(path: str, model_type: type[_ModelT]) -> _ModelT: + try: + raw = ( + sys.stdin.read() + if path == "-" + else Path(path).read_text(encoding="utf-8") + ) + return model_type.model_validate_json(raw) + except (OSError, PydanticValidationError) as exc: + raise ValidationError( + f"Invalid {model_type.__name__} artifact '{path}': {exc}" + ) from exc + + +def _model_json(model: BaseModel) -> str: + return json.dumps( + model.model_dump(mode="json"), + indent=2, + ensure_ascii=False, + sort_keys=True, + ) + + +def _verification_json(status: str, result: BaseModel) -> str: + return json.dumps( + { + "reconciliation": result.model_dump(mode="json"), + "status": status, + }, + indent=2, + ensure_ascii=False, + sort_keys=True, + ) + + +def _verification_text(status: str, result: BaseModel) -> str: + differences = getattr(result, "differences") + unverified_paths = getattr(result, "unverified_paths") + lines = [ + f"Status: {status}", + ( + f"Contract: {getattr(result, 'contract_id')}@" + f"{getattr(result, 'contract_version')}" + ), + f"Observation source: {getattr(result, 'observation_source_identifier')}", + f"Observation fingerprint: {getattr(result, 'observation_fingerprint')}", + ] + if differences: + lines.append("Differences:") + for difference in differences: + detail = f" - {difference.reason_code.value} {difference.path}" + if difference.expected is not None or difference.observed is not None: + detail += ( + f" expected={difference.expected!r}" + f" observed={difference.observed!r}" + ) + lines.append(detail) + else: + lines.append("Differences: none") + + if unverified_paths: + lines.append("Unverified paths:") + lines.extend(f" - {path}" for path in unverified_paths) + else: + lines.append("Unverified paths: none") + return "\n".join(lines) diff --git a/semapact/platforms/databricks/deployment.py b/semapact/platforms/databricks/deployment.py index fb8303a5..d19d2dd0 100644 --- a/semapact/platforms/databricks/deployment.py +++ b/semapact/platforms/databricks/deployment.py @@ -62,19 +62,17 @@ def __init__( *, client: Any, runtime_provider: RuntimeProvider, - warehouse_id: str, + warehouse_id: str | None = None, poll_interval_seconds: float = 1.0, max_poll_attempts: int = 300, ) -> None: - if not warehouse_id.strip(): - raise ValueError("warehouse_id is required for Databricks deployment") if poll_interval_seconds < 0: raise ValueError("poll_interval_seconds must be non-negative") if max_poll_attempts < 1: raise ValueError("max_poll_attempts must be positive") self._client = client self._runtime_provider = runtime_provider - self._warehouse_id = warehouse_id.strip() + self._warehouse_id = warehouse_id.strip() if warehouse_id and warehouse_id.strip() else None self._poll_interval_seconds = poll_interval_seconds self._max_poll_attempts = max_poll_attempts @@ -249,6 +247,10 @@ def _observe_plan_scope(self, plan: DeploymentPlan) -> ObservedPlatformState: return self._runtime_provider.observe(bindings=bindings) def _execute_statement(self, statement: str) -> None: + if self._warehouse_id is None: + raise ValidationError( + "Databricks deployment execution requires a SQL warehouse_id" + ) response = self._client.statement_execution.execute_statement( statement=statement, warehouse_id=self._warehouse_id, diff --git a/semapact/platforms/runtime_registry.py b/semapact/platforms/runtime_registry.py index 436d3b92..a88e7328 100644 --- a/semapact/platforms/runtime_registry.py +++ b/semapact/platforms/runtime_registry.py @@ -84,10 +84,15 @@ def create_runtime_provider_registry( def create_deployment_adapter( platform: str, *, - warehouse_id: str, + warehouse_id: str | None = None, contract_server: Server | None = None, ) -> DeploymentAdapter: - """Compose the selected write adapter and provider clients lazily.""" + """Compose the selected write adapter and provider clients lazily. + + A SQL warehouse is execution configuration, not a prerequisite for read-only + validation or preview. Databricks execution fails closed if mutation is attempted + without a warehouse ID. + """ normalized = platform.strip().casefold() if normalized != "databricks": raise ValidationError( diff --git a/semapact/services/__init__.py b/semapact/services/__init__.py index 161e41e3..a70d2991 100644 --- a/semapact/services/__init__.py +++ b/semapact/services/__init__.py @@ -1,5 +1,6 @@ """UI-independent application services for SemaPact workflows.""" +from semapact.services.deployment_service import DeploymentService from semapact.services.governance_service import ( GovernanceAnalysis, GovernanceProposal, @@ -12,6 +13,7 @@ from semapact.services.version_authority_service import VersionAuthorityService __all__ = [ + "DeploymentService", "GovernanceAnalysis", "GovernanceProposal", "GovernanceService", diff --git a/semapact/services/deployment_service.py b/semapact/services/deployment_service.py new file mode 100644 index 00000000..1ea49132 --- /dev/null +++ b/semapact/services/deployment_service.py @@ -0,0 +1,93 @@ +"""Application service for provider-neutral deployment workflows.""" + +from __future__ import annotations + +from semapact.contractops import AppliedContractRelease +from semapact.deployment import ( + DeploymentAdapter, + DeploymentAuthorization, + DeploymentPlan, + DeploymentPreview, + DeploymentTarget, + build_deployment_plan, + verify_deployment_convergence, +) +from semapact.deployment.models import validate_deployment_plan_identity +from semapact.exceptions import ValidationError +from semapact.observation import RuntimeProvider +from semapact.reconciliation import ReconciliationResult +from semapact.runtime import RuntimeAssetSpec + + +class DeploymentService: + """Compose existing deployment domain/provider boundaries for interfaces. + + The service owns orchestration only. It does not re-run governance, approval, + versioning, deployment translation, or reconciliation rules. + """ + + def plan( + self, + release: AppliedContractRelease, + target: DeploymentTarget, + ) -> DeploymentPlan: + return build_deployment_plan(release, target) + + def preview( + self, + plan: DeploymentPlan, + *, + runtime_provider: RuntimeProvider, + adapter: DeploymentAdapter, + ) -> DeploymentPreview: + """Observe the exact plan scope and delegate native translation to adapter.""" + validate_deployment_plan_identity(plan) + _validate_component_key(runtime_provider.key, plan.target.platform, "runtime provider") + _validate_component_key(adapter.key, plan.target.platform, "deployment adapter") + + assets = _runtime_assets_from_plan(plan) + bindings = runtime_provider.resolve_bindings( + runtime_target=plan.target.runtime_target, + assets=assets, + ) + observation = runtime_provider.observe(bindings=bindings) + return adapter.preview(plan, observation) + + def execute( + self, + plan: DeploymentPlan, + preview: DeploymentPreview, + authorization: DeploymentAuthorization, + *, + adapter: DeploymentAdapter, + ) -> None: + """Delegate the exact authorized side effect to the provider adapter.""" + _validate_component_key(adapter.key, plan.target.platform, "deployment adapter") + adapter.execute(plan, preview, authorization) + + def verify( + self, + plan: DeploymentPlan, + *, + runtime_provider: RuntimeProvider, + ) -> ReconciliationResult: + """Verify exact plan convergence through the existing M1 bridge.""" + return verify_deployment_convergence(plan, runtime_provider) + + +def _runtime_assets_from_plan(plan: DeploymentPlan) -> tuple[RuntimeAssetSpec, ...]: + return tuple( + RuntimeAssetSpec( + governed_asset=action.governed_asset, + physical_name=action.physical_name, + ) + for action in plan.actions + ) + + +def _validate_component_key(actual: str, expected: str, component: str) -> None: + if actual.strip().casefold() != expected.strip().casefold(): + raise ValidationError( + f"{component.capitalize()} does not match DeploymentPlan platform: " + f"{actual!r} != {expected!r}" + ) diff --git a/tests/interfaces/test_deployment_cmd.py b/tests/interfaces/test_deployment_cmd.py new file mode 100644 index 00000000..a7397446 --- /dev/null +++ b/tests/interfaces/test_deployment_cmd.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import json +import sys + +from open_data_contract_standard.model import OpenDataContractStandard, SchemaObject, SchemaProperty +import pytest + +from semapact.contractops import AppliedContractRelease +from semapact.exceptions import ValidationError +from semapact.interfaces import cli +from semapact.interfaces.commands import deployment_cmd +from semapact.interfaces.commands.deployment_cmd import DeploymentCommandResult +from semapact.interfaces.outcomes import ProcessOutcome + + +def _release() -> AppliedContractRelease: + contract = OpenDataContractStandard( + apiVersion="v3.1.0", + kind="DataContract", + id="orders-product", + name="Orders", + version="1.2.0", + status="active", + schema=[ + SchemaObject( + name="orders", + physicalName="orders_runtime", + properties=[ + SchemaProperty( + name="id", + physicalName="order_id", + type="integer", + physicalType="BIGINT", + required=True, + ) + ], + ) + ], + ) + return AppliedContractRelease( + applied_release_id="applied-release:test", + contract_id="orders-product", + decision_id="decision:test", + change_set_id="change-set:test", + release_plan_id="release-plan:test", + version_resolution_id="version-resolution:test", + release_revision_ref="rev:released", + selected_version="1.2.0", + authorization_id="authorization:test", + released_contract_json=json.dumps( + contract.model_dump(mode="json", by_alias=True, exclude_none=True), + sort_keys=True, + separators=(",", ":"), + ), + ) + + +def test_deployment_parser_exposes_four_explicit_phases() -> None: + parser = cli._build_parser() + + plan = parser.parse_args( + [ + "deployment", + "plan", + "--release", + "release.json", + "--platform", + "databricks", + "--runtime", + "main.silver", + ] + ) + preview = parser.parse_args( + ["deployment", "preview", "--plan", "plan.json"] + ) + execute = parser.parse_args( + [ + "deployment", + "execute", + "--plan", + "plan.json", + "--preview", + "preview.json", + "--authorization", + "authorization.json", + "--warehouse-id", + "warehouse-1", + ] + ) + verify = parser.parse_args( + ["deployment", "verify", "--plan", "plan.json", "--output", "json"] + ) + + assert plan.deployment_command == "plan" + assert preview.deployment_command == "preview" + assert not hasattr(preview, "warehouse_id") + assert execute.deployment_command == "execute" + assert execute.warehouse_id == "warehouse-1" + assert verify.deployment_command == "verify" + assert verify.output == "json" + + +def test_plan_command_outputs_canonical_deployment_plan(tmp_path) -> None: + release_path = tmp_path / "release.json" + release_path.write_text(_release().model_dump_json(), encoding="utf-8") + args = cli._build_parser().parse_args( + [ + "deployment", + "plan", + "--release", + str(release_path), + "--platform", + "databricks", + "--runtime", + "main.silver", + "--server", + "production", + ] + ) + + result = deployment_cmd.run_deployment_plan(args) + payload = json.loads(result.output) + + assert result.outcome is ProcessOutcome.SUCCESS + assert payload["applied_release_id"] == "applied-release:test" + assert payload["target"] == { + "platform": "databricks", + "runtime_target": "main.silver", + "server_name": "production", + } + assert payload["actions"][0]["governed_asset"] == "orders" + assert payload["actions"][0]["physical_name"] == "orders_runtime" + + +def test_invalid_artifact_is_a_validation_failure(tmp_path) -> None: + invalid = tmp_path / "invalid.json" + invalid.write_text("{}", encoding="utf-8") + + with pytest.raises(ValidationError, match="Invalid DeploymentPlan artifact"): + deployment_cmd._load_model(str(invalid), deployment_cmd.DeploymentPlan) + + +@pytest.mark.parametrize( + "outcome,expected_exit", + [ + (ProcessOutcome.SUCCESS, 0), + (ProcessOutcome.RUNTIME_DRIFT, 6), + (ProcessOutcome.RUNTIME_INDETERMINATE, 7), + ], +) +def test_main_preserves_verification_outcome_semantics( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + outcome: ProcessOutcome, + expected_exit: int, +) -> None: + monkeypatch.setattr( + deployment_cmd, + "run_deployment_verify", + lambda args: DeploymentCommandResult(output="verification", outcome=outcome), + ) + monkeypatch.setattr( + sys, + "argv", + ["semapact", "deployment", "verify", "--plan", "plan.json"], + ) + + assert cli.main() == expected_exit + assert capsys.readouterr().out.strip() == "verification" diff --git a/tests/test_deployment_service.py b/tests/test_deployment_service.py new file mode 100644 index 00000000..c0509140 --- /dev/null +++ b/tests/test_deployment_service.py @@ -0,0 +1,271 @@ +from __future__ import annotations + +from datetime import datetime, timezone + +import pytest +from open_data_contract_standard.model import SchemaObject, SchemaProperty + +from semapact.deployment import ( + DeploymentAction, + DeploymentActionKind, + DeploymentAuthorization, + DeploymentPlan, + DeploymentPreview, + DeploymentTarget, + NativeOperation, + NativeOperationKind, +) +from semapact.deployment.models import ( + compute_deployment_authorization_id, + compute_deployment_plan_id, + compute_deployment_preview_id, +) +from semapact.exceptions import ValidationError +from semapact.observation import ( + ObservedAsset, + ObservedAssetIdentity, + ObservedPlatformState, + ObservedProperty, + ObservedPropertyIdentity, + RuntimeAssetBinding, + with_observed_state_fingerprint, +) +from semapact.reconciliation import RuntimeDriftStatus, classify_reconciliation_status +from semapact.services.deployment_service import DeploymentService + +CAPTURED_AT = datetime(2026, 9, 10, 10, 0, tzinfo=timezone.utc) + + +class FakeRuntimeProvider: + key = "databricks" + + def __init__(self, observation: ObservedPlatformState) -> None: + self.observation = observation + self.resolve_calls = 0 + self.observe_calls = 0 + self.bindings: tuple[RuntimeAssetBinding, ...] = () + + def resolve_bindings(self, *, runtime_target: str, assets): + assert runtime_target == "main.silver" + self.resolve_calls += 1 + self.bindings = tuple( + RuntimeAssetBinding( + governed_asset=asset.governed_asset, + observed_asset=ObservedAssetIdentity( + platform="databricks", + namespace=("main", "silver"), + asset=asset.physical_name, + ), + ) + for asset in assets + ) + return self.bindings + + def observe(self, *, bindings): + assert tuple(bindings) == self.bindings + self.observe_calls += 1 + return self.observation + + +class FakeDeploymentAdapter: + key = "databricks" + + def __init__(self, preview: DeploymentPreview) -> None: + self.preview_result = preview + self.preview_calls = 0 + self.execute_calls = 0 + self.executed = None + + def validate(self, plan: DeploymentPlan) -> None: + pass + + def preview(self, plan: DeploymentPlan, observed_state: ObservedPlatformState): + self.preview_calls += 1 + assert observed_state.fingerprint == self.preview_result.observation_fingerprint + return self.preview_result + + def execute(self, plan, preview, authorization) -> None: + self.execute_calls += 1 + self.executed = (plan, preview, authorization) + + +def _plan() -> DeploymentPlan: + schema = SchemaObject( + name="orders", + physicalName="orders_v2", + properties=[ + SchemaProperty( + name="order_id", + physicalName="order_pk", + type="integer", + physicalType="BIGINT", + required=True, + ) + ], + ) + action = DeploymentAction( + kind=DeploymentActionKind.ENSURE_ASSET_STATE, + governed_asset="orders", + physical_name="orders_v2", + desired_state_json=schema.model_dump_json(by_alias=True, exclude_none=True), + ) + target = DeploymentTarget(platform="databricks", runtime_target="main.silver") + plan_id = compute_deployment_plan_id( + applied_release_id="release-1", + contract_id="orders-contract", + release_plan_id="release-plan-1", + released_revision_ref="abc123", + selected_version="1.2.3", + target=target, + actions=(action,), + ) + return DeploymentPlan( + deployment_plan_id=plan_id, + applied_release_id="release-1", + contract_id="orders-contract", + release_plan_id="release-plan-1", + released_revision_ref="abc123", + selected_version="1.2.3", + target=target, + actions=(action,), + ) + + +def _observation() -> ObservedPlatformState: + asset_identity = ObservedAssetIdentity( + platform="databricks", + namespace=("main", "silver"), + asset="orders_v2", + ) + return with_observed_state_fingerprint( + ObservedPlatformState( + platform="databricks", + source_identifier="https://adb.example", + captured_at=CAPTURED_AT, + assets=( + ObservedAsset( + identity=asset_identity, + asset_type="MANAGED", + properties=( + ObservedProperty( + identity=ObservedPropertyIdentity( + asset=asset_identity, + property="order_pk", + ), + physical_type="BIGINT", + nullable=False, + ), + ), + ), + ), + fingerprint=None, + ) + ) + + +def _preview(plan: DeploymentPlan, observation: ObservedPlatformState) -> DeploymentPreview: + operation = NativeOperation( + kind=NativeOperationKind.NO_OP, + governed_asset="orders", + ) + assert observation.fingerprint is not None + preview_id = compute_deployment_preview_id( + deployment_plan_id=plan.deployment_plan_id, + platform="databricks", + runtime_target=plan.target.runtime_target, + source_identifier=observation.source_identifier, + observation_fingerprint=observation.fingerprint, + operations=(operation,), + ) + return DeploymentPreview( + deployment_preview_id=preview_id, + deployment_plan_id=plan.deployment_plan_id, + platform="databricks", + runtime_target=plan.target.runtime_target, + source_identifier=observation.source_identifier, + observation_fingerprint=observation.fingerprint, + operations=(operation,), + ) + + +def _authorization(plan: DeploymentPlan) -> DeploymentAuthorization: + authorization_id = compute_deployment_authorization_id( + contract_ops_authorization_id="contractops-auth-1", + deployment_plan_id=plan.deployment_plan_id, + applied_release_id=plan.applied_release_id, + allowed=True, + ) + return DeploymentAuthorization( + deployment_authorization_id=authorization_id, + contract_ops_authorization_id="contractops-auth-1", + deployment_plan_id=plan.deployment_plan_id, + applied_release_id=plan.applied_release_id, + allowed=True, + ) + + +def test_preview_orchestrates_observation_without_execution() -> None: + plan = _plan() + observation = _observation() + provider = FakeRuntimeProvider(observation) + adapter = FakeDeploymentAdapter(_preview(plan, observation)) + + result = DeploymentService().preview( + plan, + runtime_provider=provider, + adapter=adapter, + ) + + assert result == adapter.preview_result + assert provider.resolve_calls == 1 + assert provider.observe_calls == 1 + assert adapter.preview_calls == 1 + assert adapter.execute_calls == 0 + + +def test_execute_delegates_exact_canonical_artifacts() -> None: + plan = _plan() + observation = _observation() + preview = _preview(plan, observation) + authorization = _authorization(plan) + adapter = FakeDeploymentAdapter(preview) + + DeploymentService().execute( + plan, + preview, + authorization, + adapter=adapter, + ) + + assert adapter.execute_calls == 1 + assert adapter.executed == (plan, preview, authorization) + + +def test_verify_reuses_existing_m1_reconciliation() -> None: + plan = _plan() + provider = FakeRuntimeProvider(_observation()) + + result = DeploymentService().verify(plan, runtime_provider=provider) + + assert classify_reconciliation_status(result) is RuntimeDriftStatus.IN_SYNC + assert result.contract_id == plan.contract_id + assert result.contract_version == plan.selected_version + + +def test_preview_provider_mismatch_fails_before_observation() -> None: + plan = _plan() + observation = _observation() + provider = FakeRuntimeProvider(observation) + provider.key = "snowflake" + adapter = FakeDeploymentAdapter(_preview(plan, observation)) + + with pytest.raises(ValidationError, match="Runtime provider does not match"): + DeploymentService().preview( + plan, + runtime_provider=provider, + adapter=adapter, + ) + + assert provider.resolve_calls == 0 + assert provider.observe_calls == 0 + assert adapter.preview_calls == 0 From 6b26e9adb065ecbfe9b25b7416cb931a861e878e Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Fri, 11 Sep 2026 15:57:40 +1000 Subject: [PATCH 13/35] feat(contractops): expose canonical release planning CLI (#217) * feat(contractops): add release planning result model * feat(contractops): add canonical release planning service * feat(contractops): export release planning service * feat(contractops): expose canonical release planning CLI * feat(contractops): route canonical release plan command * test(contractops): cover canonical release planning service * test(cli): cover canonical release plan command * docs(contractops): document canonical release CLI * docs(deployment): document CLI and Databricks capability * docs: align README with ContractOps deployment * refactor(application): separate services and use-case models * refactor(application): migrate interface consumers * test(application): target canonical governance module * docs(agents): remove roadmap from devops skill * docs(agents): remove roadmap from draft skill * docs(agents): make architecture skill roadmap-neutral * docs(agents): remove future wording from lifecycle skill * docs(agents): make model ownership rule roadmap-neutral --- .agents/skills/devops-workflow/SKILL.md | 82 +-- .agents/skills/draft-workflow/SKILL.md | 41 +- .agents/skills/lifecycle-policy/SKILL.md | 2 +- .agents/skills/semapact-system/SKILL.md | 155 +++-- .agents/skills/service-layer/SKILL.md | 138 +++-- AGENTS.md | 111 ++-- ARCHITECTURE.md | 532 +++++------------- README.md | 150 ++++- docs/contractops_phases.md | 55 +- docs/deployment_plans.md | 91 ++- semapact/application/__init__.py | 1 + semapact/application/models/__init__.py | 12 + semapact/application/models/governance.py | 27 + semapact/application/models/reconciliation.py | 22 + semapact/application/models/release.py | 18 + semapact/application/services/__init__.py | 5 + semapact/application/services/deployment.py | 93 +++ semapact/application/services/governance.py | 111 ++++ .../application/services/reconciliation.py | 71 +++ .../application/services/release_planning.py | 56 ++ .../application/services/version_authority.py | 84 +++ semapact/interfaces/cli.py | 34 +- .../interfaces/commands/deployment_cmd.py | 2 +- semapact/interfaces/commands/merge_cmd.py | 2 +- semapact/interfaces/commands/plan_cmd.py | 29 +- semapact/interfaces/commands/reconcile_cmd.py | 3 +- semapact/interfaces/commands/release_cmd.py | 80 ++- semapact/services/__init__.py | 22 +- semapact/services/deployment_service.py | 94 +--- semapact/services/governance_service.py | 133 +---- semapact/services/reconciliation_service.py | 90 +-- semapact/services/release_models.py | 5 + semapact/services/release_planning_service.py | 5 + .../services/version_authority_service.py | 85 +-- tests/interfaces/test_release_plan_cmd.py | 102 ++++ tests/test_application_architecture.py | 40 ++ tests/test_governance_proposal.py | 4 +- tests/test_release_planning_service.py | 119 ++++ 38 files changed, 1597 insertions(+), 1109 deletions(-) create mode 100644 semapact/application/__init__.py create mode 100644 semapact/application/models/__init__.py create mode 100644 semapact/application/models/governance.py create mode 100644 semapact/application/models/reconciliation.py create mode 100644 semapact/application/models/release.py create mode 100644 semapact/application/services/__init__.py create mode 100644 semapact/application/services/deployment.py create mode 100644 semapact/application/services/governance.py create mode 100644 semapact/application/services/reconciliation.py create mode 100644 semapact/application/services/release_planning.py create mode 100644 semapact/application/services/version_authority.py create mode 100644 semapact/services/release_models.py create mode 100644 semapact/services/release_planning_service.py create mode 100644 tests/interfaces/test_release_plan_cmd.py create mode 100644 tests/test_application_architecture.py create mode 100644 tests/test_release_planning_service.py diff --git a/.agents/skills/devops-workflow/SKILL.md b/.agents/skills/devops-workflow/SKILL.md index c744a462..479e5c3f 100644 --- a/.agents/skills/devops-workflow/SKILL.md +++ b/.agents/skills/devops-workflow/SKILL.md @@ -1,67 +1,77 @@ --- name: devops-workflow -description: Defines GitOps workflow for contract promotion including branch creation, pull requests, CI/CD, and versioning. +description: Defines current GitOps delivery rules for governed contract changes, pull requests, CI/CD, and versioning. --- # DevOps Workflow -SemaPact follows a GitOps-based promotion model. +SemaPact uses GitOps boundaries for governed contract delivery. ------------------------------------------------ -FLOW - -Draft → Promote → PR → CI/CD → Merge → Release +CURRENT FLOW + +Analyze / Plan + ↓ +Explicit Git workflow + ↓ +PR + ↓ +CI/CD + ↓ +Merge + ↓ +Release ------------------------------------------------ RULES -- MAIN contract updated only via merge -- CI/CD validates contracts before merge -- `required_bump` is computed PER CONTRACT, not per repo -- feature -> main determines `required_bump` but does NOT change contract version -- release flow applies the explicit version/tag per contract after merge -- repo-level automation may batch many contracts, but it must orchestrate them as independent per-contract release units -- multi-contract release automation should use an explicit manifest because each contract may carry its own release tag/version -- a healthy repo-level flow is: `classify-repo -> build-manifest -> create-prs` -- suggested release versions are always computed from the last released contract version and the highest current bump requirement, not by chaining unreleased bumps -- if `required_bump` is `none`, the contract should not be version-bumped by default and should be skipped in batch release manifests unless a team explicitly chooses otherwise +- MAIN contract is updated only through a reviewed merge path. +- CI/CD validates contracts before merge. +- `required_bump` is computed PER CONTRACT, not per repo. +- feature -> main classification does NOT directly change contract version. +- release version changes occur only through an explicit release path. +- repo-level automation may batch many contracts, but it must orchestrate them as independent per-contract release units. +- multi-contract release automation uses an explicit manifest because each contract may carry its own release tag/version. +- the supported compatibility repo flow is `classify-repo -> build-manifest -> create-prs`. +- suggested release versions are computed from the last released contract version and the highest current bump requirement, not by chaining unreleased bumps. +- if `required_bump` is `none`, the contract is not version-bumped by default and is skipped in batch release manifests unless explicitly selected. ------------------------------------------------ -PREFERRED AUTOMATION +AUTOMATION BOUNDARIES Feature -> Main: -- run `release classify` for single-contract repos -- run `release classify-repo` for multi-contract repos -- fail or warn based on the returned per-contract `required_bump` -- do not change `contract.version` +- run `release classify` for analysis-only classification where that compatibility workflow is used; +- run `release classify-repo` for repo-level compatibility classification; +- do not change `contract.version` during classification. -Main -> Release: +Canonical release planning: -- run `release build-manifest` -- review or edit the generated per-contract manifest -- run `release create-prs` +- use `release plan` to produce the exact `GovernanceDecision`, `ChangeSet`, `ReleasePlan`, and `VersionResolution` artifacts; +- bind planning to explicit base/candidate revision references; +- do not reinterpret governance or version policy in CI scripts. -Merge Build: +Compatibility release workflow: -- re-run validation and classification on merged main if needed -- publish summaries or audit artifacts -- keep `contract.version` unchanged until release +- `release build-manifest` produces explicit per-contract release tasks; +- `release create-prs` consumes those tasks and creates independent release PRs; +- compatibility helpers must not become a second ContractOps implementation. ------------------------------------------------- -FUTURE +Merge Build: -- auto PR creation -- automated validation pipelines -- audit logs +- re-run validation/classification when required by the workflow; +- publish only supported summaries or artifacts; +- keep `contract.version` unchanged outside the explicit release path. ------------------------------------------------ FORBIDDEN -- direct writes to main branch -- bypassing CI/CD +- direct writes to main branch; +- bypassing CI/CD or governance authorization; +- embedding lifecycle/version policy inside CI scripts; +- treating repo-level batching as a shared contract-version authority. ------------------------------------------------ GOAL -Ensure safe, auditable, and deterministic contract delivery. +Ensure safe, auditable, deterministic contract delivery while keeping Git/CI orchestration separate from domain policy. diff --git a/.agents/skills/draft-workflow/SKILL.md b/.agents/skills/draft-workflow/SKILL.md index fc820961..946391d5 100644 --- a/.agents/skills/draft-workflow/SKILL.md +++ b/.agents/skills/draft-workflow/SKILL.md @@ -1,14 +1,14 @@ --- name: draft-workflow -description: Defines draft-based editing and change workflow for contracts including save, analyze, and promotion steps. Use when implementing or reviewing SemaPact draft retrieval, draft persistence, draft validation, main-vs-draft analysis, and future promotion flow. Apply this skill when main contracts must stay protected while drafts persist independently and provide safe iterative editing. +description: Defines current draft-based editing and change workflow for governed contracts. Use when implementing or reviewing draft retrieval, persistence, validation, and main-vs-draft analysis. --- # Draft & Change Workflow -SemaPact uses a draft-based editing model. +SemaPact uses a draft-based editing model so presentation paths do not overwrite canonical governed state. ------------------------------------------------ -FLOW +CURRENT FLOW Load MAIN ↓ @@ -16,47 +16,46 @@ Create or Load DRAFT ↓ Edit Draft ↓ -Analyze Draft vs Main +Validate / Analyze Draft vs Main ↓ Save Draft - ↓ -Promote (future) ------------------------------------------------ DRAFT STORAGE -Recommended: +Current draft storage convention: `.semapact/drafts/{user}/{contract_id}.yaml` ------------------------------------------------ RULES -- Draft must NOT overwrite main -- Draft must persist independently -- Draft must be validated before saving +- Draft must NOT overwrite main. +- Draft must persist independently from canonical main state. +- Draft must be validated before saving. +- UI/API/CLI code must not implement lifecycle or merge policy directly. +- Any path that attempts to move draft state into governed main state must use the canonical governance, authorization, and GitOps boundaries rather than writing main directly. ------------------------------------------------ SAVE `save_draft`: -- validate contract -- persist draft -- do NOT modify main contract +- validate the candidate draft; +- persist draft state only; +- do NOT modify the main contract; +- preserve non-editable governed/technical fields according to the application/domain rules. ------------------------------------------------ -PROMOTION +ANALYSIS -`promote_draft`: +Main-vs-draft analysis: -- compare main vs draft -- run governance checks -- classify required version bump PER CONTRACT -- require an explicit `release_tag` -- create PR (future) +- compares the exact governed main revision with the draft candidate; +- delegates lifecycle/breaking/deprecation policy to the canonical governance layer; +- returns analysis artifacts without mutating main or draft implicitly. ------------------------------------------------ GOAL -Enable safe, iterative contract editing with real-time feedback. +Enable safe iterative editing while keeping canonical governed state protected from presentation-layer writes. diff --git a/.agents/skills/lifecycle-policy/SKILL.md b/.agents/skills/lifecycle-policy/SKILL.md index 5c618992..be743f31 100644 --- a/.agents/skills/lifecycle-policy/SKILL.md +++ b/.agents/skills/lifecycle-policy/SKILL.md @@ -114,7 +114,7 @@ Breaking checks apply ONLY when: ## 4.2 Behavior Matrix | Lifecycle State | Breaking Checks | Auto-Deprecation | Structural Changes | Metadata Updates | -|-----------------|----------------|-----------------|-------------------|------------------| +|-----------------|----------------|-----------------|------------------|------------------| | draft | ❌ Skip | ❌ Skip | ✅ Allowed | ✅ Allowed | | active | ✅ Enforce | ✅ Apply | ⚠ Governed | ✅ Allowed | | deprecated | ❌ Skip | ❌ Skip | ❌ Forbidden | ✅ Allowed | diff --git a/.agents/skills/semapact-system/SKILL.md b/.agents/skills/semapact-system/SKILL.md index 7da2e56f..305f97e0 100644 --- a/.agents/skills/semapact-system/SKILL.md +++ b/.agents/skills/semapact-system/SKILL.md @@ -1,61 +1,106 @@ --- name: semapact-system -description: Defines the core operating model, layered architecture boundaries, and change-driven workflow of SemaPact. Apply this skill when refactoring components, implementing new modules, or deciding boundary placement. +description: Defines the core operating model, layered architecture boundaries, and change-driven workflow of SemaPact. Apply when refactoring components, implementing modules, or deciding package ownership. --- # SemaPact System Model & Architecture Rules -SemaPact is an enterprise data contract control plane and governance platform. It follows a strict layered architecture and a change-driven operating model. - -## 1. Core Workflow Principles -- **Change-Driven System:** All UI edits must happen via drafts. - - Save = save draft copy. - - Publish/Promote = run governance checks and promote draft. -- **Immutability of Main:** Main production contracts are immutable from direct UI edits. They are only updated via governed merge operations. -- **Immutability of Identity:** The contract `id` is immutable once created. -- **Release Gating:** Contract `version` is release-managed and only changes through an explicit release/promotion path. - -## 2. Layered Architecture Boundaries (CRITICAL) - -### A. Ingestion / Import Layer -- **Role:** Converts external data structures into Open Data Contract Standard (ODCS) models when a contract import is explicitly requested. -- **Rules:** - - Must remain strictly stateless and idempotent. - - **NEVER** place merge, governance, or GitOps logic inside importers. - - Contract import is distinct from platform observation; observing a platform must not implicitly create or mutate an ODCS contract. - -### B. Governed Contract Model -- **Role:** Single canonical representation of governed desired contract state. -- **Rules:** - - The ODCS YAML/Pydantic model is the single source of truth for governed contract state. - - External platform state must not become governed truth merely because it was observed. - -### C. Platform Observation Model -- **Role:** Represents point-in-time external platform state for assurance and reconciliation workflows. -- **Rules:** - - `ObservedPlatformState` is a read-side model, not an alternative canonical contract format. - - Core observation models must remain platform-neutral; provider hierarchy belongs in adapter-local mapping into a generic ordered `namespace`. - - Platform-local identity must remain distinct from ODCS contract identity. - - Provider adapters may reuse official platform SDK access, but must not route observation through ODCS import/projection. - - Observation must not invoke lifecycle merge, governance evaluation, release mutation, or platform writeback. - - Rich metadata, constraints, relationships, and lineage are evidence enrichments, not prerequisites for the minimal observed-state model. - - Converting observed state into an ODCS contract is an explicit import workflow, never an implicit observation side effect. - -### D. Lifecycle Governance Layer -- **Role:** Handles breaking change checks, deprecation rules, merge policies, and version bump calculations. -- **Rules:** - - This is the **ONLY** place where contract lifecycle logic is allowed. - - It must remain fully decoupled from the UI, ingestion, and platform observation layers. - -### E. Export Layer -- **Role:** Converts contracts to downstream assets (Great Expectations suites, Spark DDL, Graph cypher). -- **Rules:** - - Exporters must be read-only and **NEVER** modify the original contracts. - -### F. Orchestration Layer -- **Role:** Coordinates multi-step workflows (e.g. import → merge → export → PR). -- **Rules:** - - Coordinates execution paths but must NOT contain custom business logic. - -### G. DevOps Layer -- **Role:** Automates PR creation, version bumps, release manifest building, and metadata auditing. +SemaPact is a change-driven ODCS lifecycle-governance and production-assurance control plane. + +## 1. Core workflow principles + +- governed contract state is canonical ODCS; +- main state is not directly overwritten from presentation paths; +- contract identity is immutable once governed; +- release version changes only through explicit release flow; +- side effects are operation-scoped: APPLY, PUBLISH, and DEPLOY are distinct; +- execution success is not convergence proof. + +## 2. Layered dependency direction + +```text +interfaces (CLI/API/UI) + ↓ +application services + ↓ +domain packages and narrow ports + ↑ +platform/provider adapters +``` + +Dependencies must not point from domain packages into application or presentation code. + +### A. Import/ingestion + +Converts external structures into ODCS only when explicit import is requested. Importers are stateless/idempotent and contain no lifecycle or CI/CD policy. + +### B. Governed contract model + +ODCS is authoritative desired contract state. External observations do not become governed truth implicitly. + +### C. Lifecycle/governance domain + +Owns identity, lifecycle semantics, breaking/deprecation policy, change classification, and deterministic governance decisions. + +### D. ContractOps domain + +Owns deterministic release artifacts, authorization, APPLY/PUBLISH contracts, and exact artifact association. Downstream phases consume earlier artifacts; they do not rerun governance/version classification. + +### E. Deployment/runtime domain + +Owns provider-neutral DeploymentPlan/authorization/preview contracts and neutral runtime asset projection. Provider-native operations belong behind deployment adapters. + +### F. Observation/reconciliation domain + +Owns platform-neutral observed state and deterministic desired-vs-observed comparison. Reconciliation does not infer deployment causality or mutate runtime. + +### G. Application layer + +Location: `semapact/application/`. + +- `application/models/` owns use-case result DTOs that compose domain artifacts; +- `application/services/` owns thin interface-independent orchestration; +- it may construct semantic context/configuration needed by a workflow; +- it must not own lifecycle/governance/version/deployment/reconciliation rules. + +`semapact/services/` is compatibility-only and must not receive new implementation. + +### H. Interfaces + +Parse request values, validate/load external artifacts at the interface edge, call application/domain boundaries, render output, and map process outcomes. Interfaces do not own business rules. + +### I. Platform adapters + +Provider SDK/client and physical-platform translation live under `semapact/platforms/`. Concrete construction belongs in composition/application/platform code, never domain logic. + +## 3. Model ownership rule + +Do not organize models by the fact that they are "data". Organize them by meaning: + +```text +ODCS governed contract → ODCS model +GovernanceDecision / ReleasePlan → owning domain package +ReleasePlanningResult → application/models +Databricks table/statement representation → platforms/databricks +persistence/history record → dedicated persistence/history boundary +``` + +Avoid generic root-level `schema`, `models`, or `data_models` dumping grounds. + +## 4. Interface/port rule + +Create interfaces only at genuine replaceable or side-effecting seams. Pure deterministic planners/value objects remain functions/models rather than acquiring ports for symmetry. + +## 5. Change discipline + +When moving ownership: + +1. establish the new canonical import path; +2. migrate internal callers; +3. retain a thin compatibility re-export when public/backward compatibility matters; +4. add architecture/import tests so compatibility wrappers cannot become a second implementation; +5. update contributor-facing architecture rules in the same change. + +## 6. Public architecture documentation rule + +Public agent skills describe only current invariants, supported behavior, package ownership, coding constraints, and compatibility contracts. Product sequencing, unimplemented capability plans, internal technical debt, and roadmap material do not belong in public skills. diff --git a/.agents/skills/service-layer/SKILL.md b/.agents/skills/service-layer/SKILL.md index 4a8b7795..39d8b66e 100644 --- a/.agents/skills/service-layer/SKILL.md +++ b/.agents/skills/service-layer/SKILL.md @@ -1,98 +1,92 @@ --- name: service-layer -description: Defines the UI-independent SemaPact application service layer in `semapact/services/`. Use when interfaces such as CLI, UI, or API need to translate request inputs into domain context and delegate contract loading, draft management, validation, permissions, or governance orchestration without owning business rules. +description: Defines SemaPact's interface-independent application layer in `semapact/application/`. Use when CLI, UI, API, or CI workflows need typed use-case orchestration without owning domain rules. --- -# Service Layer +# Application Service Layer -This is the application boundary between interfaces and system logic. +The application layer sits between interfaces and domain/provider ports. ------------------------------------------------- -RESPONSIBILITIES +```text +interfaces + ↓ +application/services + ↓ +domain functions + ports + ↑ +platform/provider adapters +``` -- normalize interface request values into formal domain inputs -- own workflow-scoped context construction such as `ChangeContext` -- load main contracts and manage drafts where applicable -- enforce permissions where applicable -- validate contracts through shared core components -- delegate governance analysis to lifecycle/governance components +## Package ownership ------------------------------------------------- -STRICT RULES +```text +semapact/application/ +├── models/ # use-case/application result DTOs only +└── services/ # thin workflow orchestration only +``` -- CLI, UI, and API layers must NOT implement business logic -- interfaces must NOT construct or regenerate governance-semantic context directly -- service must NOT depend on presentation modules -- service methods should accept and return `OpenDataContractStandard`, formal Pydantic models, or formal dataclasses -- do NOT fall back to raw `dict[str, Any]` merely to accommodate an interface -- governance-semantic dates must not silently default from wall-clock time +Canonical domain artifacts remain in their owning packages. For example, `GovernanceDecision`, `ChangeSet`, `ReleasePlan`, `VersionResolution`, `DeploymentPlan`, `DeploymentPreview`, and `ReconciliationResult` do not move into application models. ------------------------------------------------- -ALLOWED DEPENDENCIES +Application result objects that aggregate those artifacts, such as `GovernanceProposal`, `GovernanceAnalysis`, `RuntimeReconciliation`, or `ReleasePlanningResult`, belong in `application/models`. -- `semapact.core` -- `semapact.governance` -- `semapact.lifecycle` -- `semapact.utils` +`semapact/services/` is a backward-compatibility import surface only. Do not add new models or behavior there. ------------------------------------------------- -CURRENT API +## Responsibilities -Governance: -- `GovernanceService.create_context(effective_date)` -- `GovernanceService.evaluate(base, candidate, effective_date=...)` -- `GovernanceService.merge_and_evaluate(source, governed, effective_date=...)` +- normalize interface request values into explicit domain inputs; +- create workflow-scoped context such as `ChangeContext` once; +- compose existing deterministic domain functions and narrow ports; +- select configuration needed by a use case; +- return typed domain artifacts or typed application result DTOs. -Future draft/application services may expose: -- `list_contracts(user)` -- `get_contract(contract_id)` -- `get_draft(contract_id, user)` -- `save_draft(contract, user)` -- `promote_draft(contract_id, user)` +## Strict rules ------------------------------------------------- -IMPLEMENTATION GUIDANCE +- CLI/UI/API must not implement business rules; +- application services must not re-diff or reinterpret downstream governance/version/deployment/reconciliation artifacts; +- service modules must not define unrelated application DTOs inline; place reusable use-case results in `application/models`; +- services must not depend on presentation modules; +- do not use raw `dict[str, Any]` as an internal service contract when a formal model/dataclass exists; +- governance-semantic dates must not silently default from wall-clock time; +- optional provider SDK/client construction belongs in composition/platform code and remains lazy where possible. -Keep services thin. +## Model placement decision -Preferred flow: +Before creating a model, ask what it represents: -1. receive already-collected interface request values -2. normalize those values into explicit domain inputs -3. delegate validation, merge, lifecycle, and governance rules to their owning layers -4. return typed service results +- governed business/domain fact → owning domain package; +- application use-case result/composition → `application/models`; +- provider/physical representation → provider package under `platforms`; +- persistence/history record → dedicated persistence/history boundary; +- presentation-only rendering state → interface package. -For deterministic governance context: +Do not create a root `schema`, `models`, or `data_models` directory simply to collect unrelated types. -```text -interface request - ↓ -GovernanceService creates ChangeContext once - ↓ -merge / evaluator consume the same context -``` +## Preferred flow ------------------------------------------------- -FORBIDDEN +1. interface collects request values; +2. application service resolves workflow context/configuration; +3. service delegates to existing domain rules/ports; +4. service returns typed results; +5. interface renders or maps the result to process outcomes. -- duplicating lifecycle or breaking-change policy in services -- bypassing governance gates -- resolving governance-semantic dates from `date.today()` / `datetime.now()` inside lower layers -- importing presentation frameworks from services -- overwriting canonical main contracts directly from an interface path +## Forbidden ------------------------------------------------- -REVIEW CHECKLIST +- duplicating lifecycle or breaking-change policy in application services; +- bypassing governance/authorization gates; +- provider-specific DDL or SDK logic inside application models/services; +- presentation imports from application code; +- new implementation under `semapact/services/`. -1. Is the service independent of CLI/UI/API implementation details? -2. Does the interface pass request values rather than constructing domain context itself? -3. Does the service delegate lifecycle and governance rules instead of duplicating them? -4. Is one resolved context reused throughout a single workflow? -5. Are existing semantic dates preserved instead of overwritten? -6. Are operational timestamps kept separate from governance context? +## Review checklist ------------------------------------------------- -Read these first-class Agent Skills when needed: +1. Does each model have one semantic owner? +2. Is orchestration under `application/services`, separate from reusable result DTOs? +3. Do interfaces only parse/render/delegate? +4. Does the service reuse one resolved context throughout a workflow? +5. Are domain rules delegated rather than duplicated? +6. Are optional provider dependencies still lazy? +7. Are compatibility imports wrappers rather than a second implementation? -- [semapact-system](../../semapact-system/SKILL.md) for system architecture rules -- [lifecycle-policy](../../lifecycle-policy/SKILL.md) for contract lifecycle and deprecation logic +Read also: +- [semapact-system](../semapact-system/SKILL.md) +- [lifecycle-policy](../lifecycle-policy/SKILL.md) diff --git a/AGENTS.md b/AGENTS.md index af60a863..3aa7d0c6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,95 +1,74 @@ # 🤖 SemaPact AI Agent Guidelines -Welcome, AI Agent! You are working on **SemaPact**, an open-source, enterprise-level lifecycle governance platform for Open Data Contracts (ODCS). - -To ensure high-quality and consistent code generation, please adhere to the following rules: +SemaPact is an open-source, enterprise lifecycle-governance and production-assurance platform for Open Data Contracts (ODCS). ## 1. Architectural Alignment -SemaPact is a change-driven (not CRUD) system that enforces GitOps workflows, immutable main contracts, and user-scoped drafts. -- **Always read [`ARCHITECTURE.md`](./ARCHITECTURE.md)** before designing new features, adding state, or modifying core modules. - -## 2. Load Your Skills -We maintain specialized instructions for you in the `.agents/skills/` directory. -- **Always read [`.agents/README.md`](./.agents/README.md)** at the start of a session to understand the available skills. -- Load the specific `SKILL.md` file relevant to your current task (e.g., if you are touching UI, read the `streamlit-ui` skill; if touching validation, read `lifecycle-policy`). -## 3. Core Principles -1. **Defensive Coding**: All new code must be fully type-hinted and handle edge cases gracefully. Do not swallow exceptions silently. -2. **Configuration over Environment Variables**: Favor adding user configuration to `ConfigManager` (which resolves from `.semapact.yaml`) rather than hardcoding `os.environ` reads, unless it's a dynamic CI runner variable. -3. **Preserve Main Contracts**: Canonical contracts in `contracts-main` or the base path should never be overwritten blindly. Use the `merge_engine`. +SemaPact is change-driven, not CRUD. Main governed contracts are not edited blindly; lifecycle, release, authorization, deployment, and reconciliation remain separate boundaries. -## 4. Agent Working Style -As an AI contributing to an enterprise-grade open-source project, your execution must be flawless and maintainable: -1. **Plan Before Code**: Always think through the architectural implications and edge cases before writing a single line of code. If a change is complex, propose an implementation plan to the user first. -2. **Make It Simple**: Strive for elegant, minimalist solutions. Avoid over-engineering, unnecessary abstractions, or introducing heavy external dependencies unless absolutely required. -3. **Double Check Your Work**: Never assume your code works on the first try. Always double-check your syntax, type hints, and logic. Where possible, write or run tests to verify your changes. +- **Always read [`ARCHITECTURE.md`](./ARCHITECTURE.md)** before designing features, adding state, or moving code across packages. +- Preserve canonical dependency direction: interfaces → application → domain/ports; platform adapters implement external/provider boundaries. -## 5. Testing -- If you modify business logic in `semapact/core` or `semapact/lifecycle`, you must ensure backward compatibility. -- Ensure that the CLI (`semapact/interfaces/cli.py`) and TUI (`semapact/tui/app.py`) are kept in sync when introducing new configuration parameters. +### Package ownership rules -## 6. Behavioral Guidelines (CLAUDE.md) +Place code by semantic ownership, not by whichever caller happens to use it: -Behavioral guidelines to reduce common LLM coding mistakes. +- canonical domain artifacts and domain rules → their owning domain package (`governance/`, `contractops/`, `deployment/`, `reconciliation/`, `runtime/`, `lifecycle/`); +- application/use-case result DTOs that compose domain artifacts → `semapact/application/models/`; +- interface-independent workflow orchestration → `semapact/application/services/`; +- CLI/API/UI parsing and rendering → `semapact/interfaces/`; +- provider SDK/client mappings and physical platform behavior → `semapact/platforms/`; +- `semapact/services/` is compatibility-only. **Do not add new logic or models there.** -**Tradeoff:** These guidelines bias toward caution over speed. For trivial tasks, use judgment. +Do not create generic `schema/`, `models/`, or `data_models/` dumping grounds at the package root. A model belongs with its semantic owner. ODCS schema, domain artifacts, application DTOs, persistence records, and provider SDK representations are different concerns. -### 6.1 Think Before Coding +## 2. Load Your Skills -**Don't assume. Don't hide confusion. Surface tradeoffs.** +- Read [`.agents/README.md`](./.agents/README.md) at the start of a coding session. +- Load the task-specific `SKILL.md`, especially `semapact-system`, `service-layer`, `lifecycle-policy`, or UI/provider skills as applicable. -Before implementing: -- State your assumptions explicitly. If uncertain, ask. -- If multiple interpretations exist, present them - don't pick silently. -- If a simpler approach exists, say so. Push back when warranted. -- If something is unclear, stop. Name what's confusing. Ask. +## 3. Core Principles -### 6.2 Simplicity First +1. **Defensive Coding**: Fully type new code and fail closed where required evidence or authorization is incomplete. +2. **Configuration over ad-hoc environment reads**: Prefer `ConfigManager` for product configuration; environment variables are explicit overrides or CI/runtime inputs. +3. **Preserve canonical contracts**: Do not overwrite governed main contracts from interface code. +4. **One authority per rule**: Interfaces and application services must not reimplement lifecycle, governance, versioning, deployment translation, or reconciliation semantics. +5. **Exact artifacts cross side-effect boundaries**: Downstream phases consume the exact immutable artifacts produced upstream; do not silently reload mutable state and reinterpret it. -**Minimum code that solves the problem. Nothing speculative.** +## 4. Agent Working Style -- No features beyond what was asked. -- No abstractions for single-use code. -- No "flexibility" or "configurability" that wasn't requested. -- No error handling for impossible scenarios. -- If you write 200 lines and it could be 50, rewrite it. +1. **Plan before code**: inspect ownership and dependency direction before editing. +2. **Keep it simple**: avoid speculative abstractions and duplicate façade layers. +3. **Verify**: add or update tests for behavior and architecture boundaries; run the relevant suite/CI. +4. **Surgical changes**: do not refactor unrelated areas. Remove only dead code introduced by your own change unless the task explicitly calls for broader cleanup. -Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify. +## 5. Testing -### 6.3 Surgical Changes +- Domain behavior changes require deterministic unit tests. +- Application services should be tested with canonical models and fake/narrow ports rather than live providers. +- Interface tests should prove parsing, rendering, and process outcomes without duplicating domain assertions. +- When changing package ownership, add an architecture/import test so future agents do not regress the boundary. +- Keep minimal-install import safety: optional provider dependencies must remain lazy until that provider is actually composed. -**Touch only what you must. Clean up only your own mess.** +## 6. Behavioral Guidelines -When editing existing code: -- Don't "improve" adjacent code, comments, or formatting. -- Don't refactor things that aren't broken. -- Match existing style, even if you'd do it differently. -- If you notice unrelated dead code, mention it - don't delete it. +### Think before coding -When your changes create orphans: -- Remove imports/variables/functions that YOUR changes made unused. -- Don't remove pre-existing dead code unless asked. +Do not hide ambiguity. State assumptions and surface trade-offs before creating a new abstraction. -The test: Every changed line should trace directly to the user's request. +### Simplicity first -### 6.4 Goal-Driven Execution +Implement the minimum structure that gives one clear owner for each responsibility. Do not introduce an interface for a pure deterministic function merely for symmetry. -**Define success criteria. Loop until verified.** +### Goal-driven execution -Transform tasks into verifiable goals: -- "Add validation" → "Write tests for invalid inputs, then make them pass" -- "Fix the bug" → "Write a test that reproduces it, then make it pass" -- "Refactor X" → "Ensure tests pass before and after" +Translate work into verifiable outcomes, for example: -For multi-step tasks, state a brief plan: ```text -1. [Step] → verify: [check] -2. [Step] → verify: [check] -3. [Step] → verify: [check] +1. move application orchestration → imports and behavior tests pass +2. move application DTOs → module-ownership test passes +3. preserve compatibility imports → legacy import identity tests pass +4. update contributor rules → docs match the actual package tree ``` -Strong success criteria let you loop independently. Weak criteria ("make it work") require constant clarification. - ---- - -**These guidelines are working if:** fewer unnecessary changes in diffs, fewer rewrites due to overcomplication, and clarifying questions come before implementation rather than after mistakes. +These rules are working when a contributor can tell where a new model or workflow belongs without inspecting every caller. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 9ba0bdf7..59116a12 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -2,450 +2,212 @@ ## Purpose -SemaPact is an ODCS-first contract governance platform. +SemaPact is an ODCS-first, change-driven contract governance and production-assurance control plane. It separates governed contract semantics, release authorization, runtime mutation, and runtime verification so each concern has one authoritative owner. -The current implementation focuses on: +The canonical product flow is: -- canonical main contracts -- user-scoped drafts -- lifecycle governance analysis -- contract quality export -- deployment artifact export -- CLI and automation interfaces - -SemaPact is not a CRUD system. It is a change-driven system: - -- edit -> save draft -- analyze -> compare draft vs main -- promote -> future GitOps workflow - -## Current Runtime Layers - -### 1. Core - -Location: - -- `semapact/core/` - -Responsibilities: - -- load canonical ODCS contracts from supported storage -- validate ODCS contracts and quality rules -- normalize user drafts so business edits do not overwrite technical fields - -Key modules: - -- `semapact/core/loader.py` -- `semapact/core/validator.py` -- `semapact/core/draft_normalizer.py` - -### 2. Lifecycle Governance - -Location: - -- `semapact/lifecycle/` - -Responsibilities: - -- analyze main vs source contract changes -- detect breaking changes -- determine auto-deprecations -- apply lifecycle-aware merges - -Key modules: - -- `semapact/lifecycle/merge_engine.py` -- `semapact/lifecycle/policy.py` - -### 3. Utilities - -Location: - -- `semapact/utils/` - -Responsibilities: - -- YAML file IO -- YAML string parse/dump through ODCS model definitions -- input normalization helpers - -Key modules: - -- `semapact/utils/yaml_utils.py` -- `semapact/utils/schema_utils.py` - -### 4. Service Layer - -Location: - -- `semapact/services/` - -Responsibilities: - -- serve as the interface-independent application boundary into system logic -- normalize interface request values into explicit domain inputs -- own workflow-scoped governance context construction -- delegate merge, lifecycle, validation, and governance rules to their owning layers - -Key module: - -- `semapact/services/governance_service.py` - -Important boundary: - -- CLI/UI/API may collect an `effective_date` request value -- `GovernanceService` creates `ChangeContext` -- lifecycle/governance lower layers consume that context and must not regenerate it - -### 5. Platform Observation - -Location: - -- `semapact/observation/` - -Responsibilities: - -- represent point-in-time external platform state independently from governed contracts -- keep the core observed-state domain platform-neutral -- map provider-local identity into `platform + ordered namespace + asset` -- observe physical asset/property state without creating or mutating ODCS contracts - -Key modules: - -- `semapact/observation/models.py` -- `semapact/observation/databricks.py` - -Important boundary: - -- `ObservedPlatformState` is read-side state, not governed truth -- observed asset identity is platform-local and distinct from ODCS contract identity -- provider-specific hierarchy such as Databricks `catalog/schema` belongs in the adapter, not the core model -- Databricks observation consumes the official SDK `WorkspaceClient.tables.get(...) -> TableInfo` boundary instead of reimplementing the Unity Catalog REST transport -- observation projects `TableInfo` directly into observed state; it must not route through datacontract-cli's ODCS projection -- rich metadata, constraints, relationships, and lineage are follow-up evidence enrichments rather than prerequisites for the minimal observation model -- observation must not invoke lifecycle merge, governance evaluation, release mutation, or platform writeback -- explicit contract import remains a separate workflow - -### 6. Exporters - -Location: - -- `semapact/exporters/` -- `semapact/quality/` - -Responsibilities: +```text +ODCS base + candidate + ↓ +lifecycle / governance + ↓ +GovernanceDecision + ↓ +ChangeSet → ReleasePlan → VersionResolution + ↓ +ContractOpsAuthorization + ↓ +AppliedContractRelease + ↓ +DeploymentPlan → DeploymentAuthorization + ↓ +DeploymentAdapter + ↓ +runtime + ↓ +observation / reconciliation +``` -- generate Great Expectations suites from ODCS contracts -- generate SQL deployment DDL -- add limited Databricks-specific constraint enhancement where supported +Later phases consume exact artifacts from earlier phases. They do not re-run governance, lifecycle classification, version authority, or approval semantics. -Key modules: +## Dependency Direction -- `semapact/quality/ge_exporter.py` -- `semapact/exporters/sql_exporter.py` +Application and domain boundaries follow this direction: -### 7. Orchestration +```text +interfaces (CLI / API / UI / CI) + ↓ +application services + ↓ +domain functions / models / ports + ↑ +platform and external-system adapters +``` -Location: +Domain packages must not depend on `application` or `interfaces`. Provider-specific SDK/client construction stays outside domain logic. -- `semapact/orchestrator/` +## Package Ownership -Responsibilities: +### Domain packages -- coordinate non-interactive automation flows -- import -> merge -> validate -> export +Domain models live with the rules that give them meaning: -Key module: +- `semapact/lifecycle/` — canonical identity, lifecycle policy, merge/change semantics; +- `semapact/governance/` — `GovernanceDecision`, reason codes, centralized gate; +- `semapact/contractops/` — `ChangeSet`, `ReleasePlan`, `VersionResolution`, ContractOps authorization, APPLY/PUBLISH artifacts; +- `semapact/deployment/` — provider-neutral `DeploymentPlan`, `DeploymentPreview`, deployment authorization and adapter contract; +- `semapact/runtime/` — provider-neutral governed runtime asset projection; +- `semapact/observation/` — provider-neutral point-in-time runtime state; +- `semapact/reconciliation/` — desired-vs-observed comparison and `RuntimeDriftStatus`. -- `semapact/orchestrator/pipeline.py` +A domain artifact does not move into the application layer merely because an application service returns it. -### 8. Interfaces +### Application layer Location: -- `semapact/interfaces/` - -Responsibilities: - -- presentation/input adaptation only -- collect user or automation inputs -- display governance results -- call service/application boundaries - -Current interface: - -- CLI in `semapact/interfaces/cli.py` -- command adapters in `semapact/interfaces/commands/` - -## Governed Contract Model - -SemaPact assumes: - -- Open Data Contract Standard (ODCS) is the single canonical representation of governed desired contract state -- `open_data_contract_standard.model.OpenDataContractStandard` is the canonical governed contract domain model - -The system may temporarily work with Python `dict` objects at contract boundaries, but contract normalization should converge back to ODCS objects or ODCS-shaped mappings. - -Platform observations are intentionally different. `ObservedPlatformState` is a non-canonical read-side model describing what an external platform reports at a point in time. It must not replace or mutate the governed ODCS contract. - -Conceptually: - ```text -Approved ODCS Contract -= desired governed state - -ObservedPlatformState -= observed external platform state +semapact/application/ +├── models/ +└── services/ ``` -Converting external metadata into a new ODCS contract is an explicit import workflow. Observing platform state does not implicitly perform that conversion. - -## Root Contract Governance - -At the top level of the contract, SemaPact currently treats these fields specially: - -- `id` - - immutable once the governed/main contract exists - - importer-generated IDs are only used when a contract is first created outside SemaPact -- `version` - - release-managed - - must not change in the normal import/merge pipeline - - technical source versions such as Delta table versions must not overwrite the governed contract version - -Current behavior: - -- `semapact.lifecycle.merge_engine` preserves governed `id` and `version` -- `semapact.lifecycle.policy` flags root `id` changes as `id_violation` -- `semapact.lifecycle.policy` flags root version changes as `version_violation` -- `semapact.orchestrator.pipeline` blocks on `id_violation` and `version_violation` - -This means SemaPact currently supports: - -- technical schema refresh through import/merge -- governed metadata preservation +`application/services/` owns thin, interface-independent use-case orchestration. It may resolve workflow context/configuration and compose existing domain functions or ports, but it must not reimplement domain policy. -It does not yet implement: +`application/models/` owns typed use-case results that aggregate canonical domain artifacts. Examples include: -- automatic discovery of which contracts in a repo should be released together -- automatic git-tag lookup inside core/service layers +- `GovernanceAnalysis`; +- `GovernanceProposal`; +- `RuntimeReconciliation`; +- `ReleasePlanningResult`. -## Release Governance Direction +These are application DTOs, not new governance/release/deployment authorities. -Current release-version governance is intentionally **per contract**, not per repo. +### Compatibility package -This supports both: +`semapact/services/` is a backward-compatible import surface for the former package layout. It contains re-exports only and owns no models or business logic. New code must import from `semapact.application`. -- one-contract-per-repo setups -- centralized repos containing many governed contracts +### Interfaces -Current intended flow: +`semapact/interfaces/` owns parsing, loading input artifacts at the interface edge, rendering, and process-outcome mapping. Interfaces delegate to application/domain boundaries and must not independently calculate governance, version, deployment, or reconciliation results. -1. `feature -> main` - - validate and analyze one changed contract - - compute `required_bump` for that contract - - do **not** change contract `version` -2. `main -> release` - - re-evaluate the release candidate for that contract - - apply an explicit `release_tag` - - update contract `version` through the release path only +### Platform adapters -Current bump rules: +`semapact/platforms/` owns provider SDK/client integration and physical-platform translation. Databricks DDL generation/execution is provider behavior; it does not belong in ContractOps or application DTOs. -- `none` - - descriptive-only metadata changes -- `minor` - - additive or non-breaking structural changes - - newly introduced schema/property deprecations -- `major` - - lifecycle-breaking changes +### Import/export and compatibility workflows -Suggested next release versions are always computed from the last released -contract version and the highest currently required bump. They are not -calculated by chaining unreleased changes together. +- `semapact/importers/` projects explicitly imported external metadata into ODCS and contains no lifecycle policy; +- `semapact/exporters/` and `quality/` are read-only projections; +- `semapact/devops/` and parts of `core/` contain stable compatibility workflows and must not become a second canonical ContractOps implementation. -Example: +## Model Placement Rule -- last released version: `1.2.0` -- unreleased changes: one breaking removal, then one additive field -- final `required_bump`: `major` -- suggested next version: `2.0.0` +Do not create a generic root `schema/`, `models/`, or `data_models/` directory to collect unrelated objects. Decide placement from semantic ownership: -If the final `required_bump` is `none`, the suggested version remains the same -as the last released version. In that case, repo-level batch release manifest -generation skips the contract by default. +| What the object represents | Owner | +| --- | --- | +| governed ODCS contract | ODCS model | +| governance/release/deployment/reconciliation artifact | owning domain package | +| application/use-case aggregate result | `application/models/` | +| application orchestration | `application/services/` | +| provider/SDK/physical representation | `platforms//` | +| presentation-only rendering state | `interfaces/` | +| durable history/persistence record | its persistence/history boundary | -Current release tooling: +The fact that every object is “data” is not a useful architectural boundary. -- `semapact release classify` - - compute `required_bump` for one contract -- `semapact release prepare` - - prepare one promoted contract candidate with an explicit `release_tag` -- `semapact release create-pr` - - create one release PR for one contract +## Governed Identity -## Repo-Level Release Orchestration +For current governance semantics: -Some repositories contain multiple governed contracts. SemaPact supports -repo-level release orchestration helpers, but these helpers do **not** change -the versioning unit. - -Current repo-level commands: - -- `semapact release classify-repo` - - compare two contract roots - - report per-contract statuses such as `changed`, `unchanged`, `added`, and `removed` - - report `required_bump` for changed contracts only -- `semapact release build-manifest` - - generate an editable JSON array of per-contract release tasks - - suggest release tags and source branches from each contract's current version and `required_bump` -- `semapact release create-prs` - - consume an explicit batch manifest - - run independent per-contract release preparation and PR creation - -Important rule: - -- the repository is a batching boundary only -- each contract still owns its own identity, version, release tag, and release decision - -Recommended repo-level flow: - -1. `semapact release classify-repo` - - inspect changed contracts -2. `semapact release build-manifest` - - generate an editable per-contract release task list -3. review and adjust the manifest - - especially release tags and branch names -4. `semapact release create-prs` - - create one PR per contract release task - -## CI Build Modes - -Recommended CI interpretation: - -1. `pr` - - run validation and change classification - - do not change `contract.version` - - do not create release PRs -2. `merge` - - re-run validation on the merged main state - - keep `contract.version` unchanged - - publish audit or summary artifacts if needed -3. `release` - - build or review a per-contract release manifest - - apply explicit release tags only for contracts that require a bump - - create release PRs per contract - -## Draft Workflow - -Current draft workflow: - -1. load main contract -2. load existing draft or initialize draft from main -3. edit draft -4. analyze draft vs main -5. save draft -6. promote later through GitOps workflow - -Important rules: - -- UI must not overwrite the main contract -- draft persists independently -- service layer validates before saving draft -- service layer preserves non-editable contract/schema/property fields from the main contract - -Draft storage: - -- `.semapact/drafts/{user}/{contract_id}.yaml` - -## Storage Support +```text +schema identity = lowercase(schema.name) +property identity = lowercase(schema.name) + lowercase(property.name) +``` -Current canonical contract roots support: +`physicalName` is a deployment/runtime binding hint and never replaces governed logical identity. -- local filesystem paths -- ADLS2 paths -- Databricks Unity Catalog mounted volume paths +## Lifecycle and Governance -ADLS2 authentication currently supports: +Lifecycle/governance is the sole authority for change meaning. Active entities participate in governance; draft/deprecated entities are excluded where policy specifies; retired state is immutable. Interfaces, application services, adapters, and exporters must not independently reinterpret these rules. -- `SEMAPACT_ADLS_BEARER_TOKEN` -- `azure.identity.DefaultAzureCredential` +Governance produces one immutable `GovernanceDecision`. Downstream phases consume that decision and its projected artifacts rather than diffing again. -SAS URL authentication is intentionally not supported. +## ContractOps Release Boundary -## Quality and Export Boundaries +Canonical planning is: -### Contract validation +```text +base + candidate + exact revision refs + ↓ +GovernanceDecision + ChangeSet + ↓ +ReleasePlan + ↓ +VersionResolution +``` -`semapact/core/validator.py` validates: +Version selection is separate from governance classification. SemaPact-managed and Git-managed authority both resolve through the canonical version-authority boundary. -- ODCS structure -- quality rule completeness -- ODCS quality type semantics +APPLY materializes the exact released ODCS snapshot only after matching authorization. PUBLISH publishes a released artifact and is distinct from DEPLOY. -### GE export +## Deployment Boundary -`semapact/quality/ge_exporter.py`: +Deployment planning starts from an exact `AppliedContractRelease`: -- delegates suite generation to datacontract-cli -- performs GE-specific preflight on exported expectation configs -- does not execute runtime validation +```text +AppliedContractRelease + DeploymentTarget + ↓ +DeploymentPlan + ↓ +fresh runtime observation + adapter preview + ↓ +DeploymentPreview + ↓ +exact plan + preview + DeploymentAuthorization + ↓ +DeploymentAdapter.execute(...) +``` -### SQL export +`DeploymentPlan` stays provider-neutral. Provider-native CREATE/ALTER/NO_OP operations begin at the adapter boundary. Runtime mutation must fail closed when capability or evidence is insufficient. -`semapact/exporters/sql_exporter.py`: +Provider execution success is not convergence proof. -- delegates base SQL generation to datacontract-cli -- appends Databricks-only constraints for a limited supported subset of ODCS quality rules +## Observation and Reconciliation -Current supported Databricks mappings: +Observation captures platform-neutral runtime evidence. It never mutates ODCS or invokes governance. -- `nullValues mustBe 0` -> `SET NOT NULL` -- `invalidValues + validValues` -> `CHECK IN (...)` -- `invalidValues + pattern` -> `CHECK RLIKE ...` +Reconciliation compares governed desired state with fresh observation and yields the existing status vocabulary: -Precedence: +```text +IN_SYNC +DRIFT +INDETERMINATE +``` -- schema `required=True` is emitted first by datacontract-cli as `NOT NULL` -- SemaPact does not emit duplicate nullability constraints +Deployment verification reuses this same reconciliation authority rather than introducing another convergence state machine. -## Current Design Principles +## Application Services -- main governed contract is canonical and immutable from presentation paths -- ODCS is the canonical model for governed desired contract state -- platform observation is separate read-side state and cannot become governed truth implicitly -- core observation models are platform-neutral; provider-specific hierarchy belongs in adapters -- reuse official platform access/SDK layers where practical, while keeping ODCS import projection separate from observation -- service layer is the application boundary between interfaces and system logic -- lifecycle logic belongs in the lifecycle layer -- datacontract-cli is reused where possible instead of reimplemented +Application services exist only when a use case genuinely coordinates multiple domain/port calls. Current examples include governance context construction, configured version authority, canonical release planning, runtime reconciliation orchestration, and deployment orchestration. -## Authoritative Governance Invariants +Rules: -1. **Centralized Gate Enforcement**: All mutation-capable application paths must obtain an authoritative `GovernanceDecision` and enforce the appropriate `GovernanceOperation` gate before persistence, Git mutation, publication, deployment, or draft submission. -2. **Universal Retired Immutability**: A contract whose effective lifecycle is `retired` is permanently frozen. Any semantic mutation against a retired base contract produces `DecisionResult.BLOCK` with `GovernanceReasonCode.RETIRED_CONTRACT_MODIFIED`. No interface, service, exporter, merge adapter, or future draft implementation may independently reinterpret retired immutability or perform side effects before gate evaluation. -3. **Future Draft Contract**: - ```text - Load canonical contract - ↓ - Create/edit candidate draft - ↓ - GovernanceService.evaluate(...) - ↓ - GovernanceOperation.PROPOSE - ↓ - persist / submit draft - ``` - If base is `retired` and candidate differs: - - Evaluator emits `DecisionResult.BLOCK` (`RETIRED_CONTRACT_MODIFIED`) - - `PROPOSE` gate rejects draft submission - - UI read-only styling is UX only; backend governance gate is the single authority. +- keep services thin; +- reusable application result DTOs live in `application/models`, not beside service implementation; +- do not pass untyped dictionaries internally when a canonical model exists; +- do not introduce ports around pure deterministic functions only for symmetry; +- provider construction is composition/platform behavior, not domain behavior; +- optional provider dependencies remain lazy so the base installation stays import-safe. -## Known Next Steps +## Public Architecture Invariants -- separate platform discovery, observation, and contract import application workflows, starting with Databricks Unity Catalog -- add stable observed-state fingerprints and reconciliation semantics -- add governance-relevant metadata/constraint/relationship evidence independently from the minimal observation model -- add lineage as optional runtime evidence rather than a core observation dependency -- formalize draft promotion flow -- continue reducing interface-specific logic that still lives near command/editor helpers -- keep converging governed contract helper logic toward ODCS model-driven behavior +1. **Change-driven, not CRUD** — governed state evolves through explicit analysis/planning/authorization boundaries. +2. **One authority per rule** — lifecycle, governance, version selection, deployment translation, and reconciliation each have one canonical owner. +3. **Exact artifacts cross boundaries** — side effects consume exact immutable artifacts; mutable current state is not silently substituted. +4. **Operation-scoped authorization** — APPLY, PUBLISH, and DEPLOY are distinct operations; authorization for one cannot authorize another. +5. **Logical identity is stable** — `physicalName` binds runtime state but does not redefine governed identity. +6. **Execution is not convergence** — runtime state must be observed and reconciled independently. +7. **Interfaces stay thin** — CLI/API/UI parse, delegate, and render; they do not become a second business-logic implementation. +8. **Compatibility is not ownership** — legacy import paths may re-export canonical implementations but must not accumulate new logic. diff --git a/README.md b/README.md index c1e79468..aff85e2c 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ SemaPact is an open-source, change-driven governance layer for evolving data products safely. It uses the **Open Data Contract Standard (ODCS)** as its canonical governed representation and keeps lifecycle policy deterministic, reviewable, and platform-neutral. -> **SemaPact is not another metadata catalog or CRUD editor. It governs how data products are allowed to change and verifies how governed desired state compares with observed platform state.** +> **SemaPact is not another metadata catalog or CRUD editor. It governs how data products are allowed to change, turns approved contract releases into explicit deployment intent, and verifies how governed desired state compares with observed platform state.** ## Installation @@ -46,9 +46,10 @@ Validating one contract file is the easy part. Production governance becomes har - decimal precision or scale is reduced; - an active field needs to be deprecated; - a contract is retired and must become immutable; +- an approved contract release needs to become runtime state safely; - production state no longer matches the governed desired state. -SemaPact treats these as **governance and assurance problems**, not YAML editing operations. +SemaPact treats these as **governance, convergence, and assurance problems**, not YAML editing operations. ```text Current Governed Contract @@ -66,6 +67,20 @@ GovernanceDecision Governance Gate ``` +Canonical ContractOps then carries the exact governed decision forward: + +```text +GovernanceDecision +→ ChangeSet +→ ReleasePlan +→ VersionResolution +→ ContractOpsAuthorization +→ AppliedContractRelease +→ DeploymentPlan +→ DeploymentAuthorization +→ runtime mutation +``` + For production assurance: ```text @@ -75,7 +90,7 @@ ObservedPlatformState ↓ Deterministic Reconciliation ↓ -Raw Differences +IN_SYNC / DRIFT / INDETERMINATE ``` The governed desired state is an authoritative ODCS revision selected by an upstream governance / release / authorization process. `approved` is not an ODCS lifecycle status and reconciliation does not invent one. @@ -111,13 +126,17 @@ DRAFT → ACTIVE → DEPRECATED → RETIRED Lifecycle status does not itself mean that a revision has been authorized for release. +### Side effects are operation-scoped + +SemaPact distinguishes analysis and planning from side effects. APPLY, PUBLISH, and DEPLOY are separate protected operations. A publication authorization cannot be reused as runtime deployment authority. + ### Runtime-aware without becoming platform-owned -Platforms such as Databricks Unity Catalog describe what exists now. SemaPact keeps its governance kernel platform-neutral and consumes normalized observation state for assurance. +Platforms such as Databricks Unity Catalog describe what exists now. SemaPact keeps its governance kernel and deployment intent provider-neutral, while provider adapters translate supported runtime operations explicitly. ### AI can consume governance; AI does not become governance authority -Agents may consume governed contracts, decisions, reason codes, and semantic context. Deterministic governance policy remains authoritative. +Agents may consume governed contracts, decisions, reason codes, release/deployment artifacts, and semantic context. Deterministic governance policy remains authoritative. ## Current Capabilities @@ -136,7 +155,54 @@ SemaPact currently supports deterministic change analysis and lifecycle-aware po - relationship change handling; - version-policy classification; - deterministic `GovernanceDecision` artifacts; -- centralized governance gates for analyze / propose / apply / publish / CI operations. +- centralized governance gates for ANALYZE / PROPOSE / APPLY / PUBLISH / DEPLOY / CI operations. + +### Canonical ContractOps release planning + +The release planning path evaluates governance once and produces exact deterministic artifacts: + +```text +GovernanceDecision +→ ChangeSet +→ ReleasePlan +→ VersionResolution +``` + +`semapact release plan` requires explicit base and candidate revision references so downstream authorization and application can refer to the exact workflow revisions rather than mutable file paths. + +SemaPact supports two version-authority modes: + +- `semapact` — independently version contracts from their governed change requirements; +- `git` — validate an explicitly supplied product/repository release version against the governance-required minimum bump. + +See [`docs/contractops_phases.md`](docs/contractops_phases.md) and [`docs/version_authority.md`](docs/version_authority.md). + +### Governed runtime deployment + +`semapact deployment` exposes a canonical CLI/CI surface: + +```text +plan +→ DeploymentPlan + +preview +→ fresh runtime observation +→ DeploymentPreview + +execute +→ exact plan + preview + DeploymentAuthorization +→ provider execution + +verify +→ fresh observation + reconciliation +→ IN_SYNC / DRIFT / INDETERMINATE +``` + +Planning and preview are read-only. Runtime mutation occurs only through `deployment execute`, and provider execution success is not treated as convergence proof. + +The first Databricks write capability is intentionally narrow: create a missing managed Delta table, add missing nullable governed columns to an existing managed table, or perform NO_OP when the governed shape is already satisfied. Rename, existing-column type/nullability mutation, required-column addition without a safe migration strategy, DROP, and existing external/non-managed asset mutation fail closed. + +See [`docs/deployment_plans.md`](docs/deployment_plans.md). ### Databricks discovery and observation @@ -168,22 +234,24 @@ nullability Volatile envelope fields such as capture time and source location are excluded from the content fingerprint. -### Raw reconciliation +### Runtime reconciliation and convergence verification -SemaPact can deterministically compare a governed ODCS desired-state revision with `ObservedPlatformState` and report factual differences for semantics represented on both sides today: +SemaPact deterministically compares governed desired state with `ObservedPlatformState` and reports factual differences for semantics represented on both sides today: - missing / unexpected assets; - missing / unexpected properties; - physical type mismatch; - required / nullability mismatch. -Reconciliation answers **what differs**. It does not infer why the difference exists, classify deployment history, or mutate the external platform. +Reconciliation classifies the result as `IN_SYNC`, `DRIFT`, or `INDETERMINATE`. Deployment verification reuses the same reconciliation authority; it does not invent a separate convergence status machine. + +See [`docs/runtime_reconciliation.md`](docs/runtime_reconciliation.md). ## SemaPact + Databricks Unity Catalog Unity Catalog and SemaPact solve different parts of the problem. -> **Unity Catalog tells you what exists. SemaPact governs desired-state evolution and compares governed state with observed state.** +> **Unity Catalog tells you what exists. SemaPact governs desired-state evolution, executes explicitly supported governed mutations, and verifies observed state against that governed intent.** ```text Git / ODCS @@ -193,12 +261,12 @@ Governed Desired State ┌──────────┐ │ SemaPact │ └──────────┘ - ▲ │ - │ │ governance / assurance artifacts - │ ▼ -Observed Platform State - ▲ - │ + │ ▲ + │ │ observation / reconciliation + ▼ │ +Governed runtime mutation + │ │ + ▼ │ Databricks / Unity Catalog ``` @@ -240,6 +308,19 @@ semapact merge \ --effective-date 2026-09-03 ``` +### Build canonical release planning artifacts + +```bash +semapact release plan \ + --base ./contracts/orders.yaml \ + --candidate ./contracts/orders.candidate.yaml \ + --base-revision-ref git:abc123 \ + --candidate-revision-ref git:def456 \ + --effective-date 2026-09-11 +``` + +The output contains the canonical `GovernanceDecision`, `ChangeSet`, `ReleasePlan`, and `VersionResolution` artifacts. + ### Databricks integration ```bash @@ -248,13 +329,22 @@ pip install "semapact[databricks]" The Databricks SDK owns authentication-provider selection. SemaPact forwards supported connection hints rather than implementing a separate credential system. +After an exact `AppliedContractRelease` and deployment authorization have been produced, the runtime path is exposed through: + +```bash +semapact deployment plan --help +semapact deployment preview --help +semapact deployment execute --help +semapact deployment verify --help +``` + ## Optional Dependencies | Extra | Purpose | | --- | --- | | `sql` | SQL parsing and SQL-folder workflows | | `delta` | Delta table support | -| `databricks` | Databricks / Unity Catalog integration | +| `databricks` | Databricks / Unity Catalog observation and governed deployment | | `quality` | Great Expectations integration | | `graph` | Graph export support | | `llm` | Optional LLM-assisted semantic enrichment | @@ -269,10 +359,16 @@ Optional extras are intentionally separate from the base distribution. If an int ```text semapact/ - core/ # loading, validation, editor / release boundaries + core/ # loading, validation, compatibility workflow boundaries lifecycle/ # canonical identity, lifecycle and change policy governance/ # GovernanceDecision and centralized gate - services/ # application-facing governance service + contractops/ # deterministic release planning / authorization / apply / publish domain + deployment/ # provider-neutral deployment plans, authorization, preview contracts + runtime/ # provider-neutral governed runtime asset projection + application/ # interface-independent use-case models + orchestration + models/ # application result DTOs; no domain authority + services/ # thin orchestration over canonical domain rules/ports + services/ # backward-compatible imports only observation/ # platform-neutral observed state + fingerprint reconciliation/ # governed desired vs observed comparison platforms/ # provider adapters such as Databricks @@ -280,12 +376,14 @@ semapact/ exporters/ # SQL / graph and other outputs quality/ # quality intent adapters interfaces/ # CLI and user-facing boundaries - devops/ # Git / CI release helpers + devops/ # Git / CI compatibility helpers ``` A central architectural rule is: -> **Platform adapters describe external state. Governance decides what contract evolution means. Reconciliation compares governed desired state with observed state.** +> **Interfaces parse and render. Application services orchestrate. Domain packages own business meaning. Platform adapters own provider-specific effects. Compatibility packages do not become new owners.** + +See [`ARCHITECTURE.md`](ARCHITECTURE.md) for package/model placement rules. ## What SemaPact Does Not Try to Replace @@ -297,7 +395,7 @@ SemaPact is not intended to replace: - Terraform / Databricks Asset Bundles as general infrastructure tooling; - Git review and human authorization processes. -SemaPact provides a deterministic governance and assurance layer around those systems. +SemaPact provides a deterministic governance, convergence, and assurance layer around those systems. ## Development @@ -319,16 +417,16 @@ Build the Python distribution: uv build ``` -## Project Direction +## Product Model -The current delivery direction is a governed desired-state control plane: +SemaPact is organized as a governed desired-state control plane: ```text GOVERN Can this contract change be allowed? CONVERGE -Can governed desired state safely become runtime state? +Can the exact governed release safely become runtime state? ASSURE Does observed runtime state match governed desired state? @@ -337,7 +435,7 @@ PROVE What happened, why, and through which decision / release / deployment / observation? ``` -Not every stage above is complete today. The repository keeps these boundaries explicit so later deployment, drift classification, evidence, and audit capabilities can be added without collapsing responsibilities into one layer. +Governance, release planning, guarded deployment, and runtime reconciliation are implemented as separate boundaries so persistence, history, additional provider capabilities, and audit surfaces can evolve without collapsing these responsibilities into one layer. ## Open Data Contract Standard diff --git a/docs/contractops_phases.md b/docs/contractops_phases.md index 1a127935..667ce142 100644 --- a/docs/contractops_phases.md +++ b/docs/contractops_phases.md @@ -35,6 +35,17 @@ ANALYZE evaluates the candidate against the governed base contract and produces It is pure. It does not write files, create Git branches or tags, update ODCS, or mutate a runtime platform. +The CLI analysis surface is: + +```bash +semapact release classify \ + --base ./contracts/orders.yaml \ + --candidate ./contracts/orders.candidate.yaml \ + --effective-date 2026-09-11 +``` + +`release classify` is analysis-only. It reports the governance decision and required version bump, but it does not create a `ChangeSet`, `ReleasePlan`, or `VersionResolution` and does not authorize a release. + ## PLAN PLAN converts the authoritative decision into deterministic release artifacts: @@ -45,6 +56,42 @@ PLAN converts the authoritative decision into deterministic release artifacts: PLAN remains pure. +The canonical CLI planning surface is: + +```bash +semapact release plan \ + --base ./contracts/orders.yaml \ + --candidate ./contracts/orders.candidate.yaml \ + --base-revision-ref git:abc123 \ + --candidate-revision-ref git:def456 \ + --effective-date 2026-09-11 +``` + +The revision references are required because release artifacts must be bound to the exact base and candidate workflow revisions rather than to mutable file paths alone. + +When `release.versionAuthority=git`, provide the repository release reference explicitly: + +```bash +semapact release plan \ + --base ./contract.yaml \ + --candidate ./contract.candidate.yaml \ + --base-revision-ref git:abc123 \ + --candidate-revision-ref git:def456 \ + --authority-reference v1.4.0 \ + --effective-date 2026-09-11 +``` + +The JSON output is serialized directly from the canonical models and contains: + +```text +governanceDecision +changeSet +releasePlan +versionResolution +``` + +One planning pass evaluates governance once, then feeds the exact resulting artifacts forward. The CLI does not independently re-diff, reclassify, or reimplement version policy downstream. + ## AUTHORIZE AUTHORIZE evaluates whether one exact operation can cross a side-effect boundary. @@ -109,6 +156,8 @@ A release-context `ContractOpsAuthorization(operation=DEPLOY)` is not enough on Platform-specific execution belongs behind a deployment adapter. The adapter must not recompute governance, version authority, release planning, or approval semantics. +See [`deployment_plans.md`](deployment_plans.md) for the deployment CLI, Databricks capability boundary, preview integrity checks, and convergence verification semantics. + ## Failure semantics ContractOps distinguishes invalid context from denied authorization: @@ -121,6 +170,8 @@ Unexpected publisher/runtime failures are not converted into governance decision ## Compatibility helpers -`semapact.core.release.prepare_release_candidate()` remains a backward-compatible helper and is not the canonical ContractOps APPLY path because it may classify changes itself. +`semapact release prepare` and `semapact release create-pr` remain compatibility workflows for existing Git-based release processes. They are not the canonical ContractOps PLAN/APPLY/PUBLISH path and should not be treated as equivalent to `release plan` plus explicit authorization. + +`semapact.core.release.prepare_release_candidate()` likewise remains a backward-compatible helper and is not the canonical ContractOps APPLY path because it may classify changes itself. -New ContractOps flows consume the existing authoritative `GovernanceDecision`, `ReleasePlan`, and `VersionResolution` instead of recomputing them. +New ContractOps flows consume the existing authoritative `GovernanceDecision`, `ChangeSet`, `ReleasePlan`, and `VersionResolution` instead of recomputing them. diff --git a/docs/deployment_plans.md b/docs/deployment_plans.md index f199eac9..4768b358 100644 --- a/docs/deployment_plans.md +++ b/docs/deployment_plans.md @@ -39,7 +39,7 @@ Planning sees released desired state only. Without observed runtime state SemaPa Likewise, an object that exists in runtime but is absent from one contract must not be interpreted as safe to drop. The contract may not own that object. -Concrete provider-native operations therefore begin at the platform adapter boundary, where validation and preview can combine the DeploymentPlan with provider semantics and, where required, runtime evidence. +Concrete provider-native operations therefore begin at the platform adapter boundary, where validation and preview combine the DeploymentPlan with provider semantics and fresh runtime evidence. ## Identity and physical binding @@ -80,6 +80,91 @@ DeploymentTarget The plan does not contain credentials, workspace clients, SQL connections, or provider sessions. +## CLI workflow + +The deployment CLI consumes and emits canonical JSON artifacts. Planning and preview are read-only; `execute` is the runtime mutation boundary. + +### Plan + +```bash +semapact deployment plan \ + --release ./artifacts/applied-release.json \ + --platform databricks \ + --runtime main.sales +``` + +Optional `--server` preserves the selected contract-server reference as target provenance. + +The output is the canonical `DeploymentPlan` JSON. + +### Preview + +```bash +semapact deployment preview \ + --plan ./artifacts/deployment-plan.json +``` + +Preview observes the exact target scope and derives a canonical `DeploymentPreview`. It does not mutate runtime and does not require a Databricks SQL warehouse merely to inspect provider-native operations. + +### Execute + +```bash +semapact deployment execute \ + --plan ./artifacts/deployment-plan.json \ + --preview ./artifacts/deployment-preview.json \ + --authorization ./artifacts/deployment-authorization.json \ + --warehouse-id +``` + +Execution requires the exact plan, exact preview, and exact `DeploymentAuthorization`. The adapter re-observes the target, validates the observation source and fingerprint, re-derives the expected preview for integrity/freshness validation, and executes only the supplied operations when the artifacts still match. + +Provider execution success is not convergence proof. + +### Verify + +```bash +semapact deployment verify \ + --plan ./artifacts/deployment-plan.json \ + --output json +``` + +Verification performs fresh runtime observation and reuses the normal reconciliation semantics: + +| Runtime status | Exit code | +| --- | ---: | +| `IN_SYNC` | `0` | +| `DRIFT` | `6` | +| `INDETERMINATE` | `7` | + +This keeps execution status separate from convergence evidence. + +## Databricks deployment capability + +The first Databricks write slice is intentionally narrow and fail-closed. + +| Observed state | Supported behavior | +| --- | --- | +| Governed table is missing | `CREATE TABLE ... USING DELTA` as a managed table | +| Existing `MANAGED` table is missing a governed nullable column | `ALTER TABLE ... ADD COLUMNS (...)` | +| Existing `MANAGED` table already satisfies the governed shape | `NO_OP` | +| Runtime contains extra columns not governed by this contract | Leave them untouched; no inferred `DROP` | + +For this slice, `ALTER` means **only additive nullable-column change**. The adapter does not interpret `ALTER` as generic schema evolution. + +The following are rejected rather than guessed or silently converted: + +- column rename; +- existing-column physical type change; +- existing-column nullability change; +- adding a required/non-null column without an explicit safe migration/default strategy; +- `DROP` or other destructive reconciliation; +- mutation of existing external/non-managed assets; +- unsupported or ambiguous provider mappings. + +Existing external/non-managed assets remain observable through the runtime read side, but this deployment adapter does not claim mutation authority over them. + +The adapter is also not a general Databricks infrastructure engine. Workspace, catalog, schema, SQL warehouse, credentials, external locations, storage configuration, grants, jobs, and clusters are outside this deployment boundary and must be provisioned separately. + ## Authorization scope Runtime deployment is a separate protected operation from publishing a contract release artifact. @@ -88,6 +173,8 @@ A `ContractOpsAuthorization(operation=DEPLOY)` establishes release-context autho For review-required changes, structured review evidence may carry an opaque `scopeReference`. Deployment requires that scope to match the exact `deploymentPlanId`, so an approval for one target cannot be reused for another target. +A PUBLISH authorization cannot authorize DEPLOY. + ## Determinism `deploymentPlanId` is UUID5-derived from the full stable plan record: @@ -101,6 +188,8 @@ The same exact applied release and target therefore produce the same DeploymentP Action ordering is canonical even when schemas appear in a different order in source ODCS. However, DeploymentPlan does not redefine release identity: two distinct `AppliedContractRelease` artifacts remain distinct authorities even if their projected actions happen to be equivalent. +`DeploymentPreview` is likewise deterministic for the same plan and observed runtime evidence, but deterministic IDs provide artifact consistency rather than cryptographic authenticity. Execution still validates exact binding and fresh runtime evidence at the side-effect boundary. + ## Provider support belongs to the adapter DeploymentPlan intentionally does not contain generic `preconditions`, `adapterKey`, or guessed platform-specific operations. diff --git a/semapact/application/__init__.py b/semapact/application/__init__.py new file mode 100644 index 00000000..29d1373c --- /dev/null +++ b/semapact/application/__init__.py @@ -0,0 +1 @@ +"""Interface-independent application use cases for SemaPact.""" diff --git a/semapact/application/models/__init__.py b/semapact/application/models/__init__.py new file mode 100644 index 00000000..2002aa4e --- /dev/null +++ b/semapact/application/models/__init__.py @@ -0,0 +1,12 @@ +"""Typed application results composed from canonical domain artifacts.""" + +from semapact.application.models.governance import GovernanceAnalysis, GovernanceProposal +from semapact.application.models.reconciliation import RuntimeReconciliation +from semapact.application.models.release import ReleasePlanningResult + +__all__ = [ + "GovernanceAnalysis", + "GovernanceProposal", + "ReleasePlanningResult", + "RuntimeReconciliation", +] diff --git a/semapact/application/models/governance.py b/semapact/application/models/governance.py new file mode 100644 index 00000000..91a6acce --- /dev/null +++ b/semapact/application/models/governance.py @@ -0,0 +1,27 @@ +"""Application result models for governance use cases.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from semapact.change_context import ChangeContext +from semapact.contractops import ChangeSet +from semapact.governance.models import GovernanceDecision +from semapact.lifecycle.merge_engine import MergeResult + + +@dataclass(frozen=True) +class GovernanceAnalysis: + """One merge-and-governance analysis using a single resolved context.""" + + context: ChangeContext + merge_result: MergeResult + decision: GovernanceDecision + + +@dataclass(frozen=True) +class GovernanceProposal: + """One evaluated proposal represented by a ChangeSet and its decision.""" + + change_set: ChangeSet + decision: GovernanceDecision diff --git a/semapact/application/models/reconciliation.py b/semapact/application/models/reconciliation.py new file mode 100644 index 00000000..61a7af8f --- /dev/null +++ b/semapact/application/models/reconciliation.py @@ -0,0 +1,22 @@ +"""Application result models for runtime reconciliation use cases.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +from semapact.observation import RuntimeAssetBinding +from semapact.reconciliation import ReconciliationResult, RuntimeDriftStatus + + +@dataclass(frozen=True) +class RuntimeReconciliation: + """One complete read-only reconciliation of a governed data product.""" + + platform: str + runtime_target: str + runtime_source: Literal["contract", "cli"] + server_name: str | None + bindings: tuple[RuntimeAssetBinding, ...] + result: ReconciliationResult + status: RuntimeDriftStatus diff --git a/semapact/application/models/release.py b/semapact/application/models/release.py new file mode 100644 index 00000000..fa4d4546 --- /dev/null +++ b/semapact/application/models/release.py @@ -0,0 +1,18 @@ +"""Application result models for canonical release orchestration.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from semapact.contractops import ChangeSet, ReleasePlan, VersionResolution +from semapact.governance import GovernanceDecision + + +@dataclass(frozen=True) +class ReleasePlanningResult: + """Exact canonical artifacts produced by one release planning pass.""" + + change_set: ChangeSet + decision: GovernanceDecision + release_plan: ReleasePlan + version_resolution: VersionResolution diff --git a/semapact/application/services/__init__.py b/semapact/application/services/__init__.py new file mode 100644 index 00000000..35c5f684 --- /dev/null +++ b/semapact/application/services/__init__.py @@ -0,0 +1,5 @@ +"""Thin application orchestration services. + +Import concrete services from their owning modules so dependency direction remains +visible, for example ``semapact.application.services.governance``. +""" diff --git a/semapact/application/services/deployment.py b/semapact/application/services/deployment.py new file mode 100644 index 00000000..1ea49132 --- /dev/null +++ b/semapact/application/services/deployment.py @@ -0,0 +1,93 @@ +"""Application service for provider-neutral deployment workflows.""" + +from __future__ import annotations + +from semapact.contractops import AppliedContractRelease +from semapact.deployment import ( + DeploymentAdapter, + DeploymentAuthorization, + DeploymentPlan, + DeploymentPreview, + DeploymentTarget, + build_deployment_plan, + verify_deployment_convergence, +) +from semapact.deployment.models import validate_deployment_plan_identity +from semapact.exceptions import ValidationError +from semapact.observation import RuntimeProvider +from semapact.reconciliation import ReconciliationResult +from semapact.runtime import RuntimeAssetSpec + + +class DeploymentService: + """Compose existing deployment domain/provider boundaries for interfaces. + + The service owns orchestration only. It does not re-run governance, approval, + versioning, deployment translation, or reconciliation rules. + """ + + def plan( + self, + release: AppliedContractRelease, + target: DeploymentTarget, + ) -> DeploymentPlan: + return build_deployment_plan(release, target) + + def preview( + self, + plan: DeploymentPlan, + *, + runtime_provider: RuntimeProvider, + adapter: DeploymentAdapter, + ) -> DeploymentPreview: + """Observe the exact plan scope and delegate native translation to adapter.""" + validate_deployment_plan_identity(plan) + _validate_component_key(runtime_provider.key, plan.target.platform, "runtime provider") + _validate_component_key(adapter.key, plan.target.platform, "deployment adapter") + + assets = _runtime_assets_from_plan(plan) + bindings = runtime_provider.resolve_bindings( + runtime_target=plan.target.runtime_target, + assets=assets, + ) + observation = runtime_provider.observe(bindings=bindings) + return adapter.preview(plan, observation) + + def execute( + self, + plan: DeploymentPlan, + preview: DeploymentPreview, + authorization: DeploymentAuthorization, + *, + adapter: DeploymentAdapter, + ) -> None: + """Delegate the exact authorized side effect to the provider adapter.""" + _validate_component_key(adapter.key, plan.target.platform, "deployment adapter") + adapter.execute(plan, preview, authorization) + + def verify( + self, + plan: DeploymentPlan, + *, + runtime_provider: RuntimeProvider, + ) -> ReconciliationResult: + """Verify exact plan convergence through the existing M1 bridge.""" + return verify_deployment_convergence(plan, runtime_provider) + + +def _runtime_assets_from_plan(plan: DeploymentPlan) -> tuple[RuntimeAssetSpec, ...]: + return tuple( + RuntimeAssetSpec( + governed_asset=action.governed_asset, + physical_name=action.physical_name, + ) + for action in plan.actions + ) + + +def _validate_component_key(actual: str, expected: str, component: str) -> None: + if actual.strip().casefold() != expected.strip().casefold(): + raise ValidationError( + f"{component.capitalize()} does not match DeploymentPlan platform: " + f"{actual!r} != {expected!r}" + ) diff --git a/semapact/application/services/governance.py b/semapact/application/services/governance.py new file mode 100644 index 00000000..12ba4cf2 --- /dev/null +++ b/semapact/application/services/governance.py @@ -0,0 +1,111 @@ +"""Application service boundary for deterministic governance workflows.""" + +from __future__ import annotations + +from datetime import date +from typing import Sequence + +from open_data_contract_standard.model import OpenDataContractStandard + +from semapact.application.models.governance import GovernanceAnalysis, GovernanceProposal +from semapact.change_context import ChangeContext +from semapact.contractops import build_change_set_from_decision +from semapact.governance.evaluator import evaluate_governance_decision +from semapact.governance.models import GovernanceDecision +from semapact.lifecycle.merge_engine import ContractMergeEngine, MergeConflict + + +class GovernanceService: + """Own ChangeContext construction and delegate governance domain work.""" + + def __init__(self, merge_engine: ContractMergeEngine | None = None) -> None: + self._merge_engine = merge_engine or ContractMergeEngine() + + @staticmethod + def create_context(effective_date: date | str) -> ChangeContext: + """Resolve an interface/application date into the domain ChangeContext once.""" + if isinstance(effective_date, str): + try: + resolved_date = date.fromisoformat(effective_date) + except ValueError as exc: + raise ValueError("effective_date must use YYYY-MM-DD") from exc + elif isinstance(effective_date, date): + resolved_date = effective_date + else: + raise TypeError("effective_date must be a date or YYYY-MM-DD string") + + return ChangeContext(effective_date=resolved_date) + + def evaluate( + self, + base_contract: OpenDataContractStandard, + candidate_contract: OpenDataContractStandard, + *, + effective_date: date | str, + merge_conflicts: Sequence[MergeConflict] = (), + ) -> GovernanceDecision: + """Evaluate one contract change from an application-level effective date.""" + context = self.create_context(effective_date) + return evaluate_governance_decision( + base_contract, + candidate_contract, + context=context, + merge_conflicts=merge_conflicts, + ) + + def evaluate_proposal( + self, + base_contract: OpenDataContractStandard, + candidate_contract: OpenDataContractStandard, + *, + effective_date: date | str, + base_revision_ref: str, + candidate_revision_ref: str, + merge_conflicts: Sequence[MergeConflict] = (), + source: str | None = None, + actor_reference: str | None = None, + ) -> GovernanceProposal: + """Evaluate once and project the authoritative decision into a ChangeSet.""" + context = self.create_context(effective_date) + decision = evaluate_governance_decision( + base_contract, + candidate_contract, + context=context, + merge_conflicts=merge_conflicts, + ) + change_set = build_change_set_from_decision( + decision, + base_revision_ref=base_revision_ref, + candidate_revision_ref=candidate_revision_ref, + source=source, + actor_reference=actor_reference, + ) + return GovernanceProposal(change_set=change_set, decision=decision) + + def merge_and_evaluate( + self, + source_contract: OpenDataContractStandard, + business_contract: OpenDataContractStandard, + *, + effective_date: date | str, + fail_on_conflict: bool = False, + ) -> GovernanceAnalysis: + """Merge and evaluate with the exact same ChangeContext instance.""" + context = self.create_context(effective_date) + merge_result = self._merge_engine.merge( + source_contract, + business_contract, + context=context, + fail_on_conflict=fail_on_conflict, + ) + decision = evaluate_governance_decision( + business_contract, + merge_result.contract, + context=context, + merge_conflicts=merge_result.conflicts, + ) + return GovernanceAnalysis( + context=context, + merge_result=merge_result, + decision=decision, + ) diff --git a/semapact/application/services/reconciliation.py b/semapact/application/services/reconciliation.py new file mode 100644 index 00000000..bbd2c02a --- /dev/null +++ b/semapact/application/services/reconciliation.py @@ -0,0 +1,71 @@ +"""Application service for provider-neutral runtime product reconciliation.""" + +from __future__ import annotations + +from semapact.application.models.reconciliation import RuntimeReconciliation +from semapact.core.loader import ContractLoader +from semapact.observation import RuntimeProviderRegistry +from semapact.platforms.runtime_registry import ( + create_runtime_provider_registry, + resolve_runtime_location, +) +from semapact.reconciliation import ( + classify_reconciliation_status, + reconcile_governed_contract, + runtime_asset_specs_from_contract, +) + + +class ReconciliationService: + """Orchestrate load, runtime resolution, bind, observe, reconcile, and classify.""" + + def __init__( + self, + provider_registry: RuntimeProviderRegistry | None = None, + *, + contract_loader: ContractLoader | None = None, + ) -> None: + self._provider_registry = provider_registry + self._contract_loader = contract_loader or ContractLoader() + + def reconcile( + self, + *, + contract_path: str, + server_name: str | None = None, + fallback_platform: str | None = None, + fallback_runtime_target: str | None = None, + ) -> RuntimeReconciliation: + """Reconcile one governed data product against its resolved runtime location.""" + contract = self._contract_loader.load(contract_path) + location = resolve_runtime_location( + contract, + server_name=server_name, + fallback_platform=fallback_platform, + fallback_runtime_target=fallback_runtime_target, + ) + registry = self._provider_registry or create_runtime_provider_registry( + location.platform, + contract_server=location.contract_server, + ) + provider = registry.get(location.platform) + asset_specs = runtime_asset_specs_from_contract(contract) + bindings = provider.resolve_bindings( + runtime_target=location.runtime_target, + assets=asset_specs, + ) + observation = provider.observe(bindings=bindings) + result = reconcile_governed_contract( + contract, + observation, + asset_bindings=bindings, + ) + return RuntimeReconciliation( + platform=provider.key, + runtime_target=location.runtime_target, + runtime_source=location.source, + server_name=location.server_name, + bindings=bindings, + result=result, + status=classify_reconciliation_status(result), + ) diff --git a/semapact/application/services/release_planning.py b/semapact/application/services/release_planning.py new file mode 100644 index 00000000..6fd33f58 --- /dev/null +++ b/semapact/application/services/release_planning.py @@ -0,0 +1,56 @@ +"""Application orchestration for canonical ContractOps release planning.""" + +from __future__ import annotations + +from datetime import date + +from open_data_contract_standard.model import OpenDataContractStandard + +from semapact.application.models.release import ReleasePlanningResult +from semapact.application.services.governance import GovernanceService +from semapact.application.services.version_authority import VersionAuthorityService +from semapact.contractops import build_release_plan + + +class ReleasePlanningService: + """Compose governance and canonical release planning exactly once.""" + + def __init__( + self, + *, + governance_service: GovernanceService | None = None, + version_authority_service: VersionAuthorityService | None = None, + ) -> None: + self._governance = governance_service or GovernanceService() + self._version_authority = version_authority_service or VersionAuthorityService() + + def plan( + self, + base_contract: OpenDataContractStandard, + candidate_contract: OpenDataContractStandard, + *, + effective_date: date | str, + base_revision_ref: str, + candidate_revision_ref: str, + authority_reference: str | None = None, + ) -> ReleasePlanningResult: + """Produce ChangeSet → ReleasePlan → VersionResolution from one decision.""" + proposal = self._governance.evaluate_proposal( + base_contract, + candidate_contract, + effective_date=effective_date, + base_revision_ref=base_revision_ref, + candidate_revision_ref=candidate_revision_ref, + ) + release_plan = build_release_plan(proposal.change_set, proposal.decision) + version_resolution = self._version_authority.resolve( + release_plan, + base_contract, + authority_reference=authority_reference, + ) + return ReleasePlanningResult( + change_set=proposal.change_set, + decision=proposal.decision, + release_plan=release_plan, + version_resolution=version_resolution, + ) diff --git a/semapact/application/services/version_authority.py b/semapact/application/services/version_authority.py new file mode 100644 index 00000000..89682003 --- /dev/null +++ b/semapact/application/services/version_authority.py @@ -0,0 +1,84 @@ +"""Application boundary for configured contract version authority.""" + +from __future__ import annotations + +from open_data_contract_standard.model import OpenDataContractStandard +from pydantic import ValidationError as PydanticValidationError + +from semapact.contractops import ( + ReleasePlan, + VersionAuthority, + VersionAuthorityConfig, + VersionResolution, + resolve_release_version, +) +from semapact.core.config import ConfigManager +from semapact.exceptions import ReleaseValidationError, ValidationError + + +class VersionAuthorityService: + """Resolve a ReleasePlan using application configuration and released ODCS state.""" + + def __init__(self, config_manager: ConfigManager | None = None) -> None: + self._config = config_manager or ConfigManager() + + def resolve( + self, + release_plan: ReleasePlan, + released_contract: OpenDataContractStandard, + *, + authority_reference: str | None = None, + ) -> VersionResolution: + """Resolve the actual release version without mutating the contract.""" + if not isinstance(release_plan, ReleasePlan): + raise TypeError( + f"release_plan must be ReleasePlan, got {type(release_plan).__name__}" + ) + if not isinstance(released_contract, OpenDataContractStandard): + raise TypeError( + "released_contract must be OpenDataContractStandard, " + f"got {type(released_contract).__name__}" + ) + + contract_id = str(released_contract.id or "").strip() + if contract_id != release_plan.contract_id: + raise ReleaseValidationError( + "Released contract ID does not match ReleasePlan contract ID" + ) + + current_version = str(released_contract.version or "").strip() + if not current_version: + raise ReleaseValidationError( + "Released contract must define the current ODCS version" + ) + + return resolve_release_version( + release_plan, + current_version=current_version, + config=self.load_config(), + authority_reference=authority_reference, + ) + + def load_config(self) -> VersionAuthorityConfig: + """Resolve typed version-authority config from standard SemaPact config sources.""" + authority = self._config.get( + "release.versionAuthority", + env_var="SEMAPACT_RELEASE_VERSION_AUTHORITY", + default=VersionAuthority.SEMAPACT.value, + ) + tag_pattern = self._config.get( + "release.tagPattern", + env_var="SEMAPACT_RELEASE_TAG_PATTERN", + default=None, + ) + + try: + return VersionAuthorityConfig( + authority=authority, + tag_pattern=tag_pattern, + ) + except PydanticValidationError as exc: + message = exc.errors()[0].get("msg", "invalid version authority configuration") + raise ValidationError( + f"Invalid release version authority configuration: {message}" + ) from exc diff --git a/semapact/interfaces/cli.py b/semapact/interfaces/cli.py index cd66c33b..32791636 100644 --- a/semapact/interfaces/cli.py +++ b/semapact/interfaces/cli.py @@ -229,13 +229,28 @@ def _build_parser() -> argparse.ArgumentParser: release_classify_parser = release_subparsers.add_parser( "classify", - help="Classify the required version bump for one contract change set", + help="Analyze the required version bump without creating release artifacts", ) release_classify_parser.add_argument("--base", required=True) release_classify_parser.add_argument("--candidate", required=True) release_classify_parser.add_argument("--runtime-context", default="auto") _add_effective_date_argument(release_classify_parser) + release_plan_parser = release_subparsers.add_parser( + "plan", + help="Build canonical ChangeSet, ReleasePlan, and VersionResolution artifacts", + ) + release_plan_parser.add_argument("--base", required=True) + release_plan_parser.add_argument("--candidate", required=True) + release_plan_parser.add_argument("--base-revision-ref", required=True) + release_plan_parser.add_argument("--candidate-revision-ref", required=True) + release_plan_parser.add_argument( + "--authority-reference", + help="Explicit Git release reference when release.versionAuthority=git", + ) + release_plan_parser.add_argument("--runtime-context", default="auto") + _add_effective_date_argument(release_plan_parser) + release_classify_repo_parser = release_subparsers.add_parser( "classify-repo", help="Classify per-contract required bumps across two contract roots", @@ -259,7 +274,7 @@ def _build_parser() -> argparse.ArgumentParser: release_prepare_parser = release_subparsers.add_parser( "prepare", - help="Prepare one promoted contract candidate using an explicit release tag", + help="Compatibility helper: prepare a candidate using an explicit release tag", ) release_prepare_parser.add_argument("--base", required=True) release_prepare_parser.add_argument("--candidate", required=True) @@ -270,7 +285,7 @@ def _build_parser() -> argparse.ArgumentParser: release_pr_parser = release_subparsers.add_parser( "create-pr", - help="Prepare one promoted contract candidate and open a release PR", + help="Compatibility Git workflow: prepare a candidate and open a release PR", ) release_pr_parser.add_argument("--base", required=True) release_pr_parser.add_argument("--candidate", required=True) @@ -508,13 +523,22 @@ def main() -> int: if args.command == "release": from semapact.interfaces.commands.release_cmd import ( - run_release_classify, run_release_classify_repo, run_release_build_manifest, - run_release_prepare, run_release_create_pr, run_release_create_prs + run_release_build_manifest, + run_release_classify, + run_release_classify_repo, + run_release_create_pr, + run_release_create_prs, + run_release_plan, + run_release_prepare, ) if args.release_command == "classify": payload = run_release_classify(args) print(json.dumps(payload, indent=2, sort_keys=True)) return 0 + if args.release_command == "plan": + payload = run_release_plan(args) + print(json.dumps(payload, indent=2, sort_keys=True)) + return 0 if args.release_command == "classify-repo": payload = run_release_classify_repo(args) print(json.dumps(payload, indent=2, sort_keys=True)) diff --git a/semapact/interfaces/commands/deployment_cmd.py b/semapact/interfaces/commands/deployment_cmd.py index b63e4ea1..1ccc72dc 100644 --- a/semapact/interfaces/commands/deployment_cmd.py +++ b/semapact/interfaces/commands/deployment_cmd.py @@ -12,6 +12,7 @@ from pydantic import BaseModel from pydantic import ValidationError as PydanticValidationError +from semapact.application.services.deployment import DeploymentService from semapact.contractops import AppliedContractRelease from semapact.deployment import ( DeploymentAuthorization, @@ -26,7 +27,6 @@ ) from semapact.observation import RuntimeProvider from semapact.reconciliation import classify_reconciliation_status -from semapact.services.deployment_service import DeploymentService _ModelT = TypeVar("_ModelT", bound=BaseModel) diff --git a/semapact/interfaces/commands/merge_cmd.py b/semapact/interfaces/commands/merge_cmd.py index f382a365..db7c681f 100644 --- a/semapact/interfaces/commands/merge_cmd.py +++ b/semapact/interfaces/commands/merge_cmd.py @@ -1,9 +1,9 @@ import argparse from pathlib import Path +from semapact.application.services.governance import GovernanceService from semapact.core.loader import ContractLoader from semapact.governance import GovernanceOperation, enforce_governance_gate -from semapact.services import GovernanceService from semapact.utils.schema_utils import contract_to_dict from semapact.utils.yaml_utils import dump_yaml diff --git a/semapact/interfaces/commands/plan_cmd.py b/semapact/interfaces/commands/plan_cmd.py index 2f4f608b..3e3972a6 100644 --- a/semapact/interfaces/commands/plan_cmd.py +++ b/semapact/interfaces/commands/plan_cmd.py @@ -1,6 +1,8 @@ import argparse from urllib.parse import urlparse from typing import Any + +from semapact.application.services.governance import GovernanceService from semapact.governance import ( GovernanceOperation, evaluate_governance_decision, @@ -11,7 +13,7 @@ _resolve_adls_oauth_token_from_config, _split_discovered_delta_tables, ) -from semapact.services import GovernanceService + def run_plan(args: argparse.Namespace) -> None: from semapact.orchestrator.pipeline import ContractPipeline @@ -38,18 +40,24 @@ def run_plan(args: argparse.Namespace) -> None: table_uris = _parse_table_uris(args.tables) if not table_uris: from semapact.utils.storage_adapter import StorageAdapterFactory + adapter = StorageAdapterFactory.get_adapter(args.source) try: - table_uris = adapter.discover_delta_tables(args.source, credential=oauth_token) + table_uris = adapter.discover_delta_tables( + args.source, credential=oauth_token + ) except Exception as e: import logging - logging.getLogger("semapact").warning(f"Failed to auto-discover delta tables: {e}") + + logging.getLogger("semapact").warning( + f"Failed to auto-discover delta tables: {e}" + ) table_uris = [] import_source, table_uris = _split_discovered_delta_tables( args.source, table_uris, ) - + if oauth_token: import_args["oauth_bearer_token"] = oauth_token if table_uris: @@ -57,7 +65,6 @@ def run_plan(args: argparse.Namespace) -> None: elif args.tables: import_args["tables"] = args.tables - # Import temporary contract from source imported = pipeline.import_schema( source_type=args.type, source=import_source, @@ -65,11 +72,7 @@ def run_plan(args: argparse.Namespace) -> None: uc_token=args.token, import_args=import_args if import_args else None, ) - - # Load base contract (governed target) base_contract = pipeline.loader.load(args.base) - - # Merge them (to normalize and evaluate breaks) merge_result = pipeline.merge_contract_updates( imported, base_contract, @@ -77,8 +80,6 @@ def run_plan(args: argparse.Namespace) -> None: fail_on_conflict=False, ) merged = merge_result.contract - - # Evaluate decision & ANALYZE operation gate (always allowed for analysis) decision = evaluate_governance_decision( base_contract, merged, @@ -90,7 +91,10 @@ def run_plan(args: argparse.Namespace) -> None: if not decision.evidence.has_changes: print("🟢 No changes detected.") else: - print(f"📊 Governance Decision: {decision.decision.value} (Gate: {gate_res.reason})") + print( + f"📊 Governance Decision: {decision.decision.value} " + f"(Gate: {gate_res.reason})" + ) for reason in decision.reasons: print(f" • [{reason.code}] {reason.path or 'root'}: {reason.message}") @@ -101,4 +105,3 @@ def run_plan(args: argparse.Namespace) -> None: print(f"\n⚠️ Action Required: Additive changes require version bump {bump}.") elif bump == "MAJOR": print(f"\n⚠️ Action Required: Breaking changes require version bump {bump}.") - diff --git a/semapact/interfaces/commands/reconcile_cmd.py b/semapact/interfaces/commands/reconcile_cmd.py index 83e0e091..d50e2088 100644 --- a/semapact/interfaces/commands/reconcile_cmd.py +++ b/semapact/interfaces/commands/reconcile_cmd.py @@ -7,8 +7,9 @@ import json from typing import Literal +from semapact.application.models.reconciliation import RuntimeReconciliation +from semapact.application.services.reconciliation import ReconciliationService from semapact.interfaces.outcomes import ProcessOutcome, outcome_from_reconciliation_status -from semapact.services.reconciliation_service import ReconciliationService, RuntimeReconciliation @dataclass(frozen=True) diff --git a/semapact/interfaces/commands/release_cmd.py b/semapact/interfaces/commands/release_cmd.py index df9cd723..dd70b21e 100644 --- a/semapact/interfaces/commands/release_cmd.py +++ b/semapact/interfaces/commands/release_cmd.py @@ -2,19 +2,22 @@ import json from pathlib import Path from typing import Any + +from semapact.application.services.governance import GovernanceService +from semapact.application.services.release_planning import ReleasePlanningService from semapact.interfaces.commands.utils import ( _build_git_config, _get_repo_path, ) -from semapact.services import GovernanceService + def run_release_classify(args: argparse.Namespace) -> dict[str, Any]: + """Analyze one change without creating canonical release artifacts.""" + from dataclasses import asdict + from semapact.core.loader import ContractLoader - from semapact.core.release import suggest_release_version - from semapact.governance import ( - GovernanceOperation, - evaluate_governance_gate, - ) + from semapact.governance import GovernanceOperation, evaluate_governance_gate + from semapact.versioning import increment_version loader = ContractLoader(runtime_context=args.runtime_context) base_contract = loader.load(args.base) @@ -27,36 +30,60 @@ def run_release_classify(args: argparse.Namespace) -> dict[str, Any]: ) evaluate_governance_gate(decision, GovernanceOperation.ANALYZE) - from dataclasses import asdict - current_version = str(base_contract.version or "") + required_bump = decision.required_version_bump breaking_list = [asdict(bc) for bc in decision.policy.breaking_changes] reasons_list = [r.message for r in decision.reasons] or ["No contract changes detected"] + if decision.evidence.has_changes and required_bump in {"minor", "major"}: + suggested_next_version = increment_version(current_version, required_bump) + else: + suggested_next_version = current_version + return { "contractId": str(base_contract.id or ""), "currentVersion": current_version, "candidateVersion": str(candidate_contract.version or ""), "hasChanges": decision.evidence.has_changes, - "requiredBump": decision.required_version_bump, - "suggestedNextVersion": ( - suggest_release_version(current_version, decision.required_version_bump) - if decision.evidence.has_changes and decision.required_version_bump != "none" - else current_version - ), + "requiredBump": required_bump, + "suggestedNextVersion": suggested_next_version, "reasons": reasons_list, "breakingChanges": breaking_list, "governanceDecision": decision.model_dump(mode="json"), } + +def run_release_plan(args: argparse.Namespace) -> dict[str, Any]: + """Produce exact canonical M2 planning artifacts from one governance pass.""" + from semapact.core.loader import ContractLoader + + loader = ContractLoader(runtime_context=args.runtime_context) + base_contract = loader.load(args.base) + candidate_contract = loader.load(args.candidate) + + result = ReleasePlanningService().plan( + base_contract, + candidate_contract, + effective_date=args.effective_date, + base_revision_ref=args.base_revision_ref, + candidate_revision_ref=args.candidate_revision_ref, + authority_reference=args.authority_reference, + ) + return { + "governanceDecision": result.decision.model_dump(mode="json"), + "changeSet": result.change_set.model_dump(mode="json"), + "releasePlan": result.release_plan.model_dump(mode="json"), + "versionResolution": result.version_resolution.model_dump(mode="json"), + } + + def run_release_prepare(args: argparse.Namespace) -> dict[str, Any]: + """Compatibility release-tag helper retained for existing Git workflows.""" from dataclasses import asdict + from semapact.core.loader import ContractLoader from semapact.core.release import apply_release_candidate - from semapact.governance import ( - GovernanceOperation, - enforce_governance_gate, - ) + from semapact.governance import GovernanceOperation, enforce_governance_gate from semapact.utils.schema_utils import contract_to_dict from semapact.utils.yaml_utils import dump_yaml @@ -91,6 +118,7 @@ def run_release_prepare(args: argparse.Namespace) -> dict[str, Any]: "governanceDecision": decision.model_dump(mode="json"), } + def run_release_classify_repo(args: argparse.Namespace) -> dict[str, Any]: from semapact.devops.release_workflow import ( classify_contracts_in_repo, @@ -103,15 +131,14 @@ def run_release_classify_repo(args: argparse.Namespace) -> dict[str, Any]: candidate_root=args.candidate_root, context=change_context, ) - return { - "contracts": [repository_change_to_dict(item) for item in results], - } + return {"contracts": [repository_change_to_dict(item) for item in results]} + def run_release_build_manifest(args: argparse.Namespace) -> dict[str, Any]: from semapact.devops.release_workflow import ( - build_batch_release_manifest, - batch_task_to_dict, batch_manifest_build_to_dict, + batch_task_to_dict, + build_batch_release_manifest, ) change_context = GovernanceService.create_context(args.effective_date) @@ -134,7 +161,9 @@ def run_release_build_manifest(args: argparse.Namespace) -> dict[str, Any]: payload["output"] = str(output_path) return payload + def run_release_create_pr(args: argparse.Namespace) -> dict[str, Any]: + """Compatibility Git publication workflow retained until a publisher is selected.""" from semapact.core.loader import ContractLoader from semapact.devops.release_workflow import create_release_pull_request @@ -161,11 +190,12 @@ def run_release_create_pr(args: argparse.Namespace) -> dict[str, Any]: ) return payload + def run_release_create_prs(args: argparse.Namespace) -> dict[str, Any]: from semapact.devops.release_workflow import ( - load_batch_release_tasks, - create_release_pull_requests_from_manifest, batch_task_to_dict, + create_release_pull_requests_from_manifest, + load_batch_release_tasks, ) config = _build_git_config(args) diff --git a/semapact/services/__init__.py b/semapact/services/__init__.py index a70d2991..a9db5846 100644 --- a/semapact/services/__init__.py +++ b/semapact/services/__init__.py @@ -1,16 +1,20 @@ -"""UI-independent application services for SemaPact workflows.""" +"""Backward-compatible imports for the pre-application package layout. -from semapact.services.deployment_service import DeploymentService -from semapact.services.governance_service import ( +New code must import application services and application result models from +``semapact.application``. This package owns no business logic or data models. +""" + +from semapact.application.models import ( GovernanceAnalysis, GovernanceProposal, - GovernanceService, -) -from semapact.services.reconciliation_service import ( - ReconciliationService, + ReleasePlanningResult, RuntimeReconciliation, ) -from semapact.services.version_authority_service import VersionAuthorityService +from semapact.application.services.deployment import DeploymentService +from semapact.application.services.governance import GovernanceService +from semapact.application.services.reconciliation import ReconciliationService +from semapact.application.services.release_planning import ReleasePlanningService +from semapact.application.services.version_authority import VersionAuthorityService __all__ = [ "DeploymentService", @@ -18,6 +22,8 @@ "GovernanceProposal", "GovernanceService", "ReconciliationService", + "ReleasePlanningResult", + "ReleasePlanningService", "RuntimeReconciliation", "VersionAuthorityService", ] diff --git a/semapact/services/deployment_service.py b/semapact/services/deployment_service.py index 1ea49132..64806ad9 100644 --- a/semapact/services/deployment_service.py +++ b/semapact/services/deployment_service.py @@ -1,93 +1,5 @@ -"""Application service for provider-neutral deployment workflows.""" +"""Compatibility import for the former deployment service module.""" -from __future__ import annotations +from semapact.application.services.deployment import DeploymentService -from semapact.contractops import AppliedContractRelease -from semapact.deployment import ( - DeploymentAdapter, - DeploymentAuthorization, - DeploymentPlan, - DeploymentPreview, - DeploymentTarget, - build_deployment_plan, - verify_deployment_convergence, -) -from semapact.deployment.models import validate_deployment_plan_identity -from semapact.exceptions import ValidationError -from semapact.observation import RuntimeProvider -from semapact.reconciliation import ReconciliationResult -from semapact.runtime import RuntimeAssetSpec - - -class DeploymentService: - """Compose existing deployment domain/provider boundaries for interfaces. - - The service owns orchestration only. It does not re-run governance, approval, - versioning, deployment translation, or reconciliation rules. - """ - - def plan( - self, - release: AppliedContractRelease, - target: DeploymentTarget, - ) -> DeploymentPlan: - return build_deployment_plan(release, target) - - def preview( - self, - plan: DeploymentPlan, - *, - runtime_provider: RuntimeProvider, - adapter: DeploymentAdapter, - ) -> DeploymentPreview: - """Observe the exact plan scope and delegate native translation to adapter.""" - validate_deployment_plan_identity(plan) - _validate_component_key(runtime_provider.key, plan.target.platform, "runtime provider") - _validate_component_key(adapter.key, plan.target.platform, "deployment adapter") - - assets = _runtime_assets_from_plan(plan) - bindings = runtime_provider.resolve_bindings( - runtime_target=plan.target.runtime_target, - assets=assets, - ) - observation = runtime_provider.observe(bindings=bindings) - return adapter.preview(plan, observation) - - def execute( - self, - plan: DeploymentPlan, - preview: DeploymentPreview, - authorization: DeploymentAuthorization, - *, - adapter: DeploymentAdapter, - ) -> None: - """Delegate the exact authorized side effect to the provider adapter.""" - _validate_component_key(adapter.key, plan.target.platform, "deployment adapter") - adapter.execute(plan, preview, authorization) - - def verify( - self, - plan: DeploymentPlan, - *, - runtime_provider: RuntimeProvider, - ) -> ReconciliationResult: - """Verify exact plan convergence through the existing M1 bridge.""" - return verify_deployment_convergence(plan, runtime_provider) - - -def _runtime_assets_from_plan(plan: DeploymentPlan) -> tuple[RuntimeAssetSpec, ...]: - return tuple( - RuntimeAssetSpec( - governed_asset=action.governed_asset, - physical_name=action.physical_name, - ) - for action in plan.actions - ) - - -def _validate_component_key(actual: str, expected: str, component: str) -> None: - if actual.strip().casefold() != expected.strip().casefold(): - raise ValidationError( - f"{component.capitalize()} does not match DeploymentPlan platform: " - f"{actual!r} != {expected!r}" - ) +__all__ = ["DeploymentService"] diff --git a/semapact/services/governance_service.py b/semapact/services/governance_service.py index 653238e1..3b1ada83 100644 --- a/semapact/services/governance_service.py +++ b/semapact/services/governance_service.py @@ -1,131 +1,6 @@ -"""Application service boundary for deterministic governance workflows.""" +"""Compatibility imports for the former governance service module.""" -from __future__ import annotations +from semapact.application.models.governance import GovernanceAnalysis, GovernanceProposal +from semapact.application.services.governance import GovernanceService -from dataclasses import dataclass -from datetime import date -from typing import Sequence - -from open_data_contract_standard.model import OpenDataContractStandard - -from semapact.change_context import ChangeContext -from semapact.contractops import ChangeSet, build_change_set_from_decision -from semapact.governance.evaluator import evaluate_governance_decision -from semapact.governance.models import GovernanceDecision -from semapact.lifecycle.merge_engine import ContractMergeEngine, MergeConflict, MergeResult - - -@dataclass(frozen=True) -class GovernanceAnalysis: - """One merge-and-governance analysis using a single resolved context.""" - - context: ChangeContext - merge_result: MergeResult - decision: GovernanceDecision - - -@dataclass(frozen=True) -class GovernanceProposal: - """One evaluated proposal represented by a ChangeSet and its decision.""" - - change_set: ChangeSet - decision: GovernanceDecision - - -class GovernanceService: - """Own ChangeContext construction and delegate governance domain work.""" - - def __init__(self, merge_engine: ContractMergeEngine | None = None) -> None: - self._merge_engine = merge_engine or ContractMergeEngine() - - @staticmethod - def create_context(effective_date: date | str) -> ChangeContext: - """Resolve an interface/application date into the domain ChangeContext once.""" - if isinstance(effective_date, str): - try: - resolved_date = date.fromisoformat(effective_date) - except ValueError as exc: - raise ValueError("effective_date must use YYYY-MM-DD") from exc - elif isinstance(effective_date, date): - resolved_date = effective_date - else: - raise TypeError("effective_date must be a date or YYYY-MM-DD string") - - return ChangeContext(effective_date=resolved_date) - - def evaluate( - self, - base_contract: OpenDataContractStandard, - candidate_contract: OpenDataContractStandard, - *, - effective_date: date | str, - merge_conflicts: Sequence[MergeConflict] = (), - ) -> GovernanceDecision: - """Evaluate one contract change from an application-level effective date.""" - context = self.create_context(effective_date) - return evaluate_governance_decision( - base_contract, - candidate_contract, - context=context, - merge_conflicts=merge_conflicts, - ) - - def evaluate_proposal( - self, - base_contract: OpenDataContractStandard, - candidate_contract: OpenDataContractStandard, - *, - effective_date: date | str, - base_revision_ref: str, - candidate_revision_ref: str, - merge_conflicts: Sequence[MergeConflict] = (), - source: str | None = None, - actor_reference: str | None = None, - ) -> GovernanceProposal: - """Evaluate once and project the authoritative decision changes into a ChangeSet.""" - context = self.create_context(effective_date) - decision = evaluate_governance_decision( - base_contract, - candidate_contract, - context=context, - merge_conflicts=merge_conflicts, - ) - change_set = build_change_set_from_decision( - decision, - base_revision_ref=base_revision_ref, - candidate_revision_ref=candidate_revision_ref, - source=source, - actor_reference=actor_reference, - ) - return GovernanceProposal( - change_set=change_set, - decision=decision, - ) - - def merge_and_evaluate( - self, - source_contract: OpenDataContractStandard, - business_contract: OpenDataContractStandard, - *, - effective_date: date | str, - fail_on_conflict: bool = False, - ) -> GovernanceAnalysis: - """Merge and evaluate with the exact same ChangeContext instance.""" - context = self.create_context(effective_date) - merge_result = self._merge_engine.merge( - source_contract, - business_contract, - context=context, - fail_on_conflict=fail_on_conflict, - ) - decision = evaluate_governance_decision( - business_contract, - merge_result.contract, - context=context, - merge_conflicts=merge_result.conflicts, - ) - return GovernanceAnalysis( - context=context, - merge_result=merge_result, - decision=decision, - ) +__all__ = ["GovernanceAnalysis", "GovernanceProposal", "GovernanceService"] diff --git a/semapact/services/reconciliation_service.py b/semapact/services/reconciliation_service.py index 70b81619..1f52bc0a 100644 --- a/semapact/services/reconciliation_service.py +++ b/semapact/services/reconciliation_service.py @@ -1,88 +1,6 @@ -"""Application service for provider-neutral runtime product reconciliation.""" +"""Compatibility imports for the former reconciliation service module.""" -from __future__ import annotations +from semapact.application.models.reconciliation import RuntimeReconciliation +from semapact.application.services.reconciliation import ReconciliationService -from dataclasses import dataclass -from typing import Literal - -from semapact.core.loader import ContractLoader -from semapact.observation import RuntimeAssetBinding, RuntimeProviderRegistry -from semapact.platforms.runtime_registry import ( - create_runtime_provider_registry, - resolve_runtime_location, -) -from semapact.reconciliation import ( - ReconciliationResult, - RuntimeDriftStatus, - classify_reconciliation_status, - reconcile_governed_contract, - runtime_asset_specs_from_contract, -) - - -@dataclass(frozen=True) -class RuntimeReconciliation: - """One complete read-only reconciliation of a governed data product.""" - - platform: str - runtime_target: str - runtime_source: Literal["contract", "cli"] - server_name: str | None - bindings: tuple[RuntimeAssetBinding, ...] - result: ReconciliationResult - status: RuntimeDriftStatus - - -class ReconciliationService: - """Orchestrate load, runtime resolution, bind, observe, reconcile, and classify.""" - - def __init__( - self, - provider_registry: RuntimeProviderRegistry | None = None, - *, - contract_loader: ContractLoader | None = None, - ) -> None: - self._provider_registry = provider_registry - self._contract_loader = contract_loader or ContractLoader() - - def reconcile( - self, - *, - contract_path: str, - server_name: str | None = None, - fallback_platform: str | None = None, - fallback_runtime_target: str | None = None, - ) -> RuntimeReconciliation: - """Reconcile one governed data product against its resolved runtime location.""" - contract = self._contract_loader.load(contract_path) - location = resolve_runtime_location( - contract, - server_name=server_name, - fallback_platform=fallback_platform, - fallback_runtime_target=fallback_runtime_target, - ) - registry = self._provider_registry or create_runtime_provider_registry( - location.platform, - contract_server=location.contract_server, - ) - provider = registry.get(location.platform) - asset_specs = runtime_asset_specs_from_contract(contract) - bindings = provider.resolve_bindings( - runtime_target=location.runtime_target, - assets=asset_specs, - ) - observation = provider.observe(bindings=bindings) - result = reconcile_governed_contract( - contract, - observation, - asset_bindings=bindings, - ) - return RuntimeReconciliation( - platform=provider.key, - runtime_target=location.runtime_target, - runtime_source=location.source, - server_name=location.server_name, - bindings=bindings, - result=result, - status=classify_reconciliation_status(result), - ) +__all__ = ["ReconciliationService", "RuntimeReconciliation"] diff --git a/semapact/services/release_models.py b/semapact/services/release_models.py new file mode 100644 index 00000000..39b1a9f0 --- /dev/null +++ b/semapact/services/release_models.py @@ -0,0 +1,5 @@ +"""Compatibility import for the former release application-model module.""" + +from semapact.application.models.release import ReleasePlanningResult + +__all__ = ["ReleasePlanningResult"] diff --git a/semapact/services/release_planning_service.py b/semapact/services/release_planning_service.py new file mode 100644 index 00000000..e78cc0c7 --- /dev/null +++ b/semapact/services/release_planning_service.py @@ -0,0 +1,5 @@ +"""Compatibility import for the former release-planning service module.""" + +from semapact.application.services.release_planning import ReleasePlanningService + +__all__ = ["ReleasePlanningService"] diff --git a/semapact/services/version_authority_service.py b/semapact/services/version_authority_service.py index 89682003..825a2671 100644 --- a/semapact/services/version_authority_service.py +++ b/semapact/services/version_authority_service.py @@ -1,84 +1,5 @@ -"""Application boundary for configured contract version authority.""" +"""Compatibility import for the former version-authority service module.""" -from __future__ import annotations +from semapact.application.services.version_authority import VersionAuthorityService -from open_data_contract_standard.model import OpenDataContractStandard -from pydantic import ValidationError as PydanticValidationError - -from semapact.contractops import ( - ReleasePlan, - VersionAuthority, - VersionAuthorityConfig, - VersionResolution, - resolve_release_version, -) -from semapact.core.config import ConfigManager -from semapact.exceptions import ReleaseValidationError, ValidationError - - -class VersionAuthorityService: - """Resolve a ReleasePlan using application configuration and released ODCS state.""" - - def __init__(self, config_manager: ConfigManager | None = None) -> None: - self._config = config_manager or ConfigManager() - - def resolve( - self, - release_plan: ReleasePlan, - released_contract: OpenDataContractStandard, - *, - authority_reference: str | None = None, - ) -> VersionResolution: - """Resolve the actual release version without mutating the contract.""" - if not isinstance(release_plan, ReleasePlan): - raise TypeError( - f"release_plan must be ReleasePlan, got {type(release_plan).__name__}" - ) - if not isinstance(released_contract, OpenDataContractStandard): - raise TypeError( - "released_contract must be OpenDataContractStandard, " - f"got {type(released_contract).__name__}" - ) - - contract_id = str(released_contract.id or "").strip() - if contract_id != release_plan.contract_id: - raise ReleaseValidationError( - "Released contract ID does not match ReleasePlan contract ID" - ) - - current_version = str(released_contract.version or "").strip() - if not current_version: - raise ReleaseValidationError( - "Released contract must define the current ODCS version" - ) - - return resolve_release_version( - release_plan, - current_version=current_version, - config=self.load_config(), - authority_reference=authority_reference, - ) - - def load_config(self) -> VersionAuthorityConfig: - """Resolve typed version-authority config from standard SemaPact config sources.""" - authority = self._config.get( - "release.versionAuthority", - env_var="SEMAPACT_RELEASE_VERSION_AUTHORITY", - default=VersionAuthority.SEMAPACT.value, - ) - tag_pattern = self._config.get( - "release.tagPattern", - env_var="SEMAPACT_RELEASE_TAG_PATTERN", - default=None, - ) - - try: - return VersionAuthorityConfig( - authority=authority, - tag_pattern=tag_pattern, - ) - except PydanticValidationError as exc: - message = exc.errors()[0].get("msg", "invalid version authority configuration") - raise ValidationError( - f"Invalid release version authority configuration: {message}" - ) from exc +__all__ = ["VersionAuthorityService"] diff --git a/tests/interfaces/test_release_plan_cmd.py b/tests/interfaces/test_release_plan_cmd.py new file mode 100644 index 00000000..f372cc01 --- /dev/null +++ b/tests/interfaces/test_release_plan_cmd.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import json +import sys + +import pytest + +from semapact.interfaces import cli +from semapact.interfaces.commands import release_cmd + + +def test_release_plan_parser_requires_explicit_revision_refs() -> None: + args = cli._build_parser().parse_args( + [ + "release", + "plan", + "--base", + "contracts/base.yaml", + "--candidate", + "contracts/candidate.yaml", + "--base-revision-ref", + "git:base-123", + "--candidate-revision-ref", + "git:candidate-456", + "--effective-date", + "2026-09-11", + ] + ) + + assert args.command == "release" + assert args.release_command == "plan" + assert args.base_revision_ref == "git:base-123" + assert args.candidate_revision_ref == "git:candidate-456" + assert args.authority_reference is None + + +def test_release_plan_parser_accepts_git_authority_reference() -> None: + args = cli._build_parser().parse_args( + [ + "release", + "plan", + "--base", + "base.yaml", + "--candidate", + "candidate.yaml", + "--base-revision-ref", + "git:base", + "--candidate-revision-ref", + "git:candidate", + "--authority-reference", + "v2.0.0", + "--effective-date", + "2026-09-11", + ] + ) + + assert args.authority_reference == "v2.0.0" + + +def test_main_routes_release_plan_to_release_command_adapter( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + expected = { + "changeSet": {"change_set_id": "change-set:test"}, + "releasePlan": {"release_plan_id": "release-plan:test"}, + "versionResolution": {"version_resolution_id": "version:test"}, + "governanceDecision": {"decision_id": "decision:test"}, + } + monkeypatch.setattr(release_cmd, "run_release_plan", lambda args: expected) + monkeypatch.setattr( + sys, + "argv", + [ + "semapact", + "release", + "plan", + "--base", + "base.yaml", + "--candidate", + "candidate.yaml", + "--base-revision-ref", + "git:base", + "--candidate-revision-ref", + "git:candidate", + "--effective-date", + "2026-09-11", + ], + ) + + assert cli.main() == 0 + assert json.loads(capsys.readouterr().out) == expected + + +def test_legacy_release_helpers_are_labeled_as_compatibility_paths() -> None: + help_text = cli._build_parser().format_help() + release_parser = cli._build_parser()._subparsers._group_actions[0].choices["release"] + release_help = release_parser.format_help() + + assert "release" in help_text + assert "Compatibility helper" in release_help + assert "Compatibility Git workflow" in release_help diff --git a/tests/test_application_architecture.py b/tests/test_application_architecture.py new file mode 100644 index 00000000..df72b707 --- /dev/null +++ b/tests/test_application_architecture.py @@ -0,0 +1,40 @@ +"""Architecture tests for application/domain/package ownership boundaries.""" + +from semapact.application.models.governance import GovernanceAnalysis, GovernanceProposal +from semapact.application.models.reconciliation import RuntimeReconciliation +from semapact.application.models.release import ReleasePlanningResult +from semapact.application.services.deployment import DeploymentService +from semapact.application.services.governance import GovernanceService +from semapact.application.services.reconciliation import ReconciliationService +from semapact.application.services.release_planning import ReleasePlanningService +from semapact.application.services.version_authority import VersionAuthorityService +from semapact.services import ( + DeploymentService as LegacyDeploymentService, + GovernanceAnalysis as LegacyGovernanceAnalysis, + GovernanceProposal as LegacyGovernanceProposal, + GovernanceService as LegacyGovernanceService, + ReconciliationService as LegacyReconciliationService, + ReleasePlanningResult as LegacyReleasePlanningResult, + ReleasePlanningService as LegacyReleasePlanningService, + RuntimeReconciliation as LegacyRuntimeReconciliation, + VersionAuthorityService as LegacyVersionAuthorityService, +) + + +def test_application_result_models_have_explicit_model_ownership() -> None: + assert GovernanceAnalysis.__module__ == "semapact.application.models.governance" + assert GovernanceProposal.__module__ == "semapact.application.models.governance" + assert RuntimeReconciliation.__module__ == "semapact.application.models.reconciliation" + assert ReleasePlanningResult.__module__ == "semapact.application.models.release" + + +def test_legacy_services_package_is_compatibility_only() -> None: + assert LegacyGovernanceAnalysis is GovernanceAnalysis + assert LegacyGovernanceProposal is GovernanceProposal + assert LegacyRuntimeReconciliation is RuntimeReconciliation + assert LegacyReleasePlanningResult is ReleasePlanningResult + assert LegacyGovernanceService is GovernanceService + assert LegacyReconciliationService is ReconciliationService + assert LegacyDeploymentService is DeploymentService + assert LegacyReleasePlanningService is ReleasePlanningService + assert LegacyVersionAuthorityService is VersionAuthorityService diff --git a/tests/test_governance_proposal.py b/tests/test_governance_proposal.py index 108c7ade..90ab2690 100644 --- a/tests/test_governance_proposal.py +++ b/tests/test_governance_proposal.py @@ -9,11 +9,11 @@ SchemaProperty, ) -import semapact.services.governance_service as governance_service_module +import semapact.application.services.governance as governance_service_module +from semapact.application.services.governance import GovernanceService from semapact.change_context import ChangeContext from semapact.governance.models import GovernanceDecision from semapact.lifecycle.merge_engine import MergeConflict -from semapact.services import GovernanceService def _contract(physical_type: str) -> OpenDataContractStandard: diff --git a/tests/test_release_planning_service.py b/tests/test_release_planning_service.py new file mode 100644 index 00000000..0211bcdb --- /dev/null +++ b/tests/test_release_planning_service.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +from open_data_contract_standard.model import ( + OpenDataContractStandard, + SchemaObject, + SchemaProperty, +) + +from semapact.contractops import VersionAuthorityConfig, resolve_release_version +from semapact.services import GovernanceService, ReleasePlanningService + + +class CountingGovernanceService: + def __init__(self) -> None: + self.calls = 0 + self._delegate = GovernanceService() + + def evaluate_proposal(self, *args, **kwargs): + self.calls += 1 + return self._delegate.evaluate_proposal(*args, **kwargs) + + +class FixedVersionAuthorityService: + def __init__(self) -> None: + self.calls = 0 + + def resolve(self, release_plan, released_contract, *, authority_reference=None): + self.calls += 1 + assert authority_reference is None + return resolve_release_version( + release_plan, + current_version=str(released_contract.version), + config=VersionAuthorityConfig(), + ) + + +def _contract(*, name: str) -> OpenDataContractStandard: + return OpenDataContractStandard( + apiVersion="v3.1.0", + kind="DataContract", + id="orders-product", + name=name, + version="1.2.3", + status="active", + schema=[ + SchemaObject( + name="orders", + properties=[ + SchemaProperty( + name="id", + logicalType="string", + physicalType="varchar(255)", + required=True, + ) + ], + ) + ], + ) + + +def test_release_planning_composes_one_decision_into_exact_m2_artifacts() -> None: + governance = CountingGovernanceService() + version_authority = FixedVersionAuthorityService() + service = ReleasePlanningService( + governance_service=governance, + version_authority_service=version_authority, + ) + + result = service.plan( + _contract(name="Orders old"), + _contract(name="Orders new"), + effective_date="2026-09-11", + base_revision_ref="git:base-123", + candidate_revision_ref="git:candidate-456", + ) + + assert governance.calls == 1 + assert version_authority.calls == 1 + assert result.change_set.contract_id == "orders-product" + assert result.change_set.base_revision_ref == "git:base-123" + assert result.change_set.candidate_revision_ref == "git:candidate-456" + assert result.release_plan.change_set_id == result.change_set.change_set_id + assert result.release_plan.decision_id == result.decision.decision_id + assert result.release_plan.release_revision_ref == "git:candidate-456" + assert result.version_resolution.release_plan_id == result.release_plan.release_plan_id + assert result.version_resolution.current_version == "1.2.3" + assert result.version_resolution.selected_version == "1.2.4" + + +def test_release_planning_is_deterministic_for_same_exact_inputs() -> None: + service = ReleasePlanningService( + governance_service=GovernanceService(), + version_authority_service=FixedVersionAuthorityService(), + ) + base = _contract(name="Orders old") + candidate = _contract(name="Orders new") + + first = service.plan( + base, + candidate, + effective_date="2026-09-11", + base_revision_ref="git:base-123", + candidate_revision_ref="git:candidate-456", + ) + second = service.plan( + base, + candidate, + effective_date="2026-09-11", + base_revision_ref="git:base-123", + candidate_revision_ref="git:candidate-456", + ) + + assert first == second + assert first.change_set.change_set_id == second.change_set.change_set_id + assert first.release_plan.release_plan_id == second.release_plan.release_plan_id + assert ( + first.version_resolution.version_resolution_id + == second.version_resolution.version_resolution_id + ) From 628f7ba386411a4fdf8cc579ec438eea5264188f Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Fri, 11 Sep 2026 16:49:20 +1000 Subject: [PATCH 14/35] test(contractops): add deterministic cross-boundary golden scenarios Freeze the canonical ContractOps release/deployment chain with deterministic cross-boundary regression scenarios while preserving existing domain behavior. --- tests/test_contractops_golden_scenarios.py | 608 +++++++++++++++++++++ 1 file changed, 608 insertions(+) create mode 100644 tests/test_contractops_golden_scenarios.py diff --git a/tests/test_contractops_golden_scenarios.py b/tests/test_contractops_golden_scenarios.py new file mode 100644 index 00000000..9d0f850e --- /dev/null +++ b/tests/test_contractops_golden_scenarios.py @@ -0,0 +1,608 @@ +from __future__ import annotations + +from datetime import date, datetime, timezone +from types import SimpleNamespace + +import pytest +from open_data_contract_standard.model import ( + OpenDataContractStandard, + SchemaObject, + SchemaProperty, +) + +from semapact.change_context import ChangeContext +from semapact.contractops import ( + ReviewAuthorizationEvidence, + ReviewEvidenceAction, + VersionAuthority, + VersionAuthorityConfig, + apply_contract_release, + authorize_contract_operation, + build_change_set_from_decision, + build_release_plan, + resolve_release_version, +) +from semapact.deployment import ( + DeploymentTarget, + authorize_deployment, + build_deployment_plan, + verify_deployment_convergence, +) +from semapact.exceptions import ( + ContractOpsAuthorizationError, + GovernanceBlockedError, + ReleaseValidationError, + ValidationError, +) +from semapact.governance import DecisionResult, evaluate_governance_decision +from semapact.governance.gate import GovernanceOperation +from semapact.observation.fingerprint import with_observed_state_fingerprint +from semapact.observation.models import ( + ObservedAsset, + ObservedAssetIdentity, + ObservedPlatformState, + ObservedProperty, + ObservedPropertyIdentity, +) +from semapact.observation.providers import RuntimeAssetBinding +from semapact.platforms.databricks.deployment import DatabricksDeploymentAdapter +from semapact.reconciliation import RuntimeDriftStatus, classify_reconciliation_status + + +CONTEXT = ChangeContext(effective_date=date(2026, 9, 11)) +CAPTURED_AT = datetime(2026, 9, 11, 6, 0, tzinfo=timezone.utc) + + +def _contract( + *, + contract_id: str = "orders-product", + contract_name: str = "orders", + include_created_at: bool = False, +) -> OpenDataContractStandard: + properties = [ + SchemaProperty( + name="id", + logicalType="string", + physicalType="varchar(255)", + required=True, + ) + ] + if include_created_at: + properties.append( + SchemaProperty( + name="created_at", + logicalType="timestamp", + physicalType="timestamp", + required=False, + ) + ) + return OpenDataContractStandard( + apiVersion="v3.1.0", + kind="DataContract", + id=contract_id, + name=contract_name, + version="1.0.0", + status="active", + schema=[SchemaObject(name="orders", properties=properties)], + ) + + +def _release_artifacts(candidate: OpenDataContractStandard): + base = _contract(contract_name="orders-old") + decision = evaluate_governance_decision(base, candidate, context=CONTEXT) + change_set = build_change_set_from_decision( + decision, + base_revision_ref="git:base", + candidate_revision_ref="git:candidate", + source="golden-test", + actor_reference="service:ci", + ) + release_plan = build_release_plan(change_set, decision) + version_resolution = resolve_release_version( + release_plan, + current_version="1.0.0", + config=VersionAuthorityConfig(), + ) + return decision, change_set, release_plan, version_resolution + + +def _review_evidence( + decision, + change_set, + release_plan, + version_resolution, + *, + operation: GovernanceOperation, + scope_reference: str | None = None, +) -> ReviewAuthorizationEvidence: + return ReviewAuthorizationEvidence( + evidence_reference=f"approval:{operation.value.lower()}", + decision_id=decision.decision_id, + change_set_id=change_set.change_set_id, + release_plan_id=release_plan.release_plan_id, + version_resolution_id=version_resolution.version_resolution_id, + operation=operation, + action=ReviewEvidenceAction.APPROVE, + scope_reference=scope_reference, + ) + + +def _apply_release(candidate: OpenDataContractStandard): + decision, change_set, release_plan, version_resolution = _release_artifacts(candidate) + evidence = None + if decision.decision is DecisionResult.REVIEW: + evidence = _review_evidence( + decision, + change_set, + release_plan, + version_resolution, + operation=GovernanceOperation.APPLY, + ) + authorization = authorize_contract_operation( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.APPLY, + evidence=evidence, + ) + release = apply_contract_release( + candidate, + candidate_revision_ref="git:candidate", + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + authorization=authorization, + ) + return decision, change_set, release_plan, version_resolution, authorization, release + + +def _allow_chain(): + candidate = _contract(contract_name="orders-new") + ( + decision, + change_set, + release_plan, + version_resolution, + apply_authorization, + release, + ) = _apply_release(candidate) + deployment_plan = build_deployment_plan( + release, + DeploymentTarget( + platform="databricks", + runtime_target="main.silver", + server_name="production", + ), + ) + deploy_authorization = authorize_contract_operation( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.DEPLOY, + ) + deployment_authorization = authorize_deployment( + deployment_plan, + release, + deploy_authorization, + ) + return SimpleNamespace( + candidate=candidate, + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + apply_authorization=apply_authorization, + release=release, + deployment_plan=deployment_plan, + deploy_authorization=deploy_authorization, + deployment_authorization=deployment_authorization, + ) + + +def _observed_state( + *, + present: bool, + physical_type: str | None = "varchar(255)", + nullable: bool | None = False, +) -> ObservedPlatformState: + identity = ObservedAssetIdentity( + platform="databricks", + namespace=("main", "silver"), + asset="orders", + ) + assets = () + if present: + assets = ( + ObservedAsset( + identity=identity, + asset_type="MANAGED", + properties=( + ObservedProperty( + identity=ObservedPropertyIdentity( + asset=identity, + property="id", + ), + physical_type=physical_type, + nullable=nullable, + ), + ), + ), + ) + return with_observed_state_fingerprint( + ObservedPlatformState( + platform="databricks", + source_identifier="workspace:golden", + assets=assets, + captured_at=CAPTURED_AT, + fingerprint=None, + ) + ) + + +class _RuntimeProvider: + key = "databricks" + + def __init__(self, state: ObservedPlatformState) -> None: + self.state = state + self.observe_calls = 0 + + def resolve_bindings(self, *, runtime_target, assets): + assert runtime_target == "main.silver" + return tuple( + RuntimeAssetBinding( + governed_asset=asset.governed_asset, + observed_asset=ObservedAssetIdentity( + platform="databricks", + namespace=("main", "silver"), + asset=asset.physical_name, + ), + ) + for asset in assets + ) + + def observe(self, *, bindings): + self.observe_calls += 1 + return self.state + + +class _Statements: + def __init__(self) -> None: + self.calls: list[str] = [] + + def execute_statement(self, *, statement, warehouse_id, wait_timeout): + assert warehouse_id == "warehouse-golden" + self.calls.append(statement) + return SimpleNamespace( + statement_id="statement-golden", + status=SimpleNamespace(state="SUCCEEDED", error=None), + ) + + def get_statement(self, statement_id): + raise AssertionError(f"unexpected statement polling: {statement_id}") + + +class _Client: + def __init__(self) -> None: + self.statement_execution = _Statements() + + +def _adapter(provider: _RuntimeProvider): + client = _Client() + adapter = DatabricksDeploymentAdapter( + client=client, + runtime_provider=provider, + warehouse_id="warehouse-golden", + poll_interval_seconds=0, + ) + return adapter, client + + +def _allow_semantic_projection(chain) -> dict[str, object]: + action = chain.deployment_plan.actions[0] + return { + "decision": chain.decision.decision.value, + "requiredVersionBump": chain.decision.required_version_bump, + "changeCount": len(chain.change_set.changes), + "releasePreconditions": [ + item.value for item in chain.release_plan.preconditions + ], + "versionAuthority": chain.version_resolution.authority.value, + "selectedVersion": chain.version_resolution.selected_version, + "actualBump": chain.version_resolution.actual_bump, + "apply": { + "operation": chain.apply_authorization.operation.value, + "allowed": chain.apply_authorization.allowed, + "reason": chain.apply_authorization.reason.value, + }, + "appliedContract": { + "name": chain.release.to_contract().name, + "version": chain.release.to_contract().version, + }, + "deployment": { + "operation": chain.deploy_authorization.operation.value, + "allowed": chain.deploy_authorization.allowed, + "target": chain.deployment_plan.target.model_dump(mode="json"), + "actions": [ + { + "kind": action.kind.value, + "governedAsset": action.governed_asset, + "physicalName": action.physical_name, + } + ], + }, + } + + +def test_allow_chain_has_stable_cross_boundary_golden_semantics() -> None: + first = _allow_chain() + second = _allow_chain() + + assert _allow_semantic_projection(first) == { + "decision": "ALLOW", + "requiredVersionBump": "none", + "changeCount": 1, + "releasePreconditions": [], + "versionAuthority": "semapact", + "selectedVersion": "1.0.1", + "actualBump": "patch", + "apply": { + "operation": "APPLY", + "allowed": True, + "reason": "allowed_by_governance", + }, + "appliedContract": {"name": "orders-new", "version": "1.0.1"}, + "deployment": { + "operation": "DEPLOY", + "allowed": True, + "target": { + "platform": "databricks", + "runtime_target": "main.silver", + "server_name": "production", + }, + "actions": [ + { + "kind": "ENSURE_ASSET_STATE", + "governedAsset": "orders", + "physicalName": "orders", + } + ], + }, + } + + assert first.decision.decision_id == second.decision.decision_id + assert first.change_set.change_set_id == second.change_set.change_set_id + assert first.release_plan.release_plan_id == second.release_plan.release_plan_id + assert ( + first.version_resolution.version_resolution_id + == second.version_resolution.version_resolution_id + ) + assert first.apply_authorization.authorization_id == second.apply_authorization.authorization_id + assert first.release.applied_release_id == second.release.applied_release_id + assert ( + first.deployment_plan.deployment_plan_id + == second.deployment_plan.deployment_plan_id + ) + assert ( + first.deployment_authorization.deployment_authorization_id + == second.deployment_authorization.deployment_authorization_id + ) + assert first.candidate.version == "1.0.0" + + +def test_review_requires_exact_apply_and_deployment_authorization() -> None: + candidate = _contract(contract_name="orders-old", include_created_at=True) + decision, change_set, release_plan, version_resolution = _release_artifacts(candidate) + + assert decision.decision is DecisionResult.REVIEW + + denied_apply = authorize_contract_operation( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.APPLY, + ) + assert denied_apply.allowed is False + with pytest.raises(ContractOpsAuthorizationError): + apply_contract_release( + candidate, + candidate_revision_ref="git:candidate", + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + authorization=denied_apply, + ) + + apply_evidence = _review_evidence( + decision, + change_set, + release_plan, + version_resolution, + operation=GovernanceOperation.APPLY, + ) + apply_authorization = authorize_contract_operation( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.APPLY, + evidence=apply_evidence, + ) + release = apply_contract_release( + candidate, + candidate_revision_ref="git:candidate", + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + authorization=apply_authorization, + ) + plan = build_deployment_plan( + release, + DeploymentTarget(platform="databricks", runtime_target="main.silver"), + ) + + unscoped_deploy_evidence = _review_evidence( + decision, + change_set, + release_plan, + version_resolution, + operation=GovernanceOperation.DEPLOY, + ) + unscoped_deploy = authorize_contract_operation( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.DEPLOY, + evidence=unscoped_deploy_evidence, + ) + with pytest.raises(ReleaseValidationError, match="scoped"): + authorize_deployment(plan, release, unscoped_deploy) + + scoped_deploy_evidence = _review_evidence( + decision, + change_set, + release_plan, + version_resolution, + operation=GovernanceOperation.DEPLOY, + scope_reference=plan.deployment_plan_id, + ) + scoped_deploy = authorize_contract_operation( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.DEPLOY, + evidence=scoped_deploy_evidence, + ) + deployment_authorization = authorize_deployment(plan, release, scoped_deploy) + + assert decision.decision is DecisionResult.REVIEW + assert apply_authorization.allowed is True + assert scoped_deploy.allowed is True + assert deployment_authorization.allowed is True + + +def test_block_cannot_enter_release_chain() -> None: + base = _contract() + candidate = _contract(contract_id="other-product") + decision = evaluate_governance_decision(base, candidate, context=CONTEXT) + change_set = build_change_set_from_decision( + decision, + base_revision_ref="git:base", + candidate_revision_ref="git:candidate", + ) + + assert decision.decision is DecisionResult.BLOCK + with pytest.raises(GovernanceBlockedError): + build_release_plan(change_set, decision) + + +def test_version_authorities_preserve_same_required_bump() -> None: + candidate = _contract(contract_name="orders-old", include_created_at=True) + decision, _, release_plan, _ = _release_artifacts(candidate) + + semapact_resolution = resolve_release_version( + release_plan, + current_version="1.0.0", + config=VersionAuthorityConfig(authority=VersionAuthority.SEMAPACT), + ) + git_resolution = resolve_release_version( + release_plan, + current_version="1.0.0", + config=VersionAuthorityConfig( + authority=VersionAuthority.GIT, + tag_pattern="v{version}", + ), + authority_reference="v1.2.0", + ) + + assert decision.required_version_bump == "minor" + assert semapact_resolution.required_version_bump == "minor" + assert git_resolution.required_version_bump == "minor" + assert semapact_resolution.selected_version == "1.1.0" + assert git_resolution.selected_version == "1.2.0" + assert semapact_resolution.authority_reference is None + assert git_resolution.authority_reference == "v1.2.0" + + +def test_publish_authorization_cannot_authorize_runtime_deployment() -> None: + chain = _allow_chain() + publish_authorization = authorize_contract_operation( + chain.decision, + chain.change_set, + chain.release_plan, + chain.version_resolution, + GovernanceOperation.PUBLISH, + ) + + with pytest.raises(ReleaseValidationError, match="DEPLOY authorization"): + authorize_deployment( + chain.deployment_plan, + chain.release, + publish_authorization, + ) + + +def test_execute_success_is_separate_from_runtime_convergence() -> None: + chain = _allow_chain() + missing_state = _observed_state(present=False) + provider = _RuntimeProvider(missing_state) + adapter, client = _adapter(provider) + + first_preview = adapter.preview(chain.deployment_plan, missing_state) + second_preview = adapter.preview(chain.deployment_plan, missing_state) + assert first_preview == second_preview + assert client.statement_execution.calls == [] + + adapter.execute( + chain.deployment_plan, + first_preview, + chain.deployment_authorization, + ) + assert len(client.statement_execution.calls) == 1 + + statement_count = len(client.statement_execution.calls) + drift_result = verify_deployment_convergence(chain.deployment_plan, provider) + assert classify_reconciliation_status(drift_result) is RuntimeDriftStatus.DRIFT + assert len(client.statement_execution.calls) == statement_count + + provider.state = _observed_state(present=True) + in_sync_result = verify_deployment_convergence(chain.deployment_plan, provider) + assert classify_reconciliation_status(in_sync_result) is RuntimeDriftStatus.IN_SYNC + assert len(client.statement_execution.calls) == statement_count + + provider.state = _observed_state( + present=True, + physical_type=None, + nullable=None, + ) + indeterminate_result = verify_deployment_convergence(chain.deployment_plan, provider) + assert ( + classify_reconciliation_status(indeterminate_result) + is RuntimeDriftStatus.INDETERMINATE + ) + assert len(client.statement_execution.calls) == statement_count + + +def test_stale_preview_fails_before_native_mutation() -> None: + chain = _allow_chain() + missing_state = _observed_state(present=False) + provider = _RuntimeProvider(missing_state) + adapter, client = _adapter(provider) + preview = adapter.preview(chain.deployment_plan, missing_state) + + provider.state = _observed_state(present=True) + with pytest.raises(ValidationError, match="no longer equals|Runtime state changed"): + adapter.execute( + chain.deployment_plan, + preview, + chain.deployment_authorization, + ) + + assert client.statement_execution.calls == [] From 66d1df362fb4976a2a8aeace459042515dc2d195 Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Sat, 12 Sep 2026 07:19:21 +1000 Subject: [PATCH 15/35] refactor(contractops): harden artifact and deployment integrity * refactor(contractops): centralize artifact identity validation * refactor(contractops): reuse canonical ChangeSet identity * refactor(contractops): reuse canonical ReleasePlan identity * refactor(contractops): reuse canonical version identity * refactor(contractops): reuse canonical authorization identity * refactor(contractops): fail closed on artifact integrity * refactor(contractops): validate execution artifact integrity * refactor(deployment): validate upstream artifact integrity * refactor(deployment): validate applied release before planning * refactor(deployment): bind plans to exact runtime source * refactor(deployment): enforce exact runtime source in preview * refactor(deployment): enforce authorized runtime source * refactor(deployment): verify exact runtime source * refactor(governance): depend on canonical versioning primitives * feat(deployment): bind CLI plans to runtime source * refactor(contractops): expose artifact integrity validators * refactor(contractops): validate deterministic identity on rehydration * refactor(contractops): validate identity-bearing models on rehydration * feat(deployment): require exact runtime source at planning * docs(deployment): document exact runtime source binding * test(deployment): bind Databricks plans to runtime source * test(deployment): use exact release and source identities * test(deployment): enforce source-bound orchestration * test(deployment): verify exact runtime source binding * test(architecture): use canonical release and exact deployment sources * test(contractops): build valid deterministic release plans * test(contractops): use deterministic plans in version service tests * test(deployment): cover source-bound CLI artifacts * fix(contractops): preserve blocked authorization semantics * test(contractops): distinguish integrity from context mismatch * test(contractops): bind golden deployment source * test(contractops): cover artifact rehydration integrity * test(contractops): reject tampered persisted artifacts --- docs/deployment_plans.md | 20 +- semapact/application/services/deployment.py | 9 + semapact/contractops/__init__.py | 14 + semapact/contractops/authorization.py | 42 +-- semapact/contractops/changeset.py | 28 +- semapact/contractops/context.py | 31 +- semapact/contractops/execution.py | 58 ++-- semapact/contractops/integrity.py | 318 ++++++++++++++++++ semapact/contractops/models.py | 9 + semapact/contractops/release_plan.py | 33 +- semapact/contractops/version_authority.py | 33 +- semapact/deployment/authorization.py | 6 + semapact/deployment/models.py | 13 +- semapact/deployment/planner.py | 2 + semapact/deployment/verification.py | 4 + semapact/governance/models.py | 3 +- semapact/interfaces/cli.py | 8 +- .../interfaces/commands/deployment_cmd.py | 1 + semapact/platforms/databricks/deployment.py | 8 + tests/interfaces/test_deployment_cmd.py | 50 +-- tests/test_architecture_consolidation.py | 76 ++++- .../test_contractops_artifact_rehydration.py | 153 +++++++++ tests/test_contractops_authorization.py | 40 ++- tests/test_contractops_golden_scenarios.py | 8 +- tests/test_deployment_databricks.py | 59 +++- tests/test_deployment_plan.py | 52 ++- tests/test_deployment_service.py | 28 +- tests/test_deployment_verification.py | 19 +- tests/test_version_authority.py | 24 +- tests/test_version_authority_service.py | 19 +- 30 files changed, 942 insertions(+), 226 deletions(-) create mode 100644 semapact/contractops/integrity.py create mode 100644 tests/test_contractops_artifact_rehydration.py diff --git a/docs/deployment_plans.md b/docs/deployment_plans.md index 4768b358..82a585b3 100644 --- a/docs/deployment_plans.md +++ b/docs/deployment_plans.md @@ -73,10 +73,13 @@ A plan requires an explicit target: DeploymentTarget ├── platform ├── runtimeTarget +├── sourceReference └── serverName? # optional provenance ``` -`platform` is the downstream adapter dispatch key. `runtimeTarget` is an opaque provider-local target descriptor at this layer. +`platform` is the downstream adapter dispatch key. `runtimeTarget` is an opaque provider-local product target. `sourceReference` identifies the exact runtime source/end point against which the plan is authorized; for Databricks this is the workspace host used by runtime observation. It is an identity reference, never a credential. + +Preview, execution, and verification fail closed when fresh runtime evidence comes from a different source than the plan's `sourceReference`. This prevents an authorization for the same catalog/schema name from being reused against another workspace. The plan does not contain credentials, workspace clients, SQL connections, or provider sessions. @@ -90,10 +93,11 @@ The deployment CLI consumes and emits canonical JSON artifacts. Planning and pre semapact deployment plan \ --release ./artifacts/applied-release.json \ --platform databricks \ - --runtime main.sales + --runtime main.sales \ + --source-reference https://dbc-example.cloud.databricks.com ``` -Optional `--server` preserves the selected contract-server reference as target provenance. +`--source-reference` must match the stable source identity reported by the runtime provider. Optional `--server` preserves the selected contract-server reference as target provenance. The output is the canonical `DeploymentPlan` JSON. @@ -108,6 +112,8 @@ Preview observes the exact target scope and derives a canonical `DeploymentPrevi ### Execute +For a preview containing CREATE or ALTER operations, provide the SQL warehouse used for mutation: + ```bash semapact deployment execute \ --plan ./artifacts/deployment-plan.json \ @@ -116,7 +122,9 @@ semapact deployment execute \ --warehouse-id ``` -Execution requires the exact plan, exact preview, and exact `DeploymentAuthorization`. The adapter re-observes the target, validates the observation source and fingerprint, re-derives the expected preview for integrity/freshness validation, and executes only the supplied operations when the artifacts still match. +For an all-`NO_OP` preview, `--warehouse-id` may be omitted because no native mutation is executed. Any CREATE/ALTER attempt without a warehouse fails closed. + +Execution requires the exact plan, exact preview, and exact `DeploymentAuthorization`. The adapter re-observes the target, validates the authorized runtime source and observation fingerprint, re-derives the expected preview for integrity/freshness validation, and executes only the supplied operations when the artifacts still match. Provider execution success is not convergence proof. @@ -171,7 +179,7 @@ Runtime deployment is a separate protected operation from publishing a contract A `ContractOpsAuthorization(operation=DEPLOY)` establishes release-context authorization. Before runtime mutation, it must be bound to the exact `DeploymentPlan` as a `DeploymentAuthorization`. -For review-required changes, structured review evidence may carry an opaque `scopeReference`. Deployment requires that scope to match the exact `deploymentPlanId`, so an approval for one target cannot be reused for another target. +For review-required changes, structured review evidence may carry an opaque `scopeReference`. Deployment requires that scope to match the exact `deploymentPlanId`, so an approval for one exact platform/runtime/source target cannot be reused for another target. A PUBLISH authorization cannot authorize DEPLOY. @@ -180,7 +188,7 @@ A PUBLISH authorization cannot authorize DEPLOY. `deploymentPlanId` is UUID5-derived from the full stable plan record: - exact `AppliedContractRelease` identity and provenance; -- exact deployment target; +- exact deployment target, including runtime source reference; - canonical actions ordered by governed asset identity; - plan schema version. diff --git a/semapact/application/services/deployment.py b/semapact/application/services/deployment.py index 1ea49132..1776997a 100644 --- a/semapact/application/services/deployment.py +++ b/semapact/application/services/deployment.py @@ -51,6 +51,7 @@ def preview( assets=assets, ) observation = runtime_provider.observe(bindings=bindings) + _validate_source_reference(observation.source_identifier, plan.target.source_reference) return adapter.preview(plan, observation) def execute( @@ -91,3 +92,11 @@ def _validate_component_key(actual: str, expected: str, component: str) -> None: f"{component.capitalize()} does not match DeploymentPlan platform: " f"{actual!r} != {expected!r}" ) + + +def _validate_source_reference(actual: str, expected: str) -> None: + if actual.strip() != expected.strip(): + raise ValidationError( + "Runtime observation source does not match DeploymentPlan source reference: " + f"{actual!r} != {expected!r}" + ) diff --git a/semapact/contractops/__init__.py b/semapact/contractops/__init__.py index 4123e916..b95e5f8f 100644 --- a/semapact/contractops/__init__.py +++ b/semapact/contractops/__init__.py @@ -14,6 +14,14 @@ AppliedContractRelease, PublicationResult, ) +from semapact.contractops.integrity import ( + validate_applied_release_identity, + validate_change_set_identity, + validate_contractops_authorization_identity, + validate_publication_result_identity, + validate_release_plan_identity, + validate_version_resolution_identity, +) from semapact.contractops.models import ( AuthorizationReason, ChangeSet, @@ -54,4 +62,10 @@ "extract_version_from_release_reference", "publish_contract_release", "resolve_release_version", + "validate_applied_release_identity", + "validate_change_set_identity", + "validate_contractops_authorization_identity", + "validate_publication_result_identity", + "validate_release_plan_identity", + "validate_version_resolution_identity", ] diff --git a/semapact/contractops/authorization.py b/semapact/contractops/authorization.py index 97de58ee..820eaa81 100644 --- a/semapact/contractops/authorization.py +++ b/semapact/contractops/authorization.py @@ -2,9 +2,11 @@ from __future__ import annotations -import uuid - from semapact.contractops.context import validate_release_context +from semapact.contractops.integrity import ( + SEMAPACT_CONTRACTOPS_AUTHORIZATION_NAMESPACE, + compute_contractops_authorization_id, +) from semapact.contractops.models import ( AuthorizationReason, ChangeSet, @@ -16,12 +18,6 @@ ) from semapact.governance.gate import GovernanceOperation, evaluate_governance_gate from semapact.governance.models import GovernanceDecision -from semapact.utils.deterministic import deterministic_uuid5 - - -SEMAPACT_CONTRACTOPS_AUTHORIZATION_NAMESPACE = uuid.UUID( - "b6218d0c-3f9d-44a2-8d68-e3b0ee170948" -) def authorize_contract_operation( @@ -192,25 +188,17 @@ def _build_authorization( evidence_action = evidence.action if evidence is not None else None scope_reference = evidence.scope_reference if evidence is not None else None - stable_record = { - "decision_id": decision.decision_id, - "change_set_id": change_set.change_set_id, - "release_plan_id": release_plan.release_plan_id, - "version_resolution_id": version_resolution.version_resolution_id, - "operation": operation.value, - "allowed": allowed, - "reason": reason.value, - "evidence_reference": evidence_reference, - "evidence_action": evidence_action.value if evidence_action is not None else None, - } - # Preserve all pre-#122 authorization IDs byte-for-byte when no downstream - # scope was supplied. Scoped authorizations intentionally gain a new identity. - if scope_reference is not None: - stable_record["scope_reference"] = scope_reference - - authorization_id = deterministic_uuid5( - SEMAPACT_CONTRACTOPS_AUTHORIZATION_NAMESPACE, - stable_record, + authorization_id = compute_contractops_authorization_id( + decision_id=decision.decision_id, + change_set_id=change_set.change_set_id, + release_plan_id=release_plan.release_plan_id, + version_resolution_id=version_resolution.version_resolution_id, + operation=operation, + allowed=allowed, + reason=reason.value, + evidence_reference=evidence_reference, + evidence_action=(evidence_action.value if evidence_action is not None else None), + scope_reference=scope_reference, ) return ContractOpsAuthorization( diff --git a/semapact/contractops/changeset.py b/semapact/contractops/changeset.py index 24e134d0..2faa7fc7 100644 --- a/semapact/contractops/changeset.py +++ b/semapact/contractops/changeset.py @@ -2,17 +2,16 @@ from __future__ import annotations -import uuid from collections.abc import Sequence from semapact.change_context import ChangeContext +from semapact.contractops.integrity import ( + SEMAPACT_CHANGESET_NAMESPACE, + compute_change_set_id, +) from semapact.contractops.models import ChangeSet from semapact.governance.models import GovernanceDecision from semapact.lifecycle.changes import GovernanceChange, governance_change_sort_key -from semapact.utils.deterministic import deterministic_uuid5 - - -SEMAPACT_CHANGESET_NAMESPACE = uuid.UUID("3ea0f6d8-28ca-4bb4-94f5-ea1f0f48cb84") def build_change_set( @@ -45,16 +44,15 @@ def build_change_set( cleaned_source = _optional_text(source) cleaned_actor_reference = _optional_text(actor_reference) - identity_payload = { - "contract_id": cleaned_contract_id, - "base_revision_ref": cleaned_base_ref, - "candidate_revision_ref": cleaned_candidate_ref, - "context": context.model_dump(mode="json"), - "changes": [change.model_dump(mode="json") for change in canonical_changes], - "source": cleaned_source, - "actor_reference": cleaned_actor_reference, - } - change_set_id = deterministic_uuid5(SEMAPACT_CHANGESET_NAMESPACE, identity_payload) + change_set_id = compute_change_set_id( + contract_id=cleaned_contract_id, + base_revision_ref=cleaned_base_ref, + candidate_revision_ref=cleaned_candidate_ref, + changes=canonical_changes, + context=context, + source=cleaned_source, + actor_reference=cleaned_actor_reference, + ) return ChangeSet( change_set_id=change_set_id, diff --git a/semapact/contractops/context.py b/semapact/contractops/context.py index 833dc711..37df8ec2 100644 --- a/semapact/contractops/context.py +++ b/semapact/contractops/context.py @@ -2,8 +2,19 @@ from __future__ import annotations -from semapact.contractops.models import ChangeSet, ReleasePlan, VersionResolution +from semapact.contractops.integrity import ( + validate_change_set_identity, + validate_release_plan_identity, + validate_version_resolution_identity, +) +from semapact.contractops.models import ( + ChangeSet, + ReleasePlan, + ReleasePrecondition, + VersionResolution, +) from semapact.exceptions import ReleaseValidationError +from semapact.governance.gate import GovernanceOperation, evaluate_governance_gate from semapact.governance.models import GovernanceDecision @@ -12,6 +23,8 @@ def validate_proposal_context( change_set: ChangeSet, ) -> None: """Fail closed unless ChangeSet exactly projects one GovernanceDecision.""" + validate_change_set_identity(change_set) + if change_set.contract_id != decision.contract_id: raise ReleaseValidationError( "ChangeSet and GovernanceDecision contract IDs do not match" @@ -34,6 +47,8 @@ def validate_release_context( ) -> None: """Fail closed unless immutable artifacts describe one exact release context.""" validate_proposal_context(decision, change_set) + validate_release_plan_identity(release_plan) + validate_version_resolution_identity(version_resolution) if release_plan.contract_id != change_set.contract_id: raise ReleaseValidationError("ReleasePlan and ChangeSet contract IDs do not match") @@ -52,6 +67,20 @@ def validate_release_context( "ReleasePlan required version bump does not match GovernanceDecision" ) + publish_gate = evaluate_governance_gate(decision, GovernanceOperation.PUBLISH) + if publish_gate.reason == "review_required": + expected_preconditions = (ReleasePrecondition.REVIEW_AUTHORIZATION_REQUIRED,) + else: + # ALLOW and BLOCK carry no review precondition. BLOCK cannot be produced by + # the canonical planner, but authorization must still preserve its existing + # fail-closed result if a structurally valid historical/context artifact is + # supplied; review evidence can never override that decision. + expected_preconditions = () + if release_plan.preconditions != expected_preconditions: + raise ReleaseValidationError( + "ReleasePlan preconditions do not match authoritative governance disposition" + ) + if version_resolution.release_plan_id != release_plan.release_plan_id: raise ReleaseValidationError( "VersionResolution does not reference the supplied ReleasePlan" diff --git a/semapact/contractops/execution.py b/semapact/contractops/execution.py index a6bda1c6..aa276509 100644 --- a/semapact/contractops/execution.py +++ b/semapact/contractops/execution.py @@ -2,13 +2,20 @@ from __future__ import annotations -import uuid from typing import Protocol from open_data_contract_standard.model import OpenDataContractStandard from semapact.contractops.context import validate_release_context from semapact.contractops.execution_models import AppliedContractRelease, PublicationResult +from semapact.contractops.integrity import ( + SEMAPACT_APPLIED_RELEASE_NAMESPACE, + SEMAPACT_PUBLICATION_NAMESPACE, + compute_applied_release_id, + compute_publication_id, + validate_applied_release_identity, + validate_contractops_authorization_identity, +) from semapact.contractops.models import ( ChangeSet, ContractOpsAuthorization, @@ -18,18 +25,10 @@ from semapact.exceptions import ContractOpsAuthorizationError, ReleaseValidationError from semapact.governance.gate import GovernanceOperation from semapact.governance.models import GovernanceDecision -from semapact.utils.deterministic import canonical_compact_json, deterministic_uuid5 +from semapact.utils.deterministic import canonical_compact_json from semapact.versioning import normalize_semver -SEMAPACT_APPLIED_RELEASE_NAMESPACE = uuid.UUID( - "a5de2e65-aee7-48ac-9cb6-6a785f4cdf33" -) -SEMAPACT_PUBLICATION_NAMESPACE = uuid.UUID( - "f776fc77-b37f-43ef-bf6d-d8dcf38d264f" -) - - class ContractReleasePublisher(Protocol): """Narrow external publication port for one applied contract release.""" @@ -105,20 +104,16 @@ def apply_contract_release( released_contract.version = selected_version released_contract_json = _canonical_contract_json(released_contract) - stable_record = { - "contract_id": release_plan.contract_id, - "decision_id": decision.decision_id, - "change_set_id": change_set.change_set_id, - "release_plan_id": release_plan.release_plan_id, - "version_resolution_id": version_resolution.version_resolution_id, - "release_revision_ref": release_plan.release_revision_ref, - "selected_version": selected_version, - "authorization_id": authorization.authorization_id, - "released_contract_json": released_contract_json, - } - applied_release_id = deterministic_uuid5( - SEMAPACT_APPLIED_RELEASE_NAMESPACE, - stable_record, + applied_release_id = compute_applied_release_id( + contract_id=release_plan.contract_id, + decision_id=decision.decision_id, + change_set_id=change_set.change_set_id, + release_plan_id=release_plan.release_plan_id, + version_resolution_id=version_resolution.version_resolution_id, + release_revision_ref=release_plan.release_revision_ref, + selected_version=selected_version, + authorization_id=authorization.authorization_id, + released_contract_json=released_contract_json, ) return AppliedContractRelease( @@ -154,6 +149,7 @@ def publish_contract_release( if not hasattr(publisher, "publish") or not callable(publisher.publish): raise TypeError("publisher must provide a callable publish(release) method") + validate_applied_release_identity(release) _validate_publication_authorization(release, authorization) publication_reference = publisher.publish(release) @@ -165,14 +161,10 @@ def publish_contract_release( "publisher.publish() returned an empty publication reference" ) - stable_record = { - "applied_release_id": release.applied_release_id, - "authorization_id": authorization.authorization_id, - "publication_reference": publication_reference, - } - publication_id = deterministic_uuid5( - SEMAPACT_PUBLICATION_NAMESPACE, - stable_record, + publication_id = compute_publication_id( + applied_release_id=release.applied_release_id, + authorization_id=authorization.authorization_id, + publication_reference=publication_reference, ) return PublicationResult( publication_id=publication_id, @@ -196,6 +188,7 @@ def _validate_authorization( "authorization must be ContractOpsAuthorization, " f"got {type(authorization).__name__}" ) + validate_contractops_authorization_identity(authorization) if authorization.operation is not operation: raise ReleaseValidationError( f"Authorization operation must be {operation.value}, " @@ -222,6 +215,7 @@ def _validate_publication_authorization( release: AppliedContractRelease, authorization: ContractOpsAuthorization, ) -> None: + validate_contractops_authorization_identity(authorization) if authorization.operation is not GovernanceOperation.PUBLISH: raise ReleaseValidationError( "Publication requires operation-scoped PUBLISH authorization" diff --git a/semapact/contractops/integrity.py b/semapact/contractops/integrity.py new file mode 100644 index 00000000..1b606459 --- /dev/null +++ b/semapact/contractops/integrity.py @@ -0,0 +1,318 @@ +"""Deterministic identity computation and fail-closed validation for ContractOps artifacts. + +GovernanceDecision is intentionally excluded: its historical decision ID depends on +source-contract fingerprints that are not contained in the serialized decision itself. +All downstream ContractOps artifacts are self-describing and can therefore validate +their deterministic identities after persistence/deserialization. +""" + +from __future__ import annotations + +import json +import uuid +from collections.abc import Sequence + +from semapact.change_context import ChangeContext +from semapact.contractops.execution_models import AppliedContractRelease, PublicationResult +from semapact.contractops.models import ( + ChangeSet, + ContractOpsAuthorization, + ReleasePlan, + ReleasePrecondition, + VersionAuthority, + VersionResolution, +) +from semapact.exceptions import ReleaseValidationError +from semapact.governance.gate import GovernanceOperation +from semapact.lifecycle.changes import GovernanceChange, governance_change_sort_key +from semapact.utils.deterministic import canonical_compact_json, deterministic_uuid5 +from semapact.versioning import ActualVersionBump, RequiredBump + + +SEMAPACT_CHANGESET_NAMESPACE = uuid.UUID("3ea0f6d8-28ca-4bb4-94f5-ea1f0f48cb84") +SEMAPACT_RELEASE_PLAN_NAMESPACE = uuid.UUID("7d2ad1de-c196-4f12-b1af-fdf79105eb04") +SEMAPACT_VERSION_RESOLUTION_NAMESPACE = uuid.UUID( + "4a2bd29d-2e44-44a0-9fc9-a1f4c81a2108" +) +SEMAPACT_CONTRACTOPS_AUTHORIZATION_NAMESPACE = uuid.UUID( + "b6218d0c-3f9d-44a2-8d68-e3b0ee170948" +) +SEMAPACT_APPLIED_RELEASE_NAMESPACE = uuid.UUID( + "a5de2e65-aee7-48ac-9cb6-6a785f4cdf33" +) +SEMAPACT_PUBLICATION_NAMESPACE = uuid.UUID( + "f776fc77-b37f-43ef-bf6d-d8dcf38d264f" +) + + +def validate_contractops_artifact_identity(artifact: object) -> None: + """Validate deterministic identity for self-describing ContractOps artifacts. + + Non-identity configuration/evidence models are intentionally ignored. This hook + is called by ``ContractOpsModel.model_post_init`` so persisted artifacts cannot + be rehydrated with IDs that do not match their immutable content. + """ + if isinstance(artifact, ChangeSet): + validate_change_set_identity(artifact) + elif isinstance(artifact, ReleasePlan): + validate_release_plan_identity(artifact) + elif isinstance(artifact, VersionResolution): + validate_version_resolution_identity(artifact) + elif isinstance(artifact, ContractOpsAuthorization): + validate_contractops_authorization_identity(artifact) + elif isinstance(artifact, AppliedContractRelease): + validate_applied_release_identity(artifact) + elif isinstance(artifact, PublicationResult): + validate_publication_result_identity(artifact) + + +def compute_change_set_id( + *, + contract_id: str, + base_revision_ref: str, + candidate_revision_ref: str, + changes: Sequence[GovernanceChange], + context: ChangeContext, + source: str | None, + actor_reference: str | None, +) -> str: + canonical_changes = tuple(sorted(tuple(changes), key=governance_change_sort_key)) + payload = { + "contract_id": contract_id, + "base_revision_ref": base_revision_ref, + "candidate_revision_ref": candidate_revision_ref, + "context": context.model_dump(mode="json"), + "changes": [change.model_dump(mode="json") for change in canonical_changes], + "source": source, + "actor_reference": actor_reference, + } + return deterministic_uuid5(SEMAPACT_CHANGESET_NAMESPACE, payload) + + +def validate_change_set_identity(change_set: ChangeSet) -> None: + canonical_changes = tuple(sorted(change_set.changes, key=governance_change_sort_key)) + if change_set.changes != canonical_changes: + raise ReleaseValidationError("ChangeSet changes are not in canonical order") + expected = compute_change_set_id( + contract_id=change_set.contract_id, + base_revision_ref=change_set.base_revision_ref, + candidate_revision_ref=change_set.candidate_revision_ref, + changes=change_set.changes, + context=change_set.context, + source=change_set.source, + actor_reference=change_set.actor_reference, + ) + _require_identity(change_set.change_set_id, expected, "ChangeSet") + + +def compute_release_plan_id( + *, + contract_id: str, + change_set_id: str, + decision_id: str, + release_revision_ref: str, + required_version_bump: RequiredBump, + preconditions: Sequence[ReleasePrecondition], +) -> str: + payload = { + "contract_id": contract_id, + "change_set_id": change_set_id, + "decision_id": decision_id, + "release_revision_ref": release_revision_ref, + "required_version_bump": required_version_bump, + "preconditions": [item.value for item in preconditions], + } + return deterministic_uuid5(SEMAPACT_RELEASE_PLAN_NAMESPACE, payload) + + +def validate_release_plan_identity(release_plan: ReleasePlan) -> None: + expected = compute_release_plan_id( + contract_id=release_plan.contract_id, + change_set_id=release_plan.change_set_id, + decision_id=release_plan.decision_id, + release_revision_ref=release_plan.release_revision_ref, + required_version_bump=release_plan.required_version_bump, + preconditions=release_plan.preconditions, + ) + _require_identity(release_plan.release_plan_id, expected, "ReleasePlan") + + +def compute_version_resolution_id( + *, + release_plan_id: str, + contract_id: str, + release_revision_ref: str, + authority: VersionAuthority, + current_version: str, + required_version_bump: RequiredBump, + selected_version: str, + actual_bump: ActualVersionBump, + authority_reference: str | None, +) -> str: + payload = { + "release_plan_id": release_plan_id, + "contract_id": contract_id, + "release_revision_ref": release_revision_ref, + "authority": authority.value, + "current_version": current_version, + "required_version_bump": required_version_bump, + "selected_version": selected_version, + "actual_bump": actual_bump, + "authority_reference": authority_reference, + } + return deterministic_uuid5(SEMAPACT_VERSION_RESOLUTION_NAMESPACE, payload) + + +def validate_version_resolution_identity(resolution: VersionResolution) -> None: + expected = compute_version_resolution_id( + release_plan_id=resolution.release_plan_id, + contract_id=resolution.contract_id, + release_revision_ref=resolution.release_revision_ref, + authority=resolution.authority, + current_version=resolution.current_version, + required_version_bump=resolution.required_version_bump, + selected_version=resolution.selected_version, + actual_bump=resolution.actual_bump, + authority_reference=resolution.authority_reference, + ) + _require_identity(resolution.version_resolution_id, expected, "VersionResolution") + + +def compute_contractops_authorization_id( + *, + decision_id: str, + change_set_id: str, + release_plan_id: str, + version_resolution_id: str, + operation: GovernanceOperation, + allowed: bool, + reason: str, + evidence_reference: str | None, + evidence_action: str | None, + scope_reference: str | None, +) -> str: + payload: dict[str, object] = { + "decision_id": decision_id, + "change_set_id": change_set_id, + "release_plan_id": release_plan_id, + "version_resolution_id": version_resolution_id, + "operation": operation.value, + "allowed": allowed, + "reason": reason, + "evidence_reference": evidence_reference, + "evidence_action": evidence_action, + } + # Compatibility invariant: scope_reference did not participate in pre-#122 IDs + # when it was absent. Keep that byte-for-byte behavior. + if scope_reference is not None: + payload["scope_reference"] = scope_reference + return deterministic_uuid5(SEMAPACT_CONTRACTOPS_AUTHORIZATION_NAMESPACE, payload) + + +def validate_contractops_authorization_identity( + authorization: ContractOpsAuthorization, +) -> None: + expected = compute_contractops_authorization_id( + decision_id=authorization.decision_id, + change_set_id=authorization.change_set_id, + release_plan_id=authorization.release_plan_id, + version_resolution_id=authorization.version_resolution_id, + operation=authorization.operation, + allowed=authorization.allowed, + reason=authorization.reason.value, + evidence_reference=authorization.evidence_reference, + evidence_action=( + authorization.evidence_action.value + if authorization.evidence_action is not None + else None + ), + scope_reference=authorization.scope_reference, + ) + _require_identity( + authorization.authorization_id, + expected, + "ContractOpsAuthorization", + ) + + +def compute_applied_release_id( + *, + contract_id: str, + decision_id: str, + change_set_id: str, + release_plan_id: str, + version_resolution_id: str, + release_revision_ref: str, + selected_version: str, + authorization_id: str, + released_contract_json: str, +) -> str: + payload = { + "contract_id": contract_id, + "decision_id": decision_id, + "change_set_id": change_set_id, + "release_plan_id": release_plan_id, + "version_resolution_id": version_resolution_id, + "release_revision_ref": release_revision_ref, + "selected_version": selected_version, + "authorization_id": authorization_id, + "released_contract_json": released_contract_json, + } + return deterministic_uuid5(SEMAPACT_APPLIED_RELEASE_NAMESPACE, payload) + + +def validate_applied_release_identity(release: AppliedContractRelease) -> None: + _require_canonical_json(release.released_contract_json, "AppliedContractRelease snapshot") + expected = compute_applied_release_id( + contract_id=release.contract_id, + decision_id=release.decision_id, + change_set_id=release.change_set_id, + release_plan_id=release.release_plan_id, + version_resolution_id=release.version_resolution_id, + release_revision_ref=release.release_revision_ref, + selected_version=release.selected_version, + authorization_id=release.authorization_id, + released_contract_json=release.released_contract_json, + ) + _require_identity(release.applied_release_id, expected, "AppliedContractRelease") + + +def compute_publication_id( + *, + applied_release_id: str, + authorization_id: str, + publication_reference: str, +) -> str: + return deterministic_uuid5( + SEMAPACT_PUBLICATION_NAMESPACE, + { + "applied_release_id": applied_release_id, + "authorization_id": authorization_id, + "publication_reference": publication_reference, + }, + ) + + +def validate_publication_result_identity(result: PublicationResult) -> None: + expected = compute_publication_id( + applied_release_id=result.applied_release_id, + authorization_id=result.authorization_id, + publication_reference=result.publication_reference, + ) + _require_identity(result.publication_id, expected, "PublicationResult") + + +def _require_identity(actual: str, expected: str, artifact: str) -> None: + if actual != expected: + raise ReleaseValidationError( + f"{artifact} deterministic identity does not match its content" + ) + + +def _require_canonical_json(value: str, artifact: str) -> None: + try: + parsed = json.loads(value) + except json.JSONDecodeError as exc: # defensive; model validation normally catches this + raise ReleaseValidationError(f"{artifact} is not valid JSON") from exc + if canonical_compact_json(parsed) != value: + raise ReleaseValidationError(f"{artifact} must use canonical compact JSON") diff --git a/semapact/contractops/models.py b/semapact/contractops/models.py index a2fa6d91..1135b1d0 100644 --- a/semapact/contractops/models.py +++ b/semapact/contractops/models.py @@ -3,6 +3,7 @@ from __future__ import annotations from enum import Enum +from typing import Any from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator @@ -17,6 +18,14 @@ class ContractOpsModel(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") + def model_post_init(self, __context: Any) -> None: + # Import lazily so the integrity module can depend on concrete ContractOps + # model classes without creating a module-import cycle. Identity-bearing + # artifacts therefore fail closed both on construction and rehydration. + from semapact.contractops.integrity import validate_contractops_artifact_identity + + validate_contractops_artifact_identity(self) + class ChangeSet(ContractOpsModel): """Deterministic proposal over exact governed contract revisions. diff --git a/semapact/contractops/release_plan.py b/semapact/contractops/release_plan.py index c4ba0abc..665b477e 100644 --- a/semapact/contractops/release_plan.py +++ b/semapact/contractops/release_plan.py @@ -2,9 +2,11 @@ from __future__ import annotations -import uuid - from semapact.contractops.context import validate_proposal_context +from semapact.contractops.integrity import ( + SEMAPACT_RELEASE_PLAN_NAMESPACE, + compute_release_plan_id, +) from semapact.contractops.models import ChangeSet, ReleasePlan, ReleasePrecondition from semapact.exceptions import ReleaseValidationError from semapact.governance.gate import ( @@ -13,10 +15,6 @@ evaluate_governance_gate, ) from semapact.governance.models import GovernanceDecision -from semapact.utils.deterministic import deterministic_uuid5 - - -SEMAPACT_RELEASE_PLAN_NAMESPACE = uuid.UUID("7d2ad1de-c196-4f12-b1af-fdf79105eb04") def build_release_plan( @@ -28,7 +26,7 @@ def build_release_plan( The function never re-evaluates contract changes, policy, or version classification. BLOCK decisions cannot be planned. REVIEW decisions remain REVIEW and carry an explicit authorization precondition for later ContractOps - authorization (#120). + authorization. """ if not isinstance(change_set, ChangeSet): raise TypeError( @@ -41,7 +39,7 @@ def build_release_plan( validate_proposal_context(decision, change_set) - # Release planning is a pure PROPOSE operation. Reuse the authoritative M0 gate + # Release planning is a pure PROPOSE operation. Reuse the authoritative gate # rather than duplicating ALLOW/REVIEW/BLOCK mapping in ContractOps. enforce_governance_gate(decision, GovernanceOperation.PROPOSE) @@ -54,19 +52,18 @@ def build_release_plan( elif publish_gate.reason == "review_required": preconditions = (ReleasePrecondition.REVIEW_AUTHORIZATION_REQUIRED,) else: - # PROPOSE already rejects BLOCK. Reaching this state would mean the M0 gate + # PROPOSE already rejects BLOCK. Reaching this state would mean the gate # returned internally inconsistent results for the same immutable decision. raise RuntimeError("Governance gate returned inconsistent release eligibility") - stable_record = { - "contract_id": change_set.contract_id, - "change_set_id": change_set.change_set_id, - "decision_id": decision.decision_id, - "release_revision_ref": change_set.candidate_revision_ref, - "required_version_bump": decision.required_version_bump, - "preconditions": [item.value for item in preconditions], - } - release_plan_id = deterministic_uuid5(SEMAPACT_RELEASE_PLAN_NAMESPACE, stable_record) + release_plan_id = compute_release_plan_id( + contract_id=change_set.contract_id, + change_set_id=change_set.change_set_id, + decision_id=decision.decision_id, + release_revision_ref=change_set.candidate_revision_ref, + required_version_bump=decision.required_version_bump, + preconditions=preconditions, + ) return ReleasePlan( release_plan_id=release_plan_id, diff --git a/semapact/contractops/version_authority.py b/semapact/contractops/version_authority.py index dddc2c5a..06035bd1 100644 --- a/semapact/contractops/version_authority.py +++ b/semapact/contractops/version_authority.py @@ -3,8 +3,11 @@ from __future__ import annotations import re -import uuid +from semapact.contractops.integrity import ( + SEMAPACT_VERSION_RESOLUTION_NAMESPACE, + compute_version_resolution_id, +) from semapact.contractops.models import ( ReleasePlan, VersionAuthority, @@ -12,7 +15,6 @@ VersionResolution, ) from semapact.exceptions import ReleaseValidationError -from semapact.utils.deterministic import deterministic_uuid5 from semapact.versioning import ( ActualVersionBump, RequiredBump, @@ -23,9 +25,6 @@ ) -SEMAPACT_VERSION_RESOLUTION_NAMESPACE = uuid.UUID( - "4a2bd29d-2e44-44a0-9fc9-a1f4c81a2108" -) _VERSION_TOKEN = "{version}" @@ -94,20 +93,16 @@ def resolve_release_version( else: # pragma: no cover - enum exhaustiveness guard raise RuntimeError(f"Unsupported version authority: {config.authority}") - stable_record = { - "release_plan_id": release_plan.release_plan_id, - "contract_id": release_plan.contract_id, - "release_revision_ref": release_plan.release_revision_ref, - "authority": config.authority.value, - "current_version": canonical_current, - "required_version_bump": release_plan.required_version_bump, - "selected_version": selected_version, - "actual_bump": actual_bump, - "authority_reference": normalized_reference, - } - version_resolution_id = deterministic_uuid5( - SEMAPACT_VERSION_RESOLUTION_NAMESPACE, - stable_record, + version_resolution_id = compute_version_resolution_id( + release_plan_id=release_plan.release_plan_id, + contract_id=release_plan.contract_id, + release_revision_ref=release_plan.release_revision_ref, + authority=config.authority, + current_version=canonical_current, + required_version_bump=release_plan.required_version_bump, + selected_version=selected_version, + actual_bump=actual_bump, + authority_reference=normalized_reference, ) return VersionResolution( diff --git a/semapact/deployment/authorization.py b/semapact/deployment/authorization.py index 6f475fdb..70f822e0 100644 --- a/semapact/deployment/authorization.py +++ b/semapact/deployment/authorization.py @@ -3,6 +3,10 @@ from __future__ import annotations from semapact.contractops.execution_models import AppliedContractRelease +from semapact.contractops.integrity import ( + validate_applied_release_identity, + validate_contractops_authorization_identity, +) from semapact.contractops.models import AuthorizationReason, ContractOpsAuthorization from semapact.deployment.models import ( DeploymentAuthorization, @@ -33,6 +37,8 @@ def authorize_deployment( ) validate_deployment_plan_identity(plan) + validate_applied_release_identity(release) + validate_contractops_authorization_identity(authorization) if authorization.operation is not GovernanceOperation.DEPLOY: raise ReleaseValidationError( diff --git a/semapact/deployment/models.py b/semapact/deployment/models.py index 857468ec..a32cefe8 100644 --- a/semapact/deployment/models.py +++ b/semapact/deployment/models.py @@ -45,10 +45,11 @@ class NativeOperationKind(str, Enum): class DeploymentTarget(DeploymentModel): - """Explicit runtime target for one DeploymentPlan.""" + """Exact runtime target for one DeploymentPlan, excluding credentials.""" platform: str runtime_target: str + source_reference: str server_name: str | None = None @field_validator("platform") @@ -59,12 +60,12 @@ def _normalize_platform(cls, value: str) -> str: raise ValueError("platform must not be empty") return cleaned - @field_validator("runtime_target") + @field_validator("runtime_target", "source_reference") @classmethod - def _normalize_runtime_target(cls, value: str) -> str: + def _normalize_required_target_text(cls, value: str) -> str: cleaned = value.strip() if not cleaned: - raise ValueError("runtime_target must not be empty") + raise ValueError("runtime target fields must not be empty") return cleaned @field_validator("server_name") @@ -128,7 +129,7 @@ class DeploymentPlan(DeploymentModel): selected_version: str target: DeploymentTarget actions: tuple[DeploymentAction, ...] - plan_version: Literal["1"] = "1" + plan_version: Literal["2"] = "2" @field_validator( "deployment_plan_id", @@ -252,7 +253,7 @@ def compute_deployment_plan_id( selected_version: str, target: DeploymentTarget, actions: Sequence[DeploymentAction], - plan_version: str = "1", + plan_version: str = "2", ) -> str: return deterministic_uuid5( SEMAPACT_DEPLOYMENT_PLAN_NAMESPACE, diff --git a/semapact/deployment/planner.py b/semapact/deployment/planner.py index 8aec8394..10894c1f 100644 --- a/semapact/deployment/planner.py +++ b/semapact/deployment/planner.py @@ -5,6 +5,7 @@ from open_data_contract_standard.model import SchemaObject from semapact.contractops.execution_models import AppliedContractRelease +from semapact.contractops.integrity import validate_applied_release_identity from semapact.deployment.models import ( DeploymentAction, DeploymentActionKind, @@ -35,6 +36,7 @@ def build_deployment_plan( f"target must be DeploymentTarget, got {type(target).__name__}" ) + validate_applied_release_identity(release) contract = release.to_contract() asset_specs = runtime_asset_specs_from_contract(contract) specs_by_asset = {spec.governed_asset: spec for spec in asset_specs} diff --git a/semapact/deployment/verification.py b/semapact/deployment/verification.py index 050dab7e..93185b57 100644 --- a/semapact/deployment/verification.py +++ b/semapact/deployment/verification.py @@ -51,6 +51,10 @@ def verify_deployment_convergence( raise ValidationError( "Observed runtime platform does not match DeploymentPlan platform" ) + if observation.source_identifier.strip() != plan.target.source_reference: + raise ValidationError( + "Observed runtime source does not match DeploymentPlan source reference" + ) return reconcile_governed_contract( desired_contract, diff --git a/semapact/governance/models.py b/semapact/governance/models.py index ecf2b309..00fad906 100644 --- a/semapact/governance/models.py +++ b/semapact/governance/models.py @@ -8,10 +8,10 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator from semapact.change_context import ChangeContext -from semapact.core.release import RequiredBump from semapact.governance_codes import GovernanceReasonCode, GovernanceSeverity from semapact.lifecycle.changes import GovernanceChange from semapact.lifecycle.policy import BreakingChange +from semapact.versioning import RequiredBump class DecisionResult(str, Enum): @@ -110,4 +110,3 @@ def _validate_allow_invariants(self) -> GovernanceDecision: f"ALLOW decision invariant violation: evidence.merge_conflicts_count must be 0, got {self.evidence.merge_conflicts_count}" ) return self - diff --git a/semapact/interfaces/cli.py b/semapact/interfaces/cli.py index 32791636..0aa135f7 100644 --- a/semapact/interfaces/cli.py +++ b/semapact/interfaces/cli.py @@ -371,6 +371,11 @@ def _build_parser() -> argparse.ArgumentParser: deployment_plan_parser.add_argument("--release", required=True) deployment_plan_parser.add_argument("--platform", required=True) deployment_plan_parser.add_argument("--runtime", required=True) + deployment_plan_parser.add_argument( + "--source-reference", + required=True, + help="Stable runtime source identity (for Databricks, the workspace host; never credentials)", + ) deployment_plan_parser.add_argument("--server") deployment_preview_parser = deployment_subparsers.add_parser( @@ -386,8 +391,7 @@ def _build_parser() -> argparse.ArgumentParser: deployment_execute_parser.add_argument("--authorization", required=True) deployment_execute_parser.add_argument( "--warehouse-id", - required=True, - help="Databricks SQL warehouse used only for runtime mutation", + help="Databricks SQL warehouse required only when preview contains mutation operations", ) deployment_verify_parser = deployment_subparsers.add_parser( diff --git a/semapact/interfaces/commands/deployment_cmd.py b/semapact/interfaces/commands/deployment_cmd.py index 1ccc72dc..f8adbaf3 100644 --- a/semapact/interfaces/commands/deployment_cmd.py +++ b/semapact/interfaces/commands/deployment_cmd.py @@ -46,6 +46,7 @@ def run_deployment_plan(args: argparse.Namespace) -> DeploymentCommandResult: target = DeploymentTarget( platform=args.platform, runtime_target=args.runtime, + source_reference=args.source_reference, server_name=args.server, ) plan = DeploymentService().plan(release, target) diff --git a/semapact/platforms/databricks/deployment.py b/semapact/platforms/databricks/deployment.py index d19d2dd0..58c8911c 100644 --- a/semapact/platforms/databricks/deployment.py +++ b/semapact/platforms/databricks/deployment.py @@ -208,6 +208,10 @@ def execute( raise ValidationError("DeploymentPreview platform does not match adapter") if preview.runtime_target != plan.target.runtime_target: raise ValidationError("DeploymentPreview target does not match DeploymentPlan") + if preview.source_identifier != plan.target.source_reference: + raise ValidationError( + "DeploymentPreview runtime source does not match DeploymentPlan source reference" + ) current = self._observe_plan_scope(plan) if current.source_identifier != preview.source_identifier: @@ -290,6 +294,10 @@ def _validate_observation( raise ValidationError("Databricks preview requires Databricks runtime evidence") if not observed_state.source_identifier.strip(): raise ValidationError("Runtime observation source_identifier is required") + if observed_state.source_identifier != plan.target.source_reference: + raise ValidationError( + "Runtime observation source does not match DeploymentPlan source reference" + ) if observed_state.fingerprint is None: raise ValidationError("Runtime observation fingerprint is required") if observed_state.fingerprint != fingerprint_observed_state(observed_state): diff --git a/tests/interfaces/test_deployment_cmd.py b/tests/interfaces/test_deployment_cmd.py index a7397446..0ed96f05 100644 --- a/tests/interfaces/test_deployment_cmd.py +++ b/tests/interfaces/test_deployment_cmd.py @@ -7,12 +7,15 @@ import pytest from semapact.contractops import AppliedContractRelease +from semapact.contractops.integrity import compute_applied_release_id from semapact.exceptions import ValidationError from semapact.interfaces import cli from semapact.interfaces.commands import deployment_cmd from semapact.interfaces.commands.deployment_cmd import DeploymentCommandResult from semapact.interfaces.outcomes import ProcessOutcome +SOURCE_REFERENCE = "https://workspace.example" + def _release() -> AppliedContractRelease: contract = OpenDataContractStandard( @@ -38,21 +41,25 @@ def _release() -> AppliedContractRelease: ) ], ) + released_contract_json = json.dumps( + contract.model_dump(mode="json", by_alias=True, exclude_none=True), + sort_keys=True, + separators=(",", ":"), + ) + fields = { + "contract_id": "orders-product", + "decision_id": "decision:test", + "change_set_id": "change-set:test", + "release_plan_id": "release-plan:test", + "version_resolution_id": "version-resolution:test", + "release_revision_ref": "rev:released", + "selected_version": "1.2.0", + "authorization_id": "authorization:test", + "released_contract_json": released_contract_json, + } return AppliedContractRelease( - applied_release_id="applied-release:test", - contract_id="orders-product", - decision_id="decision:test", - change_set_id="change-set:test", - release_plan_id="release-plan:test", - version_resolution_id="version-resolution:test", - release_revision_ref="rev:released", - selected_version="1.2.0", - authorization_id="authorization:test", - released_contract_json=json.dumps( - contract.model_dump(mode="json", by_alias=True, exclude_none=True), - sort_keys=True, - separators=(",", ":"), - ), + applied_release_id=compute_applied_release_id(**fields), + **fields, ) @@ -69,6 +76,8 @@ def test_deployment_parser_exposes_four_explicit_phases() -> None: "databricks", "--runtime", "main.silver", + "--source-reference", + SOURCE_REFERENCE, ] ) preview = parser.parse_args( @@ -84,8 +93,6 @@ def test_deployment_parser_exposes_four_explicit_phases() -> None: "preview.json", "--authorization", "authorization.json", - "--warehouse-id", - "warehouse-1", ] ) verify = parser.parse_args( @@ -93,17 +100,19 @@ def test_deployment_parser_exposes_four_explicit_phases() -> None: ) assert plan.deployment_command == "plan" + assert plan.source_reference == SOURCE_REFERENCE assert preview.deployment_command == "preview" assert not hasattr(preview, "warehouse_id") assert execute.deployment_command == "execute" - assert execute.warehouse_id == "warehouse-1" + assert execute.warehouse_id is None assert verify.deployment_command == "verify" assert verify.output == "json" def test_plan_command_outputs_canonical_deployment_plan(tmp_path) -> None: release_path = tmp_path / "release.json" - release_path.write_text(_release().model_dump_json(), encoding="utf-8") + release = _release() + release_path.write_text(release.model_dump_json(), encoding="utf-8") args = cli._build_parser().parse_args( [ "deployment", @@ -114,6 +123,8 @@ def test_plan_command_outputs_canonical_deployment_plan(tmp_path) -> None: "databricks", "--runtime", "main.silver", + "--source-reference", + SOURCE_REFERENCE, "--server", "production", ] @@ -123,10 +134,11 @@ def test_plan_command_outputs_canonical_deployment_plan(tmp_path) -> None: payload = json.loads(result.output) assert result.outcome is ProcessOutcome.SUCCESS - assert payload["applied_release_id"] == "applied-release:test" + assert payload["applied_release_id"] == release.applied_release_id assert payload["target"] == { "platform": "databricks", "runtime_target": "main.silver", + "source_reference": SOURCE_REFERENCE, "server_name": "production", } assert payload["actions"][0]["governed_asset"] == "orders" diff --git a/tests/test_architecture_consolidation.py b/tests/test_architecture_consolidation.py index e298b120..6a27e5ef 100644 --- a/tests/test_architecture_consolidation.py +++ b/tests/test_architecture_consolidation.py @@ -13,11 +13,11 @@ from semapact.change_context import ChangeContext from semapact.contractops import ( - AppliedContractRelease, AuthorizationReason, ReviewAuthorizationEvidence, ReviewEvidenceAction, VersionAuthorityConfig, + apply_contract_release, authorize_contract_operation, build_change_set_from_decision, build_release_plan, @@ -94,23 +94,31 @@ def _review_release_chain(): current_version="1.0.0", config=VersionAuthorityConfig(), ) - - released_contract = candidate.model_copy(deep=True) - released_contract.version = version_resolution.selected_version - released_json = canonical_compact_json( - released_contract.model_dump(mode="json", by_alias=True, exclude_none=True) - ) - release = AppliedContractRelease( - applied_release_id="applied:review-test", - contract_id=release_plan.contract_id, + apply_evidence = ReviewAuthorizationEvidence( + evidence_reference="approval:apply", decision_id=decision.decision_id, change_set_id=change_set.change_set_id, release_plan_id=release_plan.release_plan_id, version_resolution_id=version_resolution.version_resolution_id, - release_revision_ref=release_plan.release_revision_ref, - selected_version=version_resolution.selected_version, - authorization_id="authorization:apply-test", - released_contract_json=released_json, + operation=GovernanceOperation.APPLY, + action=ReviewEvidenceAction.APPROVE, + ) + apply_authorization = authorize_contract_operation( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.APPLY, + evidence=apply_evidence, + ) + release = apply_contract_release( + candidate, + candidate_revision_ref="rev:candidate", + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + authorization=apply_authorization, ) return decision, change_set, release_plan, version_resolution, release @@ -149,7 +157,11 @@ def test_review_deploy_authorization_is_bound_to_exact_deployment_plan() -> None decision, change_set, release_plan, version_resolution, release = _review_release_chain() production_plan = build_deployment_plan( release, - DeploymentTarget(platform="databricks", runtime_target="main.production"), + DeploymentTarget( + platform="databricks", + runtime_target="main.production", + source_reference="https://production-workspace.example", + ), ) evidence = ReviewAuthorizationEvidence( @@ -185,17 +197,47 @@ def test_review_deploy_authorization_is_bound_to_exact_deployment_plan() -> None staging_plan = build_deployment_plan( release, - DeploymentTarget(platform="databricks", runtime_target="main.staging"), + DeploymentTarget( + platform="databricks", + runtime_target="main.staging", + source_reference="https://staging-workspace.example", + ), ) with pytest.raises(ReleaseValidationError, match="not scoped to this DeploymentPlan"): authorize_deployment(staging_plan, release, contractops_authorization) +def test_same_runtime_namespace_on_another_source_requires_distinct_plan() -> None: + _, _, _, _, release = _review_release_chain() + workspace_a = build_deployment_plan( + release, + DeploymentTarget( + platform="databricks", + runtime_target="main.production", + source_reference="https://workspace-a.example", + ), + ) + workspace_b = build_deployment_plan( + release, + DeploymentTarget( + platform="databricks", + runtime_target="main.production", + source_reference="https://workspace-b.example", + ), + ) + + assert workspace_a.deployment_plan_id != workspace_b.deployment_plan_id + + def test_publish_authorization_cannot_be_reused_for_runtime_deploy() -> None: decision, change_set, release_plan, version_resolution, release = _review_release_chain() plan = build_deployment_plan( release, - DeploymentTarget(platform="databricks", runtime_target="main.production"), + DeploymentTarget( + platform="databricks", + runtime_target="main.production", + source_reference="https://production-workspace.example", + ), ) evidence = ReviewAuthorizationEvidence( evidence_reference="approval:publish", diff --git a/tests/test_contractops_artifact_rehydration.py b/tests/test_contractops_artifact_rehydration.py new file mode 100644 index 00000000..64d1690b --- /dev/null +++ b/tests/test_contractops_artifact_rehydration.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +import json +from datetime import date + +import pytest +from open_data_contract_standard.model import ( + OpenDataContractStandard, + SchemaObject, + SchemaProperty, +) +from pydantic import ValidationError as PydanticValidationError + +from semapact.change_context import ChangeContext +from semapact.contractops import ( + AppliedContractRelease, + ChangeSet, + ContractOpsAuthorization, + PublicationResult, + ReleasePlan, + VersionAuthorityConfig, + VersionResolution, + apply_contract_release, + authorize_contract_operation, + build_change_set_from_decision, + build_release_plan, + publish_contract_release, + resolve_release_version, +) +from semapact.governance import evaluate_governance_decision +from semapact.governance.gate import GovernanceOperation + + +CONTEXT = ChangeContext(effective_date=date(2026, 9, 11)) + + +def _contract(*, name: str) -> OpenDataContractStandard: + return OpenDataContractStandard( + apiVersion="v3.1.0", + kind="DataContract", + id="orders-product", + name=name, + version="1.0.0", + status="active", + schema=[ + SchemaObject( + name="orders", + properties=[ + SchemaProperty( + name="id", + logicalType="string", + physicalType="varchar(255)", + required=True, + ) + ], + ) + ], + ) + + +class _Publisher: + def publish(self, release: AppliedContractRelease) -> str: + return f"registry:{release.applied_release_id}" + + +def _artifact_chain(): + base = _contract(name="orders-old") + candidate = _contract(name="orders-new") + decision = evaluate_governance_decision(base, candidate, context=CONTEXT) + change_set = build_change_set_from_decision( + decision, + base_revision_ref="git:base", + candidate_revision_ref="git:candidate", + source="rehydration-test", + actor_reference="service:ci", + ) + release_plan = build_release_plan(change_set, decision) + version_resolution = resolve_release_version( + release_plan, + current_version="1.0.0", + config=VersionAuthorityConfig(), + ) + apply_authorization = authorize_contract_operation( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.APPLY, + ) + applied_release = apply_contract_release( + candidate, + candidate_revision_ref="git:candidate", + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + authorization=apply_authorization, + ) + publish_authorization = authorize_contract_operation( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.PUBLISH, + ) + publication = publish_contract_release( + applied_release, + authorization=publish_authorization, + publisher=_Publisher(), + ) + return ( + change_set, + release_plan, + version_resolution, + apply_authorization, + applied_release, + publication, + ) + + +def test_identity_bearing_artifacts_round_trip_through_json() -> None: + artifacts = _artifact_chain() + + for artifact in artifacts: + rehydrated = type(artifact).model_validate_json(artifact.model_dump_json()) + assert rehydrated == artifact + + +def test_rehydration_rejects_content_with_stale_deterministic_identity() -> None: + ( + change_set, + release_plan, + version_resolution, + authorization, + applied_release, + publication, + ) = _artifact_chain() + + cases = ( + (ChangeSet, change_set, "candidate_revision_ref", "git:other"), + (ReleasePlan, release_plan, "release_revision_ref", "git:other"), + (VersionResolution, version_resolution, "release_revision_ref", "git:other"), + (ContractOpsAuthorization, authorization, "decision_id", "decision:other"), + (AppliedContractRelease, applied_release, "decision_id", "decision:other"), + (PublicationResult, publication, "publication_reference", "registry:other"), + ) + + for model, artifact, field, tampered_value in cases: + payload = artifact.model_dump(mode="json") + payload[field] = tampered_value + persisted_json = json.dumps(payload, separators=(",", ":"), sort_keys=True) + with pytest.raises(PydanticValidationError, match="deterministic identity"): + model.model_validate_json(persisted_json) diff --git a/tests/test_contractops_authorization.py b/tests/test_contractops_authorization.py index 04e4ddb9..91f41b49 100644 --- a/tests/test_contractops_authorization.py +++ b/tests/test_contractops_authorization.py @@ -21,6 +21,7 @@ build_release_plan, resolve_release_version, ) +from semapact.contractops.integrity import compute_release_plan_id from semapact.exceptions import ReleaseValidationError from semapact.governance import DecisionResult, evaluate_governance_decision from semapact.governance.gate import GovernanceOperation @@ -84,16 +85,20 @@ def _release_context(kind: str): ) if decision.decision is DecisionResult.BLOCK: - # BLOCK cannot normally produce a ReleasePlan. Constructing an internally - # associated plan here proves that explicit review evidence still cannot - # override the authoritative M0 BLOCK result. + # BLOCK cannot be emitted by the canonical planner. A structurally valid + # associated artifact is built only to preserve the authorization contract: + # explicit review evidence still cannot override authoritative BLOCK. + fields = { + "contract_id": change_set.contract_id, + "change_set_id": change_set.change_set_id, + "decision_id": decision.decision_id, + "release_revision_ref": change_set.candidate_revision_ref, + "required_version_bump": decision.required_version_bump, + "preconditions": (), + } release_plan = ReleasePlan( - release_plan_id="release-plan-block-test", - contract_id=change_set.contract_id, - change_set_id=change_set.change_set_id, - decision_id=decision.decision_id, - release_revision_ref=change_set.candidate_revision_ref, - required_version_bump=decision.required_version_bump, + release_plan_id=compute_release_plan_id(**fields), + **fields, ) else: release_plan = build_release_plan(change_set, decision) @@ -282,14 +287,17 @@ def test_block_cannot_be_overridden_by_approval_evidence() -> None: def test_invalid_release_context_fails_closed_before_authorization() -> None: decision, change_set, release_plan, version_resolution = _release_context("review") + fields = { + "contract_id": release_plan.contract_id, + "change_set_id": "change-set:other", + "decision_id": release_plan.decision_id, + "release_revision_ref": release_plan.release_revision_ref, + "required_version_bump": release_plan.required_version_bump, + "preconditions": release_plan.preconditions, + } invalid_plan = ReleasePlan( - release_plan_id=release_plan.release_plan_id, - contract_id=release_plan.contract_id, - change_set_id="change-set:other", - decision_id=release_plan.decision_id, - release_revision_ref=release_plan.release_revision_ref, - required_version_bump=release_plan.required_version_bump, - preconditions=release_plan.preconditions, + release_plan_id=compute_release_plan_id(**fields), + **fields, ) with pytest.raises(ReleaseValidationError, match="supplied ChangeSet"): diff --git a/tests/test_contractops_golden_scenarios.py b/tests/test_contractops_golden_scenarios.py index 9d0f850e..1527b947 100644 --- a/tests/test_contractops_golden_scenarios.py +++ b/tests/test_contractops_golden_scenarios.py @@ -173,6 +173,7 @@ def _allow_chain(): DeploymentTarget( platform="databricks", runtime_target="main.silver", + source_reference="workspace:golden", server_name="production", ), ) @@ -360,6 +361,7 @@ def test_allow_chain_has_stable_cross_boundary_golden_semantics() -> None: "target": { "platform": "databricks", "runtime_target": "main.silver", + "source_reference": "workspace:golden", "server_name": "production", }, "actions": [ @@ -443,7 +445,11 @@ def test_review_requires_exact_apply_and_deployment_authorization() -> None: ) plan = build_deployment_plan( release, - DeploymentTarget(platform="databricks", runtime_target="main.silver"), + DeploymentTarget( + platform="databricks", + runtime_target="main.silver", + source_reference="workspace:golden", + ), ) unscoped_deploy_evidence = _review_evidence( diff --git a/tests/test_deployment_databricks.py b/tests/test_deployment_databricks.py index a70744b1..e72c0b07 100644 --- a/tests/test_deployment_databricks.py +++ b/tests/test_deployment_databricks.py @@ -32,6 +32,7 @@ from semapact.platforms.databricks.deployment import DatabricksDeploymentAdapter CAPTURED_AT = datetime(2026, 9, 10, 5, 0, tzinfo=timezone.utc) +SOURCE_REFERENCE = "workspace-a" def _property(name: str, physical_type: str, *, required: bool = False) -> SchemaProperty: @@ -44,7 +45,7 @@ def _property(name: str, physical_type: str, *, required: bool = False) -> Schem ) -def _plan(*properties: SchemaProperty) -> DeploymentPlan: +def _plan(*properties: SchemaProperty, source_reference: str = SOURCE_REFERENCE) -> DeploymentPlan: schema = SchemaObject( name="orders", physicalName="orders", @@ -61,7 +62,11 @@ def _plan(*properties: SchemaProperty) -> DeploymentPlan: separators=(",", ":"), ), ) - target = DeploymentTarget(platform="databricks", runtime_target="main.silver") + target = DeploymentTarget( + platform="databricks", + runtime_target="main.silver", + source_reference=source_reference, + ) plan_id = compute_deployment_plan_id( applied_release_id="applied:test", contract_id="orders-product", @@ -102,7 +107,7 @@ def _authorization(plan: DeploymentPlan, allowed: bool = True) -> DeploymentAuth def _state( *columns: tuple[str, str, bool], asset_type: str = "MANAGED", - source: str = "workspace-a", + source: str = SOURCE_REFERENCE, present: bool = True, ) -> ObservedPlatformState: identity = ObservedAssetIdentity( @@ -179,13 +184,13 @@ def __init__(self) -> None: self.statement_execution = _Statements() -def _adapter(state: ObservedPlatformState): +def _adapter(state: ObservedPlatformState, *, warehouse_id: str | None = "warehouse-1"): client = _Client() provider = _Provider(state) adapter = DatabricksDeploymentAdapter( client=client, runtime_provider=provider, - warehouse_id="warehouse-1", + warehouse_id=warehouse_id, poll_interval_seconds=0, ) return adapter, provider, client @@ -263,9 +268,18 @@ def test_non_managed_asset_and_unsafe_type_fail_closed() -> None: adapter.validate(malicious) +def test_preview_rejects_cross_source_runtime_evidence() -> None: + plan = _plan(_property("id", "BIGINT", required=True)) + other_workspace = _state(("id", "bigint", False), source="workspace-b") + adapter, _, _ = _adapter(other_workspace) + + with pytest.raises(ValidationError, match="source reference"): + adapter.preview(plan, other_workspace) + + def test_execute_fails_closed_for_denied_stale_and_cross_source() -> None: plan = _plan(_property("id", "BIGINT", required=True)) - before = _state(("id", "bigint", False), source="workspace-a") + before = _state(("id", "bigint", False)) adapter, provider, _ = _adapter(before) preview = adapter.preview(plan, before) @@ -275,7 +289,6 @@ def test_execute_fails_closed_for_denied_stale_and_cross_source() -> None: provider.state = _state( ("id", "bigint", False), ("other", "string", True), - source="workspace-a", ) with pytest.raises(ValidationError, match="Runtime state changed"): adapter.execute(plan, preview, _authorization(plan)) @@ -332,3 +345,35 @@ def test_execute_runs_exact_preview_statement() -> None: assert client.statement_execution.calls == [ "ALTER TABLE `main`.`silver`.`orders` ADD COLUMNS (`note` STRING)" ] + + +def test_no_op_execute_does_not_require_warehouse() -> None: + plan = _plan(_property("id", "BIGINT", required=True)) + current = _state(("id", "bigint", False)) + adapter, _, client = _adapter(current, warehouse_id=None) + preview = adapter.preview(plan, current) + + assert preview.operations[0].kind is NativeOperationKind.NO_OP + adapter.execute(plan, preview, _authorization(plan)) + assert client.statement_execution.calls == [] + + +def test_mutation_execute_without_warehouse_fails_closed() -> None: + plan = _plan( + _property("id", "BIGINT", required=True), + _property("note", "STRING"), + ) + current = _state(("id", "bigint", False)) + adapter, _, client = _adapter(current, warehouse_id=None) + preview = adapter.preview(plan, current) + + with pytest.raises(ValidationError, match="warehouse_id"): + adapter.execute(plan, preview, _authorization(plan)) + assert client.statement_execution.calls == [] + + +def test_runtime_source_participates_in_plan_identity() -> None: + plan_a = _plan(_property("id", "BIGINT", required=True), source_reference="workspace-a") + plan_b = _plan(_property("id", "BIGINT", required=True), source_reference="workspace-b") + + assert plan_a.deployment_plan_id != plan_b.deployment_plan_id diff --git a/tests/test_deployment_plan.py b/tests/test_deployment_plan.py index 05e705c4..f044f8ed 100644 --- a/tests/test_deployment_plan.py +++ b/tests/test_deployment_plan.py @@ -11,6 +11,7 @@ from pydantic import ValidationError as PydanticValidationError from semapact.contractops import AppliedContractRelease +from semapact.contractops.integrity import compute_applied_release_id from semapact.deployment import ( DeploymentAction, DeploymentActionKind, @@ -41,7 +42,6 @@ def _schema( def _release( *, schemas: list[SchemaObject] | None = None, - applied_release_id: str = "applied-release:test", ) -> AppliedContractRelease: contract = OpenDataContractStandard( apiVersion="v3.1.0", @@ -58,17 +58,20 @@ def _release( separators=(",", ":"), ensure_ascii=False, ) + fields = { + "contract_id": "orders-product", + "decision_id": "decision:test", + "change_set_id": "change-set:test", + "release_plan_id": "release-plan:test", + "version_resolution_id": "version-resolution:test", + "release_revision_ref": "rev:released", + "selected_version": "1.2.0", + "authorization_id": "authorization:test", + "released_contract_json": released_contract_json, + } return AppliedContractRelease( - applied_release_id=applied_release_id, - contract_id="orders-product", - decision_id="decision:test", - change_set_id="change-set:test", - release_plan_id="release-plan:test", - version_resolution_id="version-resolution:test", - release_revision_ref="rev:released", - selected_version="1.2.0", - authorization_id="authorization:test", - released_contract_json=released_contract_json, + applied_release_id=compute_applied_release_id(**fields), + **fields, ) @@ -76,6 +79,7 @@ def _target() -> DeploymentTarget: return DeploymentTarget( platform="Databricks", runtime_target="main.analytics", + source_reference="https://workspace.example", server_name="production", ) @@ -102,7 +106,8 @@ def test_plan_preserves_exact_applied_release_provenance() -> None: assert plan.release_plan_id == release.release_plan_id assert plan.released_revision_ref == release.release_revision_ref assert plan.selected_version == release.selected_version - assert plan.plan_version == "1" + assert plan.plan_version == "2" + assert plan.target.source_reference == "https://workspace.example" def test_actions_are_provider_neutral_ensure_state_intents() -> None: @@ -154,6 +159,7 @@ def test_target_is_explicit_and_changes_plan_identity() -> None: DeploymentTarget( platform="databricks", runtime_target="main.staging", + source_reference="https://staging-workspace.example", server_name="staging", ), ) @@ -163,14 +169,28 @@ def test_target_is_explicit_and_changes_plan_identity() -> None: assert staging.target.runtime_target == "main.staging" +def test_runtime_source_changes_plan_identity_for_same_runtime_target() -> None: + release = _release() + first = build_deployment_plan(release, _target()) + second = build_deployment_plan( + release, + DeploymentTarget( + platform="databricks", + runtime_target="main.analytics", + source_reference="https://other-workspace.example", + server_name="production", + ), + ) + + assert first.deployment_plan_id != second.deployment_plan_id + + def test_schema_order_is_canonicalized_within_each_exact_release() -> None: first_release = _release( schemas=[_schema("zeta"), _schema("alpha")], - applied_release_id="applied-release:first", ) second_release = _release( schemas=[_schema("alpha"), _schema("zeta")], - applied_release_id="applied-release:second", ) first = build_deployment_plan(first_release, _target()) @@ -178,8 +198,8 @@ def test_schema_order_is_canonicalized_within_each_exact_release() -> None: assert [action.governed_asset for action in first.actions] == ["alpha", "zeta"] assert [action.governed_asset for action in second.actions] == ["alpha", "zeta"] - # Exact release identity remains authoritative; #115 does not collapse two - # distinct AppliedContractRelease artifacts into one plan identity. + # Exact release snapshot identity remains authoritative; two releases with + # differently serialized source schema order remain distinct authorities. assert first.deployment_plan_id != second.deployment_plan_id diff --git a/tests/test_deployment_service.py b/tests/test_deployment_service.py index c0509140..7b904a97 100644 --- a/tests/test_deployment_service.py +++ b/tests/test_deployment_service.py @@ -34,6 +34,7 @@ from semapact.services.deployment_service import DeploymentService CAPTURED_AT = datetime(2026, 9, 10, 10, 0, tzinfo=timezone.utc) +SOURCE_REFERENCE = "https://adb.example" class FakeRuntimeProvider: @@ -109,7 +110,11 @@ def _plan() -> DeploymentPlan: physical_name="orders_v2", desired_state_json=schema.model_dump_json(by_alias=True, exclude_none=True), ) - target = DeploymentTarget(platform="databricks", runtime_target="main.silver") + target = DeploymentTarget( + platform="databricks", + runtime_target="main.silver", + source_reference=SOURCE_REFERENCE, + ) plan_id = compute_deployment_plan_id( applied_release_id="release-1", contract_id="orders-contract", @@ -131,7 +136,7 @@ def _plan() -> DeploymentPlan: ) -def _observation() -> ObservedPlatformState: +def _observation(*, source: str = SOURCE_REFERENCE) -> ObservedPlatformState: asset_identity = ObservedAssetIdentity( platform="databricks", namespace=("main", "silver"), @@ -140,7 +145,7 @@ def _observation() -> ObservedPlatformState: return with_observed_state_fingerprint( ObservedPlatformState( platform="databricks", - source_identifier="https://adb.example", + source_identifier=source, captured_at=CAPTURED_AT, assets=( ObservedAsset( @@ -223,6 +228,23 @@ def test_preview_orchestrates_observation_without_execution() -> None: assert adapter.execute_calls == 0 +def test_preview_rejects_runtime_source_mismatch() -> None: + plan = _plan() + observation = _observation(source="https://other-workspace.example") + provider = FakeRuntimeProvider(observation) + adapter = FakeDeploymentAdapter(_preview(plan, observation)) + + with pytest.raises(ValidationError, match="source reference"): + DeploymentService().preview( + plan, + runtime_provider=provider, + adapter=adapter, + ) + + assert provider.observe_calls == 1 + assert adapter.preview_calls == 0 + + def test_execute_delegates_exact_canonical_artifacts() -> None: plan = _plan() observation = _observation() diff --git a/tests/test_deployment_verification.py b/tests/test_deployment_verification.py index 78970ba3..c01a2bad 100644 --- a/tests/test_deployment_verification.py +++ b/tests/test_deployment_verification.py @@ -26,6 +26,7 @@ from semapact.reconciliation import RuntimeDriftStatus, classify_reconciliation_status CAPTURED_AT = datetime(2026, 9, 10, 7, 0, tzinfo=timezone.utc) +SOURCE_REFERENCE = "https://adb.example" class FakeRuntimeProvider: @@ -75,7 +76,11 @@ def _plan() -> DeploymentPlan: physical_name="orders_v2", desired_state_json=schema.model_dump_json(by_alias=True, exclude_none=True), ) - target = DeploymentTarget(platform="databricks", runtime_target="main.silver") + target = DeploymentTarget( + platform="databricks", + runtime_target="main.silver", + source_reference=SOURCE_REFERENCE, + ) plan_id = compute_deployment_plan_id( applied_release_id="release-1", contract_id="orders-contract", @@ -101,6 +106,7 @@ def _observation( *, physical_type: str | None = "BIGINT", nullable: bool | None = False, + source_identifier: str = SOURCE_REFERENCE, ) -> ObservedPlatformState: asset_identity = ObservedAssetIdentity( platform="databricks", @@ -109,7 +115,7 @@ def _observation( ) state = ObservedPlatformState( platform="databricks", - source_identifier="https://adb.example", + source_identifier=source_identifier, captured_at=CAPTURED_AT, assets=( ObservedAsset( @@ -178,6 +184,15 @@ def test_provider_platform_mismatch_fails_closed() -> None: verify_deployment_convergence(_plan(), provider) +def test_runtime_source_mismatch_fails_closed() -> None: + provider = FakeRuntimeProvider( + _observation(source_identifier="https://other-workspace.example") + ) + + with pytest.raises(ValidationError, match="source reference"): + verify_deployment_convergence(_plan(), provider) + + def test_tampered_deployment_plan_identity_fails_closed() -> None: plan = _plan().model_copy(update={"selected_version": "9.9.9"}) diff --git a/tests/test_version_authority.py b/tests/test_version_authority.py index 5e327138..22021309 100644 --- a/tests/test_version_authority.py +++ b/tests/test_version_authority.py @@ -10,23 +10,27 @@ extract_version_from_release_reference, resolve_release_version, ) -from semapact.core.release import ActualVersionBump, RequiredBump +from semapact.contractops.integrity import compute_release_plan_id from semapact.exceptions import ReleaseValidationError +from semapact.versioning import ActualVersionBump, RequiredBump def _plan( required_bump: RequiredBump, *, contract_id: str = "orders-product", - release_plan_id: str = "release-plan-1", ) -> ReleasePlan: + fields = { + "contract_id": contract_id, + "change_set_id": f"change-set:{contract_id}", + "decision_id": f"decision:{contract_id}", + "release_revision_ref": "rev:candidate", + "required_version_bump": required_bump, + "preconditions": (), + } return ReleasePlan( - release_plan_id=release_plan_id, - contract_id=contract_id, - change_set_id="change-set-1", - decision_id="decision-1", - release_revision_ref="rev:candidate", - required_version_bump=required_bump, + release_plan_id=compute_release_plan_id(**fields), + **fields, ) @@ -59,12 +63,12 @@ def test_semapact_authority_selects_smallest_valid_release_version( def test_semapact_authority_versions_contracts_independently() -> None: orders = resolve_release_version( - _plan("minor", contract_id="orders", release_plan_id="release-orders"), + _plan("minor", contract_id="orders"), current_version="1.2.3", config=VersionAuthorityConfig(), ) customers = resolve_release_version( - _plan("none", contract_id="customers", release_plan_id="release-customers"), + _plan("none", contract_id="customers"), current_version="4.7.2", config=VersionAuthorityConfig(), ) diff --git a/tests/test_version_authority_service.py b/tests/test_version_authority_service.py index 7b5bb904..793f3f8c 100644 --- a/tests/test_version_authority_service.py +++ b/tests/test_version_authority_service.py @@ -3,10 +3,11 @@ import pytest from semapact.contractops import ReleasePlan, VersionAuthority +from semapact.contractops.integrity import compute_release_plan_id from semapact.core.config import ConfigManager -from semapact.core.release import RequiredBump from semapact.exceptions import ReleaseValidationError, ValidationError from semapact.services import VersionAuthorityService +from semapact.versioning import RequiredBump @pytest.fixture(autouse=True) @@ -29,13 +30,17 @@ def _config(data: dict[str, object] | None = None) -> ConfigManager: def _plan(required_bump: RequiredBump = "minor") -> ReleasePlan: + fields = { + "contract_id": "orders-product", + "change_set_id": "change-set-1", + "decision_id": "decision-1", + "release_revision_ref": "rev:candidate", + "required_version_bump": required_bump, + "preconditions": (), + } return ReleasePlan( - release_plan_id="release-plan-1", - contract_id="orders-product", - change_set_id="change-set-1", - decision_id="decision-1", - release_revision_ref="rev:candidate", - required_version_bump=required_bump, + release_plan_id=compute_release_plan_id(**fields), + **fields, ) From 9d6f0a504a8bd000d1fca7a51dd35965b16f6f78 Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Sat, 12 Sep 2026 09:11:55 +1000 Subject: [PATCH 16/35] feat(history): add immutable governance repository foundation (#220) * feat(history): add repository boundary * feat(history): define governance history repository * feat(history): add governance history service * feat(history): add git history adapter package * feat(history): implement immutable git working-tree repository * test(history): cover immutable git repository semantics * docs(history): document persistence boundary * refactor(history): segregate typed persistence ports * refactor(history): export narrow repository capabilities * refactor(history): remove pass-through history service * test(history): exercise segregated persistence ports * docs(history): document segregated repository ports * docs(history): clarify shared Git storage kernel * docs(history): refine persistence layering --- docs/governance_history.md | 66 +++++ semapact/history/__init__.py | 23 ++ semapact/history/repository.py | 56 ++++ semapact/platforms/git/__init__.py | 5 + semapact/platforms/git/history_repository.py | 270 +++++++++++++++++++ tests/test_history_repository.py | 149 ++++++++++ 6 files changed, 569 insertions(+) create mode 100644 docs/governance_history.md create mode 100644 semapact/history/__init__.py create mode 100644 semapact/history/repository.py create mode 100644 semapact/platforms/git/__init__.py create mode 100644 semapact/platforms/git/history_repository.py create mode 100644 tests/test_history_repository.py diff --git a/docs/governance_history.md b/docs/governance_history.md new file mode 100644 index 00000000..50eab6ec --- /dev/null +++ b/docs/governance_history.md @@ -0,0 +1,66 @@ +# Governance history persistence + +SemaPact persists governance history without moving domain authority into storage. +Canonical artifacts remain owned by their existing domains; history adapters only +store and retrieve those exact immutable models. + +The persistence boundary is capability-oriented: + +```text +GovernanceDecision ChangeSet + ↓ ↓ +DecisionHistoryRepository ChangeSetHistoryRepository + \ / + \ / + GitWorkingTreeHistoryRepository + ↓ + .semapact/history/ +``` + +Callers depend only on the narrow typed repository capability they need. A physical +backend may implement several repository protocols through one shared storage kernel. +This avoids a single growing history interface while still allowing Git, SQLite, and +Delta implementations to share backend-specific mechanics. + +The Git working-tree adapter writes deterministic JSON files into a SemaPact-owned +path. It does not create Git commits, branches, or pull requests. Normal GitOps +workflows may version those files after SemaPact writes them. + +No application service is introduced merely to proxy repository methods. A history +query/application service belongs above these ports only when a use case actually +coordinates multiple artifact types, for example evolution-chain reconstruction or a +cross-artifact timeline. + +## Persistence semantics + +For supported artifacts: + +- writing identical content under the same artifact ID is idempotent; +- writing different content under an existing artifact ID fails closed; +- reads rehydrate and validate the canonical domain model; +- the embedded artifact ID must match the requested/file identity; +- malformed or invalid persisted content fails closed; +- missing IDs produce an explicit history not-found error; +- contract-scoped listings are deterministic. + +`ChangeContext` remains part of `ChangeSet` and round-trips with it. History state is +not written into canonical ODCS contracts. + +M2 deterministic identities remain authoritative. Persistence does not generate a +replacement identity and does not reinterpret lifecycle, governance, version, +authorization, deployment, or reconciliation semantics. + +## Backend extension rule + +Logical ports remain typed by domain artifact. Physical representation belongs to the +backend adapter: + +```text +Git → canonical JSON files +SQLite → relational rows/schema +Delta → append-oriented Delta rows/schema +``` + +Backend-specific row/file models must not replace or leak into the canonical domain +artifacts. Future artifact types add their own narrow persistence capability rather +than expanding one universal repository interface. diff --git a/semapact/history/__init__.py b/semapact/history/__init__.py new file mode 100644 index 00000000..65070d9a --- /dev/null +++ b/semapact/history/__init__.py @@ -0,0 +1,23 @@ +"""Durable governance-history boundary. + +History persists canonical domain artifacts without becoming a second source of +governance, ContractOps, deployment, or reconciliation semantics. +""" + +from semapact.history.repository import ( + ChangeSetHistoryRepository, + DecisionHistoryRepository, + HistoryConflictError, + HistoryCorruptionError, + HistoryNotFoundError, + HistoryRepositoryError, +) + +__all__ = [ + "ChangeSetHistoryRepository", + "DecisionHistoryRepository", + "HistoryConflictError", + "HistoryCorruptionError", + "HistoryNotFoundError", + "HistoryRepositoryError", +] diff --git a/semapact/history/repository.py b/semapact/history/repository.py new file mode 100644 index 00000000..31d96bed --- /dev/null +++ b/semapact/history/repository.py @@ -0,0 +1,56 @@ +"""Storage-neutral typed persistence ports for governance history.""" + +from __future__ import annotations + +from typing import Protocol + +from semapact.contractops import ChangeSet +from semapact.governance import GovernanceDecision + + +class HistoryRepositoryError(RuntimeError): + """Base error for durable governance-history access.""" + + +class HistoryNotFoundError(HistoryRepositoryError): + """Requested historical artifact does not exist.""" + + +class HistoryConflictError(HistoryRepositoryError): + """An immutable artifact ID already exists with different content.""" + + +class HistoryCorruptionError(HistoryRepositoryError): + """Persisted or supplied historical content fails canonical validation.""" + + +class DecisionHistoryRepository(Protocol): + """Persistence capability for canonical GovernanceDecision history only.""" + + def put_decision(self, decision: GovernanceDecision) -> None: + """Persist one immutable GovernanceDecision idempotently.""" + ... + + def get_decision(self, decision_id: str) -> GovernanceDecision: + """Load one GovernanceDecision by its exact artifact ID.""" + ... + + def list_decisions(self, contract_id: str) -> tuple[GovernanceDecision, ...]: + """List decisions for one contract in deterministic artifact-ID order.""" + ... + + +class ChangeSetHistoryRepository(Protocol): + """Persistence capability for canonical ChangeSet history only.""" + + def put_change_set(self, change_set: ChangeSet) -> None: + """Persist one immutable ChangeSet idempotently.""" + ... + + def get_change_set(self, change_set_id: str) -> ChangeSet: + """Load one ChangeSet by its exact artifact ID.""" + ... + + def list_change_sets(self, contract_id: str) -> tuple[ChangeSet, ...]: + """List ChangeSets for one contract in deterministic artifact-ID order.""" + ... diff --git a/semapact/platforms/git/__init__.py b/semapact/platforms/git/__init__.py new file mode 100644 index 00000000..eaa930b2 --- /dev/null +++ b/semapact/platforms/git/__init__.py @@ -0,0 +1,5 @@ +"""Git working-tree persistence adapters.""" + +from semapact.platforms.git.history_repository import GitWorkingTreeHistoryRepository + +__all__ = ["GitWorkingTreeHistoryRepository"] diff --git a/semapact/platforms/git/history_repository.py b/semapact/platforms/git/history_repository.py new file mode 100644 index 00000000..807080b7 --- /dev/null +++ b/semapact/platforms/git/history_repository.py @@ -0,0 +1,270 @@ +"""Immutable governance-history persistence in a Git working tree. + +The adapter owns file layout only. It does not invoke Git, create commits, or create +pull requests; normal GitOps tooling can version the deterministic files it writes. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import TypeVar + +from pydantic import BaseModel, ValidationError as PydanticValidationError + +from semapact.contractops import ChangeSet +from semapact.governance import GovernanceDecision +from semapact.history import ( + HistoryConflictError, + HistoryCorruptionError, + HistoryNotFoundError, +) +from semapact.utils.deterministic import canonical_compact_json + + +T = TypeVar("T", bound=BaseModel) +_SAFE_ARTIFACT_ID = re.compile(r"^[A-Za-z0-9._-]+$") + + +class GitWorkingTreeHistoryRepository: + """Shared Git backend implementing narrow typed history capabilities. + + Public methods satisfy artifact-specific repository protocols while the private + helpers below own the common JSON/file persistence mechanics. + """ + + def __init__( + self, + repository_root: str | Path, + *, + state_directory: str | Path = ".semapact/history", + ) -> None: + self._history_root = Path(repository_root) / Path(state_directory) + + def put_decision(self, decision: GovernanceDecision) -> None: + self._put( + kind="decisions", + artifact_id=decision.decision_id, + artifact=decision, + model_type=GovernanceDecision, + id_attribute="decision_id", + ) + + def get_decision(self, decision_id: str) -> GovernanceDecision: + return self._get( + kind="decisions", + artifact_id=decision_id, + model_type=GovernanceDecision, + id_attribute="decision_id", + ) + + def list_decisions(self, contract_id: str) -> tuple[GovernanceDecision, ...]: + contract_id = _required_text(contract_id, "contract_id") + records = self._list( + kind="decisions", + model_type=GovernanceDecision, + id_attribute="decision_id", + ) + return tuple(record for record in records if record.contract_id == contract_id) + + def put_change_set(self, change_set: ChangeSet) -> None: + self._put( + kind="change_sets", + artifact_id=change_set.change_set_id, + artifact=change_set, + model_type=ChangeSet, + id_attribute="change_set_id", + ) + + def get_change_set(self, change_set_id: str) -> ChangeSet: + return self._get( + kind="change_sets", + artifact_id=change_set_id, + model_type=ChangeSet, + id_attribute="change_set_id", + ) + + def list_change_sets(self, contract_id: str) -> tuple[ChangeSet, ...]: + contract_id = _required_text(contract_id, "contract_id") + records = self._list( + kind="change_sets", + model_type=ChangeSet, + id_attribute="change_set_id", + ) + return tuple(record for record in records if record.contract_id == contract_id) + + def _put( + self, + *, + kind: str, + artifact_id: str, + artifact: T, + model_type: type[T], + id_attribute: str, + ) -> None: + artifact_id = _safe_artifact_id(artifact_id) + canonical = self._validated_canonical_json( + artifact, + model_type=model_type, + expected_id=artifact_id, + id_attribute=id_attribute, + ) + path = self._artifact_path(kind, artifact_id) + path.parent.mkdir(parents=True, exist_ok=True) + + if path.exists(): + self._require_idempotent_existing( + path, + canonical=canonical, + model_type=model_type, + expected_id=artifact_id, + id_attribute=id_attribute, + ) + return + + try: + with path.open("x", encoding="utf-8", newline="\n") as handle: + handle.write(canonical) + except FileExistsError: + # Another writer won the create race. Treat identical content as the + # same idempotent write and fail closed on any conflict. + self._require_idempotent_existing( + path, + canonical=canonical, + model_type=model_type, + expected_id=artifact_id, + id_attribute=id_attribute, + ) + + def _get( + self, + *, + kind: str, + artifact_id: str, + model_type: type[T], + id_attribute: str, + ) -> T: + artifact_id = _safe_artifact_id(artifact_id) + path = self._artifact_path(kind, artifact_id) + if not path.is_file(): + raise HistoryNotFoundError( + f"{model_type.__name__} {artifact_id!r} was not found" + ) + return self._read_validated( + path, + model_type=model_type, + expected_id=artifact_id, + id_attribute=id_attribute, + ) + + def _list( + self, + *, + kind: str, + model_type: type[T], + id_attribute: str, + ) -> tuple[T, ...]: + directory = self._history_root / kind + if not directory.is_dir(): + return () + + records: list[T] = [] + for path in sorted(directory.glob("*.json"), key=lambda item: item.name): + artifact_id = _safe_artifact_id(path.stem) + records.append( + self._read_validated( + path, + model_type=model_type, + expected_id=artifact_id, + id_attribute=id_attribute, + ) + ) + return tuple(records) + + def _require_idempotent_existing( + self, + path: Path, + *, + canonical: str, + model_type: type[T], + expected_id: str, + id_attribute: str, + ) -> None: + existing = self._read_validated( + path, + model_type=model_type, + expected_id=expected_id, + id_attribute=id_attribute, + ) + existing_canonical = canonical_compact_json(existing.model_dump(mode="json")) + if existing_canonical != canonical: + raise HistoryConflictError( + f"{model_type.__name__} {expected_id!r} already exists with different content" + ) + + def _read_validated( + self, + path: Path, + *, + model_type: type[T], + expected_id: str, + id_attribute: str, + ) -> T: + try: + raw = path.read_text(encoding="utf-8") + artifact = model_type.model_validate_json(raw) + except (OSError, PydanticValidationError, ValueError) as exc: + raise HistoryCorruptionError( + f"Persisted {model_type.__name__} {expected_id!r} is invalid" + ) from exc + + actual_id = getattr(artifact, id_attribute) + if actual_id != expected_id: + raise HistoryCorruptionError( + f"Persisted {model_type.__name__} identity does not match its file identity" + ) + return artifact + + def _validated_canonical_json( + self, + artifact: T, + *, + model_type: type[T], + expected_id: str, + id_attribute: str, + ) -> str: + if not isinstance(artifact, model_type): + raise TypeError( + f"artifact must be {model_type.__name__}, got {type(artifact).__name__}" + ) + try: + canonical = canonical_compact_json(artifact.model_dump(mode="json")) + validated = model_type.model_validate_json(canonical) + except (PydanticValidationError, ValueError) as exc: + raise HistoryCorruptionError( + f"Supplied {model_type.__name__} {expected_id!r} is invalid" + ) from exc + if getattr(validated, id_attribute) != expected_id: + raise HistoryCorruptionError( + f"Supplied {model_type.__name__} identity does not match its artifact ID" + ) + return canonical + + def _artifact_path(self, kind: str, artifact_id: str) -> Path: + return self._history_root / kind / f"{artifact_id}.json" + + +def _safe_artifact_id(value: str) -> str: + cleaned = _required_text(value, "artifact_id") + if not _SAFE_ARTIFACT_ID.fullmatch(cleaned): + raise ValueError("artifact_id contains characters unsafe for history file names") + return cleaned + + +def _required_text(value: str, field_name: str) -> str: + if not isinstance(value, str): + raise TypeError(f"{field_name} must be str") + cleaned = value.strip() + if not cleaned: + raise ValueError(f"{field_name} must not be empty") + return cleaned diff --git a/tests/test_history_repository.py b/tests/test_history_repository.py new file mode 100644 index 00000000..c69ac4f4 --- /dev/null +++ b/tests/test_history_repository.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +from datetime import date +from pathlib import Path + +import pytest +from open_data_contract_standard.model import ( + OpenDataContractStandard, + SchemaObject, + SchemaProperty, +) + +from semapact.change_context import ChangeContext +from semapact.contractops import build_change_set_from_decision +from semapact.governance import GovernanceDecision, evaluate_governance_decision +from semapact.history import ( + ChangeSetHistoryRepository, + DecisionHistoryRepository, + HistoryConflictError, + HistoryCorruptionError, + HistoryNotFoundError, +) +from semapact.platforms.git import GitWorkingTreeHistoryRepository + + +CONTEXT = ChangeContext(effective_date=date(2026, 9, 12)) + + +def _contract(*, name: str) -> OpenDataContractStandard: + return OpenDataContractStandard( + apiVersion="v3.1.0", + kind="DataContract", + id="orders-product", + name=name, + version="1.0.0", + status="active", + schema=[ + SchemaObject( + name="orders", + properties=[ + SchemaProperty( + name="id", + logicalType="string", + physicalType="varchar(255)", + required=True, + ) + ], + ) + ], + ) + + +def _decision(*, candidate_name: str) -> GovernanceDecision: + return evaluate_governance_decision( + _contract(name="orders-base"), + _contract(name=candidate_name), + context=CONTEXT, + ) + + +def test_artifacts_round_trip_through_segregated_repository_ports( + tmp_path: Path, +) -> None: + backend = GitWorkingTreeHistoryRepository(tmp_path) + decisions: DecisionHistoryRepository = backend + change_sets: ChangeSetHistoryRepository = backend + decision = _decision(candidate_name="orders-candidate") + change_set = build_change_set_from_decision( + decision, + base_revision_ref="git:base", + candidate_revision_ref="git:candidate", + source="history-test", + actor_reference="service:ci", + ) + + decisions.put_decision(decision) + change_sets.put_change_set(change_set) + + assert decisions.get_decision(decision.decision_id) == decision + assert change_sets.get_change_set(change_set.change_set_id) == change_set + assert change_sets.get_change_set(change_set.change_set_id).context == CONTEXT + + +def test_identical_writes_are_idempotent(tmp_path: Path) -> None: + repository = GitWorkingTreeHistoryRepository(tmp_path) + decision = _decision(candidate_name="orders-candidate") + + repository.put_decision(decision) + repository.put_decision(decision) + + assert repository.get_decision(decision.decision_id) == decision + + +def test_conflicting_content_under_existing_id_fails_closed(tmp_path: Path) -> None: + repository = GitWorkingTreeHistoryRepository(tmp_path) + decision = _decision(candidate_name="orders-candidate") + repository.put_decision(decision) + + conflicting = decision.model_copy(update={"contract_id": "different-contract"}) + + with pytest.raises(HistoryConflictError, match="different content"): + repository.put_decision(conflicting) + + +def test_corrupted_persisted_change_set_fails_closed(tmp_path: Path) -> None: + repository = GitWorkingTreeHistoryRepository(tmp_path) + decision = _decision(candidate_name="orders-candidate") + change_set = build_change_set_from_decision( + decision, + base_revision_ref="git:base", + candidate_revision_ref="git:candidate", + ) + repository.put_change_set(change_set) + + path = ( + tmp_path + / ".semapact" + / "history" + / "change_sets" + / f"{change_set.change_set_id}.json" + ) + path.write_text("{}", encoding="utf-8") + + with pytest.raises(HistoryCorruptionError, match="is invalid"): + repository.get_change_set(change_set.change_set_id) + + +def test_missing_artifact_has_explicit_not_found_error(tmp_path: Path) -> None: + repository = GitWorkingTreeHistoryRepository(tmp_path) + + with pytest.raises(HistoryNotFoundError, match="was not found"): + repository.get_decision("00000000-0000-0000-0000-000000000000") + + +def test_contract_scoped_lists_are_deterministic(tmp_path: Path) -> None: + repository = GitWorkingTreeHistoryRepository(tmp_path) + decisions = ( + _decision(candidate_name="orders-zeta"), + _decision(candidate_name="orders-alpha"), + ) + for decision in reversed(decisions): + repository.put_decision(decision) + + listed = repository.list_decisions("orders-product") + + assert [item.decision_id for item in listed] == sorted( + item.decision_id for item in decisions + ) + assert repository.list_decisions("other-contract") == () From 50e23c6b37dff7cf9efb9ad7d65d2de12c940b6e Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Sat, 12 Sep 2026 15:12:57 +1000 Subject: [PATCH 17/35] feat(revision): add immutable contract revision identity (#222) * feat(history): add canonical contract serialization * feat(history): add immutable contract revisions * feat(history): add revision persistence ports * feat(history): export revision history contracts * feat(history): persist contract revisions in Git backend * refactor(contractops): reuse canonical contract serialization * test(history): cover contract revision identity * docs(history): document contract revision identity * refactor(history): narrow revision validation errors * test(history): lock revision identity protocol * refactor(history): enforce canonical fingerprint text * refactor(revision): separate revision domain models * refactor(revision): isolate revision integrity rules * refactor(revision): separate revision builders * refactor(revision): define revision domain API * refactor(history): depend on revision domain models * refactor(history): keep history as persistence boundary * refactor(history): delegate revision integrity to domain * test(revision): assert clean model and integrity boundaries * docs(architecture): make revision domain ownership explicit * docs(history): clarify revision and persistence ownership * refactor(revision): remove history-owned revision implementation * refactor(revision): validate provenance at construction * refactor(history): narrow dependency to revision models * refactor(history): narrow revision adapter dependencies * refactor(revision): reuse canonical ODCS contract model * refactor(revision): validate identity from ODCS model * refactor(revision): snapshot canonical ODCS state * refactor(history): query revisions through ODCS contract * test(revision): cover canonical ODCS reuse * docs(revision): document canonical ODCS reuse * docs(architecture): require canonical ODCS model reuse * fix(revision): serialize nested ODCS with aliases * refactor(revision): keep persistence serialization out of model * fix(history): persist nested models with canonical aliases * test(revision): round-trip ODCS aliases explicitly * refactor(odcs): add canonical serialization package * refactor(odcs): move canonical serialization out of utils * refactor(revision): use ODCS serialization authority * refactor(revision): depend on ODCS serialization package * refactor(contractops): use ODCS serialization authority * refactor(odcs): remove ambiguous contracts utility --- ARCHITECTURE.md | 23 +- docs/governance_history.md | 92 +++++++- semapact/contractops/execution.py | 9 +- semapact/history/__init__.py | 10 +- semapact/history/repository.py | 36 +++ semapact/odcs/__init__.py | 5 + semapact/odcs/serialization.py | 22 ++ semapact/platforms/git/history_repository.py | 103 ++++++++- semapact/revision/__init__.py | 30 +++ semapact/revision/builders.py | 55 +++++ semapact/revision/integrity.py | 105 +++++++++ semapact/revision/models.py | 49 ++++ tests/test_contract_revisions.py | 223 +++++++++++++++++++ 13 files changed, 731 insertions(+), 31 deletions(-) create mode 100644 semapact/odcs/__init__.py create mode 100644 semapact/odcs/serialization.py create mode 100644 semapact/revision/__init__.py create mode 100644 semapact/revision/builders.py create mode 100644 semapact/revision/integrity.py create mode 100644 semapact/revision/models.py create mode 100644 tests/test_contract_revisions.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 59116a12..203273d6 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -54,13 +54,14 @@ Domain models live with the rules that give them meaning: - `semapact/lifecycle/` — canonical identity, lifecycle policy, merge/change semantics; - `semapact/governance/` — `GovernanceDecision`, reason codes, centralized gate; +- `semapact/revision/` — immutable governed contract content identity and source-provenance links; - `semapact/contractops/` — `ChangeSet`, `ReleasePlan`, `VersionResolution`, ContractOps authorization, APPLY/PUBLISH artifacts; - `semapact/deployment/` — provider-neutral `DeploymentPlan`, `DeploymentPreview`, deployment authorization and adapter contract; - `semapact/runtime/` — provider-neutral governed runtime asset projection; - `semapact/observation/` — provider-neutral point-in-time runtime state; - `semapact/reconciliation/` — desired-vs-observed comparison and `RuntimeDriftStatus`. -A domain artifact does not move into the application layer merely because an application service returns it. +A domain artifact does not move into the application or persistence layer merely because an application service returns it or a history backend stores it. ### Application layer @@ -91,9 +92,13 @@ These are application DTOs, not new governance/release/deployment authorities. `semapact/interfaces/` owns parsing, loading input artifacts at the interface edge, rendering, and process-outcome mapping. Interfaces delegate to application/domain boundaries and must not independently calculate governance, version, deployment, or reconciliation results. +### History persistence + +`semapact/history/` owns storage-neutral typed persistence/query ports and persistence errors. It stores canonical artifacts from their owning domains but does not redefine their models, identity formulas, or business semantics. + ### Platform adapters -`semapact/platforms/` owns provider SDK/client integration and physical-platform translation. Databricks DDL generation/execution is provider behavior; it does not belong in ContractOps or application DTOs. +`semapact/platforms/` owns provider SDK/client integration and physical-platform translation. Databricks DDL generation/execution is provider behavior; it does not belong in ContractOps or application DTOs. Storage backend layout and physical persistence mechanics likewise belong to the corresponding platform adapter. ### Import/export and compatibility workflows @@ -108,15 +113,24 @@ Do not create a generic root `schema/`, `models/`, or `data_models/` directory t | What the object represents | Owner | | --- | --- | | governed ODCS contract | ODCS model | +| governed contract revision identity/provenance | `semapact/revision/` | | governance/release/deployment/reconciliation artifact | owning domain package | | application/use-case aggregate result | `application/models/` | | application orchestration | `application/services/` | +| storage-neutral history persistence/query capability | `semapact/history/` | | provider/SDK/physical representation | `platforms//` | | presentation-only rendering state | `interfaces/` | -| durable history/persistence record | its persistence/history boundary | The fact that every object is “data” is not a useful architectural boundary. +## Canonical Contract Model + +SemaPact reuses `OpenDataContractStandard` as the canonical logical contract model. It must not create a second contract representation merely to support governance, revision history, persistence, or deployment. + +Domain artifacts may reference or wrap the canonical ODCS model while adding only semantics owned by that domain. For example, `ContractRevision` adds content identity around an exact `OpenDataContractStandard`; it does not duplicate `contract.id`, `contract.version`, schema fields, or a serialized contract copy as parallel logical fields. + +Canonical JSON may be derived transiently for deterministic hashing, signatures, persistence, or transport. That serialization is not a second logical contract model. + ## Governed Identity For current governance semantics: @@ -204,10 +218,11 @@ Rules: ## Public Architecture Invariants 1. **Change-driven, not CRUD** — governed state evolves through explicit analysis/planning/authorization boundaries. -2. **One authority per rule** — lifecycle, governance, version selection, deployment translation, and reconciliation each have one canonical owner. +2. **One authority per rule** — lifecycle, governance, revision identity, version selection, deployment translation, and reconciliation each have one canonical owner. 3. **Exact artifacts cross boundaries** — side effects consume exact immutable artifacts; mutable current state is not silently substituted. 4. **Operation-scoped authorization** — APPLY, PUBLISH, and DEPLOY are distinct operations; authorization for one cannot authorize another. 5. **Logical identity is stable** — `physicalName` binds runtime state but does not redefine governed identity. 6. **Execution is not convergence** — runtime state must be observed and reconciled independently. 7. **Interfaces stay thin** — CLI/API/UI parse, delegate, and render; they do not become a second business-logic implementation. 8. **Compatibility is not ownership** — legacy import paths may re-export canonical implementations but must not accumulate new logic. +9. **One canonical contract model** — SemaPact reuses ODCS rather than maintaining a parallel contract schema. diff --git a/docs/governance_history.md b/docs/governance_history.md index 50eab6ec..87da80c5 100644 --- a/docs/governance_history.md +++ b/docs/governance_history.md @@ -7,14 +7,15 @@ store and retrieve those exact immutable models. The persistence boundary is capability-oriented: ```text -GovernanceDecision ChangeSet - ↓ ↓ -DecisionHistoryRepository ChangeSetHistoryRepository - \ / - \ / - GitWorkingTreeHistoryRepository - ↓ - .semapact/history/ +GovernanceDecision ChangeSet ContractRevision RevisionSource + ↓ ↓ ↓ ↓ +DecisionHistory ChangeSetHistory RevisionHistory RevisionSourceHistory +Repository Repository Repository Repository + \ | | / + \ | | / + GitWorkingTreeHistoryRepository + ↓ + .semapact/history/ ``` Callers depend only on the narrow typed repository capability they need. A physical @@ -31,15 +32,82 @@ query/application service belongs above these ports only when a use case actuall coordinates multiple artifact types, for example evolution-chain reconstruction or a cross-artifact timeline. +## Contract revision identity + +`ContractRevision` is a revision-domain artifact owned by `semapact/revision/`. +`semapact/history/` only exposes persistence capabilities for it. Persisting a domain +artifact does not transfer semantic ownership to the persistence layer. + +The revision package keeps structure, identity rules, and construction separate: + +```text +revision/models.py + ContractRevision / ContractRevisionSource + +revision/integrity.py + fingerprint + UUID formulas + integrity validation + +revision/builders.py + build canonical revision/provenance artifacts +``` + +SemaPact does not define another contract model. `ContractRevision.contract` uses the +same `OpenDataContractStandard` model used by the ODCS/datacontract-cli stack. The +revision envelope adds only SemaPact-owned identity: + +```text +ContractRevision +├── revision_id +├── content_fingerprint +└── contract: OpenDataContractStandard +``` + +`contract.id`, `contract.version`, schema, quality, servers, and all other ODCS fields +remain authoritative inside the ODCS model. They are not duplicated as revision fields. +Likewise, canonical JSON is not stored as a second logical contract representation; it +is derived transiently when computing or validating revision identity. + +Revision identity is computed from the exact ODCS state: + +```text +OpenDataContractStandard + ↓ +canonical JSON bytes + ↓ +SHA-256 content fingerprint + ↓ +UUIDv5 revision ID +``` + +The canonical JSON representation uses ODCS alias names, excludes absent (`None`) +fields, preserves list order, and sorts object keys through SemaPact's canonical JSON +serializer. The fixed ContractRevision UUID namespace is part of the identity protocol; +changing it is an identity-schema migration. + +A Git SHA, branch, tag, registry URI, or other external source reference does not +participate in revision identity. Source provenance is represented as a separate +immutable `ContractRevisionSource` link: + +```text +ContractRevision + ├── source A + ├── source B + └── source C +``` + +Therefore the same canonical contract content has the same revision ID wherever it is +observed, while all known source references can still be retained independently. + ## Persistence semantics For supported artifacts: - writing identical content under the same artifact ID is idempotent; - writing different content under an existing artifact ID fails closed; -- reads rehydrate and validate the canonical domain model; +- reads rehydrate the canonical domain model and invoke domain integrity validation + before trusting persisted content; - the embedded artifact ID must match the requested/file identity; -- malformed or invalid persisted content fails closed; +- malformed or semantically inconsistent persisted content fails closed; - missing IDs produce an explicit history not-found error; - contract-scoped listings are deterministic. @@ -47,8 +115,8 @@ For supported artifacts: not written into canonical ODCS contracts. M2 deterministic identities remain authoritative. Persistence does not generate a -replacement identity and does not reinterpret lifecycle, governance, version, -authorization, deployment, or reconciliation semantics. +replacement identity and does not reinterpret lifecycle, governance, revision, +version, authorization, deployment, or reconciliation semantics. ## Backend extension rule diff --git a/semapact/contractops/execution.py b/semapact/contractops/execution.py index aa276509..93b3ec4b 100644 --- a/semapact/contractops/execution.py +++ b/semapact/contractops/execution.py @@ -25,7 +25,7 @@ from semapact.exceptions import ContractOpsAuthorizationError, ReleaseValidationError from semapact.governance.gate import GovernanceOperation from semapact.governance.models import GovernanceDecision -from semapact.utils.deterministic import canonical_compact_json +from semapact.odcs.serialization import canonical_contract_json from semapact.versioning import normalize_semver @@ -102,7 +102,7 @@ def apply_contract_release( released_contract = candidate_contract.model_copy(deep=True) released_contract.version = selected_version - released_contract_json = _canonical_contract_json(released_contract) + released_contract_json = canonical_contract_json(released_contract) applied_release_id = compute_applied_release_id( contract_id=release_plan.contract_id, @@ -246,8 +246,3 @@ def _canonical_version(version: str, *, field_name: str) -> str: f"{field_name} must use canonical major.minor.patch form" ) return canonical - - -def _canonical_contract_json(contract: OpenDataContractStandard) -> str: - payload = contract.model_dump(mode="json", by_alias=True, exclude_none=True) - return canonical_compact_json(payload) diff --git a/semapact/history/__init__.py b/semapact/history/__init__.py index 65070d9a..34673499 100644 --- a/semapact/history/__init__.py +++ b/semapact/history/__init__.py @@ -1,11 +1,13 @@ -"""Durable governance-history boundary. +"""Durable governance-history persistence boundary. -History persists canonical domain artifacts without becoming a second source of -governance, ContractOps, deployment, or reconciliation semantics. +History stores canonical domain artifacts without becoming a second source of +revision, governance, ContractOps, deployment, or reconciliation semantics. """ from semapact.history.repository import ( ChangeSetHistoryRepository, + ContractRevisionHistoryRepository, + ContractRevisionSourceHistoryRepository, DecisionHistoryRepository, HistoryConflictError, HistoryCorruptionError, @@ -15,6 +17,8 @@ __all__ = [ "ChangeSetHistoryRepository", + "ContractRevisionHistoryRepository", + "ContractRevisionSourceHistoryRepository", "DecisionHistoryRepository", "HistoryConflictError", "HistoryCorruptionError", diff --git a/semapact/history/repository.py b/semapact/history/repository.py index 31d96bed..feb089e4 100644 --- a/semapact/history/repository.py +++ b/semapact/history/repository.py @@ -6,6 +6,7 @@ from semapact.contractops import ChangeSet from semapact.governance import GovernanceDecision +from semapact.revision.models import ContractRevision, ContractRevisionSource class HistoryRepositoryError(RuntimeError): @@ -54,3 +55,38 @@ def get_change_set(self, change_set_id: str) -> ChangeSet: def list_change_sets(self, contract_id: str) -> tuple[ChangeSet, ...]: """List ChangeSets for one contract in deterministic artifact-ID order.""" ... + + +class ContractRevisionHistoryRepository(Protocol): + """Persistence capability for immutable ContractRevision history only.""" + + def put_revision(self, revision: ContractRevision) -> None: + """Persist one immutable ContractRevision idempotently.""" + ... + + def get_revision(self, revision_id: str) -> ContractRevision: + """Load one ContractRevision by its exact content-derived ID.""" + ... + + def list_revisions(self, contract_id: str) -> tuple[ContractRevision, ...]: + """List revisions for one contract in deterministic artifact-ID order.""" + ... + + +class ContractRevisionSourceHistoryRepository(Protocol): + """Persistence capability for immutable revision provenance links only.""" + + def put_revision_source(self, source: ContractRevisionSource) -> None: + """Persist one revision-to-source provenance link idempotently.""" + ... + + def get_revision_source(self, source_link_id: str) -> ContractRevisionSource: + """Load one revision provenance link by exact ID.""" + ... + + def list_revision_sources( + self, + revision_id: str, + ) -> tuple[ContractRevisionSource, ...]: + """List all source links for one revision in deterministic ID order.""" + ... diff --git a/semapact/odcs/__init__.py b/semapact/odcs/__init__.py new file mode 100644 index 00000000..19ee2838 --- /dev/null +++ b/semapact/odcs/__init__.py @@ -0,0 +1,5 @@ +"""ODCS-specific helpers owned by SemaPact.""" + +from semapact.odcs.serialization import canonical_contract_json + +__all__ = ["canonical_contract_json"] diff --git a/semapact/odcs/serialization.py b/semapact/odcs/serialization.py new file mode 100644 index 00000000..504516d5 --- /dev/null +++ b/semapact/odcs/serialization.py @@ -0,0 +1,22 @@ +"""Deterministic serialization for the canonical ODCS contract model.""" + +from __future__ import annotations + +from open_data_contract_standard.model import OpenDataContractStandard + +from semapact.utils.deterministic import canonical_compact_json + + +def canonical_contract_json(contract: OpenDataContractStandard) -> str: + """Return the stable compact JSON representation of one ODCS contract. + + This function defines deterministic serialization only. The canonical logical + contract model remains ``OpenDataContractStandard``. + """ + if not isinstance(contract, OpenDataContractStandard): + raise TypeError( + "contract must be OpenDataContractStandard, " + f"got {type(contract).__name__}" + ) + payload = contract.model_dump(mode="json", by_alias=True, exclude_none=True) + return canonical_compact_json(payload) diff --git a/semapact/platforms/git/history_repository.py b/semapact/platforms/git/history_repository.py index 807080b7..095517fe 100644 --- a/semapact/platforms/git/history_repository.py +++ b/semapact/platforms/git/history_repository.py @@ -7,6 +7,7 @@ from __future__ import annotations import re +from collections.abc import Callable from pathlib import Path from typing import TypeVar @@ -19,6 +20,11 @@ HistoryCorruptionError, HistoryNotFoundError, ) +from semapact.revision.integrity import ( + validate_contract_revision_identity, + validate_contract_revision_source_identity, +) +from semapact.revision.models import ContractRevision, ContractRevisionSource from semapact.utils.deterministic import canonical_compact_json @@ -30,7 +36,8 @@ class GitWorkingTreeHistoryRepository: """Shared Git backend implementing narrow typed history capabilities. Public methods satisfy artifact-specific repository protocols while the private - helpers below own the common JSON/file persistence mechanics. + helpers own common JSON/file persistence mechanics. Domain integrity remains in + each owning domain and is injected when persistence rehydrates that artifact. """ def __init__( @@ -93,6 +100,71 @@ def list_change_sets(self, contract_id: str) -> tuple[ChangeSet, ...]: ) return tuple(record for record in records if record.contract_id == contract_id) + def put_revision(self, revision: ContractRevision) -> None: + self._put( + kind="contract_revisions", + artifact_id=revision.revision_id, + artifact=revision, + model_type=ContractRevision, + id_attribute="revision_id", + integrity_validator=validate_contract_revision_identity, + ) + + def get_revision(self, revision_id: str) -> ContractRevision: + return self._get( + kind="contract_revisions", + artifact_id=revision_id, + model_type=ContractRevision, + id_attribute="revision_id", + integrity_validator=validate_contract_revision_identity, + ) + + def list_revisions(self, contract_id: str) -> tuple[ContractRevision, ...]: + contract_id = _required_text(contract_id, "contract_id") + records = self._list( + kind="contract_revisions", + model_type=ContractRevision, + id_attribute="revision_id", + integrity_validator=validate_contract_revision_identity, + ) + return tuple( + record + for record in records + if str(record.contract.id or "") == contract_id + ) + + def put_revision_source(self, source: ContractRevisionSource) -> None: + self._put( + kind="contract_revision_sources", + artifact_id=source.source_link_id, + artifact=source, + model_type=ContractRevisionSource, + id_attribute="source_link_id", + integrity_validator=validate_contract_revision_source_identity, + ) + + def get_revision_source(self, source_link_id: str) -> ContractRevisionSource: + return self._get( + kind="contract_revision_sources", + artifact_id=source_link_id, + model_type=ContractRevisionSource, + id_attribute="source_link_id", + integrity_validator=validate_contract_revision_source_identity, + ) + + def list_revision_sources( + self, + revision_id: str, + ) -> tuple[ContractRevisionSource, ...]: + revision_id = _required_text(revision_id, "revision_id") + records = self._list( + kind="contract_revision_sources", + model_type=ContractRevisionSource, + id_attribute="source_link_id", + integrity_validator=validate_contract_revision_source_identity, + ) + return tuple(record for record in records if record.revision_id == revision_id) + def _put( self, *, @@ -101,6 +173,7 @@ def _put( artifact: T, model_type: type[T], id_attribute: str, + integrity_validator: Callable[[T], None] | None = None, ) -> None: artifact_id = _safe_artifact_id(artifact_id) canonical = self._validated_canonical_json( @@ -108,6 +181,7 @@ def _put( model_type=model_type, expected_id=artifact_id, id_attribute=id_attribute, + integrity_validator=integrity_validator, ) path = self._artifact_path(kind, artifact_id) path.parent.mkdir(parents=True, exist_ok=True) @@ -119,6 +193,7 @@ def _put( model_type=model_type, expected_id=artifact_id, id_attribute=id_attribute, + integrity_validator=integrity_validator, ) return @@ -126,14 +201,13 @@ def _put( with path.open("x", encoding="utf-8", newline="\n") as handle: handle.write(canonical) except FileExistsError: - # Another writer won the create race. Treat identical content as the - # same idempotent write and fail closed on any conflict. self._require_idempotent_existing( path, canonical=canonical, model_type=model_type, expected_id=artifact_id, id_attribute=id_attribute, + integrity_validator=integrity_validator, ) def _get( @@ -143,6 +217,7 @@ def _get( artifact_id: str, model_type: type[T], id_attribute: str, + integrity_validator: Callable[[T], None] | None = None, ) -> T: artifact_id = _safe_artifact_id(artifact_id) path = self._artifact_path(kind, artifact_id) @@ -155,6 +230,7 @@ def _get( model_type=model_type, expected_id=artifact_id, id_attribute=id_attribute, + integrity_validator=integrity_validator, ) def _list( @@ -163,6 +239,7 @@ def _list( kind: str, model_type: type[T], id_attribute: str, + integrity_validator: Callable[[T], None] | None = None, ) -> tuple[T, ...]: directory = self._history_root / kind if not directory.is_dir(): @@ -177,6 +254,7 @@ def _list( model_type=model_type, expected_id=artifact_id, id_attribute=id_attribute, + integrity_validator=integrity_validator, ) ) return tuple(records) @@ -189,14 +267,16 @@ def _require_idempotent_existing( model_type: type[T], expected_id: str, id_attribute: str, + integrity_validator: Callable[[T], None] | None = None, ) -> None: existing = self._read_validated( path, model_type=model_type, expected_id=expected_id, id_attribute=id_attribute, + integrity_validator=integrity_validator, ) - existing_canonical = canonical_compact_json(existing.model_dump(mode="json")) + existing_canonical = _canonical_model_json(existing) if existing_canonical != canonical: raise HistoryConflictError( f"{model_type.__name__} {expected_id!r} already exists with different content" @@ -209,10 +289,13 @@ def _read_validated( model_type: type[T], expected_id: str, id_attribute: str, + integrity_validator: Callable[[T], None] | None = None, ) -> T: try: raw = path.read_text(encoding="utf-8") artifact = model_type.model_validate_json(raw) + if integrity_validator is not None: + integrity_validator(artifact) except (OSError, PydanticValidationError, ValueError) as exc: raise HistoryCorruptionError( f"Persisted {model_type.__name__} {expected_id!r} is invalid" @@ -232,14 +315,17 @@ def _validated_canonical_json( model_type: type[T], expected_id: str, id_attribute: str, + integrity_validator: Callable[[T], None] | None = None, ) -> str: if not isinstance(artifact, model_type): raise TypeError( f"artifact must be {model_type.__name__}, got {type(artifact).__name__}" ) try: - canonical = canonical_compact_json(artifact.model_dump(mode="json")) + canonical = _canonical_model_json(artifact) validated = model_type.model_validate_json(canonical) + if integrity_validator is not None: + integrity_validator(validated) except (PydanticValidationError, ValueError) as exc: raise HistoryCorruptionError( f"Supplied {model_type.__name__} {expected_id!r} is invalid" @@ -254,6 +340,13 @@ def _artifact_path(self, kind: str, artifact_id: str) -> Path: return self._history_root / kind / f"{artifact_id}.json" +def _canonical_model_json(artifact: BaseModel) -> str: + """Serialize persisted models with aliases so nested ODCS models round-trip.""" + return canonical_compact_json( + artifact.model_dump(mode="json", by_alias=True) + ) + + def _safe_artifact_id(value: str) -> str: cleaned = _required_text(value, "artifact_id") if not _SAFE_ARTIFACT_ID.fullmatch(cleaned): diff --git a/semapact/revision/__init__.py b/semapact/revision/__init__.py new file mode 100644 index 00000000..aff80b51 --- /dev/null +++ b/semapact/revision/__init__.py @@ -0,0 +1,30 @@ +"""Governed contract revision identity domain.""" + +from semapact.revision.builders import ( + build_contract_revision, + link_contract_revision_source, +) +from semapact.revision.integrity import ( + SEMAPACT_CONTRACT_REVISION_NAMESPACE, + SEMAPACT_CONTRACT_REVISION_SOURCE_NAMESPACE, + compute_contract_content_fingerprint, + compute_contract_revision_id, + compute_contract_revision_source_id, + validate_contract_revision_identity, + validate_contract_revision_source_identity, +) +from semapact.revision.models import ContractRevision, ContractRevisionSource + +__all__ = [ + "ContractRevision", + "ContractRevisionSource", + "SEMAPACT_CONTRACT_REVISION_NAMESPACE", + "SEMAPACT_CONTRACT_REVISION_SOURCE_NAMESPACE", + "build_contract_revision", + "compute_contract_content_fingerprint", + "compute_contract_revision_id", + "compute_contract_revision_source_id", + "link_contract_revision_source", + "validate_contract_revision_identity", + "validate_contract_revision_source_identity", +] diff --git a/semapact/revision/builders.py b/semapact/revision/builders.py new file mode 100644 index 00000000..42e51e1b --- /dev/null +++ b/semapact/revision/builders.py @@ -0,0 +1,55 @@ +"""Construction helpers for canonical contract revision artifacts.""" + +from __future__ import annotations + +from open_data_contract_standard.model import OpenDataContractStandard + +from semapact.odcs.serialization import canonical_contract_json +from semapact.revision.integrity import ( + compute_contract_content_fingerprint, + compute_contract_revision_id, + compute_contract_revision_source_id, + validate_contract_revision_identity, + validate_contract_revision_source_identity, +) +from semapact.revision.models import ContractRevision, ContractRevisionSource + + +def build_contract_revision(contract: OpenDataContractStandard) -> ContractRevision: + """Build a revision envelope around one exact canonical ODCS contract snapshot.""" + if not isinstance(contract, OpenDataContractStandard): + raise TypeError( + "contract must be OpenDataContractStandard, " + f"got {type(contract).__name__}" + ) + + canonical = canonical_contract_json(contract) + contract_snapshot = OpenDataContractStandard.model_validate_json(canonical) + fingerprint = compute_contract_content_fingerprint(canonical) + + revision = ContractRevision( + revision_id=compute_contract_revision_id(fingerprint), + content_fingerprint=fingerprint, + contract=contract_snapshot, + ) + validate_contract_revision_identity(revision) + return revision + + +def link_contract_revision_source( + revision: ContractRevision, + *, + source_reference: str, +) -> ContractRevisionSource: + """Build one immutable provenance link without changing revision identity.""" + validate_contract_revision_identity(revision) + source = ContractRevisionSource( + source_link_id=compute_contract_revision_source_id( + revision_id=revision.revision_id, + source_reference=source_reference, + ), + revision_id=revision.revision_id, + source_reference=source_reference, + ) + validate_contract_revision_source_identity(source) + return source diff --git a/semapact/revision/integrity.py b/semapact/revision/integrity.py new file mode 100644 index 00000000..6d1e551a --- /dev/null +++ b/semapact/revision/integrity.py @@ -0,0 +1,105 @@ +"""Deterministic identity rules for governed contract revisions.""" + +from __future__ import annotations + +import hashlib +import re +import uuid + +from semapact.odcs.serialization import canonical_contract_json +from semapact.revision.models import ContractRevision, ContractRevisionSource +from semapact.utils.deterministic import deterministic_uuid5 + + +SEMAPACT_CONTRACT_REVISION_NAMESPACE = uuid.UUID( + "53dc6172-6f46-46b1-a9d6-f7d967b0635c" +) +SEMAPACT_CONTRACT_REVISION_SOURCE_NAMESPACE = uuid.UUID( + "81b9275f-02b6-447b-b219-c355119a78d5" +) +_SHA256_HEX = re.compile(r"^[0-9a-f]{64}$") + + +def compute_contract_content_fingerprint(canonical_json: str) -> str: + """Return SHA-256 over the exact canonical contract JSON bytes.""" + if not isinstance(canonical_json, str): + raise TypeError("canonical_json must be str") + if not canonical_json: + raise ValueError("canonical_json must not be empty") + return hashlib.sha256(canonical_json.encode("utf-8")).hexdigest() + + +def compute_contract_revision_id(content_fingerprint: str) -> str: + """Return the stable UUIDv5 identity for one content fingerprint.""" + _require_sha256_hex(content_fingerprint) + return deterministic_uuid5( + SEMAPACT_CONTRACT_REVISION_NAMESPACE, + {"contentFingerprint": content_fingerprint}, + ) + + +def compute_contract_revision_source_id( + *, + revision_id: str, + source_reference: str, +) -> str: + """Return deterministic identity for one revision-to-source provenance link.""" + revision_id = _require_canonical_text(revision_id, "revision_id") + source_reference = _require_canonical_text(source_reference, "source_reference") + return deterministic_uuid5( + SEMAPACT_CONTRACT_REVISION_SOURCE_NAMESPACE, + { + "revisionId": revision_id, + "sourceReference": source_reference, + }, + ) + + +def validate_contract_revision_identity(revision: ContractRevision) -> None: + """Fail if a revision's derived identity does not match its exact ODCS state.""" + if not isinstance(revision, ContractRevision): + raise TypeError( + f"revision must be ContractRevision, got {type(revision).__name__}" + ) + + _require_canonical_text(str(revision.contract.id or ""), "contract.id") + _require_canonical_text(str(revision.contract.version or ""), "contract.version") + + canonical = canonical_contract_json(revision.contract) + expected_fingerprint = compute_contract_content_fingerprint(canonical) + if expected_fingerprint != revision.content_fingerprint: + raise ValueError("content_fingerprint does not match canonical contract content") + + expected_revision_id = compute_contract_revision_id(expected_fingerprint) + if expected_revision_id != revision.revision_id: + raise ValueError("revision_id does not match deterministic content identity") + + +def validate_contract_revision_source_identity(source: ContractRevisionSource) -> None: + """Fail if a provenance link ID does not match its revision/source pair.""" + if not isinstance(source, ContractRevisionSource): + raise TypeError( + "source must be ContractRevisionSource, " + f"got {type(source).__name__}" + ) + expected = compute_contract_revision_source_id( + revision_id=source.revision_id, + source_reference=source.source_reference, + ) + if expected != source.source_link_id: + raise ValueError("source_link_id does not match deterministic provenance identity") + + +def _require_sha256_hex(value: str) -> None: + if not isinstance(value, str): + raise TypeError("content_fingerprint must be str") + if value != value.strip().lower() or not _SHA256_HEX.fullmatch(value): + raise ValueError("content_fingerprint must be a lowercase SHA-256 hex digest") + + +def _require_canonical_text(value: str, field_name: str) -> str: + if not isinstance(value, str): + raise TypeError(f"{field_name} must be str") + if not value or value != value.strip(): + raise ValueError(f"{field_name} must be non-empty canonical text") + return value diff --git a/semapact/revision/models.py b/semapact/revision/models.py new file mode 100644 index 00000000..a5068578 --- /dev/null +++ b/semapact/revision/models.py @@ -0,0 +1,49 @@ +"""Immutable domain models for governed contract revision identity.""" + +from __future__ import annotations + +from open_data_contract_standard.model import OpenDataContractStandard +from pydantic import BaseModel, ConfigDict, field_validator + + +class ContractRevision(BaseModel): + """Content-identity envelope around one exact canonical ODCS contract state. + + The ODCS model remains the canonical contract representation. This model adds only + SemaPact-owned revision identity and does not duplicate contract ID, version, or a + serialized contract copy. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + revision_id: str + content_fingerprint: str + contract: OpenDataContractStandard + + @field_validator("revision_id", "content_fingerprint") + @classmethod + def _require_canonical_text(cls, value: str) -> str: + return _canonical_text(value) + + +class ContractRevisionSource(BaseModel): + """Immutable provenance link from one revision to one source reference.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + source_link_id: str + revision_id: str + source_reference: str + + @field_validator("source_link_id", "revision_id", "source_reference") + @classmethod + def _require_canonical_text(cls, value: str) -> str: + return _canonical_text(value) + + +def _canonical_text(value: str) -> str: + if not isinstance(value, str): + raise TypeError("value must be str") + if not value or value != value.strip(): + raise ValueError("value must be non-empty canonical text") + return value diff --git a/tests/test_contract_revisions.py b/tests/test_contract_revisions.py new file mode 100644 index 00000000..a0c1e8b7 --- /dev/null +++ b/tests/test_contract_revisions.py @@ -0,0 +1,223 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest +from open_data_contract_standard.model import ( + OpenDataContractStandard, + SchemaObject, + SchemaProperty, +) + +from semapact.history import ( + ContractRevisionHistoryRepository, + ContractRevisionSourceHistoryRepository, + HistoryCorruptionError, +) +from semapact.platforms.git import GitWorkingTreeHistoryRepository +from semapact.revision import ( + ContractRevision, + build_contract_revision, + compute_contract_content_fingerprint, + compute_contract_revision_id, + compute_contract_revision_source_id, + link_contract_revision_source, + validate_contract_revision_identity, +) +from semapact.utils.deterministic import canonical_compact_json + + +def _contract( + *, + name: str = "orders", + version: str = "1.0.0", + physical_type: str = "varchar(255)", +) -> OpenDataContractStandard: + return OpenDataContractStandard( + apiVersion="v3.1.0", + kind="DataContract", + id="orders-product", + name=name, + version=version, + status="active", + schema=[ + SchemaObject( + name="orders", + properties=[ + SchemaProperty( + name="id", + logicalType="string", + physicalType=physical_type, + required=True, + ) + ], + ) + ], + ) + + +def test_revision_identity_protocol_has_golden_values() -> None: + canonical = '{"id":"x"}' + fingerprint = compute_contract_content_fingerprint(canonical) + + assert fingerprint == "5e2b92cc57ce618dfbb54844a31775e4b95c6fb552ee6bf5a068133c12d2ad90" + revision_id = compute_contract_revision_id(fingerprint) + assert revision_id == "88b0f51f-8208-528e-92c7-75d2b357a751" + assert ( + compute_contract_revision_source_id( + revision_id=revision_id, + source_reference="git:commit:abc123", + ) + == "8598e362-fef7-58e9-87c1-a582d511728d" + ) + + +def test_revision_reuses_odcs_contract_model_without_duplicate_contract_fields() -> None: + contract = _contract() + revision = build_contract_revision(contract) + + assert isinstance(revision.contract, OpenDataContractStandard) + assert revision.contract == contract + assert revision.contract is not contract + assert set(ContractRevision.model_fields) == { + "revision_id", + "content_fingerprint", + "contract", + } + + +def test_identical_content_has_stable_revision_identity() -> None: + first = build_contract_revision(_contract()) + second = build_contract_revision(_contract()) + + assert first == second + assert first.revision_id == second.revision_id + assert first.content_fingerprint == second.content_fingerprint + assert first.revision_id != str(first.contract.version or "") + + +def test_governed_content_change_changes_revision_identity() -> None: + before = build_contract_revision(_contract()) + after = build_contract_revision(_contract(physical_type="string")) + + assert before.content_fingerprint != after.content_fingerprint + assert before.revision_id != after.revision_id + + +def test_semantic_version_is_part_of_exact_contract_content_not_revision_alias() -> None: + first = build_contract_revision(_contract(version="1.0.0")) + second = build_contract_revision(_contract(version="1.0.1")) + + assert str(first.contract.version) == "1.0.0" + assert str(second.contract.version) == "1.0.1" + assert first.revision_id != second.revision_id + assert first.revision_id not in { + str(first.contract.version), + str(second.contract.version), + } + + +def test_source_provenance_does_not_change_revision_identity() -> None: + revision = build_contract_revision(_contract()) + + branch_source = link_contract_revision_source( + revision, + source_reference="git:refs/heads/main@abc123", + ) + tag_source = link_contract_revision_source( + revision, + source_reference="git:refs/tags/v1.0.0@def456", + ) + + assert branch_source.revision_id == revision.revision_id + assert tag_source.revision_id == revision.revision_id + assert branch_source.source_link_id != tag_source.source_link_id + assert branch_source.source_reference != tag_source.source_reference + + +def test_revision_model_structure_is_separate_from_derived_identity_validation() -> None: + revision = build_contract_revision(_contract()) + payload = revision.model_dump(mode="json", by_alias=True) + payload["content_fingerprint"] = "0" * 64 + + structurally_valid = ContractRevision.model_validate(payload) + + with pytest.raises(ValueError, match="content_fingerprint"): + validate_contract_revision_identity(structurally_valid) + + +def test_revision_integrity_detects_changed_nested_contract_content() -> None: + revision = build_contract_revision(_contract()) + changed_contract = revision.contract.model_copy(deep=True) + changed_contract.name = "changed-orders" + tampered = revision.model_copy(update={"contract": changed_contract}) + + with pytest.raises(ValueError, match="content_fingerprint"): + validate_contract_revision_identity(tampered) + + +def test_source_link_builder_rejects_invalid_revision_identity() -> None: + revision = build_contract_revision(_contract()) + tampered = revision.model_copy(update={"content_fingerprint": "0" * 64}) + + with pytest.raises(ValueError, match="content_fingerprint"): + link_contract_revision_source( + tampered, + source_reference="git:commit:abc123", + ) + + +def test_revision_and_sources_round_trip_through_typed_history_ports( + tmp_path: Path, +) -> None: + backend = GitWorkingTreeHistoryRepository(tmp_path) + revisions: ContractRevisionHistoryRepository = backend + sources: ContractRevisionSourceHistoryRepository = backend + revision = build_contract_revision(_contract()) + source_a = link_contract_revision_source( + revision, + source_reference="git:commit:abc123", + ) + source_b = link_contract_revision_source( + revision, + source_reference="artifact:registry:orders/1.0.0", + ) + + revisions.put_revision(revision) + revisions.put_revision(revision) + sources.put_revision_source(source_b) + sources.put_revision_source(source_a) + + assert revisions.get_revision(revision.revision_id) == revision + assert revisions.list_revisions("orders-product") == (revision,) + assert revisions.list_revisions("other-contract") == () + assert sources.get_revision_source(source_a.source_link_id) == source_a + source_ids = [ + item.source_link_id + for item in sources.list_revision_sources(revision.revision_id) + ] + assert source_ids == sorted([source_a.source_link_id, source_b.source_link_id]) + + +def test_semantically_corrupted_persisted_revision_fails_closed(tmp_path: Path) -> None: + repository = GitWorkingTreeHistoryRepository(tmp_path) + revision = build_contract_revision(_contract()) + repository.put_revision(revision) + + path = ( + tmp_path + / ".semapact" + / "history" + / "contract_revisions" + / f"{revision.revision_id}.json" + ) + tampered = revision.model_copy(update={"content_fingerprint": "0" * 64}) + path.write_text( + canonical_compact_json( + tampered.model_dump(mode="json", by_alias=True) + ), + encoding="utf-8", + ) + + with pytest.raises(HistoryCorruptionError, match="is invalid"): + repository.get_revision(revision.revision_id) From 266bcefa527798665e999ddbfbd0597332d76fba Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Sat, 12 Sep 2026 20:57:32 +1000 Subject: [PATCH 18/35] feat(history): record complete changeset proposal history (#223) * feat(history): add proposal decision link model * feat(history): add proposal link persistence capability * feat(history): export proposal history link capability * feat(history): persist changeset decision links * feat(history): add proposal history orchestration * test(history): cover durable changeset proposal history * test(history): use canonical mismatched changeset fixture --- semapact/application/services/history.py | 102 +++++++++++ semapact/history/__init__.py | 4 + semapact/history/models.py | 28 +++ semapact/history/repository.py | 16 ++ semapact/platforms/git/history_repository.py | 37 +++- tests/test_changeset_history.py | 176 +++++++++++++++++++ 6 files changed, 360 insertions(+), 3 deletions(-) create mode 100644 semapact/application/services/history.py create mode 100644 semapact/history/models.py create mode 100644 tests/test_changeset_history.py diff --git a/semapact/application/services/history.py b/semapact/application/services/history.py new file mode 100644 index 00000000..b1ab1d0d --- /dev/null +++ b/semapact/application/services/history.py @@ -0,0 +1,102 @@ +"""Application orchestration for durable proposal history.""" + +from __future__ import annotations + +from semapact.application.models.governance import GovernanceProposal +from semapact.contractops.integrity import validate_change_set_identity +from semapact.history import ( + ChangeSetDecisionLink, + ChangeSetDecisionLinkHistoryRepository, + ChangeSetHistoryRepository, + ContractRevisionHistoryRepository, + DecisionHistoryRepository, +) +from semapact.revision.integrity import validate_contract_revision_identity +from semapact.revision.models import ContractRevision + + +class ProposalHistoryService: + """Record one already-evaluated proposal without recomputing domain semantics.""" + + def __init__( + self, + *, + revisions: ContractRevisionHistoryRepository, + change_sets: ChangeSetHistoryRepository, + decisions: DecisionHistoryRepository, + decision_links: ChangeSetDecisionLinkHistoryRepository, + ) -> None: + self._revisions = revisions + self._change_sets = change_sets + self._decisions = decisions + self._decision_links = decision_links + + def record_proposal( + self, + proposal: GovernanceProposal, + *, + base_revision: ContractRevision, + candidate_revision: ContractRevision, + ) -> ChangeSetDecisionLink: + """Persist the exact revision → ChangeSet → decision proposal history chain. + + This boundary validates cross-artifact references only. It never re-diffs + contracts or re-runs governance, lifecycle policy, or revision construction. + """ + if not isinstance(proposal, GovernanceProposal): + raise TypeError( + f"proposal must be GovernanceProposal, got {type(proposal).__name__}" + ) + validate_contract_revision_identity(base_revision) + validate_contract_revision_identity(candidate_revision) + validate_change_set_identity(proposal.change_set) + _validate_proposal_links( + proposal, + base_revision=base_revision, + candidate_revision=candidate_revision, + ) + + # Persist independently valid artifacts first. The relationship marker is + # written last so a partial failure cannot leave a dangling audit link. + self._revisions.put_revision(base_revision) + self._revisions.put_revision(candidate_revision) + self._change_sets.put_change_set(proposal.change_set) + self._decisions.put_decision(proposal.decision) + + link = ChangeSetDecisionLink( + change_set_id=proposal.change_set.change_set_id, + decision_id=proposal.decision.decision_id, + ) + self._decision_links.put_change_set_decision_link(link) + return link + + +def _validate_proposal_links( + proposal: GovernanceProposal, + *, + base_revision: ContractRevision, + candidate_revision: ContractRevision, +) -> None: + change_set = proposal.change_set + decision = proposal.decision + + base_contract_id = str(base_revision.contract.id or "").strip() + candidate_contract_id = str(candidate_revision.contract.id or "").strip() + if not base_contract_id or not candidate_contract_id: + raise ValueError("proposal history requires non-empty revision contract IDs") + if base_contract_id != candidate_contract_id: + raise ValueError("base and candidate revisions must belong to the same contract") + if change_set.contract_id != base_contract_id or decision.contract_id != base_contract_id: + raise ValueError("proposal artifacts do not reference the same contract") + + if change_set.base_revision_ref != base_revision.revision_id: + raise ValueError("ChangeSet base_revision_ref must equal base ContractRevision ID") + if change_set.candidate_revision_ref != candidate_revision.revision_id: + raise ValueError( + "ChangeSet candidate_revision_ref must equal candidate ContractRevision ID" + ) + + if change_set.context != decision.context: + raise ValueError("ChangeSet context does not match GovernanceDecision context") + if change_set.changes != decision.changes: + raise ValueError("ChangeSet changes do not match GovernanceDecision changes") diff --git a/semapact/history/__init__.py b/semapact/history/__init__.py index 34673499..e5420c09 100644 --- a/semapact/history/__init__.py +++ b/semapact/history/__init__.py @@ -4,7 +4,9 @@ revision, governance, ContractOps, deployment, or reconciliation semantics. """ +from semapact.history.models import ChangeSetDecisionLink from semapact.history.repository import ( + ChangeSetDecisionLinkHistoryRepository, ChangeSetHistoryRepository, ContractRevisionHistoryRepository, ContractRevisionSourceHistoryRepository, @@ -16,6 +18,8 @@ ) __all__ = [ + "ChangeSetDecisionLink", + "ChangeSetDecisionLinkHistoryRepository", "ChangeSetHistoryRepository", "ContractRevisionHistoryRepository", "ContractRevisionSourceHistoryRepository", diff --git a/semapact/history/models.py b/semapact/history/models.py new file mode 100644 index 00000000..eb0b4d97 --- /dev/null +++ b/semapact/history/models.py @@ -0,0 +1,28 @@ +"""Immutable history-owned relationship models. + +These models record cross-artifact audit relationships without redefining the +canonical domain artifacts they connect. +""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, field_validator + + +class ChangeSetDecisionLink(BaseModel): + """Audit provenance linking one ChangeSet to one GovernanceDecision outcome.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + change_set_id: str + decision_id: str + + @field_validator("change_set_id", "decision_id") + @classmethod + def _require_non_empty_text(cls, value: str) -> str: + if not isinstance(value, str): + raise TypeError("history link identifiers must be strings") + cleaned = value.strip() + if not cleaned: + raise ValueError("history link identifiers must not be empty") + return cleaned diff --git a/semapact/history/repository.py b/semapact/history/repository.py index feb089e4..beb87d06 100644 --- a/semapact/history/repository.py +++ b/semapact/history/repository.py @@ -6,6 +6,7 @@ from semapact.contractops import ChangeSet from semapact.governance import GovernanceDecision +from semapact.history.models import ChangeSetDecisionLink from semapact.revision.models import ContractRevision, ContractRevisionSource @@ -57,6 +58,21 @@ def list_change_sets(self, contract_id: str) -> tuple[ChangeSet, ...]: ... +class ChangeSetDecisionLinkHistoryRepository(Protocol): + """Persistence capability for ChangeSet-to-decision audit provenance only.""" + + def put_change_set_decision_link(self, link: ChangeSetDecisionLink) -> None: + """Persist one immutable ChangeSet-to-decision link idempotently.""" + ... + + def list_change_set_decision_links( + self, + change_set_id: str, + ) -> tuple[ChangeSetDecisionLink, ...]: + """List governance outcomes linked to one ChangeSet in deterministic order.""" + ... + + class ContractRevisionHistoryRepository(Protocol): """Persistence capability for immutable ContractRevision history only.""" diff --git a/semapact/platforms/git/history_repository.py b/semapact/platforms/git/history_repository.py index 095517fe..f6aa6791 100644 --- a/semapact/platforms/git/history_repository.py +++ b/semapact/platforms/git/history_repository.py @@ -16,6 +16,7 @@ from semapact.contractops import ChangeSet from semapact.governance import GovernanceDecision from semapact.history import ( + ChangeSetDecisionLink, HistoryConflictError, HistoryCorruptionError, HistoryNotFoundError, @@ -100,6 +101,38 @@ def list_change_sets(self, contract_id: str) -> tuple[ChangeSet, ...]: ) return tuple(record for record in records if record.contract_id == contract_id) + def put_change_set_decision_link(self, link: ChangeSetDecisionLink) -> None: + if not isinstance(link, ChangeSetDecisionLink): + raise TypeError( + "link must be ChangeSetDecisionLink, " + f"got {type(link).__name__}" + ) + change_set_id = _safe_artifact_id(link.change_set_id) + self._put( + kind=f"change_set_decisions/{change_set_id}", + artifact_id=link.decision_id, + artifact=link, + model_type=ChangeSetDecisionLink, + id_attribute="decision_id", + ) + + def list_change_set_decision_links( + self, + change_set_id: str, + ) -> tuple[ChangeSetDecisionLink, ...]: + change_set_id = _safe_artifact_id(change_set_id) + records = self._list( + kind=f"change_set_decisions/{change_set_id}", + model_type=ChangeSetDecisionLink, + id_attribute="decision_id", + ) + for record in records: + if record.change_set_id != change_set_id: + raise HistoryCorruptionError( + "Persisted ChangeSetDecisionLink does not match its ChangeSet path" + ) + return records + def put_revision(self, revision: ContractRevision) -> None: self._put( kind="contract_revisions", @@ -342,9 +375,7 @@ def _artifact_path(self, kind: str, artifact_id: str) -> Path: def _canonical_model_json(artifact: BaseModel) -> str: """Serialize persisted models with aliases so nested ODCS models round-trip.""" - return canonical_compact_json( - artifact.model_dump(mode="json", by_alias=True) - ) + return canonical_compact_json(artifact.model_dump(mode="json", by_alias=True)) def _safe_artifact_id(value: str) -> str: diff --git a/tests/test_changeset_history.py b/tests/test_changeset_history.py new file mode 100644 index 00000000..1abd3157 --- /dev/null +++ b/tests/test_changeset_history.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest +from open_data_contract_standard.model import ( + OpenDataContractStandard, + SchemaObject, + SchemaProperty, +) + +from semapact.application.services.governance import GovernanceService +from semapact.application.services.history import ProposalHistoryService +from semapact.contractops import build_change_set +from semapact.history import ( + ChangeSetDecisionLinkHistoryRepository, + ChangeSetHistoryRepository, + ContractRevisionHistoryRepository, + DecisionHistoryRepository, +) +from semapact.platforms.git import GitWorkingTreeHistoryRepository +from semapact.revision import build_contract_revision + + +def _contract(*, name: str) -> OpenDataContractStandard: + return OpenDataContractStandard( + apiVersion="v3.1.0", + kind="DataContract", + id="orders-product", + name=name, + version="1.2.3", + status="active", + schema=[ + SchemaObject( + name="orders", + properties=[ + SchemaProperty( + name="id", + logicalType="string", + physicalType="varchar(255)", + required=True, + ) + ], + ) + ], + ) + + +def _proposal(*, effective_date: str = "2026-09-12"): + base_revision = build_contract_revision(_contract(name="Orders old")) + candidate_revision = build_contract_revision(_contract(name="Orders new")) + proposal = GovernanceService().evaluate_proposal( + base_revision.contract, + candidate_revision.contract, + effective_date=effective_date, + base_revision_ref=base_revision.revision_id, + candidate_revision_ref=candidate_revision.revision_id, + source="test", + actor_reference="user:test", + ) + return proposal, base_revision, candidate_revision + + +def _service(tmp_path: Path): + backend = GitWorkingTreeHistoryRepository(tmp_path) + revisions: ContractRevisionHistoryRepository = backend + change_sets: ChangeSetHistoryRepository = backend + decisions: DecisionHistoryRepository = backend + links: ChangeSetDecisionLinkHistoryRepository = backend + service = ProposalHistoryService( + revisions=revisions, + change_sets=change_sets, + decisions=decisions, + decision_links=links, + ) + return service, backend + + +def test_records_exact_proposal_chain_without_recomputing_domain_artifacts( + tmp_path: Path, +) -> None: + service, backend = _service(tmp_path) + proposal, base_revision, candidate_revision = _proposal() + + link = service.record_proposal( + proposal, + base_revision=base_revision, + candidate_revision=candidate_revision, + ) + # Same exact history write is idempotent. + assert ( + service.record_proposal( + proposal, + base_revision=base_revision, + candidate_revision=candidate_revision, + ) + == link + ) + + assert backend.get_revision(base_revision.revision_id) == base_revision + assert backend.get_revision(candidate_revision.revision_id) == candidate_revision + assert backend.get_change_set(proposal.change_set.change_set_id) == proposal.change_set + assert backend.get_decision(proposal.decision.decision_id) == proposal.decision + assert backend.list_change_set_decision_links(proposal.change_set.change_set_id) == ( + link, + ) + assert link.change_set_id == proposal.change_set.change_set_id + assert link.decision_id == proposal.decision.decision_id + + +def test_changeset_context_round_trips_and_different_contexts_do_not_overwrite( + tmp_path: Path, +) -> None: + service, backend = _service(tmp_path) + first, first_base, first_candidate = _proposal(effective_date="2026-09-12") + second, second_base, second_candidate = _proposal(effective_date="2026-09-13") + + service.record_proposal( + first, + base_revision=first_base, + candidate_revision=first_candidate, + ) + service.record_proposal( + second, + base_revision=second_base, + candidate_revision=second_candidate, + ) + + assert first.change_set.change_set_id != second.change_set.change_set_id + assert backend.get_change_set(first.change_set.change_set_id).context == first.change_set.context + assert backend.get_change_set(second.change_set.change_set_id).context == second.change_set.context + assert len(backend.list_change_sets("orders-product")) == 2 + + +def test_rejects_changeset_that_does_not_reference_exact_contract_revisions( + tmp_path: Path, +) -> None: + service, _ = _service(tmp_path) + proposal, base_revision, candidate_revision = _proposal() + original = proposal.change_set + mismatched = build_change_set( + contract_id=original.contract_id, + base_revision_ref="git:base", + candidate_revision_ref=original.candidate_revision_ref, + changes=original.changes, + context=original.context, + source=original.source, + actor_reference=original.actor_reference, + ) + broken_proposal = proposal.__class__(change_set=mismatched, decision=proposal.decision) + + with pytest.raises(ValueError, match="base_revision_ref"): + service.record_proposal( + broken_proposal, + base_revision=base_revision, + candidate_revision=candidate_revision, + ) + + +def test_rejects_decision_that_is_not_the_outcome_of_the_changeset( + tmp_path: Path, +) -> None: + service, _ = _service(tmp_path) + proposal, base_revision, candidate_revision = _proposal() + other, _, _ = _proposal(effective_date="2026-09-13") + broken_proposal = proposal.__class__( + change_set=proposal.change_set, + decision=other.decision, + ) + + with pytest.raises(ValueError, match="context"): + service.record_proposal( + broken_proposal, + base_revision=base_revision, + candidate_revision=candidate_revision, + ) From 238b139a2a2aa529dfab77ef5df56b11eb931d2d Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Sun, 13 Sep 2026 09:59:34 +1000 Subject: [PATCH 19/35] feat(history): record finalized contract releases (#224) * feat(history): add immutable release record * feat(history): add release record identity * feat(history): add release history ports * feat(history): export release history contracts * feat(history): persist release plans and records * feat(history): add release history orchestration * test(history): cover finalized release records * refactor(history): enforce release record invariants * test(history): use a real governed release change * test(history): authorize review-required release fixture --- .../application/services/release_history.py | 214 +++++++++++++ semapact/history/__init__.py | 7 +- semapact/history/integrity.py | 81 +++++ semapact/history/models.py | 100 +++++- semapact/history/repository.py | 40 ++- semapact/platforms/git/history_repository.py | 89 +++++- tests/test_release_history.py | 302 ++++++++++++++++++ 7 files changed, 820 insertions(+), 13 deletions(-) create mode 100644 semapact/application/services/release_history.py create mode 100644 semapact/history/integrity.py create mode 100644 tests/test_release_history.py diff --git a/semapact/application/services/release_history.py b/semapact/application/services/release_history.py new file mode 100644 index 00000000..bd8305b7 --- /dev/null +++ b/semapact/application/services/release_history.py @@ -0,0 +1,214 @@ +"""Application orchestration for finalized contract release history.""" + +from __future__ import annotations + +from semapact.contractops.context import validate_release_context +from semapact.contractops.execution_models import AppliedContractRelease +from semapact.contractops.integrity import ( + validate_applied_release_identity, + validate_contractops_authorization_identity, +) +from semapact.contractops.models import ( + ContractOpsAuthorization, + ReleasePlan, + VersionResolution, +) +from semapact.governance.gate import GovernanceOperation +from semapact.history import ( + ChangeSetDecisionLinkHistoryRepository, + ChangeSetHistoryRepository, + ContractRevisionHistoryRepository, + DecisionHistoryRepository, + ReleasePlanHistoryRepository, + ReleaseRecord, + ReleaseRecordHistoryRepository, +) +from semapact.history.integrity import compute_release_record_id +from semapact.revision import build_contract_revision +from semapact.revision.integrity import validate_contract_revision_identity + + +class ReleaseHistoryService: + """Record one already-applied release without recomputing release semantics.""" + + def __init__( + self, + *, + revisions: ContractRevisionHistoryRepository, + change_sets: ChangeSetHistoryRepository, + decisions: DecisionHistoryRepository, + decision_links: ChangeSetDecisionLinkHistoryRepository, + release_plans: ReleasePlanHistoryRepository, + release_records: ReleaseRecordHistoryRepository, + ) -> None: + self._revisions = revisions + self._change_sets = change_sets + self._decisions = decisions + self._decision_links = decision_links + self._release_plans = release_plans + self._release_records = release_records + + def record_release( + self, + *, + release_plan: ReleasePlan, + version_resolution: VersionResolution, + authorization: ContractOpsAuthorization, + applied_release: AppliedContractRelease, + ) -> ReleaseRecord: + """Persist the exact proposal → plan → released-revision audit chain. + + All supplied ContractOps artifacts must already exist as valid outputs from + their owning M2 stages. This use case validates linkage only; it never reruns + governance, version authority, authorization, or APPLY. + """ + decision = self._decisions.get_decision(release_plan.decision_id) + change_set = self._change_sets.get_change_set(release_plan.change_set_id) + candidate_revision = self._revisions.get_revision(release_plan.release_revision_ref) + + _require_decision_link( + self._decision_links, + change_set_id=change_set.change_set_id, + decision_id=decision.decision_id, + ) + validate_contract_revision_identity(candidate_revision) + validate_release_context( + decision, + change_set, + release_plan, + version_resolution, + ) + _validate_candidate_revision(candidate_revision, release_plan, version_resolution) + _validate_apply_authorization( + authorization, + release_plan=release_plan, + version_resolution=version_resolution, + ) + _validate_applied_release( + applied_release, + release_plan=release_plan, + version_resolution=version_resolution, + authorization=authorization, + ) + + released_revision = build_contract_revision(applied_release.to_contract()) + if released_revision.revision_id == candidate_revision.revision_id: + raise ValueError( + "released revision must differ from candidate revision after APPLY versioning" + ) + + record_id = compute_release_record_id( + contract_id=release_plan.contract_id, + contract_version=version_resolution.selected_version, + decision_id=release_plan.decision_id, + change_set_id=release_plan.change_set_id, + release_plan_id=release_plan.release_plan_id, + version_resolution_id=version_resolution.version_resolution_id, + authorization_id=authorization.authorization_id, + applied_release_id=applied_release.applied_release_id, + released_revision_id=released_revision.revision_id, + required_version_bump=version_resolution.required_version_bump, + actual_version_bump=version_resolution.actual_bump, + version_authority=version_resolution.authority.value, + authority_reference=version_resolution.authority_reference, + review_evidence_reference=authorization.evidence_reference, + review_evidence_action=( + authorization.evidence_action.value + if authorization.evidence_action is not None + else None + ), + ) + record = ReleaseRecord( + release_record_id=record_id, + contract_id=release_plan.contract_id, + contract_version=version_resolution.selected_version, + decision_id=release_plan.decision_id, + change_set_id=release_plan.change_set_id, + release_plan_id=release_plan.release_plan_id, + version_resolution_id=version_resolution.version_resolution_id, + authorization_id=authorization.authorization_id, + applied_release_id=applied_release.applied_release_id, + released_revision_id=released_revision.revision_id, + required_version_bump=version_resolution.required_version_bump, + actual_version_bump=version_resolution.actual_bump, + version_authority=version_resolution.authority, + authority_reference=version_resolution.authority_reference, + review_evidence_reference=authorization.evidence_reference, + review_evidence_action=authorization.evidence_action, + ) + + # Persist referenced artifacts before the final record so a partial failure + # cannot leave a ReleaseRecord pointing at absent release/revision history. + self._release_plans.put_release_plan(release_plan) + self._revisions.put_revision(released_revision) + self._release_records.put_release_record(record) + return record + + +def _require_decision_link( + repository: ChangeSetDecisionLinkHistoryRepository, + *, + change_set_id: str, + decision_id: str, +) -> None: + links = repository.list_change_set_decision_links(change_set_id) + if not any(link.decision_id == decision_id for link in links): + raise ValueError( + "release history requires the ChangeSet-to-GovernanceDecision history link" + ) + + +def _validate_candidate_revision( + candidate_revision, + release_plan: ReleasePlan, + version_resolution: VersionResolution, +) -> None: + contract_id = str(candidate_revision.contract.id or "").strip() + contract_version = str(candidate_revision.contract.version or "").strip() + if contract_id != release_plan.contract_id: + raise ValueError("candidate ContractRevision does not match ReleasePlan contract") + if contract_version != version_resolution.current_version: + raise ValueError( + "candidate ContractRevision version does not match VersionResolution current_version" + ) + + +def _validate_apply_authorization( + authorization: ContractOpsAuthorization, + *, + release_plan: ReleasePlan, + version_resolution: VersionResolution, +) -> None: + validate_contractops_authorization_identity(authorization) + if authorization.operation is not GovernanceOperation.APPLY: + raise ValueError("release history requires APPLY-scoped authorization") + if not authorization.allowed: + raise ValueError("release history requires an allowed APPLY authorization") + if ( + authorization.decision_id != release_plan.decision_id + or authorization.change_set_id != release_plan.change_set_id + or authorization.release_plan_id != release_plan.release_plan_id + or authorization.version_resolution_id != version_resolution.version_resolution_id + ): + raise ValueError("APPLY authorization does not match the exact release context") + + +def _validate_applied_release( + applied_release: AppliedContractRelease, + *, + release_plan: ReleasePlan, + version_resolution: VersionResolution, + authorization: ContractOpsAuthorization, +) -> None: + validate_applied_release_identity(applied_release) + if ( + applied_release.contract_id != release_plan.contract_id + or applied_release.decision_id != release_plan.decision_id + or applied_release.change_set_id != release_plan.change_set_id + or applied_release.release_plan_id != release_plan.release_plan_id + or applied_release.version_resolution_id != version_resolution.version_resolution_id + or applied_release.release_revision_ref != release_plan.release_revision_ref + or applied_release.selected_version != version_resolution.selected_version + or applied_release.authorization_id != authorization.authorization_id + ): + raise ValueError("AppliedContractRelease does not match the exact release context") diff --git a/semapact/history/__init__.py b/semapact/history/__init__.py index e5420c09..6ef65a82 100644 --- a/semapact/history/__init__.py +++ b/semapact/history/__init__.py @@ -4,7 +4,7 @@ revision, governance, ContractOps, deployment, or reconciliation semantics. """ -from semapact.history.models import ChangeSetDecisionLink +from semapact.history.models import ChangeSetDecisionLink, ReleaseRecord from semapact.history.repository import ( ChangeSetDecisionLinkHistoryRepository, ChangeSetHistoryRepository, @@ -15,6 +15,8 @@ HistoryCorruptionError, HistoryNotFoundError, HistoryRepositoryError, + ReleasePlanHistoryRepository, + ReleaseRecordHistoryRepository, ) __all__ = [ @@ -28,4 +30,7 @@ "HistoryCorruptionError", "HistoryNotFoundError", "HistoryRepositoryError", + "ReleasePlanHistoryRepository", + "ReleaseRecord", + "ReleaseRecordHistoryRepository", ] diff --git a/semapact/history/integrity.py b/semapact/history/integrity.py new file mode 100644 index 00000000..4a4206ef --- /dev/null +++ b/semapact/history/integrity.py @@ -0,0 +1,81 @@ +"""Deterministic identity for history-owned audit records.""" + +from __future__ import annotations + +import uuid + +from semapact.history.models import ReleaseRecord +from semapact.utils.deterministic import deterministic_uuid5 + + +SEMAPACT_RELEASE_RECORD_NAMESPACE = uuid.UUID( + "9f750d1c-8f0a-491a-b861-8e349fc351cb" +) + + +def compute_release_record_id( + *, + contract_id: str, + contract_version: str, + decision_id: str, + change_set_id: str, + release_plan_id: str, + version_resolution_id: str, + authorization_id: str, + applied_release_id: str, + released_revision_id: str, + required_version_bump: str, + actual_version_bump: str, + version_authority: str, + authority_reference: str | None, + review_evidence_reference: str | None, + review_evidence_action: str | None, +) -> str: + """Derive one stable identity from the complete immutable release audit record.""" + return deterministic_uuid5( + SEMAPACT_RELEASE_RECORD_NAMESPACE, + { + "contract_id": contract_id, + "contract_version": contract_version, + "decision_id": decision_id, + "change_set_id": change_set_id, + "release_plan_id": release_plan_id, + "version_resolution_id": version_resolution_id, + "authorization_id": authorization_id, + "applied_release_id": applied_release_id, + "released_revision_id": released_revision_id, + "required_version_bump": required_version_bump, + "actual_version_bump": actual_version_bump, + "version_authority": version_authority, + "authority_reference": authority_reference, + "review_evidence_reference": review_evidence_reference, + "review_evidence_action": review_evidence_action, + }, + ) + + +def validate_release_record_identity(record: ReleaseRecord) -> None: + """Fail closed when a persisted release record ID does not match its content.""" + expected = compute_release_record_id( + contract_id=record.contract_id, + contract_version=record.contract_version, + decision_id=record.decision_id, + change_set_id=record.change_set_id, + release_plan_id=record.release_plan_id, + version_resolution_id=record.version_resolution_id, + authorization_id=record.authorization_id, + applied_release_id=record.applied_release_id, + released_revision_id=record.released_revision_id, + required_version_bump=record.required_version_bump, + actual_version_bump=record.actual_version_bump, + version_authority=record.version_authority.value, + authority_reference=record.authority_reference, + review_evidence_reference=record.review_evidence_reference, + review_evidence_action=( + record.review_evidence_action.value + if record.review_evidence_action is not None + else None + ), + ) + if record.release_record_id != expected: + raise ValueError("ReleaseRecord deterministic identity does not match its content") diff --git a/semapact/history/models.py b/semapact/history/models.py index eb0b4d97..b7e0b233 100644 --- a/semapact/history/models.py +++ b/semapact/history/models.py @@ -1,4 +1,4 @@ -"""Immutable history-owned relationship models. +"""Immutable history-owned audit models. These models record cross-artifact audit relationships without redefining the canonical domain artifacts they connect. @@ -6,23 +6,105 @@ from __future__ import annotations -from pydantic import BaseModel, ConfigDict, field_validator +from pydantic import BaseModel, ConfigDict, field_validator, model_validator +from semapact.contractops.models import ReviewEvidenceAction, VersionAuthority +from semapact.versioning import ActualVersionBump, RequiredBump -class ChangeSetDecisionLink(BaseModel): - """Audit provenance linking one ChangeSet to one GovernanceDecision outcome.""" + +class HistoryModel(BaseModel): + """Shared immutable base for history-owned logical records.""" model_config = ConfigDict(frozen=True, extra="forbid") + +class ChangeSetDecisionLink(HistoryModel): + """Audit provenance linking one ChangeSet to one GovernanceDecision outcome.""" + change_set_id: str decision_id: str @field_validator("change_set_id", "decision_id") @classmethod def _require_non_empty_text(cls, value: str) -> str: - if not isinstance(value, str): - raise TypeError("history link identifiers must be strings") + return _required_text(value) + + +class ReleaseRecord(HistoryModel): + """Durable audit projection for one finalized governed contract release. + + Canonical ContractOps artifacts remain authoritative for planning, versioning, + authorization, and APPLY semantics. This record freezes the stable links and + final facts required to reconstruct one released semantic version. + """ + + release_record_id: str + contract_id: str + contract_version: str + decision_id: str + change_set_id: str + release_plan_id: str + version_resolution_id: str + authorization_id: str + applied_release_id: str + released_revision_id: str + required_version_bump: RequiredBump + actual_version_bump: ActualVersionBump + version_authority: VersionAuthority + authority_reference: str | None = None + review_evidence_reference: str | None = None + review_evidence_action: ReviewEvidenceAction | None = None + + @field_validator( + "release_record_id", + "contract_id", + "contract_version", + "decision_id", + "change_set_id", + "release_plan_id", + "version_resolution_id", + "authorization_id", + "applied_release_id", + "released_revision_id", + ) + @classmethod + def _require_release_text(cls, value: str) -> str: + return _required_text(value) + + @field_validator( + "authority_reference", + "review_evidence_reference", + ) + @classmethod + def _normalize_optional_text(cls, value: str | None) -> str | None: + if value is None: + return None cleaned = value.strip() - if not cleaned: - raise ValueError("history link identifiers must not be empty") - return cleaned + return cleaned or None + + @model_validator(mode="after") + def _validate_provenance_pairs(self) -> ReleaseRecord: + if self.version_authority is VersionAuthority.GIT: + if self.authority_reference is None: + raise ValueError("git release history requires authority_reference") + elif self.authority_reference is not None: + raise ValueError( + "SemaPact release history must not contain authority_reference" + ) + + has_reference = self.review_evidence_reference is not None + has_action = self.review_evidence_action is not None + if has_reference != has_action: + raise ValueError( + "review evidence reference and action must either both be present or both be absent" + ) + return self + + +def _required_text(value: str) -> str: + if not isinstance(value, str): + raise TypeError("history identifiers must be strings") + cleaned = value.strip() + if not cleaned: + raise ValueError("history identifiers must not be empty") + return cleaned diff --git a/semapact/history/repository.py b/semapact/history/repository.py index beb87d06..cd88c893 100644 --- a/semapact/history/repository.py +++ b/semapact/history/repository.py @@ -4,9 +4,9 @@ from typing import Protocol -from semapact.contractops import ChangeSet +from semapact.contractops import ChangeSet, ReleasePlan from semapact.governance import GovernanceDecision -from semapact.history.models import ChangeSetDecisionLink +from semapact.history.models import ChangeSetDecisionLink, ReleaseRecord from semapact.revision.models import ContractRevision, ContractRevisionSource @@ -106,3 +106,39 @@ def list_revision_sources( ) -> tuple[ContractRevisionSource, ...]: """List all source links for one revision in deterministic ID order.""" ... + + +class ReleasePlanHistoryRepository(Protocol): + """Persistence capability for canonical ReleasePlan history only.""" + + def put_release_plan(self, release_plan: ReleasePlan) -> None: + """Persist one immutable ReleasePlan idempotently.""" + ... + + def get_release_plan(self, release_plan_id: str) -> ReleasePlan: + """Load one ReleasePlan by exact deterministic artifact ID.""" + ... + + +class ReleaseRecordHistoryRepository(Protocol): + """Persistence capability for finalized release audit records only.""" + + def put_release_record(self, record: ReleaseRecord) -> None: + """Persist one immutable ReleaseRecord idempotently.""" + ... + + def get_release_record(self, release_record_id: str) -> ReleaseRecord: + """Load one ReleaseRecord by exact deterministic artifact ID.""" + ... + + def list_release_records(self, contract_id: str) -> tuple[ReleaseRecord, ...]: + """List release records for one contract in deterministic artifact-ID order.""" + ... + + def get_release_record_by_version( + self, + contract_id: str, + contract_version: str, + ) -> ReleaseRecord: + """Load the unique finalized release for one contract semantic version.""" + ... diff --git a/semapact/platforms/git/history_repository.py b/semapact/platforms/git/history_repository.py index f6aa6791..89244927 100644 --- a/semapact/platforms/git/history_repository.py +++ b/semapact/platforms/git/history_repository.py @@ -13,14 +13,17 @@ from pydantic import BaseModel, ValidationError as PydanticValidationError -from semapact.contractops import ChangeSet +from semapact.contractops import ChangeSet, ReleasePlan +from semapact.contractops.integrity import validate_release_plan_identity from semapact.governance import GovernanceDecision from semapact.history import ( ChangeSetDecisionLink, HistoryConflictError, HistoryCorruptionError, HistoryNotFoundError, + ReleaseRecord, ) +from semapact.history.integrity import validate_release_record_identity from semapact.revision.integrity import ( validate_contract_revision_identity, validate_contract_revision_source_identity, @@ -198,6 +201,90 @@ def list_revision_sources( ) return tuple(record for record in records if record.revision_id == revision_id) + def put_release_plan(self, release_plan: ReleasePlan) -> None: + self._put( + kind="release_plans", + artifact_id=release_plan.release_plan_id, + artifact=release_plan, + model_type=ReleasePlan, + id_attribute="release_plan_id", + integrity_validator=validate_release_plan_identity, + ) + + def get_release_plan(self, release_plan_id: str) -> ReleasePlan: + return self._get( + kind="release_plans", + artifact_id=release_plan_id, + model_type=ReleasePlan, + id_attribute="release_plan_id", + integrity_validator=validate_release_plan_identity, + ) + + def put_release_record(self, record: ReleaseRecord) -> None: + if not isinstance(record, ReleaseRecord): + raise TypeError( + f"record must be ReleaseRecord, got {type(record).__name__}" + ) + validate_release_record_identity(record) + for existing in self.list_release_records(record.contract_id): + if ( + existing.contract_version == record.contract_version + and existing.release_record_id != record.release_record_id + ): + raise HistoryConflictError( + "A different ReleaseRecord already exists for " + f"{record.contract_id!r} version {record.contract_version!r}" + ) + self._put( + kind="release_records", + artifact_id=record.release_record_id, + artifact=record, + model_type=ReleaseRecord, + id_attribute="release_record_id", + integrity_validator=validate_release_record_identity, + ) + + def get_release_record(self, release_record_id: str) -> ReleaseRecord: + return self._get( + kind="release_records", + artifact_id=release_record_id, + model_type=ReleaseRecord, + id_attribute="release_record_id", + integrity_validator=validate_release_record_identity, + ) + + def list_release_records(self, contract_id: str) -> tuple[ReleaseRecord, ...]: + contract_id = _required_text(contract_id, "contract_id") + records = self._list( + kind="release_records", + model_type=ReleaseRecord, + id_attribute="release_record_id", + integrity_validator=validate_release_record_identity, + ) + return tuple(record for record in records if record.contract_id == contract_id) + + def get_release_record_by_version( + self, + contract_id: str, + contract_version: str, + ) -> ReleaseRecord: + contract_id = _required_text(contract_id, "contract_id") + contract_version = _required_text(contract_version, "contract_version") + matches = tuple( + record + for record in self.list_release_records(contract_id) + if record.contract_version == contract_version + ) + if not matches: + raise HistoryNotFoundError( + f"ReleaseRecord for {contract_id!r} version {contract_version!r} was not found" + ) + if len(matches) != 1: + raise HistoryCorruptionError( + f"Multiple ReleaseRecords exist for {contract_id!r} version {contract_version!r}" + ) + return matches[0] + def _put( self, *, diff --git a/tests/test_release_history.py b/tests/test_release_history.py new file mode 100644 index 00000000..45424ec0 --- /dev/null +++ b/tests/test_release_history.py @@ -0,0 +1,302 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest +from open_data_contract_standard.model import ( + OpenDataContractStandard, + SchemaObject, + SchemaProperty, +) + +from semapact.application.services.governance import GovernanceService +from semapact.application.services.history import ProposalHistoryService +from semapact.application.services.release_history import ReleaseHistoryService +from semapact.contractops import ( + ReviewAuthorizationEvidence, + ReviewEvidenceAction, + VersionAuthorityConfig, + apply_contract_release, + authorize_contract_operation, + build_release_plan, + resolve_release_version, +) +from semapact.governance.gate import GovernanceOperation +from semapact.history import HistoryConflictError +from semapact.platforms.git import GitWorkingTreeHistoryRepository +from semapact.revision import build_contract_revision + + +def _contract(*, include_note: bool = False) -> OpenDataContractStandard: + properties = [ + SchemaProperty( + name="id", + logicalType="string", + physicalType="varchar(255)", + required=True, + ) + ] + if include_note: + properties.append( + SchemaProperty( + name="note", + logicalType="string", + physicalType="varchar(255)", + required=False, + ) + ) + + return OpenDataContractStandard( + apiVersion="v3.1.0", + kind="DataContract", + id="orders-product", + name="Orders", + version="1.2.3", + status="active", + schema=[ + SchemaObject( + name="orders", + properties=properties, + ) + ], + ) + + +def _services(tmp_path: Path): + backend = GitWorkingTreeHistoryRepository(tmp_path) + proposal_history = ProposalHistoryService( + revisions=backend, + change_sets=backend, + decisions=backend, + decision_links=backend, + ) + release_history = ReleaseHistoryService( + revisions=backend, + change_sets=backend, + decisions=backend, + decision_links=backend, + release_plans=backend, + release_records=backend, + ) + return proposal_history, release_history, backend + + +def _release_bundle(*, source: str): + base_revision = build_contract_revision(_contract()) + candidate_revision = build_contract_revision(_contract(include_note=True)) + proposal = GovernanceService().evaluate_proposal( + base_revision.contract, + candidate_revision.contract, + effective_date="2026-09-12", + base_revision_ref=base_revision.revision_id, + candidate_revision_ref=candidate_revision.revision_id, + source=source, + actor_reference=f"actor:{source}", + ) + release_plan = build_release_plan(proposal.change_set, proposal.decision) + version_resolution = resolve_release_version( + release_plan, + current_version="1.2.3", + config=VersionAuthorityConfig(), + ) + apply_review = ReviewAuthorizationEvidence( + evidence_reference=f"review:{source}", + decision_id=proposal.decision.decision_id, + change_set_id=proposal.change_set.change_set_id, + release_plan_id=release_plan.release_plan_id, + version_resolution_id=version_resolution.version_resolution_id, + operation=GovernanceOperation.APPLY, + action=ReviewEvidenceAction.APPROVE, + ) + authorization = authorize_contract_operation( + proposal.decision, + proposal.change_set, + release_plan, + version_resolution, + GovernanceOperation.APPLY, + evidence=apply_review, + ) + applied_release = apply_contract_release( + candidate_revision.contract, + candidate_revision_ref=candidate_revision.revision_id, + decision=proposal.decision, + change_set=proposal.change_set, + release_plan=release_plan, + version_resolution=version_resolution, + authorization=authorization, + ) + return ( + proposal, + base_revision, + candidate_revision, + release_plan, + version_resolution, + authorization, + applied_release, + ) + + +def _record_bundle(tmp_path: Path, *, source: str = "test"): + proposal_history, release_history, backend = _services(tmp_path) + ( + proposal, + base_revision, + candidate_revision, + release_plan, + version_resolution, + authorization, + applied_release, + ) = _release_bundle(source=source) + proposal_history.record_proposal( + proposal, + base_revision=base_revision, + candidate_revision=candidate_revision, + ) + record = release_history.record_release( + release_plan=release_plan, + version_resolution=version_resolution, + authorization=authorization, + applied_release=applied_release, + ) + return ( + record, + proposal, + candidate_revision, + release_plan, + version_resolution, + authorization, + applied_release, + release_history, + backend, + ) + + +def test_records_release_against_exact_plan_and_released_revision(tmp_path: Path) -> None: + ( + record, + proposal, + candidate_revision, + release_plan, + version_resolution, + authorization, + applied_release, + _, + backend, + ) = _record_bundle(tmp_path) + + assert backend.get_release_plan(release_plan.release_plan_id) == release_plan + assert backend.get_release_record(record.release_record_id) == record + assert ( + backend.get_release_record_by_version( + "orders-product", + version_resolution.selected_version, + ) + == record + ) + released_revision = backend.get_revision(record.released_revision_id) + assert released_revision.revision_id != candidate_revision.revision_id + assert str(released_revision.contract.version) == version_resolution.selected_version + assert record.decision_id == proposal.decision.decision_id + assert record.change_set_id == proposal.change_set.change_set_id + assert record.version_resolution_id == version_resolution.version_resolution_id + assert record.authorization_id == authorization.authorization_id + assert record.applied_release_id == applied_release.applied_release_id + assert record.required_version_bump == version_resolution.required_version_bump + assert record.actual_version_bump == version_resolution.actual_bump + assert record.version_authority == version_resolution.authority + assert record.review_evidence_reference == authorization.evidence_reference + assert record.review_evidence_action == authorization.evidence_action + + +def test_exact_release_history_write_is_idempotent(tmp_path: Path) -> None: + ( + first, + _, + _, + release_plan, + version_resolution, + authorization, + applied_release, + release_history, + backend, + ) = _record_bundle(tmp_path) + + second = release_history.record_release( + release_plan=release_plan, + version_resolution=version_resolution, + authorization=authorization, + applied_release=applied_release, + ) + + assert second == first + assert backend.list_release_records("orders-product") == (first,) + + +def test_conflicting_release_for_same_contract_version_fails_closed(tmp_path: Path) -> None: + _record_bundle(tmp_path, source="first") + proposal_history, release_history, _ = _services(tmp_path) + ( + proposal, + base_revision, + candidate_revision, + release_plan, + version_resolution, + authorization, + applied_release, + ) = _release_bundle(source="second") + proposal_history.record_proposal( + proposal, + base_revision=base_revision, + candidate_revision=candidate_revision, + ) + + with pytest.raises(HistoryConflictError, match="version"): + release_history.record_release( + release_plan=release_plan, + version_resolution=version_resolution, + authorization=authorization, + applied_release=applied_release, + ) + + +def test_release_history_rejects_wrong_apply_authorization(tmp_path: Path) -> None: + proposal_history, release_history, _ = _services(tmp_path) + ( + proposal, + base_revision, + candidate_revision, + release_plan, + version_resolution, + _, + applied_release, + ) = _release_bundle(source="release") + proposal_history.record_proposal( + proposal, + base_revision=base_revision, + candidate_revision=candidate_revision, + ) + publish_review = ReviewAuthorizationEvidence( + evidence_reference="review:publish", + decision_id=proposal.decision.decision_id, + change_set_id=proposal.change_set.change_set_id, + release_plan_id=release_plan.release_plan_id, + version_resolution_id=version_resolution.version_resolution_id, + operation=GovernanceOperation.PUBLISH, + action=ReviewEvidenceAction.APPROVE, + ) + publish_authorization = authorize_contract_operation( + proposal.decision, + proposal.change_set, + release_plan, + version_resolution, + GovernanceOperation.PUBLISH, + evidence=publish_review, + ) + + with pytest.raises(ValueError, match="APPLY"): + release_history.record_release( + release_plan=release_plan, + version_resolution=version_resolution, + authorization=publish_authorization, + applied_release=applied_release, + ) From 81cb2b214acc6b5c42d544a6f4936e2de1243cea Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Sun, 13 Sep 2026 10:37:11 +1000 Subject: [PATCH 20/35] feat(history): record deployment execution attempts Record immutable deployment execution history linked to finalized releases, exact deployment plans, previews, and authorizations while keeping runtime convergence separate. --- .../services/deployment_history.py | 191 ++++++++++++ semapact/history/__init__.py | 17 +- semapact/history/integrity.py | 62 +++- semapact/history/models.py | 80 ++++- semapact/history/repository.py | 75 ++++- semapact/platforms/git/history_repository.py | 124 +++++++- tests/test_deployment_history.py | 285 ++++++++++++++++++ 7 files changed, 826 insertions(+), 8 deletions(-) create mode 100644 semapact/application/services/deployment_history.py create mode 100644 tests/test_deployment_history.py diff --git a/semapact/application/services/deployment_history.py b/semapact/application/services/deployment_history.py new file mode 100644 index 00000000..ef96a50d --- /dev/null +++ b/semapact/application/services/deployment_history.py @@ -0,0 +1,191 @@ +"""Application orchestration for terminal deployment execution history.""" + +from __future__ import annotations + +from datetime import datetime, timezone + +from semapact.deployment import DeploymentAuthorization, DeploymentPlan, DeploymentPreview +from semapact.deployment.models import ( + validate_deployment_authorization_identity, + validate_deployment_plan_identity, + validate_deployment_preview_identity, +) +from semapact.history import ( + DeploymentAuthorizationHistoryRepository, + DeploymentPlanHistoryRepository, + DeploymentPreviewHistoryRepository, + DeploymentRecord, + DeploymentRecordHistoryRepository, + DeploymentStatus, + ReleaseRecord, + ReleaseRecordHistoryRepository, +) +from semapact.history.integrity import compute_deployment_record_id + + +class DeploymentHistoryService: + """Record already-observed provider execution outcomes without executing them.""" + + def __init__( + self, + *, + releases: ReleaseRecordHistoryRepository, + deployment_plans: DeploymentPlanHistoryRepository, + deployment_previews: DeploymentPreviewHistoryRepository, + deployment_authorizations: DeploymentAuthorizationHistoryRepository, + deployment_records: DeploymentRecordHistoryRepository, + ) -> None: + self._releases = releases + self._deployment_plans = deployment_plans + self._deployment_previews = deployment_previews + self._deployment_authorizations = deployment_authorizations + self._deployment_records = deployment_records + + def record_execution( + self, + *, + plan: DeploymentPlan, + preview: DeploymentPreview, + authorization: DeploymentAuthorization, + status: DeploymentStatus, + started_at: datetime, + completed_at: datetime, + actor_reference: str | None = None, + external_reference: str | None = None, + ) -> DeploymentRecord: + """Persist one terminal execution occurrence against exact deployment artifacts. + + The caller supplies the outcome after the provider execution boundary has + completed. This method never invokes a DeploymentAdapter and never infers + runtime convergence from execution success. + """ + _validate_types(plan, preview, authorization, status) + validate_deployment_plan_identity(plan) + validate_deployment_preview_identity(preview) + validate_deployment_authorization_identity(authorization) + + release = self._releases.get_release_record_by_version( + plan.contract_id, + plan.selected_version, + ) + _validate_release_link(release, plan) + _validate_preview_link(preview, plan) + _validate_authorization_link(authorization, plan) + + started = _utc_timestamp(started_at, "started_at") + completed = _utc_timestamp(completed_at, "completed_at") + if completed < started: + raise ValueError("completed_at must not be earlier than started_at") + + actor = _optional_text(actor_reference) + external = _optional_text(external_reference) + deployment_record_id = compute_deployment_record_id( + release_record_id=release.release_record_id, + deployment_plan_id=plan.deployment_plan_id, + deployment_preview_id=preview.deployment_preview_id, + deployment_authorization_id=authorization.deployment_authorization_id, + platform=plan.target.platform, + runtime_target=plan.target.runtime_target, + source_reference=plan.target.source_reference, + status=status.value, + started_at=started.isoformat(), + completed_at=completed.isoformat(), + actor_reference=actor, + external_reference=external, + ) + record = DeploymentRecord( + deployment_record_id=deployment_record_id, + release_record_id=release.release_record_id, + deployment_plan_id=plan.deployment_plan_id, + deployment_preview_id=preview.deployment_preview_id, + deployment_authorization_id=authorization.deployment_authorization_id, + platform=plan.target.platform, + runtime_target=plan.target.runtime_target, + source_reference=plan.target.source_reference, + status=status, + started_at=started, + completed_at=completed, + actor_reference=actor, + external_reference=external, + ) + + # Persist exact referenced artifacts before the occurrence record so a + # partial failure cannot leave history pointing at absent dependencies. + self._deployment_plans.put_deployment_plan(plan) + self._deployment_previews.put_deployment_preview(preview) + self._deployment_authorizations.put_deployment_authorization(authorization) + self._deployment_records.put_deployment_record(record) + return record + + +def _validate_types( + plan: DeploymentPlan, + preview: DeploymentPreview, + authorization: DeploymentAuthorization, + status: DeploymentStatus, +) -> None: + expected = ( + (plan, DeploymentPlan, "plan"), + (preview, DeploymentPreview, "preview"), + (authorization, DeploymentAuthorization, "authorization"), + (status, DeploymentStatus, "status"), + ) + for value, expected_type, name in expected: + if not isinstance(value, expected_type): + raise TypeError( + f"{name} must be {expected_type.__name__}, got {type(value).__name__}" + ) + + +def _validate_release_link(release: ReleaseRecord, plan: DeploymentPlan) -> None: + if ( + release.contract_id != plan.contract_id + or release.contract_version != plan.selected_version + or release.release_plan_id != plan.release_plan_id + or release.applied_release_id != plan.applied_release_id + ): + raise ValueError("DeploymentPlan does not match the finalized ReleaseRecord") + + +def _validate_preview_link(preview: DeploymentPreview, plan: DeploymentPlan) -> None: + if preview.deployment_plan_id != plan.deployment_plan_id: + raise ValueError("DeploymentPreview does not reference the supplied DeploymentPlan") + if preview.platform.strip().casefold() != plan.target.platform: + raise ValueError("DeploymentPreview platform does not match DeploymentPlan target") + if preview.runtime_target != plan.target.runtime_target: + raise ValueError("DeploymentPreview runtime target does not match DeploymentPlan") + if preview.source_identifier != plan.target.source_reference: + raise ValueError("DeploymentPreview source does not match DeploymentPlan source reference") + + +def _validate_authorization_link( + authorization: DeploymentAuthorization, + plan: DeploymentPlan, +) -> None: + if authorization.deployment_plan_id != plan.deployment_plan_id: + raise ValueError( + "DeploymentAuthorization does not reference the supplied DeploymentPlan" + ) + if authorization.applied_release_id != plan.applied_release_id: + raise ValueError( + "DeploymentAuthorization does not reference the plan's applied release" + ) + if not authorization.allowed: + raise ValueError("deployment history requires an allowed DeploymentAuthorization") + + +def _utc_timestamp(value: datetime, field_name: str) -> datetime: + if not isinstance(value, datetime): + raise TypeError(f"{field_name} must be datetime") + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError(f"{field_name} must be timezone-aware") + return value.astimezone(timezone.utc) + + +def _optional_text(value: str | None) -> str | None: + if value is None: + return None + if not isinstance(value, str): + raise TypeError("optional deployment references must be strings") + cleaned = value.strip() + return cleaned or None diff --git a/semapact/history/__init__.py b/semapact/history/__init__.py index 6ef65a82..1df69920 100644 --- a/semapact/history/__init__.py +++ b/semapact/history/__init__.py @@ -4,13 +4,22 @@ revision, governance, ContractOps, deployment, or reconciliation semantics. """ -from semapact.history.models import ChangeSetDecisionLink, ReleaseRecord +from semapact.history.models import ( + ChangeSetDecisionLink, + DeploymentRecord, + DeploymentStatus, + ReleaseRecord, +) from semapact.history.repository import ( ChangeSetDecisionLinkHistoryRepository, ChangeSetHistoryRepository, ContractRevisionHistoryRepository, ContractRevisionSourceHistoryRepository, DecisionHistoryRepository, + DeploymentAuthorizationHistoryRepository, + DeploymentPlanHistoryRepository, + DeploymentPreviewHistoryRepository, + DeploymentRecordHistoryRepository, HistoryConflictError, HistoryCorruptionError, HistoryNotFoundError, @@ -26,6 +35,12 @@ "ContractRevisionHistoryRepository", "ContractRevisionSourceHistoryRepository", "DecisionHistoryRepository", + "DeploymentAuthorizationHistoryRepository", + "DeploymentPlanHistoryRepository", + "DeploymentPreviewHistoryRepository", + "DeploymentRecord", + "DeploymentRecordHistoryRepository", + "DeploymentStatus", "HistoryConflictError", "HistoryCorruptionError", "HistoryNotFoundError", diff --git a/semapact/history/integrity.py b/semapact/history/integrity.py index 4a4206ef..4775f2b7 100644 --- a/semapact/history/integrity.py +++ b/semapact/history/integrity.py @@ -4,13 +4,16 @@ import uuid -from semapact.history.models import ReleaseRecord +from semapact.history.models import DeploymentRecord, ReleaseRecord from semapact.utils.deterministic import deterministic_uuid5 SEMAPACT_RELEASE_RECORD_NAMESPACE = uuid.UUID( "9f750d1c-8f0a-491a-b861-8e349fc351cb" ) +SEMAPACT_DEPLOYMENT_RECORD_NAMESPACE = uuid.UUID( + "93db6770-90f0-4ac5-a185-2f21e818d18e" +) def compute_release_record_id( @@ -79,3 +82,60 @@ def validate_release_record_identity(record: ReleaseRecord) -> None: ) if record.release_record_id != expected: raise ValueError("ReleaseRecord deterministic identity does not match its content") + + +def compute_deployment_record_id( + *, + release_record_id: str, + deployment_plan_id: str, + deployment_preview_id: str, + deployment_authorization_id: str, + platform: str, + runtime_target: str, + source_reference: str, + status: str, + started_at: str, + completed_at: str, + actor_reference: str | None, + external_reference: str | None, +) -> str: + """Derive one stable identity for a concrete deployment execution occurrence.""" + return deterministic_uuid5( + SEMAPACT_DEPLOYMENT_RECORD_NAMESPACE, + { + "release_record_id": release_record_id, + "deployment_plan_id": deployment_plan_id, + "deployment_preview_id": deployment_preview_id, + "deployment_authorization_id": deployment_authorization_id, + "platform": platform, + "runtime_target": runtime_target, + "source_reference": source_reference, + "status": status, + "started_at": started_at, + "completed_at": completed_at, + "actor_reference": actor_reference, + "external_reference": external_reference, + }, + ) + + +def validate_deployment_record_identity(record: DeploymentRecord) -> None: + """Fail closed when a deployment occurrence ID does not match its content.""" + expected = compute_deployment_record_id( + release_record_id=record.release_record_id, + deployment_plan_id=record.deployment_plan_id, + deployment_preview_id=record.deployment_preview_id, + deployment_authorization_id=record.deployment_authorization_id, + platform=record.platform, + runtime_target=record.runtime_target, + source_reference=record.source_reference, + status=record.status.value, + started_at=record.started_at.isoformat(), + completed_at=record.completed_at.isoformat(), + actor_reference=record.actor_reference, + external_reference=record.external_reference, + ) + if record.deployment_record_id != expected: + raise ValueError( + "DeploymentRecord deterministic identity does not match its content" + ) diff --git a/semapact/history/models.py b/semapact/history/models.py index b7e0b233..1ddeae3a 100644 --- a/semapact/history/models.py +++ b/semapact/history/models.py @@ -6,6 +6,9 @@ from __future__ import annotations +from datetime import datetime, timezone +from enum import Enum + from pydantic import BaseModel, ConfigDict, field_validator, model_validator from semapact.contractops.models import ReviewEvidenceAction, VersionAuthority @@ -77,10 +80,7 @@ def _require_release_text(cls, value: str) -> str: ) @classmethod def _normalize_optional_text(cls, value: str | None) -> str | None: - if value is None: - return None - cleaned = value.strip() - return cleaned or None + return _optional_text(value) @model_validator(mode="after") def _validate_provenance_pairs(self) -> ReleaseRecord: @@ -101,6 +101,71 @@ def _validate_provenance_pairs(self) -> ReleaseRecord: return self +class DeploymentStatus(str, Enum): + """Terminal provider-execution outcome for one deployment occurrence.""" + + SUCCEEDED = "SUCCEEDED" + FAILED = "FAILED" + + +class DeploymentRecord(HistoryModel): + """Immutable audit record for one completed deployment execution occurrence. + + This records provider execution only. ``SUCCEEDED`` never implies runtime + convergence; reconciliation remains a separate observation/history concern. + """ + + deployment_record_id: str + release_record_id: str + deployment_plan_id: str + deployment_preview_id: str + deployment_authorization_id: str + platform: str + runtime_target: str + source_reference: str + status: DeploymentStatus + started_at: datetime + completed_at: datetime + actor_reference: str | None = None + external_reference: str | None = None + + @field_validator( + "deployment_record_id", + "release_record_id", + "deployment_plan_id", + "deployment_preview_id", + "deployment_authorization_id", + "runtime_target", + "source_reference", + ) + @classmethod + def _require_deployment_text(cls, value: str) -> str: + return _required_text(value) + + @field_validator("platform") + @classmethod + def _normalize_platform(cls, value: str) -> str: + return _required_text(value).casefold() + + @field_validator("actor_reference", "external_reference") + @classmethod + def _normalize_optional_deployment_text(cls, value: str | None) -> str | None: + return _optional_text(value) + + @field_validator("started_at", "completed_at") + @classmethod + def _normalize_timestamp(cls, value: datetime) -> datetime: + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError("deployment timestamps must be timezone-aware") + return value.astimezone(timezone.utc) + + @model_validator(mode="after") + def _validate_timestamp_order(self) -> DeploymentRecord: + if self.completed_at < self.started_at: + raise ValueError("completed_at must not be earlier than started_at") + return self + + def _required_text(value: str) -> str: if not isinstance(value, str): raise TypeError("history identifiers must be strings") @@ -108,3 +173,10 @@ def _required_text(value: str) -> str: if not cleaned: raise ValueError("history identifiers must not be empty") return cleaned + + +def _optional_text(value: str | None) -> str | None: + if value is None: + return None + cleaned = value.strip() + return cleaned or None diff --git a/semapact/history/repository.py b/semapact/history/repository.py index cd88c893..6d7f496e 100644 --- a/semapact/history/repository.py +++ b/semapact/history/repository.py @@ -5,8 +5,13 @@ from typing import Protocol from semapact.contractops import ChangeSet, ReleasePlan +from semapact.deployment import DeploymentAuthorization, DeploymentPlan, DeploymentPreview from semapact.governance import GovernanceDecision -from semapact.history.models import ChangeSetDecisionLink, ReleaseRecord +from semapact.history.models import ( + ChangeSetDecisionLink, + DeploymentRecord, + ReleaseRecord, +) from semapact.revision.models import ContractRevision, ContractRevisionSource @@ -142,3 +147,71 @@ def get_release_record_by_version( ) -> ReleaseRecord: """Load the unique finalized release for one contract semantic version.""" ... + + +class DeploymentPlanHistoryRepository(Protocol): + """Persistence capability for canonical DeploymentPlan history only.""" + + def put_deployment_plan(self, plan: DeploymentPlan) -> None: + """Persist one immutable DeploymentPlan idempotently.""" + ... + + def get_deployment_plan(self, deployment_plan_id: str) -> DeploymentPlan: + """Load one DeploymentPlan by exact deterministic artifact ID.""" + ... + + +class DeploymentPreviewHistoryRepository(Protocol): + """Persistence capability for canonical DeploymentPreview history only.""" + + def put_deployment_preview(self, preview: DeploymentPreview) -> None: + """Persist one immutable DeploymentPreview idempotently.""" + ... + + def get_deployment_preview(self, deployment_preview_id: str) -> DeploymentPreview: + """Load one DeploymentPreview by exact deterministic artifact ID.""" + ... + + +class DeploymentAuthorizationHistoryRepository(Protocol): + """Persistence capability for canonical DeploymentAuthorization history only.""" + + def put_deployment_authorization( + self, + authorization: DeploymentAuthorization, + ) -> None: + """Persist one immutable DeploymentAuthorization idempotently.""" + ... + + def get_deployment_authorization( + self, + deployment_authorization_id: str, + ) -> DeploymentAuthorization: + """Load one DeploymentAuthorization by exact deterministic artifact ID.""" + ... + + +class DeploymentRecordHistoryRepository(Protocol): + """Persistence capability for terminal deployment execution occurrences only.""" + + def put_deployment_record(self, record: DeploymentRecord) -> None: + """Persist one immutable DeploymentRecord idempotently.""" + ... + + def get_deployment_record(self, deployment_record_id: str) -> DeploymentRecord: + """Load one DeploymentRecord by exact deterministic artifact ID.""" + ... + + def list_deployment_records_for_release( + self, + release_record_id: str, + ) -> tuple[DeploymentRecord, ...]: + """List deployment occurrences for one finalized release.""" + ... + + def list_deployment_records_for_plan( + self, + deployment_plan_id: str, + ) -> tuple[DeploymentRecord, ...]: + """List deployment occurrences for one exact deployment plan.""" + ... diff --git a/semapact/platforms/git/history_repository.py b/semapact/platforms/git/history_repository.py index 89244927..639e5b21 100644 --- a/semapact/platforms/git/history_repository.py +++ b/semapact/platforms/git/history_repository.py @@ -15,15 +15,25 @@ from semapact.contractops import ChangeSet, ReleasePlan from semapact.contractops.integrity import validate_release_plan_identity +from semapact.deployment import DeploymentAuthorization, DeploymentPlan, DeploymentPreview +from semapact.deployment.models import ( + validate_deployment_authorization_identity, + validate_deployment_plan_identity, + validate_deployment_preview_identity, +) from semapact.governance import GovernanceDecision from semapact.history import ( ChangeSetDecisionLink, + DeploymentRecord, HistoryConflictError, HistoryCorruptionError, HistoryNotFoundError, ReleaseRecord, ) -from semapact.history.integrity import validate_release_record_identity +from semapact.history.integrity import ( + validate_deployment_record_identity, + validate_release_record_identity, +) from semapact.revision.integrity import ( validate_contract_revision_identity, validate_contract_revision_source_identity, @@ -285,6 +295,118 @@ def get_release_record_by_version( ) return matches[0] + def put_deployment_plan(self, plan: DeploymentPlan) -> None: + self._put( + kind="deployment_plans", + artifact_id=plan.deployment_plan_id, + artifact=plan, + model_type=DeploymentPlan, + id_attribute="deployment_plan_id", + integrity_validator=validate_deployment_plan_identity, + ) + + def get_deployment_plan(self, deployment_plan_id: str) -> DeploymentPlan: + return self._get( + kind="deployment_plans", + artifact_id=deployment_plan_id, + model_type=DeploymentPlan, + id_attribute="deployment_plan_id", + integrity_validator=validate_deployment_plan_identity, + ) + + def put_deployment_preview(self, preview: DeploymentPreview) -> None: + self._put( + kind="deployment_previews", + artifact_id=preview.deployment_preview_id, + artifact=preview, + model_type=DeploymentPreview, + id_attribute="deployment_preview_id", + integrity_validator=validate_deployment_preview_identity, + ) + + def get_deployment_preview(self, deployment_preview_id: str) -> DeploymentPreview: + return self._get( + kind="deployment_previews", + artifact_id=deployment_preview_id, + model_type=DeploymentPreview, + id_attribute="deployment_preview_id", + integrity_validator=validate_deployment_preview_identity, + ) + + def put_deployment_authorization( + self, + authorization: DeploymentAuthorization, + ) -> None: + self._put( + kind="deployment_authorizations", + artifact_id=authorization.deployment_authorization_id, + artifact=authorization, + model_type=DeploymentAuthorization, + id_attribute="deployment_authorization_id", + integrity_validator=validate_deployment_authorization_identity, + ) + + def get_deployment_authorization( + self, + deployment_authorization_id: str, + ) -> DeploymentAuthorization: + return self._get( + kind="deployment_authorizations", + artifact_id=deployment_authorization_id, + model_type=DeploymentAuthorization, + id_attribute="deployment_authorization_id", + integrity_validator=validate_deployment_authorization_identity, + ) + + def put_deployment_record(self, record: DeploymentRecord) -> None: + self._put( + kind="deployment_records", + artifact_id=record.deployment_record_id, + artifact=record, + model_type=DeploymentRecord, + id_attribute="deployment_record_id", + integrity_validator=validate_deployment_record_identity, + ) + + def get_deployment_record(self, deployment_record_id: str) -> DeploymentRecord: + return self._get( + kind="deployment_records", + artifact_id=deployment_record_id, + model_type=DeploymentRecord, + id_attribute="deployment_record_id", + integrity_validator=validate_deployment_record_identity, + ) + + def list_deployment_records_for_release( + self, + release_record_id: str, + ) -> tuple[DeploymentRecord, ...]: + release_record_id = _required_text(release_record_id, "release_record_id") + records = self._list( + kind="deployment_records", + model_type=DeploymentRecord, + id_attribute="deployment_record_id", + integrity_validator=validate_deployment_record_identity, + ) + return tuple( + record for record in records if record.release_record_id == release_record_id + ) + + def list_deployment_records_for_plan( + self, + deployment_plan_id: str, + ) -> tuple[DeploymentRecord, ...]: + deployment_plan_id = _required_text(deployment_plan_id, "deployment_plan_id") + records = self._list( + kind="deployment_records", + model_type=DeploymentRecord, + id_attribute="deployment_record_id", + integrity_validator=validate_deployment_record_identity, + ) + return tuple( + record for record in records if record.deployment_plan_id == deployment_plan_id + ) + def _put( self, *, diff --git a/tests/test_deployment_history.py b/tests/test_deployment_history.py new file mode 100644 index 00000000..0279e699 --- /dev/null +++ b/tests/test_deployment_history.py @@ -0,0 +1,285 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest + +from semapact.application.services.deployment_history import DeploymentHistoryService +from semapact.contractops import VersionAuthority +from semapact.deployment import ( + DeploymentAuthorization, + DeploymentPlan, + DeploymentPreview, + DeploymentTarget, +) +from semapact.deployment.models import ( + compute_deployment_authorization_id, + compute_deployment_plan_id, + compute_deployment_preview_id, +) +from semapact.history import ( + DeploymentStatus, + HistoryCorruptionError, + ReleaseRecord, +) +from semapact.history.integrity import compute_release_record_id +from semapact.platforms.git import GitWorkingTreeHistoryRepository + + +def _release_record() -> ReleaseRecord: + fields = { + "contract_id": "orders-product", + "contract_version": "1.3.0", + "decision_id": "decision-1", + "change_set_id": "changeset-1", + "release_plan_id": "release-plan-1", + "version_resolution_id": "version-resolution-1", + "authorization_id": "apply-authorization-1", + "applied_release_id": "applied-release-1", + "released_revision_id": "released-revision-1", + "required_version_bump": "minor", + "actual_version_bump": "minor", + "version_authority": VersionAuthority.SEMAPACT, + "authority_reference": None, + "review_evidence_reference": None, + "review_evidence_action": None, + } + release_record_id = compute_release_record_id( + **{ + **fields, + "version_authority": fields["version_authority"].value, + } + ) + return ReleaseRecord(release_record_id=release_record_id, **fields) + + +def _deployment_artifacts(*, preview_runtime_target: str = "catalog.schema", allowed: bool = True): + target = DeploymentTarget( + platform="databricks", + runtime_target="catalog.schema", + source_reference="workspace:test", + ) + plan_id = compute_deployment_plan_id( + applied_release_id="applied-release-1", + contract_id="orders-product", + release_plan_id="release-plan-1", + released_revision_ref="candidate-revision-1", + selected_version="1.3.0", + target=target, + actions=(), + ) + plan = DeploymentPlan( + deployment_plan_id=plan_id, + applied_release_id="applied-release-1", + contract_id="orders-product", + release_plan_id="release-plan-1", + released_revision_ref="candidate-revision-1", + selected_version="1.3.0", + target=target, + actions=(), + ) + + preview_id = compute_deployment_preview_id( + deployment_plan_id=plan.deployment_plan_id, + platform="databricks", + runtime_target=preview_runtime_target, + source_identifier="workspace:test", + observation_fingerprint="observation-1", + operations=(), + ) + preview = DeploymentPreview( + deployment_preview_id=preview_id, + deployment_plan_id=plan.deployment_plan_id, + platform="databricks", + runtime_target=preview_runtime_target, + source_identifier="workspace:test", + observation_fingerprint="observation-1", + operations=(), + ) + + authorization_id = compute_deployment_authorization_id( + contract_ops_authorization_id="deploy-authorization-1", + deployment_plan_id=plan.deployment_plan_id, + applied_release_id=plan.applied_release_id, + allowed=allowed, + ) + authorization = DeploymentAuthorization( + deployment_authorization_id=authorization_id, + contract_ops_authorization_id="deploy-authorization-1", + deployment_plan_id=plan.deployment_plan_id, + applied_release_id=plan.applied_release_id, + allowed=allowed, + ) + return plan, preview, authorization + + +def _service(tmp_path: Path): + backend = GitWorkingTreeHistoryRepository(tmp_path) + backend.put_release_record(_release_record()) + service = DeploymentHistoryService( + releases=backend, + deployment_plans=backend, + deployment_previews=backend, + deployment_authorizations=backend, + deployment_records=backend, + ) + return service, backend + + +def test_records_terminal_deployment_occurrence(tmp_path: Path) -> None: + service, backend = _service(tmp_path) + plan, preview, authorization = _deployment_artifacts() + started = datetime(2026, 9, 13, 0, 0, tzinfo=timezone.utc) + completed = started + timedelta(seconds=12) + + record = service.record_execution( + plan=plan, + preview=preview, + authorization=authorization, + status=DeploymentStatus.SUCCEEDED, + started_at=started, + completed_at=completed, + actor_reference="agent:deploy", + external_reference="pipeline:42", + ) + + assert backend.get_deployment_plan(plan.deployment_plan_id) == plan + assert backend.get_deployment_preview(preview.deployment_preview_id) == preview + assert ( + backend.get_deployment_authorization(authorization.deployment_authorization_id) + == authorization + ) + assert backend.get_deployment_record(record.deployment_record_id) == record + assert backend.list_deployment_records_for_release(record.release_record_id) == ( + record, + ) + assert backend.list_deployment_records_for_plan(plan.deployment_plan_id) == (record,) + assert record.platform == "databricks" + assert record.runtime_target == "catalog.schema" + assert record.source_reference == "workspace:test" + assert record.status is DeploymentStatus.SUCCEEDED + + +def test_exact_deployment_history_write_is_idempotent(tmp_path: Path) -> None: + service, backend = _service(tmp_path) + plan, preview, authorization = _deployment_artifacts() + started = datetime(2026, 9, 13, 0, 0, tzinfo=timezone.utc) + completed = started + timedelta(seconds=1) + + first = service.record_execution( + plan=plan, + preview=preview, + authorization=authorization, + status=DeploymentStatus.SUCCEEDED, + started_at=started, + completed_at=completed, + ) + second = service.record_execution( + plan=plan, + preview=preview, + authorization=authorization, + status=DeploymentStatus.SUCCEEDED, + started_at=started, + completed_at=completed, + ) + + assert second == first + assert backend.list_deployment_records_for_plan(plan.deployment_plan_id) == (first,) + + +def test_failed_attempt_remains_after_later_success(tmp_path: Path) -> None: + service, backend = _service(tmp_path) + plan, preview, authorization = _deployment_artifacts() + started = datetime(2026, 9, 13, 0, 0, tzinfo=timezone.utc) + + failed = service.record_execution( + plan=plan, + preview=preview, + authorization=authorization, + status=DeploymentStatus.FAILED, + started_at=started, + completed_at=started + timedelta(seconds=2), + external_reference="pipeline:42", + ) + succeeded = service.record_execution( + plan=plan, + preview=preview, + authorization=authorization, + status=DeploymentStatus.SUCCEEDED, + started_at=started + timedelta(minutes=5), + completed_at=started + timedelta(minutes=5, seconds=3), + external_reference="pipeline:43", + ) + + records = backend.list_deployment_records_for_plan(plan.deployment_plan_id) + assert set(records) == {failed, succeeded} + assert {record.status for record in records} == { + DeploymentStatus.FAILED, + DeploymentStatus.SUCCEEDED, + } + + +def test_rejects_preview_for_different_runtime_target(tmp_path: Path) -> None: + service, _ = _service(tmp_path) + plan, preview, authorization = _deployment_artifacts( + preview_runtime_target="other.schema" + ) + + with pytest.raises(ValueError, match="runtime target"): + service.record_execution( + plan=plan, + preview=preview, + authorization=authorization, + status=DeploymentStatus.FAILED, + started_at=datetime(2026, 9, 13, tzinfo=timezone.utc), + completed_at=datetime(2026, 9, 13, 0, 0, 1, tzinfo=timezone.utc), + ) + + +def test_rejects_denied_deployment_authorization(tmp_path: Path) -> None: + service, _ = _service(tmp_path) + plan, preview, authorization = _deployment_artifacts(allowed=False) + + with pytest.raises(ValueError, match="allowed"): + service.record_execution( + plan=plan, + preview=preview, + authorization=authorization, + status=DeploymentStatus.FAILED, + started_at=datetime(2026, 9, 13, tzinfo=timezone.utc), + completed_at=datetime(2026, 9, 13, 0, 0, 1, tzinfo=timezone.utc), + ) + + +def test_rejects_naive_execution_timestamps(tmp_path: Path) -> None: + service, _ = _service(tmp_path) + plan, preview, authorization = _deployment_artifacts() + + with pytest.raises(ValueError, match="timezone-aware"): + service.record_execution( + plan=plan, + preview=preview, + authorization=authorization, + status=DeploymentStatus.SUCCEEDED, + started_at=datetime(2026, 9, 13), + completed_at=datetime(2026, 9, 13, 0, 0, 1), + ) + + +def test_tampered_deployment_record_identity_fails_closed(tmp_path: Path) -> None: + service, backend = _service(tmp_path) + plan, preview, authorization = _deployment_artifacts() + started = datetime(2026, 9, 13, tzinfo=timezone.utc) + record = service.record_execution( + plan=plan, + preview=preview, + authorization=authorization, + status=DeploymentStatus.SUCCEEDED, + started_at=started, + completed_at=started + timedelta(seconds=1), + ) + + tampered = record.model_copy(update={"status": DeploymentStatus.FAILED}) + with pytest.raises(HistoryCorruptionError, match="invalid"): + backend.put_deployment_record(tampered) From 596bfe14eee09993c815edaa931055c00f4e5c5d Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Sun, 13 Sep 2026 17:11:18 +1000 Subject: [PATCH 21/35] feat(history): preserve runtime reconciliation evidence (#226) * feat(history): add runtime evidence envelopes * feat(history): add runtime evidence identity * feat(history): add runtime evidence persistence ports * feat(history): persist runtime evidence in git backend * feat(history): export runtime history capabilities * feat(history): add runtime evidence orchestration * test(history): cover runtime observation history --- .../application/services/runtime_history.py | 191 ++++++++++++++ semapact/history/__init__.py | 8 + semapact/history/integrity.py | 83 +++++- semapact/history/models.py | 61 +++++ semapact/history/repository.py | 174 +++++-------- semapact/platforms/git/history_repository.py | 105 +++++++- tests/test_runtime_history.py | 243 ++++++++++++++++++ 7 files changed, 755 insertions(+), 110 deletions(-) create mode 100644 semapact/application/services/runtime_history.py create mode 100644 tests/test_runtime_history.py diff --git a/semapact/application/services/runtime_history.py b/semapact/application/services/runtime_history.py new file mode 100644 index 00000000..99b8038a --- /dev/null +++ b/semapact/application/services/runtime_history.py @@ -0,0 +1,191 @@ +"""Application orchestration for point-in-time runtime evidence history.""" + +from __future__ import annotations + +from semapact.history import ( + DeploymentRecord, + DeploymentRecordHistoryRepository, + ReleaseRecord, + ReleaseRecordHistoryRepository, + RuntimeObservationHistoryRepository, + RuntimeObservationRecord, + RuntimeReconciliationHistoryRepository, + RuntimeReconciliationRecord, +) +from semapact.history.integrity import ( + compute_runtime_observation_record_id, + compute_runtime_reconciliation_record_id, +) +from semapact.observation import ObservedPlatformState, fingerprint_observed_state +from semapact.reconciliation import ( + ReconciliationResult, + classify_reconciliation_status, +) + + +class RuntimeHistoryService: + """Persist canonical M1 runtime evidence with optional exact lifecycle links.""" + + def __init__( + self, + *, + observations: RuntimeObservationHistoryRepository, + reconciliations: RuntimeReconciliationHistoryRepository, + releases: ReleaseRecordHistoryRepository, + deployments: DeploymentRecordHistoryRepository, + ) -> None: + self._observations = observations + self._reconciliations = reconciliations + self._releases = releases + self._deployments = deployments + + def record_reconciliation( + self, + observation: ObservedPlatformState, + result: ReconciliationResult, + *, + release_record_id: str | None = None, + deployment_record_id: str | None = None, + ) -> RuntimeReconciliationRecord: + """Record one canonical observation/result pair without recomputing M1 semantics.""" + if not isinstance(observation, ObservedPlatformState): + raise TypeError( + "observation must be ObservedPlatformState, " + f"got {type(observation).__name__}" + ) + if not isinstance(result, ReconciliationResult): + raise TypeError( + f"result must be ReconciliationResult, got {type(result).__name__}" + ) + + _validate_observation_result_link(observation, result) + release = _load_optional_release(self._releases, release_record_id) + deployment = _load_optional_deployment( + self._deployments, + deployment_record_id, + ) + _validate_optional_history_links( + observation, + result, + release=release, + deployment=deployment, + ) + + observation_record_id = compute_runtime_observation_record_id(observation) + observation_record = RuntimeObservationRecord( + observation_record_id=observation_record_id, + observation=observation, + ) + + status = classify_reconciliation_status(result) + runtime_reconciliation_record_id = compute_runtime_reconciliation_record_id( + observation_record_id=observation_record_id, + result=result, + status=status.value, + release_record_id=( + release.release_record_id if release is not None else None + ), + deployment_record_id=( + deployment.deployment_record_id if deployment is not None else None + ), + ) + record = RuntimeReconciliationRecord( + runtime_reconciliation_record_id=runtime_reconciliation_record_id, + observation_record_id=observation_record_id, + result=result, + status=status, + release_record_id=( + release.release_record_id if release is not None else None + ), + deployment_record_id=( + deployment.deployment_record_id if deployment is not None else None + ), + ) + + # Persist the canonical observation first so the final linkage record never + # points at absent runtime evidence after a partial storage failure. + self._observations.put_runtime_observation_record(observation_record) + self._reconciliations.put_runtime_reconciliation_record(record) + return record + + +def _validate_observation_result_link( + observation: ObservedPlatformState, + result: ReconciliationResult, +) -> None: + if observation.source_identifier != result.observation_source_identifier: + raise ValueError( + "ReconciliationResult source does not match ObservedPlatformState" + ) + observation_fingerprint = observation.fingerprint or fingerprint_observed_state( + observation + ) + if observation_fingerprint != result.observation_fingerprint: + raise ValueError( + "ReconciliationResult fingerprint does not match ObservedPlatformState" + ) + + +def _load_optional_release( + repository: ReleaseRecordHistoryRepository, + release_record_id: str | None, +) -> ReleaseRecord | None: + if release_record_id is None: + return None + return repository.get_release_record(_required_text(release_record_id, "release_record_id")) + + +def _load_optional_deployment( + repository: DeploymentRecordHistoryRepository, + deployment_record_id: str | None, +) -> DeploymentRecord | None: + if deployment_record_id is None: + return None + return repository.get_deployment_record( + _required_text(deployment_record_id, "deployment_record_id") + ) + + +def _validate_optional_history_links( + observation: ObservedPlatformState, + result: ReconciliationResult, + *, + release: ReleaseRecord | None, + deployment: DeploymentRecord | None, +) -> None: + if release is not None: + if ( + release.contract_id != result.contract_id + or release.contract_version != result.contract_version + ): + raise ValueError( + "Runtime reconciliation does not match the linked ReleaseRecord" + ) + + if deployment is None: + return + if release is None: + raise ValueError( + "deployment-linked runtime history requires an explicit ReleaseRecord link" + ) + if deployment.release_record_id != release.release_record_id: + raise ValueError( + "DeploymentRecord does not belong to the linked ReleaseRecord" + ) + if deployment.platform.strip().casefold() != observation.platform.strip().casefold(): + raise ValueError( + "DeploymentRecord platform does not match ObservedPlatformState" + ) + if deployment.source_reference != observation.source_identifier: + raise ValueError( + "DeploymentRecord source does not match ObservedPlatformState" + ) + + +def _required_text(value: str, field_name: str) -> str: + if not isinstance(value, str): + raise TypeError(f"{field_name} must be str") + cleaned = value.strip() + if not cleaned: + raise ValueError(f"{field_name} must not be empty") + return cleaned diff --git a/semapact/history/__init__.py b/semapact/history/__init__.py index 1df69920..652886f6 100644 --- a/semapact/history/__init__.py +++ b/semapact/history/__init__.py @@ -9,6 +9,8 @@ DeploymentRecord, DeploymentStatus, ReleaseRecord, + RuntimeObservationRecord, + RuntimeReconciliationRecord, ) from semapact.history.repository import ( ChangeSetDecisionLinkHistoryRepository, @@ -26,6 +28,8 @@ HistoryRepositoryError, ReleasePlanHistoryRepository, ReleaseRecordHistoryRepository, + RuntimeObservationHistoryRepository, + RuntimeReconciliationHistoryRepository, ) __all__ = [ @@ -48,4 +52,8 @@ "ReleasePlanHistoryRepository", "ReleaseRecord", "ReleaseRecordHistoryRepository", + "RuntimeObservationHistoryRepository", + "RuntimeObservationRecord", + "RuntimeReconciliationHistoryRepository", + "RuntimeReconciliationRecord", ] diff --git a/semapact/history/integrity.py b/semapact/history/integrity.py index 4775f2b7..bb8e82e5 100644 --- a/semapact/history/integrity.py +++ b/semapact/history/integrity.py @@ -4,7 +4,14 @@ import uuid -from semapact.history.models import DeploymentRecord, ReleaseRecord +from semapact.history.models import ( + DeploymentRecord, + ReleaseRecord, + RuntimeObservationRecord, + RuntimeReconciliationRecord, +) +from semapact.observation import ObservedPlatformState +from semapact.reconciliation import ReconciliationResult from semapact.utils.deterministic import deterministic_uuid5 @@ -14,6 +21,12 @@ SEMAPACT_DEPLOYMENT_RECORD_NAMESPACE = uuid.UUID( "93db6770-90f0-4ac5-a185-2f21e818d18e" ) +SEMAPACT_RUNTIME_OBSERVATION_RECORD_NAMESPACE = uuid.UUID( + "8a3c8c10-e7d0-4df6-8b1a-3f087ef7e8e2" +) +SEMAPACT_RUNTIME_RECONCILIATION_RECORD_NAMESPACE = uuid.UUID( + "11969598-1f11-4c03-8ab0-f99f012a5041" +) def compute_release_record_id( @@ -139,3 +152,71 @@ def validate_deployment_record_identity(record: DeploymentRecord) -> None: raise ValueError( "DeploymentRecord deterministic identity does not match its content" ) + + +def compute_runtime_observation_record_id( + observation: ObservedPlatformState, +) -> str: + """Derive stable history identity from the exact canonical M1 observation.""" + if not isinstance(observation, ObservedPlatformState): + raise TypeError( + "observation must be ObservedPlatformState, " + f"got {type(observation).__name__}" + ) + return deterministic_uuid5( + SEMAPACT_RUNTIME_OBSERVATION_RECORD_NAMESPACE, + {"observation": observation.model_dump(mode="json")}, + ) + + +def validate_runtime_observation_record_identity( + record: RuntimeObservationRecord, +) -> None: + """Fail closed when observation history identity does not match exact evidence.""" + expected = compute_runtime_observation_record_id(record.observation) + if record.observation_record_id != expected: + raise ValueError( + "RuntimeObservationRecord deterministic identity does not match content" + ) + + +def compute_runtime_reconciliation_record_id( + *, + observation_record_id: str, + result: ReconciliationResult, + status: str, + release_record_id: str | None, + deployment_record_id: str | None, +) -> str: + """Derive stable identity for one persisted runtime reconciliation conclusion.""" + if not isinstance(result, ReconciliationResult): + raise TypeError( + f"result must be ReconciliationResult, got {type(result).__name__}" + ) + return deterministic_uuid5( + SEMAPACT_RUNTIME_RECONCILIATION_RECORD_NAMESPACE, + { + "observation_record_id": observation_record_id, + "result": result.model_dump(mode="json"), + "status": status, + "release_record_id": release_record_id, + "deployment_record_id": deployment_record_id, + }, + ) + + +def validate_runtime_reconciliation_record_identity( + record: RuntimeReconciliationRecord, +) -> None: + """Fail closed when runtime reconciliation history identity is stale/tampered.""" + expected = compute_runtime_reconciliation_record_id( + observation_record_id=record.observation_record_id, + result=record.result, + status=record.status.value, + release_record_id=record.release_record_id, + deployment_record_id=record.deployment_record_id, + ) + if record.runtime_reconciliation_record_id != expected: + raise ValueError( + "RuntimeReconciliationRecord deterministic identity does not match content" + ) diff --git a/semapact/history/models.py b/semapact/history/models.py index 1ddeae3a..b876b663 100644 --- a/semapact/history/models.py +++ b/semapact/history/models.py @@ -12,6 +12,12 @@ from pydantic import BaseModel, ConfigDict, field_validator, model_validator from semapact.contractops.models import ReviewEvidenceAction, VersionAuthority +from semapact.observation import ObservedPlatformState +from semapact.reconciliation import ( + ReconciliationResult, + RuntimeDriftStatus, + classify_reconciliation_status, +) from semapact.versioning import ActualVersionBump, RequiredBump @@ -166,6 +172,59 @@ def _validate_timestamp_order(self) -> DeploymentRecord: return self +class RuntimeObservationRecord(HistoryModel): + """Content-addressed history envelope around canonical M1 runtime evidence.""" + + observation_record_id: str + observation: ObservedPlatformState + + @field_validator("observation_record_id") + @classmethod + def _require_observation_id(cls, value: str) -> str: + return _required_text(value) + + +class RuntimeReconciliationRecord(HistoryModel): + """Immutable linkage from canonical runtime evidence to optional lifecycle context. + + ``result`` remains the canonical M1 ReconciliationResult. This record adds only + durable identity, point-in-time classification, and optional exact history links. + """ + + runtime_reconciliation_record_id: str + observation_record_id: str + result: ReconciliationResult + status: RuntimeDriftStatus + release_record_id: str | None = None + deployment_record_id: str | None = None + + @field_validator( + "runtime_reconciliation_record_id", + "observation_record_id", + ) + @classmethod + def _require_runtime_history_text(cls, value: str) -> str: + return _required_text(value) + + @field_validator("release_record_id", "deployment_record_id") + @classmethod + def _normalize_runtime_links(cls, value: str | None) -> str | None: + return _optional_text(value) + + @model_validator(mode="after") + def _validate_runtime_history_semantics(self) -> RuntimeReconciliationRecord: + expected_status = classify_reconciliation_status(self.result) + if self.status is not expected_status: + raise ValueError( + "runtime reconciliation status does not match canonical classification" + ) + if self.deployment_record_id is not None and self.release_record_id is None: + raise ValueError( + "deployment-linked runtime history requires release_record_id" + ) + return self + + def _required_text(value: str) -> str: if not isinstance(value, str): raise TypeError("history identifiers must be strings") @@ -178,5 +237,7 @@ def _required_text(value: str) -> str: def _optional_text(value: str | None) -> str | None: if value is None: return None + if not isinstance(value, str): + raise TypeError("optional history references must be strings") cleaned = value.strip() return cleaned or None diff --git a/semapact/history/repository.py b/semapact/history/repository.py index 6d7f496e..1194d213 100644 --- a/semapact/history/repository.py +++ b/semapact/history/repository.py @@ -11,6 +11,8 @@ ChangeSetDecisionLink, DeploymentRecord, ReleaseRecord, + RuntimeObservationRecord, + RuntimeReconciliationRecord, ) from semapact.revision.models import ContractRevision, ContractRevisionSource @@ -34,143 +36,83 @@ class HistoryCorruptionError(HistoryRepositoryError): class DecisionHistoryRepository(Protocol): """Persistence capability for canonical GovernanceDecision history only.""" - def put_decision(self, decision: GovernanceDecision) -> None: - """Persist one immutable GovernanceDecision idempotently.""" - ... - - def get_decision(self, decision_id: str) -> GovernanceDecision: - """Load one GovernanceDecision by its exact artifact ID.""" - ... - - def list_decisions(self, contract_id: str) -> tuple[GovernanceDecision, ...]: - """List decisions for one contract in deterministic artifact-ID order.""" - ... + def put_decision(self, decision: GovernanceDecision) -> None: ... + def get_decision(self, decision_id: str) -> GovernanceDecision: ... + def list_decisions(self, contract_id: str) -> tuple[GovernanceDecision, ...]: ... class ChangeSetHistoryRepository(Protocol): """Persistence capability for canonical ChangeSet history only.""" - def put_change_set(self, change_set: ChangeSet) -> None: - """Persist one immutable ChangeSet idempotently.""" - ... - - def get_change_set(self, change_set_id: str) -> ChangeSet: - """Load one ChangeSet by its exact artifact ID.""" - ... - - def list_change_sets(self, contract_id: str) -> tuple[ChangeSet, ...]: - """List ChangeSets for one contract in deterministic artifact-ID order.""" - ... + def put_change_set(self, change_set: ChangeSet) -> None: ... + def get_change_set(self, change_set_id: str) -> ChangeSet: ... + def list_change_sets(self, contract_id: str) -> tuple[ChangeSet, ...]: ... class ChangeSetDecisionLinkHistoryRepository(Protocol): """Persistence capability for ChangeSet-to-decision audit provenance only.""" - def put_change_set_decision_link(self, link: ChangeSetDecisionLink) -> None: - """Persist one immutable ChangeSet-to-decision link idempotently.""" - ... + def put_change_set_decision_link(self, link: ChangeSetDecisionLink) -> None: ... def list_change_set_decision_links( self, change_set_id: str, - ) -> tuple[ChangeSetDecisionLink, ...]: - """List governance outcomes linked to one ChangeSet in deterministic order.""" - ... + ) -> tuple[ChangeSetDecisionLink, ...]: ... class ContractRevisionHistoryRepository(Protocol): """Persistence capability for immutable ContractRevision history only.""" - def put_revision(self, revision: ContractRevision) -> None: - """Persist one immutable ContractRevision idempotently.""" - ... - - def get_revision(self, revision_id: str) -> ContractRevision: - """Load one ContractRevision by its exact content-derived ID.""" - ... - - def list_revisions(self, contract_id: str) -> tuple[ContractRevision, ...]: - """List revisions for one contract in deterministic artifact-ID order.""" - ... + def put_revision(self, revision: ContractRevision) -> None: ... + def get_revision(self, revision_id: str) -> ContractRevision: ... + def list_revisions(self, contract_id: str) -> tuple[ContractRevision, ...]: ... class ContractRevisionSourceHistoryRepository(Protocol): """Persistence capability for immutable revision provenance links only.""" - def put_revision_source(self, source: ContractRevisionSource) -> None: - """Persist one revision-to-source provenance link idempotently.""" - ... - - def get_revision_source(self, source_link_id: str) -> ContractRevisionSource: - """Load one revision provenance link by exact ID.""" - ... + def put_revision_source(self, source: ContractRevisionSource) -> None: ... + def get_revision_source(self, source_link_id: str) -> ContractRevisionSource: ... def list_revision_sources( self, revision_id: str, - ) -> tuple[ContractRevisionSource, ...]: - """List all source links for one revision in deterministic ID order.""" - ... + ) -> tuple[ContractRevisionSource, ...]: ... class ReleasePlanHistoryRepository(Protocol): """Persistence capability for canonical ReleasePlan history only.""" - def put_release_plan(self, release_plan: ReleasePlan) -> None: - """Persist one immutable ReleasePlan idempotently.""" - ... - - def get_release_plan(self, release_plan_id: str) -> ReleasePlan: - """Load one ReleasePlan by exact deterministic artifact ID.""" - ... + def put_release_plan(self, release_plan: ReleasePlan) -> None: ... + def get_release_plan(self, release_plan_id: str) -> ReleasePlan: ... class ReleaseRecordHistoryRepository(Protocol): """Persistence capability for finalized release audit records only.""" - def put_release_record(self, record: ReleaseRecord) -> None: - """Persist one immutable ReleaseRecord idempotently.""" - ... - - def get_release_record(self, release_record_id: str) -> ReleaseRecord: - """Load one ReleaseRecord by exact deterministic artifact ID.""" - ... - - def list_release_records(self, contract_id: str) -> tuple[ReleaseRecord, ...]: - """List release records for one contract in deterministic artifact-ID order.""" - ... + def put_release_record(self, record: ReleaseRecord) -> None: ... + def get_release_record(self, release_record_id: str) -> ReleaseRecord: ... + def list_release_records(self, contract_id: str) -> tuple[ReleaseRecord, ...]: ... def get_release_record_by_version( self, contract_id: str, contract_version: str, - ) -> ReleaseRecord: - """Load the unique finalized release for one contract semantic version.""" - ... + ) -> ReleaseRecord: ... class DeploymentPlanHistoryRepository(Protocol): """Persistence capability for canonical DeploymentPlan history only.""" - def put_deployment_plan(self, plan: DeploymentPlan) -> None: - """Persist one immutable DeploymentPlan idempotently.""" - ... - - def get_deployment_plan(self, deployment_plan_id: str) -> DeploymentPlan: - """Load one DeploymentPlan by exact deterministic artifact ID.""" - ... + def put_deployment_plan(self, plan: DeploymentPlan) -> None: ... + def get_deployment_plan(self, deployment_plan_id: str) -> DeploymentPlan: ... class DeploymentPreviewHistoryRepository(Protocol): """Persistence capability for canonical DeploymentPreview history only.""" - def put_deployment_preview(self, preview: DeploymentPreview) -> None: - """Persist one immutable DeploymentPreview idempotently.""" - ... - - def get_deployment_preview(self, deployment_preview_id: str) -> DeploymentPreview: - """Load one DeploymentPreview by exact deterministic artifact ID.""" - ... + def put_deployment_preview(self, preview: DeploymentPreview) -> None: ... + def get_deployment_preview(self, deployment_preview_id: str) -> DeploymentPreview: ... class DeploymentAuthorizationHistoryRepository(Protocol): @@ -179,39 +121,65 @@ class DeploymentAuthorizationHistoryRepository(Protocol): def put_deployment_authorization( self, authorization: DeploymentAuthorization, - ) -> None: - """Persist one immutable DeploymentAuthorization idempotently.""" - ... + ) -> None: ... def get_deployment_authorization( self, deployment_authorization_id: str, - ) -> DeploymentAuthorization: - """Load one DeploymentAuthorization by exact deterministic artifact ID.""" - ... + ) -> DeploymentAuthorization: ... class DeploymentRecordHistoryRepository(Protocol): """Persistence capability for terminal deployment execution occurrences only.""" - def put_deployment_record(self, record: DeploymentRecord) -> None: - """Persist one immutable DeploymentRecord idempotently.""" - ... - - def get_deployment_record(self, deployment_record_id: str) -> DeploymentRecord: - """Load one DeploymentRecord by exact deterministic artifact ID.""" - ... + def put_deployment_record(self, record: DeploymentRecord) -> None: ... + def get_deployment_record(self, deployment_record_id: str) -> DeploymentRecord: ... def list_deployment_records_for_release( self, release_record_id: str, - ) -> tuple[DeploymentRecord, ...]: - """List deployment occurrences for one finalized release.""" - ... + ) -> tuple[DeploymentRecord, ...]: ... def list_deployment_records_for_plan( self, deployment_plan_id: str, - ) -> tuple[DeploymentRecord, ...]: - """List deployment occurrences for one exact deployment plan.""" - ... + ) -> tuple[DeploymentRecord, ...]: ... + + +class RuntimeObservationHistoryRepository(Protocol): + """Persistence capability for canonical M1 observation evidence envelopes.""" + + def put_runtime_observation_record(self, record: RuntimeObservationRecord) -> None: ... + def get_runtime_observation_record( + self, + observation_record_id: str, + ) -> RuntimeObservationRecord: ... + + def list_runtime_observation_records( + self, + source_identifier: str, + ) -> tuple[RuntimeObservationRecord, ...]: ... + + +class RuntimeReconciliationHistoryRepository(Protocol): + """Persistence capability for point-in-time runtime reconciliation history.""" + + def put_runtime_reconciliation_record( + self, + record: RuntimeReconciliationRecord, + ) -> None: ... + + def get_runtime_reconciliation_record( + self, + runtime_reconciliation_record_id: str, + ) -> RuntimeReconciliationRecord: ... + + def list_runtime_reconciliation_records( + self, + contract_id: str, + ) -> tuple[RuntimeReconciliationRecord, ...]: ... + + def list_runtime_reconciliation_records_for_source( + self, + source_identifier: str, + ) -> tuple[RuntimeReconciliationRecord, ...]: ... diff --git a/semapact/platforms/git/history_repository.py b/semapact/platforms/git/history_repository.py index 639e5b21..b0d98a31 100644 --- a/semapact/platforms/git/history_repository.py +++ b/semapact/platforms/git/history_repository.py @@ -29,10 +29,14 @@ HistoryCorruptionError, HistoryNotFoundError, ReleaseRecord, + RuntimeObservationRecord, + RuntimeReconciliationRecord, ) from semapact.history.integrity import ( validate_deployment_record_identity, validate_release_record_identity, + validate_runtime_observation_record_identity, + validate_runtime_reconciliation_record_identity, ) from semapact.revision.integrity import ( validate_contract_revision_identity, @@ -47,12 +51,7 @@ class GitWorkingTreeHistoryRepository: - """Shared Git backend implementing narrow typed history capabilities. - - Public methods satisfy artifact-specific repository protocols while the private - helpers own common JSON/file persistence mechanics. Domain integrity remains in - each owning domain and is injected when persistence rehydrates that artifact. - """ + """Shared Git backend implementing narrow typed history capabilities.""" def __init__( self, @@ -407,6 +406,100 @@ def list_deployment_records_for_plan( record for record in records if record.deployment_plan_id == deployment_plan_id ) + def put_runtime_observation_record(self, record: RuntimeObservationRecord) -> None: + self._put( + kind="runtime_observations", + artifact_id=record.observation_record_id, + artifact=record, + model_type=RuntimeObservationRecord, + id_attribute="observation_record_id", + integrity_validator=validate_runtime_observation_record_identity, + ) + + def get_runtime_observation_record( + self, + observation_record_id: str, + ) -> RuntimeObservationRecord: + return self._get( + kind="runtime_observations", + artifact_id=observation_record_id, + model_type=RuntimeObservationRecord, + id_attribute="observation_record_id", + integrity_validator=validate_runtime_observation_record_identity, + ) + + def list_runtime_observation_records( + self, + source_identifier: str, + ) -> tuple[RuntimeObservationRecord, ...]: + source_identifier = _required_text(source_identifier, "source_identifier") + records = self._list( + kind="runtime_observations", + model_type=RuntimeObservationRecord, + id_attribute="observation_record_id", + integrity_validator=validate_runtime_observation_record_identity, + ) + return tuple( + record + for record in records + if record.observation.source_identifier == source_identifier + ) + + def put_runtime_reconciliation_record( + self, + record: RuntimeReconciliationRecord, + ) -> None: + self._put( + kind="runtime_reconciliations", + artifact_id=record.runtime_reconciliation_record_id, + artifact=record, + model_type=RuntimeReconciliationRecord, + id_attribute="runtime_reconciliation_record_id", + integrity_validator=validate_runtime_reconciliation_record_identity, + ) + + def get_runtime_reconciliation_record( + self, + runtime_reconciliation_record_id: str, + ) -> RuntimeReconciliationRecord: + return self._get( + kind="runtime_reconciliations", + artifact_id=runtime_reconciliation_record_id, + model_type=RuntimeReconciliationRecord, + id_attribute="runtime_reconciliation_record_id", + integrity_validator=validate_runtime_reconciliation_record_identity, + ) + + def list_runtime_reconciliation_records( + self, + contract_id: str, + ) -> tuple[RuntimeReconciliationRecord, ...]: + contract_id = _required_text(contract_id, "contract_id") + records = self._list( + kind="runtime_reconciliations", + model_type=RuntimeReconciliationRecord, + id_attribute="runtime_reconciliation_record_id", + integrity_validator=validate_runtime_reconciliation_record_identity, + ) + return tuple(record for record in records if record.result.contract_id == contract_id) + + def list_runtime_reconciliation_records_for_source( + self, + source_identifier: str, + ) -> tuple[RuntimeReconciliationRecord, ...]: + source_identifier = _required_text(source_identifier, "source_identifier") + records = self._list( + kind="runtime_reconciliations", + model_type=RuntimeReconciliationRecord, + id_attribute="runtime_reconciliation_record_id", + integrity_validator=validate_runtime_reconciliation_record_identity, + ) + return tuple( + record + for record in records + if record.result.observation_source_identifier == source_identifier + ) + def _put( self, *, diff --git a/tests/test_runtime_history.py b/tests/test_runtime_history.py new file mode 100644 index 00000000..af9424da --- /dev/null +++ b/tests/test_runtime_history.py @@ -0,0 +1,243 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest + +from semapact.application.services.runtime_history import RuntimeHistoryService +from semapact.contractops import VersionAuthority +from semapact.history import DeploymentRecord, DeploymentStatus, ReleaseRecord +from semapact.history.integrity import compute_deployment_record_id, compute_release_record_id +from semapact.observation import ( + ObservedAsset, + ObservedAssetIdentity, + ObservedPlatformState, + with_observed_state_fingerprint, +) +from semapact.platforms.git import GitWorkingTreeHistoryRepository +from semapact.reconciliation import ( + ReconciliationDifference, + ReconciliationDifferenceType, + ReconciliationResult, + ReconciliationSubject, + RuntimeDriftStatus, + RuntimeReasonCode, +) + + +def _observation(at: datetime, *, asset_type: str = "TABLE", source: str = "workspace:test") -> ObservedPlatformState: + return with_observed_state_fingerprint( + ObservedPlatformState( + platform="databricks", + source_identifier=source, + assets=( + ObservedAsset( + identity=ObservedAssetIdentity( + platform="databricks", + namespace=("catalog", "schema"), + asset="orders", + ), + asset_type=asset_type, + ), + ), + captured_at=at, + ) + ) + + +def _result(observation: ObservedPlatformState, *, drift: bool = False) -> ReconciliationResult: + differences = () + if drift: + differences = ( + ReconciliationDifference( + difference_type=ReconciliationDifferenceType.MISMATCH, + subject=ReconciliationSubject.PHYSICAL_TYPE, + reason_code=RuntimeReasonCode.RUNTIME_PHYSICAL_TYPE_CHANGED, + path="orders.id.physical_type", + asset_identity="orders", + property_identity="id", + expected="STRING", + observed="BIGINT", + ), + ) + assert observation.fingerprint is not None + return ReconciliationResult( + contract_id="orders-product", + contract_version="1.3.0", + observation_source_identifier=observation.source_identifier, + observation_fingerprint=observation.fingerprint, + differences=differences, + ) + + +def _release() -> ReleaseRecord: + fields = dict( + contract_id="orders-product", + contract_version="1.3.0", + decision_id="decision-1", + change_set_id="changeset-1", + release_plan_id="release-plan-1", + version_resolution_id="version-resolution-1", + authorization_id="apply-authorization-1", + applied_release_id="applied-release-1", + released_revision_id="released-revision-1", + required_version_bump="minor", + actual_version_bump="minor", + version_authority=VersionAuthority.SEMAPACT, + authority_reference=None, + review_evidence_reference=None, + review_evidence_action=None, + ) + record_id = compute_release_record_id( + **{**fields, "version_authority": fields["version_authority"].value} + ) + return ReleaseRecord(release_record_id=record_id, **fields) + + +def _deployment(release_id: str, *, source: str = "workspace:test") -> DeploymentRecord: + started = datetime(2026, 9, 13, tzinfo=timezone.utc) + completed = started + timedelta(seconds=1) + fields = dict( + release_record_id=release_id, + deployment_plan_id="deployment-plan-1", + deployment_preview_id="deployment-preview-1", + deployment_authorization_id="deployment-authorization-1", + platform="databricks", + runtime_target="catalog.schema", + source_reference=source, + status=DeploymentStatus.SUCCEEDED, + started_at=started, + completed_at=completed, + actor_reference=None, + external_reference=None, + ) + record_id = compute_deployment_record_id( + release_record_id=release_id, + deployment_plan_id=fields["deployment_plan_id"], + deployment_preview_id=fields["deployment_preview_id"], + deployment_authorization_id=fields["deployment_authorization_id"], + platform=fields["platform"], + runtime_target=fields["runtime_target"], + source_reference=fields["source_reference"], + status=fields["status"].value, + started_at=started.isoformat(), + completed_at=completed.isoformat(), + actor_reference=None, + external_reference=None, + ) + return DeploymentRecord(deployment_record_id=record_id, **fields) + + +def _service(tmp_path: Path): + backend = GitWorkingTreeHistoryRepository(tmp_path) + return RuntimeHistoryService( + observations=backend, + reconciliations=backend, + releases=backend, + deployments=backend, + ), backend + + +def test_round_trip_and_idempotency(tmp_path: Path) -> None: + service, backend = _service(tmp_path) + observation = _observation(datetime(2026, 9, 13, 1, tzinfo=timezone.utc)) + result = _result(observation) + + first = service.record_reconciliation(observation, result) + second = service.record_reconciliation(observation, result) + + assert first == second + assert backend.get_runtime_observation_record(first.observation_record_id).observation == observation + assert backend.get_runtime_reconciliation_record(first.runtime_reconciliation_record_id) == first + assert first.status is RuntimeDriftStatus.IN_SYNC + + +def test_same_semantic_state_at_different_times_keeps_both_observations(tmp_path: Path) -> None: + service, backend = _service(tmp_path) + first_observation = _observation(datetime(2026, 9, 13, 1, tzinfo=timezone.utc)) + second_observation = _observation(datetime(2026, 9, 13, 2, tzinfo=timezone.utc)) + + first = service.record_reconciliation(first_observation, _result(first_observation)) + second = service.record_reconciliation(second_observation, _result(second_observation)) + + assert first_observation.fingerprint == second_observation.fingerprint + assert first.observation_record_id != second.observation_record_id + assert len(backend.list_runtime_observation_records("workspace:test")) == 2 + + +def test_sync_then_drift_coexist(tmp_path: Path) -> None: + service, backend = _service(tmp_path) + sync_observation = _observation(datetime(2026, 9, 13, 1, tzinfo=timezone.utc)) + drift_observation = _observation( + datetime(2026, 9, 13, 2, tzinfo=timezone.utc), + asset_type="VIEW", + ) + + sync = service.record_reconciliation(sync_observation, _result(sync_observation)) + drift = service.record_reconciliation(drift_observation, _result(drift_observation, drift=True)) + + records = backend.list_runtime_reconciliation_records_for_source("workspace:test") + assert set(records) == {sync, drift} + assert {item.status for item in records} == {RuntimeDriftStatus.IN_SYNC, RuntimeDriftStatus.DRIFT} + + +def test_exact_release_and_deployment_links_are_optional(tmp_path: Path) -> None: + service, backend = _service(tmp_path) + release = _release() + deployment = _deployment(release.release_record_id) + backend.put_release_record(release) + backend.put_deployment_record(deployment) + observation = _observation(datetime(2026, 9, 13, 1, tzinfo=timezone.utc)) + + record = service.record_reconciliation( + observation, + _result(observation), + release_record_id=release.release_record_id, + deployment_record_id=deployment.deployment_record_id, + ) + + assert record.release_record_id == release.release_record_id + assert record.deployment_record_id == deployment.deployment_record_id + + +def test_deployment_success_can_coexist_with_later_drift(tmp_path: Path) -> None: + service, backend = _service(tmp_path) + release = _release() + deployment = _deployment(release.release_record_id) + backend.put_release_record(release) + backend.put_deployment_record(deployment) + observation = _observation( + datetime(2026, 9, 13, 2, tzinfo=timezone.utc), + asset_type="VIEW", + ) + + record = service.record_reconciliation( + observation, + _result(observation, drift=True), + release_record_id=release.release_record_id, + deployment_record_id=deployment.deployment_record_id, + ) + + assert deployment.status is DeploymentStatus.SUCCEEDED + assert record.status is RuntimeDriftStatus.DRIFT + + +def test_rejects_mismatched_runtime_evidence(tmp_path: Path) -> None: + service, backend = _service(tmp_path) + observation = _observation(datetime(2026, 9, 13, 1, tzinfo=timezone.utc)) + wrong_result = _result(observation).model_copy(update={"observation_fingerprint": "wrong"}) + with pytest.raises(ValueError, match="fingerprint"): + service.record_reconciliation(observation, wrong_result) + + release = _release() + deployment = _deployment(release.release_record_id, source="workspace:other") + backend.put_release_record(release) + backend.put_deployment_record(deployment) + with pytest.raises(ValueError, match="source"): + service.record_reconciliation( + observation, + _result(observation), + release_record_id=release.release_record_id, + deployment_record_id=deployment.deployment_record_id, + ) From 995485d19f3736e2fa89c91d081eff03787c5b73 Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Mon, 14 Sep 2026 22:28:11 +1000 Subject: [PATCH 22/35] feat(history): reconstruct governance evolution chain Reconstruct persisted governance evolution through stable history references without introducing a second graph or persisted chain projection. --- semapact/application/models/__init__.py | 14 + semapact/application/models/evolution.py | 75 +++ semapact/application/services/evolution.py | 512 +++++++++++++++++++++ tests/test_evolution_chain.py | 297 ++++++++++++ 4 files changed, 898 insertions(+) create mode 100644 semapact/application/models/evolution.py create mode 100644 semapact/application/services/evolution.py create mode 100644 tests/test_evolution_chain.py diff --git a/semapact/application/models/__init__.py b/semapact/application/models/__init__.py index 2002aa4e..c2c1b454 100644 --- a/semapact/application/models/__init__.py +++ b/semapact/application/models/__init__.py @@ -1,12 +1,26 @@ """Typed application results composed from canonical domain artifacts.""" +from semapact.application.models.evolution import ( + BrokenHistoryReference, + ContractEvolution, + DeploymentEvolution, + ProposalEvolution, + ReleaseEvolution, + RuntimeEvolution, +) from semapact.application.models.governance import GovernanceAnalysis, GovernanceProposal from semapact.application.models.reconciliation import RuntimeReconciliation from semapact.application.models.release import ReleasePlanningResult __all__ = [ + "BrokenHistoryReference", + "ContractEvolution", + "DeploymentEvolution", "GovernanceAnalysis", "GovernanceProposal", + "ProposalEvolution", + "ReleaseEvolution", "ReleasePlanningResult", + "RuntimeEvolution", "RuntimeReconciliation", ] diff --git a/semapact/application/models/evolution.py b/semapact/application/models/evolution.py new file mode 100644 index 00000000..655c08a5 --- /dev/null +++ b/semapact/application/models/evolution.py @@ -0,0 +1,75 @@ +"""Read models for deterministic governance-history reconstruction.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from semapact.contractops import ChangeSet +from semapact.governance import GovernanceDecision +from semapact.history import ( + DeploymentRecord, + ReleaseRecord, + RuntimeObservationRecord, + RuntimeReconciliationRecord, +) +from semapact.revision import ContractRevision + + +@dataclass(frozen=True) +class BrokenHistoryReference: + """One explicit history reference that could not be resolved consistently.""" + + source_kind: str + source_id: str + reference_field: str + target_kind: str + target_id: str + reason: str + + +@dataclass(frozen=True) +class RuntimeEvolution: + """One reconciliation occurrence with its exact observed-state evidence when present.""" + + reconciliation: RuntimeReconciliationRecord + observation: RuntimeObservationRecord | None + + +@dataclass(frozen=True) +class DeploymentEvolution: + """One deployment occurrence and runtime evidence explicitly linked to it.""" + + deployment: DeploymentRecord + runtime: tuple[RuntimeEvolution, ...] = () + + +@dataclass(frozen=True) +class ReleaseEvolution: + """One finalized release and its downstream execution/runtime history.""" + + release: ReleaseRecord + released_revision: ContractRevision | None + deployments: tuple[DeploymentEvolution, ...] = () + runtime: tuple[RuntimeEvolution, ...] = () + + +@dataclass(frozen=True) +class ProposalEvolution: + """One ChangeSet path through an optional recorded GovernanceDecision.""" + + change_set: ChangeSet + base_revision: ContractRevision | None + candidate_revision: ContractRevision | None + decision: GovernanceDecision | None + releases: tuple[ReleaseEvolution, ...] = () + + +@dataclass(frozen=True) +class ContractEvolution: + """Deterministic read-side reconstruction of one contract's persisted history.""" + + contract_id: str + proposals: tuple[ProposalEvolution, ...] = () + unlinked_releases: tuple[ReleaseEvolution, ...] = () + unlinked_runtime: tuple[RuntimeEvolution, ...] = () + broken_references: tuple[BrokenHistoryReference, ...] = () diff --git a/semapact/application/services/evolution.py b/semapact/application/services/evolution.py new file mode 100644 index 00000000..2daede25 --- /dev/null +++ b/semapact/application/services/evolution.py @@ -0,0 +1,512 @@ +"""Read-only reconstruction of persisted governance evolution history.""" + +from __future__ import annotations + +from semapact.application.models.evolution import ( + BrokenHistoryReference, + ContractEvolution, + DeploymentEvolution, + ProposalEvolution, + ReleaseEvolution, + RuntimeEvolution, +) +from semapact.contractops import ChangeSet +from semapact.governance import GovernanceDecision +from semapact.history import ( + ChangeSetDecisionLinkHistoryRepository, + ChangeSetHistoryRepository, + ContractRevisionHistoryRepository, + DecisionHistoryRepository, + DeploymentRecordHistoryRepository, + HistoryNotFoundError, + ReleaseRecord, + ReleaseRecordHistoryRepository, + RuntimeObservationHistoryRepository, + RuntimeReconciliationHistoryRepository, + RuntimeReconciliationRecord, +) +from semapact.revision import ContractRevision + + +class EvolutionChainService: + """Reconstruct one contract's history from existing stable artifact references.""" + + def __init__( + self, + *, + revisions: ContractRevisionHistoryRepository, + change_sets: ChangeSetHistoryRepository, + decisions: DecisionHistoryRepository, + decision_links: ChangeSetDecisionLinkHistoryRepository, + releases: ReleaseRecordHistoryRepository, + deployments: DeploymentRecordHistoryRepository, + observations: RuntimeObservationHistoryRepository, + runtime_reconciliations: RuntimeReconciliationHistoryRepository, + ) -> None: + self._revisions = revisions + self._change_sets = change_sets + self._decisions = decisions + self._decision_links = decision_links + self._releases = releases + self._deployments = deployments + self._observations = observations + self._runtime_reconciliations = runtime_reconciliations + + def reconstruct(self, contract_id: str) -> ContractEvolution: + """Return a deterministic read model without persisting a second history graph.""" + contract_id = _required_text(contract_id, "contract_id") + issues: list[BrokenHistoryReference] = [] + + change_sets = tuple( + sorted( + self._change_sets.list_change_sets(contract_id), + key=lambda item: item.change_set_id, + ) + ) + release_records = tuple( + sorted( + self._releases.list_release_records(contract_id), + key=lambda item: item.release_record_id, + ) + ) + runtime_records = tuple( + sorted( + self._runtime_reconciliations.list_runtime_reconciliation_records( + contract_id + ), + key=lambda item: item.runtime_reconciliation_record_id, + ) + ) + + runtime_by_id = { + record.runtime_reconciliation_record_id: self._runtime_evolution( + record, + issues, + ) + for record in runtime_records + } + runtime_by_release: dict[str, list[RuntimeReconciliationRecord]] = {} + runtime_by_deployment: dict[str, list[RuntimeReconciliationRecord]] = {} + for record in runtime_records: + if record.deployment_record_id is not None: + runtime_by_deployment.setdefault(record.deployment_record_id, []).append(record) + elif record.release_record_id is not None: + runtime_by_release.setdefault(record.release_record_id, []).append(record) + + releases_by_path: dict[tuple[str, str], list[ReleaseRecord]] = {} + for release in release_records: + releases_by_path.setdefault( + (release.change_set_id, release.decision_id), + [], + ).append(release) + + consumed_release_ids: set[str] = set() + consumed_runtime_ids: set[str] = set() + known_deployment_ids: set[str] = set() + linked_decision_pairs: set[tuple[str, str]] = set() + proposals: list[ProposalEvolution] = [] + + for change_set in change_sets: + base_revision = self._resolve_revision( + change_set, + field_name="base_revision_ref", + revision_id=change_set.base_revision_ref, + contract_id=contract_id, + issues=issues, + ) + candidate_revision = self._resolve_revision( + change_set, + field_name="candidate_revision_ref", + revision_id=change_set.candidate_revision_ref, + contract_id=contract_id, + issues=issues, + ) + links = tuple( + sorted( + self._decision_links.list_change_set_decision_links( + change_set.change_set_id + ), + key=lambda item: item.decision_id, + ) + ) + if not links: + proposals.append( + ProposalEvolution( + change_set=change_set, + base_revision=base_revision, + candidate_revision=candidate_revision, + decision=None, + ) + ) + continue + + for link in links: + pair = (change_set.change_set_id, link.decision_id) + linked_decision_pairs.add(pair) + decision = self._resolve_decision( + change_set, + link.decision_id, + contract_id=contract_id, + issues=issues, + ) + releases = tuple( + self._build_release( + release, + contract_id=contract_id, + runtime_by_id=runtime_by_id, + runtime_by_release=runtime_by_release, + runtime_by_deployment=runtime_by_deployment, + consumed_runtime_ids=consumed_runtime_ids, + known_deployment_ids=known_deployment_ids, + issues=issues, + ) + for release in sorted( + releases_by_path.get(pair, ()), + key=lambda item: item.release_record_id, + ) + ) + consumed_release_ids.update( + item.release.release_record_id for item in releases + ) + proposals.append( + ProposalEvolution( + change_set=change_set, + base_revision=base_revision, + candidate_revision=candidate_revision, + decision=decision, + releases=releases, + ) + ) + + change_set_ids = {item.change_set_id for item in change_sets} + unlinked_releases: list[ReleaseEvolution] = [] + for release in release_records: + if release.release_record_id in consumed_release_ids: + continue + pair = (release.change_set_id, release.decision_id) + if release.change_set_id not in change_set_ids: + _add_issue( + issues, + source_kind="ReleaseRecord", + source_id=release.release_record_id, + reference_field="change_set_id", + target_kind="ChangeSet", + target_id=release.change_set_id, + reason="NOT_FOUND", + ) + elif pair not in linked_decision_pairs: + _add_issue( + issues, + source_kind="ReleaseRecord", + source_id=release.release_record_id, + reference_field="decision_id", + target_kind="ChangeSetDecisionLink", + target_id=f"{release.change_set_id}:{release.decision_id}", + reason="RELATION_MISSING", + ) + unlinked_releases.append( + self._build_release( + release, + contract_id=contract_id, + runtime_by_id=runtime_by_id, + runtime_by_release=runtime_by_release, + runtime_by_deployment=runtime_by_deployment, + consumed_runtime_ids=consumed_runtime_ids, + known_deployment_ids=known_deployment_ids, + issues=issues, + ) + ) + + release_ids = {item.release_record_id for item in release_records} + unlinked_runtime: list[RuntimeEvolution] = [] + for record in runtime_records: + record_id = record.runtime_reconciliation_record_id + if record_id in consumed_runtime_ids: + continue + if ( + record.release_record_id is not None + and record.release_record_id not in release_ids + ): + _add_issue( + issues, + source_kind="RuntimeReconciliationRecord", + source_id=record_id, + reference_field="release_record_id", + target_kind="ReleaseRecord", + target_id=record.release_record_id, + reason="NOT_FOUND", + ) + if ( + record.deployment_record_id is not None + and record.deployment_record_id not in known_deployment_ids + ): + _add_issue( + issues, + source_kind="RuntimeReconciliationRecord", + source_id=record_id, + reference_field="deployment_record_id", + target_kind="DeploymentRecord", + target_id=record.deployment_record_id, + reason="NOT_FOUND", + ) + unlinked_runtime.append(runtime_by_id[record_id]) + + proposals.sort( + key=lambda item: ( + item.change_set.change_set_id, + item.decision.decision_id if item.decision is not None else "", + ) + ) + unlinked_releases.sort(key=lambda item: item.release.release_record_id) + unlinked_runtime.sort( + key=lambda item: item.reconciliation.runtime_reconciliation_record_id + ) + issues.sort( + key=lambda item: ( + item.source_kind, + item.source_id, + item.reference_field, + item.target_kind, + item.target_id, + item.reason, + ) + ) + return ContractEvolution( + contract_id=contract_id, + proposals=tuple(proposals), + unlinked_releases=tuple(unlinked_releases), + unlinked_runtime=tuple(unlinked_runtime), + broken_references=tuple(issues), + ) + + def _resolve_revision( + self, + change_set: ChangeSet, + *, + field_name: str, + revision_id: str, + contract_id: str, + issues: list[BrokenHistoryReference], + ) -> ContractRevision | None: + try: + revision = self._revisions.get_revision(revision_id) + except HistoryNotFoundError: + _add_issue( + issues, + source_kind="ChangeSet", + source_id=change_set.change_set_id, + reference_field=field_name, + target_kind="ContractRevision", + target_id=revision_id, + reason="NOT_FOUND", + ) + return None + if str(revision.contract.id or "") != contract_id: + _add_issue( + issues, + source_kind="ChangeSet", + source_id=change_set.change_set_id, + reference_field=field_name, + target_kind="ContractRevision", + target_id=revision_id, + reason="CONTRACT_MISMATCH", + ) + return None + return revision + + def _resolve_decision( + self, + change_set: ChangeSet, + decision_id: str, + *, + contract_id: str, + issues: list[BrokenHistoryReference], + ) -> GovernanceDecision | None: + try: + decision = self._decisions.get_decision(decision_id) + except HistoryNotFoundError: + _add_issue( + issues, + source_kind="ChangeSetDecisionLink", + source_id=f"{change_set.change_set_id}:{decision_id}", + reference_field="decision_id", + target_kind="GovernanceDecision", + target_id=decision_id, + reason="NOT_FOUND", + ) + return None + if decision.contract_id != contract_id: + _add_issue( + issues, + source_kind="ChangeSetDecisionLink", + source_id=f"{change_set.change_set_id}:{decision_id}", + reference_field="decision_id", + target_kind="GovernanceDecision", + target_id=decision_id, + reason="CONTRACT_MISMATCH", + ) + return None + return decision + + def _build_release( + self, + release: ReleaseRecord, + *, + contract_id: str, + runtime_by_id: dict[str, RuntimeEvolution], + runtime_by_release: dict[str, list[RuntimeReconciliationRecord]], + runtime_by_deployment: dict[str, list[RuntimeReconciliationRecord]], + consumed_runtime_ids: set[str], + known_deployment_ids: set[str], + issues: list[BrokenHistoryReference], + ) -> ReleaseEvolution: + released_revision: ContractRevision | None + try: + released_revision = self._revisions.get_revision(release.released_revision_id) + except HistoryNotFoundError: + released_revision = None + _add_issue( + issues, + source_kind="ReleaseRecord", + source_id=release.release_record_id, + reference_field="released_revision_id", + target_kind="ContractRevision", + target_id=release.released_revision_id, + reason="NOT_FOUND", + ) + else: + if ( + str(released_revision.contract.id or "") != contract_id + or str(released_revision.contract.version or "") != release.contract_version + ): + _add_issue( + issues, + source_kind="ReleaseRecord", + source_id=release.release_record_id, + reference_field="released_revision_id", + target_kind="ContractRevision", + target_id=release.released_revision_id, + reason="RELEASE_MISMATCH", + ) + released_revision = None + + deployments = tuple( + sorted( + self._deployments.list_deployment_records_for_release( + release.release_record_id + ), + key=lambda item: item.deployment_record_id, + ) + ) + deployment_paths: list[DeploymentEvolution] = [] + for deployment in deployments: + known_deployment_ids.add(deployment.deployment_record_id) + linked_runtime: list[RuntimeEvolution] = [] + for record in sorted( + runtime_by_deployment.get(deployment.deployment_record_id, ()), + key=lambda item: item.runtime_reconciliation_record_id, + ): + if record.release_record_id != release.release_record_id: + _add_issue( + issues, + source_kind="RuntimeReconciliationRecord", + source_id=record.runtime_reconciliation_record_id, + reference_field="release_record_id", + target_kind="ReleaseRecord", + target_id=str(record.release_record_id or ""), + reason="DEPLOYMENT_RELEASE_MISMATCH", + ) + continue + linked_runtime.append(runtime_by_id[record.runtime_reconciliation_record_id]) + consumed_runtime_ids.add(record.runtime_reconciliation_record_id) + deployment_paths.append( + DeploymentEvolution( + deployment=deployment, + runtime=tuple(linked_runtime), + ) + ) + + release_runtime = tuple( + runtime_by_id[record.runtime_reconciliation_record_id] + for record in sorted( + runtime_by_release.get(release.release_record_id, ()), + key=lambda item: item.runtime_reconciliation_record_id, + ) + ) + consumed_runtime_ids.update( + item.reconciliation.runtime_reconciliation_record_id for item in release_runtime + ) + return ReleaseEvolution( + release=release, + released_revision=released_revision, + deployments=tuple(deployment_paths), + runtime=release_runtime, + ) + + def _runtime_evolution( + self, + record: RuntimeReconciliationRecord, + issues: list[BrokenHistoryReference], + ) -> RuntimeEvolution: + try: + observation = self._observations.get_runtime_observation_record( + record.observation_record_id + ) + except HistoryNotFoundError: + _add_issue( + issues, + source_kind="RuntimeReconciliationRecord", + source_id=record.runtime_reconciliation_record_id, + reference_field="observation_record_id", + target_kind="RuntimeObservationRecord", + target_id=record.observation_record_id, + reason="NOT_FOUND", + ) + return RuntimeEvolution(reconciliation=record, observation=None) + + observed = observation.observation + if ( + observed.source_identifier != record.result.observation_source_identifier + or observed.fingerprint != record.result.observation_fingerprint + ): + _add_issue( + issues, + source_kind="RuntimeReconciliationRecord", + source_id=record.runtime_reconciliation_record_id, + reference_field="observation_record_id", + target_kind="RuntimeObservationRecord", + target_id=record.observation_record_id, + reason="EVIDENCE_MISMATCH", + ) + return RuntimeEvolution(reconciliation=record, observation=observation) + + +def _add_issue( + issues: list[BrokenHistoryReference], + *, + source_kind: str, + source_id: str, + reference_field: str, + target_kind: str, + target_id: str, + reason: str, +) -> None: + issues.append( + BrokenHistoryReference( + source_kind=source_kind, + source_id=source_id, + reference_field=reference_field, + target_kind=target_kind, + target_id=target_id, + reason=reason, + ) + ) + + +def _required_text(value: str, field_name: str) -> str: + if not isinstance(value, str): + raise TypeError(f"{field_name} must be str") + cleaned = value.strip() + if not cleaned: + raise ValueError(f"{field_name} must not be empty") + return cleaned diff --git a/tests/test_evolution_chain.py b/tests/test_evolution_chain.py new file mode 100644 index 00000000..47bd700a --- /dev/null +++ b/tests/test_evolution_chain.py @@ -0,0 +1,297 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from pathlib import Path + +from open_data_contract_standard.model import ( + OpenDataContractStandard, + SchemaObject, + SchemaProperty, +) + +from semapact.application.services.evolution import EvolutionChainService +from semapact.application.services.governance import GovernanceService +from semapact.application.services.history import ProposalHistoryService +from semapact.application.services.runtime_history import RuntimeHistoryService +from semapact.contractops import VersionAuthority, build_change_set +from semapact.history import DeploymentRecord, DeploymentStatus, ReleaseRecord +from semapact.history.integrity import compute_deployment_record_id, compute_release_record_id +from semapact.observation import ( + ObservedAsset, + ObservedAssetIdentity, + ObservedPlatformState, + with_observed_state_fingerprint, +) +from semapact.platforms.git import GitWorkingTreeHistoryRepository +from semapact.reconciliation import ReconciliationResult +from semapact.revision import build_contract_revision + + +def _contract(*, version: str = "1.2.3", include_note: bool = False) -> OpenDataContractStandard: + properties = [ + SchemaProperty( + name="id", + logicalType="string", + physicalType="varchar(255)", + required=True, + ) + ] + if include_note: + properties.append( + SchemaProperty( + name="note", + logicalType="string", + physicalType="varchar(255)", + required=False, + ) + ) + return OpenDataContractStandard( + apiVersion="v3.1.0", + kind="DataContract", + id="orders-product", + name="Orders", + version=version, + status="active", + schema=[SchemaObject(name="orders", properties=properties)], + ) + + +def _record_proposal(backend: GitWorkingTreeHistoryRepository): + base_revision = build_contract_revision(_contract()) + candidate_revision = build_contract_revision(_contract(include_note=True)) + proposal = GovernanceService().evaluate_proposal( + base_revision.contract, + candidate_revision.contract, + effective_date="2026-09-13", + base_revision_ref=base_revision.revision_id, + candidate_revision_ref=candidate_revision.revision_id, + source="test", + actor_reference="actor:test", + ) + ProposalHistoryService( + revisions=backend, + change_sets=backend, + decisions=backend, + decision_links=backend, + ).record_proposal( + proposal, + base_revision=base_revision, + candidate_revision=candidate_revision, + ) + return proposal, base_revision, candidate_revision + + +def _release_record(proposal, released_revision_id: str) -> ReleaseRecord: + fields = dict( + contract_id="orders-product", + contract_version="1.3.0", + decision_id=proposal.decision.decision_id, + change_set_id=proposal.change_set.change_set_id, + release_plan_id="release-plan-1", + version_resolution_id="version-resolution-1", + authorization_id="apply-authorization-1", + applied_release_id="applied-release-1", + released_revision_id=released_revision_id, + required_version_bump="minor", + actual_version_bump="minor", + version_authority=VersionAuthority.SEMAPACT, + authority_reference=None, + review_evidence_reference=None, + review_evidence_action=None, + ) + release_record_id = compute_release_record_id( + **{**fields, "version_authority": fields["version_authority"].value} + ) + return ReleaseRecord(release_record_id=release_record_id, **fields) + + +def _deployment_record(release_record_id: str) -> DeploymentRecord: + started = datetime(2026, 9, 13, 1, tzinfo=timezone.utc) + completed = started + timedelta(seconds=2) + fields = dict( + release_record_id=release_record_id, + deployment_plan_id="deployment-plan-1", + deployment_preview_id="deployment-preview-1", + deployment_authorization_id="deployment-authorization-1", + platform="databricks", + runtime_target="catalog.schema", + source_reference="workspace:test", + status=DeploymentStatus.SUCCEEDED, + started_at=started, + completed_at=completed, + actor_reference=None, + external_reference="pipeline:1", + ) + deployment_record_id = compute_deployment_record_id( + release_record_id=release_record_id, + deployment_plan_id=fields["deployment_plan_id"], + deployment_preview_id=fields["deployment_preview_id"], + deployment_authorization_id=fields["deployment_authorization_id"], + platform=fields["platform"], + runtime_target=fields["runtime_target"], + source_reference=fields["source_reference"], + status=fields["status"].value, + started_at=started.isoformat(), + completed_at=completed.isoformat(), + actor_reference=None, + external_reference=fields["external_reference"], + ) + return DeploymentRecord(deployment_record_id=deployment_record_id, **fields) + + +def _observation(at: datetime) -> ObservedPlatformState: + return with_observed_state_fingerprint( + ObservedPlatformState( + platform="databricks", + source_identifier="workspace:test", + assets=( + ObservedAsset( + identity=ObservedAssetIdentity( + platform="databricks", + namespace=("catalog", "schema"), + asset="orders", + ), + asset_type="TABLE", + ), + ), + captured_at=at, + ) + ) + + +def _result(observation: ObservedPlatformState) -> ReconciliationResult: + assert observation.fingerprint is not None + return ReconciliationResult( + contract_id="orders-product", + contract_version="1.3.0", + observation_source_identifier=observation.source_identifier, + observation_fingerprint=observation.fingerprint, + ) + + +def _service(backend: GitWorkingTreeHistoryRepository) -> EvolutionChainService: + return EvolutionChainService( + revisions=backend, + change_sets=backend, + decisions=backend, + decision_links=backend, + releases=backend, + deployments=backend, + observations=backend, + runtime_reconciliations=backend, + ) + + +def test_reconstructs_complete_governance_evolution_chain(tmp_path: Path) -> None: + backend = GitWorkingTreeHistoryRepository(tmp_path) + proposal, base_revision, candidate_revision = _record_proposal(backend) + + released_revision = build_contract_revision( + _contract(version="1.3.0", include_note=True) + ) + backend.put_revision(released_revision) + release = _release_record(proposal, released_revision.revision_id) + backend.put_release_record(release) + deployment = _deployment_record(release.release_record_id) + backend.put_deployment_record(deployment) + + observation = _observation(datetime(2026, 9, 13, 2, tzinfo=timezone.utc)) + runtime = RuntimeHistoryService( + observations=backend, + reconciliations=backend, + releases=backend, + deployments=backend, + ).record_reconciliation( + observation, + _result(observation), + release_record_id=release.release_record_id, + deployment_record_id=deployment.deployment_record_id, + ) + + chain = _service(backend).reconstruct("orders-product") + + assert chain.broken_references == () + assert chain.unlinked_releases == () + assert chain.unlinked_runtime == () + assert len(chain.proposals) == 1 + path = chain.proposals[0] + assert path.base_revision == base_revision + assert path.candidate_revision == candidate_revision + assert path.decision == proposal.decision + assert len(path.releases) == 1 + release_path = path.releases[0] + assert release_path.release == release + assert release_path.released_revision == released_revision + assert len(release_path.deployments) == 1 + deployment_path = release_path.deployments[0] + assert deployment_path.deployment == deployment + assert tuple(item.reconciliation for item in deployment_path.runtime) == (runtime,) + assert deployment_path.runtime[0].observation is not None + assert _service(backend).reconstruct("orders-product") == chain + + +def test_incomplete_changeset_history_does_not_fabricate_downstream_stages(tmp_path: Path) -> None: + backend = GitWorkingTreeHistoryRepository(tmp_path) + proposal, base_revision, candidate_revision = _record_proposal(backend) + + isolated = build_change_set( + contract_id=proposal.change_set.contract_id, + base_revision_ref=base_revision.revision_id, + candidate_revision_ref=candidate_revision.revision_id, + changes=proposal.change_set.changes, + context=proposal.change_set.context, + source="isolated", + ) + backend.put_change_set(isolated) + + chain = _service(backend).reconstruct("orders-product") + isolated_paths = [item for item in chain.proposals if item.change_set == isolated] + + assert len(isolated_paths) == 1 + assert isolated_paths[0].decision is None + assert isolated_paths[0].releases == () + + +def test_broken_revision_reference_is_reported_explicitly(tmp_path: Path) -> None: + backend = GitWorkingTreeHistoryRepository(tmp_path) + proposal, _, candidate_revision = _record_proposal(backend) + broken = build_change_set( + contract_id="orders-product", + base_revision_ref="missing-revision", + candidate_revision_ref=candidate_revision.revision_id, + changes=proposal.change_set.changes, + context=proposal.change_set.context, + source="broken", + ) + backend.put_change_set(broken) + + chain = _service(backend).reconstruct("orders-product") + path = next(item for item in chain.proposals if item.change_set == broken) + + assert path.base_revision is None + assert any( + item.source_id == broken.change_set_id + and item.reference_field == "base_revision_ref" + and item.target_id == "missing-revision" + and item.reason == "NOT_FOUND" + for item in chain.broken_references + ) + + +def test_contract_level_runtime_evidence_remains_unlinked(tmp_path: Path) -> None: + backend = GitWorkingTreeHistoryRepository(tmp_path) + observation = _observation(datetime(2026, 9, 13, 3, tzinfo=timezone.utc)) + runtime = RuntimeHistoryService( + observations=backend, + reconciliations=backend, + releases=backend, + deployments=backend, + ).record_reconciliation(observation, _result(observation)) + + chain = _service(backend).reconstruct("orders-product") + + assert chain.proposals == () + assert chain.unlinked_releases == () + assert tuple(item.reconciliation for item in chain.unlinked_runtime) == (runtime,) + assert chain.unlinked_runtime[0].observation is not None + assert chain.broken_references == () From 1ed850d758a3915c7608527d1b27d3dbd924a0f9 Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Tue, 15 Sep 2026 15:15:57 +1000 Subject: [PATCH 23/35] feat(history): harden immutable history storage * feat(history): add storage integrity diagnostics * feat(history): expose storage integrity capability * feat(history): export integrity diagnostics * feat(history): seal and inspect Git history storage * feat(history): add integrity check result model * feat(history): add integrity check service * feat(history): export integrity report * test(history): cover immutable storage integrity * fix(history): preserve corruption contract and harden path publication * test(history): cover strict path containment * fix(history): fail closed before escaped integrity scans * refactor(history): centralize git history kind metadata * refactor(history): simplify integrity inspection flow --- semapact/application/models/__init__.py | 2 + .../application/models/history_integrity.py | 27 + .../application/services/history_integrity.py | 47 + semapact/history/__init__.py | 6 + semapact/history/models.py | 33 + semapact/history/repository.py | 7 + semapact/platforms/git/history_repository.py | 1008 +++++++++++------ tests/test_history_integrity.py | 263 +++++ 8 files changed, 1069 insertions(+), 324 deletions(-) create mode 100644 semapact/application/models/history_integrity.py create mode 100644 semapact/application/services/history_integrity.py create mode 100644 tests/test_history_integrity.py diff --git a/semapact/application/models/__init__.py b/semapact/application/models/__init__.py index c2c1b454..34a27143 100644 --- a/semapact/application/models/__init__.py +++ b/semapact/application/models/__init__.py @@ -9,6 +9,7 @@ RuntimeEvolution, ) from semapact.application.models.governance import GovernanceAnalysis, GovernanceProposal +from semapact.application.models.history_integrity import HistoryIntegrityReport from semapact.application.models.reconciliation import RuntimeReconciliation from semapact.application.models.release import ReleasePlanningResult @@ -18,6 +19,7 @@ "DeploymentEvolution", "GovernanceAnalysis", "GovernanceProposal", + "HistoryIntegrityReport", "ProposalEvolution", "ReleaseEvolution", "ReleasePlanningResult", diff --git a/semapact/application/models/history_integrity.py b/semapact/application/models/history_integrity.py new file mode 100644 index 00000000..4dfbc396 --- /dev/null +++ b/semapact/application/models/history_integrity.py @@ -0,0 +1,27 @@ +"""Application read model for governance-history integrity checks.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from semapact.application.models.evolution import BrokenHistoryReference +from semapact.history import HistoryStorageIntegrityIssue + + +@dataclass(frozen=True) +class HistoryIntegrityReport: + """Physical storage and logical-reference integrity for one contract check.""" + + contract_id: str + storage_issues: tuple[HistoryStorageIntegrityIssue, ...] = () + broken_references: tuple[BrokenHistoryReference, ...] = () + references_checked: bool = True + + @property + def valid(self) -> bool: + """Return whether both storage and reference integrity were proven clean.""" + return ( + self.references_checked + and not self.storage_issues + and not self.broken_references + ) diff --git a/semapact/application/services/history_integrity.py b/semapact/application/services/history_integrity.py new file mode 100644 index 00000000..2e82c68d --- /dev/null +++ b/semapact/application/services/history_integrity.py @@ -0,0 +1,47 @@ +"""Application orchestration for governance-history integrity checks.""" + +from __future__ import annotations + +from semapact.application.models.history_integrity import HistoryIntegrityReport +from semapact.application.services.evolution import EvolutionChainService +from semapact.history import HistoryIntegrityRepository + + +class HistoryIntegrityService: + """Fail closed on physical corruption before validating logical history links.""" + + def __init__( + self, + *, + storage: HistoryIntegrityRepository, + evolution: EvolutionChainService, + ) -> None: + self._storage = storage + self._evolution = evolution + + def check_contract(self, contract_id: str) -> HistoryIntegrityReport: + """Check repository storage first, then reuse evolution reference validation.""" + contract_id = _required_text(contract_id, "contract_id") + storage_issues = self._storage.inspect_history_integrity() + if storage_issues: + return HistoryIntegrityReport( + contract_id=contract_id, + storage_issues=storage_issues, + references_checked=False, + ) + + evolution = self._evolution.reconstruct(contract_id) + return HistoryIntegrityReport( + contract_id=contract_id, + broken_references=evolution.broken_references, + references_checked=True, + ) + + +def _required_text(value: str, field_name: str) -> str: + if not isinstance(value, str): + raise TypeError(f"{field_name} must be str") + cleaned = value.strip() + if not cleaned: + raise ValueError(f"{field_name} must not be empty") + return cleaned diff --git a/semapact/history/__init__.py b/semapact/history/__init__.py index 652886f6..16d5af14 100644 --- a/semapact/history/__init__.py +++ b/semapact/history/__init__.py @@ -8,6 +8,8 @@ ChangeSetDecisionLink, DeploymentRecord, DeploymentStatus, + HistoryIntegrityIssueCode, + HistoryStorageIntegrityIssue, ReleaseRecord, RuntimeObservationRecord, RuntimeReconciliationRecord, @@ -24,6 +26,7 @@ DeploymentRecordHistoryRepository, HistoryConflictError, HistoryCorruptionError, + HistoryIntegrityRepository, HistoryNotFoundError, HistoryRepositoryError, ReleasePlanHistoryRepository, @@ -47,8 +50,11 @@ "DeploymentStatus", "HistoryConflictError", "HistoryCorruptionError", + "HistoryIntegrityIssueCode", + "HistoryIntegrityRepository", "HistoryNotFoundError", "HistoryRepositoryError", + "HistoryStorageIntegrityIssue", "ReleasePlanHistoryRepository", "ReleaseRecord", "ReleaseRecordHistoryRepository", diff --git a/semapact/history/models.py b/semapact/history/models.py index b876b663..2fbad67a 100644 --- a/semapact/history/models.py +++ b/semapact/history/models.py @@ -27,6 +27,39 @@ class HistoryModel(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") +class HistoryIntegrityIssueCode(str, Enum): + """Stable diagnostics emitted by a physical history backend integrity scan.""" + + CHECKSUM_MISSING = "CHECKSUM_MISSING" + CHECKSUM_MISMATCH = "CHECKSUM_MISMATCH" + CHECKSUM_INVALID = "CHECKSUM_INVALID" + ORPHAN_CHECKSUM = "ORPHAN_CHECKSUM" + ARTIFACT_INVALID = "ARTIFACT_INVALID" + IDENTITY_MISMATCH = "IDENTITY_MISMATCH" + PATH_PROVENANCE_MISMATCH = "PATH_PROVENANCE_MISMATCH" + UNKNOWN_ARTIFACT_LAYOUT = "UNKNOWN_ARTIFACT_LAYOUT" + + +class HistoryStorageIntegrityIssue(HistoryModel): + """One deterministic storage-integrity diagnostic without mutating history.""" + + code: HistoryIntegrityIssueCode + artifact_kind: str + storage_reference: str + artifact_id: str | None = None + detail: str + + @field_validator("artifact_kind", "storage_reference", "detail") + @classmethod + def _require_integrity_text(cls, value: str) -> str: + return _required_text(value) + + @field_validator("artifact_id") + @classmethod + def _normalize_integrity_artifact_id(cls, value: str | None) -> str | None: + return _optional_text(value) + + class ChangeSetDecisionLink(HistoryModel): """Audit provenance linking one ChangeSet to one GovernanceDecision outcome.""" diff --git a/semapact/history/repository.py b/semapact/history/repository.py index 1194d213..0ddc03a2 100644 --- a/semapact/history/repository.py +++ b/semapact/history/repository.py @@ -10,6 +10,7 @@ from semapact.history.models import ( ChangeSetDecisionLink, DeploymentRecord, + HistoryStorageIntegrityIssue, ReleaseRecord, RuntimeObservationRecord, RuntimeReconciliationRecord, @@ -33,6 +34,12 @@ class HistoryCorruptionError(HistoryRepositoryError): """Persisted or supplied historical content fails canonical validation.""" +class HistoryIntegrityRepository(Protocol): + """Backend-specific physical integrity inspection without history mutation.""" + + def inspect_history_integrity(self) -> tuple[HistoryStorageIntegrityIssue, ...]: ... + + class DecisionHistoryRepository(Protocol): """Persistence capability for canonical GovernanceDecision history only.""" diff --git a/semapact/platforms/git/history_repository.py b/semapact/platforms/git/history_repository.py index b0d98a31..e0eade53 100644 --- a/semapact/platforms/git/history_repository.py +++ b/semapact/platforms/git/history_repository.py @@ -6,15 +6,22 @@ from __future__ import annotations +import hashlib +import os import re +import secrets from collections.abc import Callable +from dataclasses import dataclass from pathlib import Path -from typing import TypeVar +from typing import Generic, TypeVar from pydantic import BaseModel, ValidationError as PydanticValidationError from semapact.contractops import ChangeSet, ReleasePlan -from semapact.contractops.integrity import validate_release_plan_identity +from semapact.contractops.integrity import ( + validate_change_set_identity, + validate_release_plan_identity, +) from semapact.deployment import DeploymentAuthorization, DeploymentPlan, DeploymentPreview from semapact.deployment.models import ( validate_deployment_authorization_identity, @@ -27,7 +34,9 @@ DeploymentRecord, HistoryConflictError, HistoryCorruptionError, + HistoryIntegrityIssueCode, HistoryNotFoundError, + HistoryStorageIntegrityIssue, ReleaseRecord, RuntimeObservationRecord, RuntimeReconciliationRecord, @@ -48,6 +57,114 @@ T = TypeVar("T", bound=BaseModel) _SAFE_ARTIFACT_ID = re.compile(r"^[A-Za-z0-9._-]+$") +_CHECKSUM_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$") + + +@dataclass(frozen=True) +class _HistoryKindSpec(Generic[T]): + """Single source of Git-layout and rehydration metadata for one history kind.""" + + directory: str + model_type: type[T] + id_attribute: str + integrity_validator: Callable[[T], None] | None = None + + +_DECISIONS = _HistoryKindSpec( + "decisions", + GovernanceDecision, + "decision_id", +) +_CHANGE_SETS = _HistoryKindSpec( + "change_sets", + ChangeSet, + "change_set_id", + validate_change_set_identity, +) +_CHANGE_SET_DECISIONS = _HistoryKindSpec( + "change_set_decisions", + ChangeSetDecisionLink, + "decision_id", +) +_CONTRACT_REVISIONS = _HistoryKindSpec( + "contract_revisions", + ContractRevision, + "revision_id", + validate_contract_revision_identity, +) +_CONTRACT_REVISION_SOURCES = _HistoryKindSpec( + "contract_revision_sources", + ContractRevisionSource, + "source_link_id", + validate_contract_revision_source_identity, +) +_RELEASE_PLANS = _HistoryKindSpec( + "release_plans", + ReleasePlan, + "release_plan_id", + validate_release_plan_identity, +) +_RELEASE_RECORDS = _HistoryKindSpec( + "release_records", + ReleaseRecord, + "release_record_id", + validate_release_record_identity, +) +_DEPLOYMENT_PLANS = _HistoryKindSpec( + "deployment_plans", + DeploymentPlan, + "deployment_plan_id", + validate_deployment_plan_identity, +) +_DEPLOYMENT_PREVIEWS = _HistoryKindSpec( + "deployment_previews", + DeploymentPreview, + "deployment_preview_id", + validate_deployment_preview_identity, +) +_DEPLOYMENT_AUTHORIZATIONS = _HistoryKindSpec( + "deployment_authorizations", + DeploymentAuthorization, + "deployment_authorization_id", + validate_deployment_authorization_identity, +) +_DEPLOYMENT_RECORDS = _HistoryKindSpec( + "deployment_records", + DeploymentRecord, + "deployment_record_id", + validate_deployment_record_identity, +) +_RUNTIME_OBSERVATIONS = _HistoryKindSpec( + "runtime_observations", + RuntimeObservationRecord, + "observation_record_id", + validate_runtime_observation_record_identity, +) +_RUNTIME_RECONCILIATIONS = _HistoryKindSpec( + "runtime_reconciliations", + RuntimeReconciliationRecord, + "runtime_reconciliation_record_id", + validate_runtime_reconciliation_record_identity, +) + +_HISTORY_KIND_SPECS: dict[str, _HistoryKindSpec[BaseModel]] = { + spec.directory: spec + for spec in ( + _DECISIONS, + _CHANGE_SETS, + _CHANGE_SET_DECISIONS, + _CONTRACT_REVISIONS, + _CONTRACT_REVISION_SOURCES, + _RELEASE_PLANS, + _RELEASE_RECORDS, + _DEPLOYMENT_PLANS, + _DEPLOYMENT_PREVIEWS, + _DEPLOYMENT_AUTHORIZATIONS, + _DEPLOYMENT_RECORDS, + _RUNTIME_OBSERVATIONS, + _RUNTIME_RECONCILIATIONS, + ) +} class GitWorkingTreeHistoryRepository: @@ -59,59 +176,45 @@ def __init__( *, state_directory: str | Path = ".semapact/history", ) -> None: - self._history_root = Path(repository_root) / Path(state_directory) + repository_path = Path(repository_root).resolve(strict=False) + state_path = Path(state_directory) + if state_path.is_absolute(): + raise ValueError("history state_directory must be repository-relative") + if ".." in state_path.parts: + raise ValueError("history state_directory must not contain '..'") + history_root = (repository_path / state_path).resolve(strict=False) + if not history_root.is_relative_to(repository_path): + raise ValueError("history state_directory must remain inside repository_root") + self._repository_root = repository_path + self._history_root = history_root def put_decision(self, decision: GovernanceDecision) -> None: - self._put( - kind="decisions", - artifact_id=decision.decision_id, - artifact=decision, - model_type=GovernanceDecision, - id_attribute="decision_id", - ) + self._put(_DECISIONS, decision.decision_id, decision) def get_decision(self, decision_id: str) -> GovernanceDecision: - return self._get( - kind="decisions", - artifact_id=decision_id, - model_type=GovernanceDecision, - id_attribute="decision_id", - ) + return self._get(_DECISIONS, decision_id) def list_decisions(self, contract_id: str) -> tuple[GovernanceDecision, ...]: contract_id = _required_text(contract_id, "contract_id") - records = self._list( - kind="decisions", - model_type=GovernanceDecision, - id_attribute="decision_id", + return tuple( + record + for record in self._list(_DECISIONS) + if record.contract_id == contract_id ) - return tuple(record for record in records if record.contract_id == contract_id) def put_change_set(self, change_set: ChangeSet) -> None: - self._put( - kind="change_sets", - artifact_id=change_set.change_set_id, - artifact=change_set, - model_type=ChangeSet, - id_attribute="change_set_id", - ) + self._put(_CHANGE_SETS, change_set.change_set_id, change_set) def get_change_set(self, change_set_id: str) -> ChangeSet: - return self._get( - kind="change_sets", - artifact_id=change_set_id, - model_type=ChangeSet, - id_attribute="change_set_id", - ) + return self._get(_CHANGE_SETS, change_set_id) def list_change_sets(self, contract_id: str) -> tuple[ChangeSet, ...]: contract_id = _required_text(contract_id, "contract_id") - records = self._list( - kind="change_sets", - model_type=ChangeSet, - id_attribute="change_set_id", + return tuple( + record + for record in self._list(_CHANGE_SETS) + if record.contract_id == contract_id ) - return tuple(record for record in records if record.contract_id == contract_id) def put_change_set_decision_link(self, link: ChangeSetDecisionLink) -> None: if not isinstance(link, ChangeSetDecisionLink): @@ -121,11 +224,10 @@ def put_change_set_decision_link(self, link: ChangeSetDecisionLink) -> None: ) change_set_id = _safe_artifact_id(link.change_set_id) self._put( - kind=f"change_set_decisions/{change_set_id}", - artifact_id=link.decision_id, - artifact=link, - model_type=ChangeSetDecisionLink, - id_attribute="decision_id", + _CHANGE_SET_DECISIONS, + link.decision_id, + link, + directory=f"{_CHANGE_SET_DECISIONS.directory}/{change_set_id}", ) def list_change_set_decision_links( @@ -134,9 +236,8 @@ def list_change_set_decision_links( ) -> tuple[ChangeSetDecisionLink, ...]: change_set_id = _safe_artifact_id(change_set_id) records = self._list( - kind=f"change_set_decisions/{change_set_id}", - model_type=ChangeSetDecisionLink, - id_attribute="decision_id", + _CHANGE_SET_DECISIONS, + directory=f"{_CHANGE_SET_DECISIONS.directory}/{change_set_id}", ) for record in records: if record.change_set_id != change_set_id: @@ -146,88 +247,41 @@ def list_change_set_decision_links( return records def put_revision(self, revision: ContractRevision) -> None: - self._put( - kind="contract_revisions", - artifact_id=revision.revision_id, - artifact=revision, - model_type=ContractRevision, - id_attribute="revision_id", - integrity_validator=validate_contract_revision_identity, - ) + self._put(_CONTRACT_REVISIONS, revision.revision_id, revision) def get_revision(self, revision_id: str) -> ContractRevision: - return self._get( - kind="contract_revisions", - artifact_id=revision_id, - model_type=ContractRevision, - id_attribute="revision_id", - integrity_validator=validate_contract_revision_identity, - ) + return self._get(_CONTRACT_REVISIONS, revision_id) def list_revisions(self, contract_id: str) -> tuple[ContractRevision, ...]: contract_id = _required_text(contract_id, "contract_id") - records = self._list( - kind="contract_revisions", - model_type=ContractRevision, - id_attribute="revision_id", - integrity_validator=validate_contract_revision_identity, - ) return tuple( record - for record in records + for record in self._list(_CONTRACT_REVISIONS) if str(record.contract.id or "") == contract_id ) def put_revision_source(self, source: ContractRevisionSource) -> None: - self._put( - kind="contract_revision_sources", - artifact_id=source.source_link_id, - artifact=source, - model_type=ContractRevisionSource, - id_attribute="source_link_id", - integrity_validator=validate_contract_revision_source_identity, - ) + self._put(_CONTRACT_REVISION_SOURCES, source.source_link_id, source) def get_revision_source(self, source_link_id: str) -> ContractRevisionSource: - return self._get( - kind="contract_revision_sources", - artifact_id=source_link_id, - model_type=ContractRevisionSource, - id_attribute="source_link_id", - integrity_validator=validate_contract_revision_source_identity, - ) + return self._get(_CONTRACT_REVISION_SOURCES, source_link_id) def list_revision_sources( self, revision_id: str, ) -> tuple[ContractRevisionSource, ...]: revision_id = _required_text(revision_id, "revision_id") - records = self._list( - kind="contract_revision_sources", - model_type=ContractRevisionSource, - id_attribute="source_link_id", - integrity_validator=validate_contract_revision_source_identity, + return tuple( + record + for record in self._list(_CONTRACT_REVISION_SOURCES) + if record.revision_id == revision_id ) - return tuple(record for record in records if record.revision_id == revision_id) def put_release_plan(self, release_plan: ReleasePlan) -> None: - self._put( - kind="release_plans", - artifact_id=release_plan.release_plan_id, - artifact=release_plan, - model_type=ReleasePlan, - id_attribute="release_plan_id", - integrity_validator=validate_release_plan_identity, - ) + self._put(_RELEASE_PLANS, release_plan.release_plan_id, release_plan) def get_release_plan(self, release_plan_id: str) -> ReleasePlan: - return self._get( - kind="release_plans", - artifact_id=release_plan_id, - model_type=ReleasePlan, - id_attribute="release_plan_id", - integrity_validator=validate_release_plan_identity, - ) + return self._get(_RELEASE_PLANS, release_plan_id) def put_release_record(self, record: ReleaseRecord) -> None: if not isinstance(record, ReleaseRecord): @@ -244,33 +298,18 @@ def put_release_record(self, record: ReleaseRecord) -> None: "A different ReleaseRecord already exists for " f"{record.contract_id!r} version {record.contract_version!r}" ) - self._put( - kind="release_records", - artifact_id=record.release_record_id, - artifact=record, - model_type=ReleaseRecord, - id_attribute="release_record_id", - integrity_validator=validate_release_record_identity, - ) + self._put(_RELEASE_RECORDS, record.release_record_id, record) def get_release_record(self, release_record_id: str) -> ReleaseRecord: - return self._get( - kind="release_records", - artifact_id=release_record_id, - model_type=ReleaseRecord, - id_attribute="release_record_id", - integrity_validator=validate_release_record_identity, - ) + return self._get(_RELEASE_RECORDS, release_record_id) def list_release_records(self, contract_id: str) -> tuple[ReleaseRecord, ...]: contract_id = _required_text(contract_id, "contract_id") - records = self._list( - kind="release_records", - model_type=ReleaseRecord, - id_attribute="release_record_id", - integrity_validator=validate_release_record_identity, + return tuple( + record + for record in self._list(_RELEASE_RECORDS) + if record.contract_id == contract_id ) - return tuple(record for record in records if record.contract_id == contract_id) def get_release_record_by_version( self, @@ -295,100 +334,48 @@ def get_release_record_by_version( return matches[0] def put_deployment_plan(self, plan: DeploymentPlan) -> None: - self._put( - kind="deployment_plans", - artifact_id=plan.deployment_plan_id, - artifact=plan, - model_type=DeploymentPlan, - id_attribute="deployment_plan_id", - integrity_validator=validate_deployment_plan_identity, - ) + self._put(_DEPLOYMENT_PLANS, plan.deployment_plan_id, plan) def get_deployment_plan(self, deployment_plan_id: str) -> DeploymentPlan: - return self._get( - kind="deployment_plans", - artifact_id=deployment_plan_id, - model_type=DeploymentPlan, - id_attribute="deployment_plan_id", - integrity_validator=validate_deployment_plan_identity, - ) + return self._get(_DEPLOYMENT_PLANS, deployment_plan_id) def put_deployment_preview(self, preview: DeploymentPreview) -> None: - self._put( - kind="deployment_previews", - artifact_id=preview.deployment_preview_id, - artifact=preview, - model_type=DeploymentPreview, - id_attribute="deployment_preview_id", - integrity_validator=validate_deployment_preview_identity, - ) + self._put(_DEPLOYMENT_PREVIEWS, preview.deployment_preview_id, preview) def get_deployment_preview(self, deployment_preview_id: str) -> DeploymentPreview: - return self._get( - kind="deployment_previews", - artifact_id=deployment_preview_id, - model_type=DeploymentPreview, - id_attribute="deployment_preview_id", - integrity_validator=validate_deployment_preview_identity, - ) + return self._get(_DEPLOYMENT_PREVIEWS, deployment_preview_id) def put_deployment_authorization( self, authorization: DeploymentAuthorization, ) -> None: self._put( - kind="deployment_authorizations", - artifact_id=authorization.deployment_authorization_id, - artifact=authorization, - model_type=DeploymentAuthorization, - id_attribute="deployment_authorization_id", - integrity_validator=validate_deployment_authorization_identity, + _DEPLOYMENT_AUTHORIZATIONS, + authorization.deployment_authorization_id, + authorization, ) def get_deployment_authorization( self, deployment_authorization_id: str, ) -> DeploymentAuthorization: - return self._get( - kind="deployment_authorizations", - artifact_id=deployment_authorization_id, - model_type=DeploymentAuthorization, - id_attribute="deployment_authorization_id", - integrity_validator=validate_deployment_authorization_identity, - ) + return self._get(_DEPLOYMENT_AUTHORIZATIONS, deployment_authorization_id) def put_deployment_record(self, record: DeploymentRecord) -> None: - self._put( - kind="deployment_records", - artifact_id=record.deployment_record_id, - artifact=record, - model_type=DeploymentRecord, - id_attribute="deployment_record_id", - integrity_validator=validate_deployment_record_identity, - ) + self._put(_DEPLOYMENT_RECORDS, record.deployment_record_id, record) def get_deployment_record(self, deployment_record_id: str) -> DeploymentRecord: - return self._get( - kind="deployment_records", - artifact_id=deployment_record_id, - model_type=DeploymentRecord, - id_attribute="deployment_record_id", - integrity_validator=validate_deployment_record_identity, - ) + return self._get(_DEPLOYMENT_RECORDS, deployment_record_id) def list_deployment_records_for_release( self, release_record_id: str, ) -> tuple[DeploymentRecord, ...]: release_record_id = _required_text(release_record_id, "release_record_id") - records = self._list( - kind="deployment_records", - model_type=DeploymentRecord, - id_attribute="deployment_record_id", - integrity_validator=validate_deployment_record_identity, - ) return tuple( - record for record in records if record.release_record_id == release_record_id + record + for record in self._list(_DEPLOYMENT_RECORDS) + if record.release_record_id == release_record_id ) def list_deployment_records_for_plan( @@ -396,52 +383,29 @@ def list_deployment_records_for_plan( deployment_plan_id: str, ) -> tuple[DeploymentRecord, ...]: deployment_plan_id = _required_text(deployment_plan_id, "deployment_plan_id") - records = self._list( - kind="deployment_records", - model_type=DeploymentRecord, - id_attribute="deployment_record_id", - integrity_validator=validate_deployment_record_identity, - ) return tuple( - record for record in records if record.deployment_plan_id == deployment_plan_id + record + for record in self._list(_DEPLOYMENT_RECORDS) + if record.deployment_plan_id == deployment_plan_id ) def put_runtime_observation_record(self, record: RuntimeObservationRecord) -> None: - self._put( - kind="runtime_observations", - artifact_id=record.observation_record_id, - artifact=record, - model_type=RuntimeObservationRecord, - id_attribute="observation_record_id", - integrity_validator=validate_runtime_observation_record_identity, - ) + self._put(_RUNTIME_OBSERVATIONS, record.observation_record_id, record) def get_runtime_observation_record( self, observation_record_id: str, ) -> RuntimeObservationRecord: - return self._get( - kind="runtime_observations", - artifact_id=observation_record_id, - model_type=RuntimeObservationRecord, - id_attribute="observation_record_id", - integrity_validator=validate_runtime_observation_record_identity, - ) + return self._get(_RUNTIME_OBSERVATIONS, observation_record_id) def list_runtime_observation_records( self, source_identifier: str, ) -> tuple[RuntimeObservationRecord, ...]: source_identifier = _required_text(source_identifier, "source_identifier") - records = self._list( - kind="runtime_observations", - model_type=RuntimeObservationRecord, - id_attribute="observation_record_id", - integrity_validator=validate_runtime_observation_record_identity, - ) return tuple( record - for record in records + for record in self._list(_RUNTIME_OBSERVATIONS) if record.observation.source_identifier == source_identifier ) @@ -450,12 +414,9 @@ def put_runtime_reconciliation_record( record: RuntimeReconciliationRecord, ) -> None: self._put( - kind="runtime_reconciliations", - artifact_id=record.runtime_reconciliation_record_id, - artifact=record, - model_type=RuntimeReconciliationRecord, - id_attribute="runtime_reconciliation_record_id", - integrity_validator=validate_runtime_reconciliation_record_identity, + _RUNTIME_RECONCILIATIONS, + record.runtime_reconciliation_record_id, + record, ) def get_runtime_reconciliation_record( @@ -463,11 +424,8 @@ def get_runtime_reconciliation_record( runtime_reconciliation_record_id: str, ) -> RuntimeReconciliationRecord: return self._get( - kind="runtime_reconciliations", - artifact_id=runtime_reconciliation_record_id, - model_type=RuntimeReconciliationRecord, - id_attribute="runtime_reconciliation_record_id", - integrity_validator=validate_runtime_reconciliation_record_identity, + _RUNTIME_RECONCILIATIONS, + runtime_reconciliation_record_id, ) def list_runtime_reconciliation_records( @@ -475,204 +433,580 @@ def list_runtime_reconciliation_records( contract_id: str, ) -> tuple[RuntimeReconciliationRecord, ...]: contract_id = _required_text(contract_id, "contract_id") - records = self._list( - kind="runtime_reconciliations", - model_type=RuntimeReconciliationRecord, - id_attribute="runtime_reconciliation_record_id", - integrity_validator=validate_runtime_reconciliation_record_identity, + return tuple( + record + for record in self._list(_RUNTIME_RECONCILIATIONS) + if record.result.contract_id == contract_id ) - return tuple(record for record in records if record.result.contract_id == contract_id) def list_runtime_reconciliation_records_for_source( self, source_identifier: str, ) -> tuple[RuntimeReconciliationRecord, ...]: source_identifier = _required_text(source_identifier, "source_identifier") - records = self._list( - kind="runtime_reconciliations", - model_type=RuntimeReconciliationRecord, - id_attribute="runtime_reconciliation_record_id", - integrity_validator=validate_runtime_reconciliation_record_identity, - ) return tuple( record - for record in records + for record in self._list(_RUNTIME_RECONCILIATIONS) if record.result.observation_source_identifier == source_identifier ) - def _put( + def inspect_history_integrity(self) -> tuple[HistoryStorageIntegrityIssue, ...]: + """Inspect physical history artifacts and checksum evidence without mutation.""" + root_issue = self._inspect_history_root() + if root_issue is not None: + return (root_issue,) + + issues = [ + *self._inspect_history_artifacts(), + *self._inspect_orphan_checksums(), + ] + return tuple(sorted(issues, key=_integrity_issue_sort_key)) + + def _inspect_history_root(self) -> HistoryStorageIntegrityIssue | None: + if not self._history_root.exists(): + return None + if self._path_is_within_history(self._history_root): + return None + return self._integrity_issue( + HistoryIntegrityIssueCode.UNKNOWN_ARTIFACT_LAYOUT, + self._history_root, + detail="resolved history root escapes repository containment", + ) + + def _inspect_history_artifacts(self) -> list[HistoryStorageIntegrityIssue]: + if not self._history_root.exists(): + return [] + + issues: list[HistoryStorageIntegrityIssue] = [] + for path in sorted(self._history_root.rglob("*.json"), key=Path.as_posix): + issues.extend(self._inspect_history_artifact(path)) + return issues + + def _inspect_history_artifact( self, + path: Path, + ) -> tuple[HistoryStorageIntegrityIssue, ...]: + if not self._path_is_within_history(path): + return ( + self._integrity_issue( + HistoryIntegrityIssueCode.UNKNOWN_ARTIFACT_LAYOUT, + path, + detail="resolved artifact path escapes history root", + ), + ) + + resolved = self._resolve_history_spec(path) + if resolved is None: + return ( + self._integrity_issue( + HistoryIntegrityIssueCode.UNKNOWN_ARTIFACT_LAYOUT, + path, + detail="artifact path does not match a canonical history layout", + ), + ) + + spec, artifact_id, parent_change_set_id = resolved + try: + raw = path.read_text(encoding="utf-8") + except OSError as exc: + return ( + self._integrity_issue( + HistoryIntegrityIssueCode.ARTIFACT_INVALID, + path, + artifact_kind=spec.directory, + artifact_id=artifact_id, + detail=f"artifact could not be read: {exc}", + ), + ) + + issues = list( + self._inspect_checksum( + path, + raw, + artifact_kind=spec.directory, + artifact_id=artifact_id, + ) + ) + + try: + artifact = spec.model_type.model_validate_json(raw) + if spec.integrity_validator is not None: + spec.integrity_validator(artifact) + except (PydanticValidationError, ValueError, TypeError) as exc: + issues.append( + self._integrity_issue( + HistoryIntegrityIssueCode.ARTIFACT_INVALID, + path, + artifact_kind=spec.directory, + artifact_id=artifact_id, + detail=f"artifact failed canonical validation: {exc}", + ) + ) + return tuple(issues) + + actual_id = getattr(artifact, spec.id_attribute) + if actual_id != artifact_id: + issues.append( + self._integrity_issue( + HistoryIntegrityIssueCode.IDENTITY_MISMATCH, + path, + artifact_kind=spec.directory, + artifact_id=artifact_id, + detail=( + f"embedded {spec.id_attribute} {actual_id!r} does not match " + f"file identity {artifact_id!r}" + ), + ) + ) + + provenance_issue = self._inspect_path_provenance( + path, + spec=spec, + artifact=artifact, + artifact_id=artifact_id, + parent_change_set_id=parent_change_set_id, + ) + if provenance_issue is not None: + issues.append(provenance_issue) + + return tuple(issues) + + def _inspect_path_provenance( + self, + path: Path, *, - kind: str, + spec: _HistoryKindSpec[BaseModel], + artifact: BaseModel, + artifact_id: str, + parent_change_set_id: str | None, + ) -> HistoryStorageIntegrityIssue | None: + if spec is not _CHANGE_SET_DECISIONS or parent_change_set_id is None: + return None + if not isinstance(artifact, ChangeSetDecisionLink): + return None + if artifact.change_set_id == parent_change_set_id: + return None + return self._integrity_issue( + HistoryIntegrityIssueCode.PATH_PROVENANCE_MISMATCH, + path, + artifact_kind=spec.directory, + artifact_id=artifact_id, + detail=( + f"embedded change_set_id {artifact.change_set_id!r} does not match " + f"path provenance {parent_change_set_id!r}" + ), + ) + + def _inspect_orphan_checksums(self) -> list[HistoryStorageIntegrityIssue]: + if not self._history_root.exists(): + return [] + + issues: list[HistoryStorageIntegrityIssue] = [] + for checksum_path in sorted( + self._history_root.rglob("*.json.sha256"), + key=Path.as_posix, + ): + if not self._path_is_within_history(checksum_path): + issues.append( + self._integrity_issue( + HistoryIntegrityIssueCode.UNKNOWN_ARTIFACT_LAYOUT, + checksum_path, + detail="resolved checksum path escapes history root", + ) + ) + continue + + artifact_path = checksum_path.with_name( + checksum_path.name.removesuffix(".sha256") + ) + if not artifact_path.is_file(): + issues.append( + self._integrity_issue( + HistoryIntegrityIssueCode.ORPHAN_CHECKSUM, + checksum_path, + detail="checksum sidecar has no corresponding JSON artifact", + ) + ) + return issues + + def _put( + self, + spec: _HistoryKindSpec[T], artifact_id: str, artifact: T, - model_type: type[T], - id_attribute: str, - integrity_validator: Callable[[T], None] | None = None, + *, + directory: str | None = None, ) -> None: artifact_id = _safe_artifact_id(artifact_id) canonical = self._validated_canonical_json( + spec, artifact, - model_type=model_type, expected_id=artifact_id, - id_attribute=id_attribute, - integrity_validator=integrity_validator, ) - path = self._artifact_path(kind, artifact_id) + path = self._artifact_path(spec, artifact_id, directory=directory) path.parent.mkdir(parents=True, exist_ok=True) + self._assert_path_within_history(path) if path.exists(): self._require_idempotent_existing( + spec, path, canonical=canonical, - model_type=model_type, expected_id=artifact_id, - id_attribute=id_attribute, - integrity_validator=integrity_validator, ) + self._ensure_checksum(path) return - try: - with path.open("x", encoding="utf-8", newline="\n") as handle: - handle.write(canonical) - except FileExistsError: + created = self._publish_create_only(path, canonical) + if not created: self._require_idempotent_existing( + spec, path, canonical=canonical, - model_type=model_type, expected_id=artifact_id, - id_attribute=id_attribute, - integrity_validator=integrity_validator, ) + self._ensure_checksum(path) def _get( self, - *, - kind: str, + spec: _HistoryKindSpec[T], artifact_id: str, - model_type: type[T], - id_attribute: str, - integrity_validator: Callable[[T], None] | None = None, + *, + directory: str | None = None, ) -> T: artifact_id = _safe_artifact_id(artifact_id) - path = self._artifact_path(kind, artifact_id) + path = self._artifact_path(spec, artifact_id, directory=directory) if not path.is_file(): raise HistoryNotFoundError( - f"{model_type.__name__} {artifact_id!r} was not found" + f"{spec.model_type.__name__} {artifact_id!r} was not found" ) - return self._read_validated( - path, - model_type=model_type, - expected_id=artifact_id, - id_attribute=id_attribute, - integrity_validator=integrity_validator, - ) + return self._read_validated(spec, path, expected_id=artifact_id) def _list( self, + spec: _HistoryKindSpec[T], *, - kind: str, - model_type: type[T], - id_attribute: str, - integrity_validator: Callable[[T], None] | None = None, + directory: str | None = None, ) -> tuple[T, ...]: - directory = self._history_root / kind - if not directory.is_dir(): + resolved_directory = directory or spec.directory + path = self._history_root / resolved_directory + self._assert_path_within_history(path) + if not path.is_dir(): return () records: list[T] = [] - for path in sorted(directory.glob("*.json"), key=lambda item: item.name): - artifact_id = _safe_artifact_id(path.stem) + for artifact_path in sorted(path.glob("*.json"), key=_path_name): + artifact_id = _safe_artifact_id(artifact_path.stem) records.append( self._read_validated( - path, - model_type=model_type, + spec, + artifact_path, expected_id=artifact_id, - id_attribute=id_attribute, - integrity_validator=integrity_validator, ) ) return tuple(records) def _require_idempotent_existing( self, + spec: _HistoryKindSpec[T], path: Path, *, canonical: str, - model_type: type[T], expected_id: str, - id_attribute: str, - integrity_validator: Callable[[T], None] | None = None, ) -> None: - existing = self._read_validated( - path, - model_type=model_type, - expected_id=expected_id, - id_attribute=id_attribute, - integrity_validator=integrity_validator, - ) + existing = self._read_validated(spec, path, expected_id=expected_id) existing_canonical = _canonical_model_json(existing) if existing_canonical != canonical: raise HistoryConflictError( - f"{model_type.__name__} {expected_id!r} already exists with different content" + f"{spec.model_type.__name__} {expected_id!r} already exists with different content" ) def _read_validated( self, + spec: _HistoryKindSpec[T], path: Path, *, - model_type: type[T], expected_id: str, - id_attribute: str, - integrity_validator: Callable[[T], None] | None = None, ) -> T: + self._assert_path_within_history(path) try: raw = path.read_text(encoding="utf-8") - artifact = model_type.model_validate_json(raw) - if integrity_validator is not None: - integrity_validator(artifact) - except (OSError, PydanticValidationError, ValueError) as exc: + self._verify_checksum_if_present(path, raw) + artifact = spec.model_type.model_validate_json(raw) + if spec.integrity_validator is not None: + spec.integrity_validator(artifact) + except HistoryCorruptionError: + raise + except (OSError, PydanticValidationError, ValueError, TypeError) as exc: raise HistoryCorruptionError( - f"Persisted {model_type.__name__} {expected_id!r} is invalid" + f"Persisted {spec.model_type.__name__} {expected_id!r} is invalid" ) from exc - actual_id = getattr(artifact, id_attribute) + actual_id = getattr(artifact, spec.id_attribute) if actual_id != expected_id: raise HistoryCorruptionError( - f"Persisted {model_type.__name__} identity does not match its file identity" + f"Persisted {spec.model_type.__name__} identity does not match its file identity" ) return artifact def _validated_canonical_json( self, + spec: _HistoryKindSpec[T], artifact: T, *, - model_type: type[T], expected_id: str, - id_attribute: str, - integrity_validator: Callable[[T], None] | None = None, ) -> str: - if not isinstance(artifact, model_type): + if not isinstance(artifact, spec.model_type): raise TypeError( - f"artifact must be {model_type.__name__}, got {type(artifact).__name__}" + f"artifact must be {spec.model_type.__name__}, got {type(artifact).__name__}" ) try: canonical = _canonical_model_json(artifact) - validated = model_type.model_validate_json(canonical) - if integrity_validator is not None: - integrity_validator(validated) - except (PydanticValidationError, ValueError) as exc: + validated = spec.model_type.model_validate_json(canonical) + if spec.integrity_validator is not None: + spec.integrity_validator(validated) + except (PydanticValidationError, ValueError, TypeError) as exc: raise HistoryCorruptionError( - f"Supplied {model_type.__name__} {expected_id!r} is invalid" + f"Supplied {spec.model_type.__name__} {expected_id!r} is invalid" ) from exc - if getattr(validated, id_attribute) != expected_id: + if getattr(validated, spec.id_attribute) != expected_id: raise HistoryCorruptionError( - f"Supplied {model_type.__name__} identity does not match its artifact ID" + f"Supplied {spec.model_type.__name__} identity does not match its artifact ID" ) return canonical - def _artifact_path(self, kind: str, artifact_id: str) -> Path: - return self._history_root / kind / f"{artifact_id}.json" + def _artifact_path( + self, + spec: _HistoryKindSpec[BaseModel], + artifact_id: str, + *, + directory: str | None = None, + ) -> Path: + path = self._history_root / (directory or spec.directory) / f"{artifact_id}.json" + self._assert_path_within_history(path) + return path + + def _ensure_checksum(self, path: Path) -> None: + self._assert_path_within_history(path) + try: + raw = path.read_text(encoding="utf-8") + except OSError as exc: + raise HistoryCorruptionError( + f"Persisted history artifact {self._storage_reference(path)!r} cannot be read" + ) from exc + + checksum_path = _checksum_path(path) + expected = _checksum_record(raw) + if checksum_path.exists(): + self._verify_checksum_if_present(path, raw) + return + + created = self._publish_create_only(checksum_path, expected) + if not created: + self._verify_checksum_if_present(path, raw) + + def _verify_checksum_if_present(self, path: Path, raw: str) -> None: + checksum_path = _checksum_path(path) + self._assert_path_within_history(checksum_path) + if not checksum_path.exists(): + return + try: + checksum_text = checksum_path.read_text(encoding="utf-8").strip() + except OSError as exc: + raise HistoryCorruptionError( + f"Persisted history checksum {self._storage_reference(checksum_path)!r} cannot be read" + ) from exc + if not _CHECKSUM_PATTERN.fullmatch(checksum_text): + raise HistoryCorruptionError( + f"Persisted history checksum {self._storage_reference(checksum_path)!r} is invalid" + ) + if checksum_text != _checksum_record(raw).strip(): + raise HistoryCorruptionError( + f"Persisted history artifact {self._storage_reference(path)!r} is invalid: " + "checksum does not match" + ) + + def _publish_create_only(self, path: Path, content: str) -> bool: + """Publish complete content atomically without replacing an existing path.""" + self._assert_path_within_history(path) + path.parent.mkdir(parents=True, exist_ok=True) + self._assert_path_within_history(path) + temp_path = path.parent / f".{path.name}.{secrets.token_hex(8)}.tmp" + self._assert_path_within_history(temp_path) + fd = os.open(temp_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o666) + try: + handle = os.fdopen(fd, "w", encoding="utf-8", newline="\n") + fd = -1 + with handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + try: + os.link(temp_path, path) + except FileExistsError: + return False + _fsync_directory(path.parent) + return True + finally: + if fd != -1: + os.close(fd) + try: + temp_path.unlink() + except FileNotFoundError: + pass + + def _inspect_checksum( + self, + path: Path, + raw: str, + *, + artifact_kind: str, + artifact_id: str, + ) -> tuple[HistoryStorageIntegrityIssue, ...]: + checksum_path = _checksum_path(path) + if not self._path_is_within_history(checksum_path): + return ( + self._integrity_issue( + HistoryIntegrityIssueCode.UNKNOWN_ARTIFACT_LAYOUT, + checksum_path, + artifact_kind=artifact_kind, + artifact_id=artifact_id, + detail="resolved checksum path escapes history root", + ), + ) + if not checksum_path.is_file(): + return ( + self._integrity_issue( + HistoryIntegrityIssueCode.CHECKSUM_MISSING, + path, + artifact_kind=artifact_kind, + artifact_id=artifact_id, + detail="artifact has no SHA-256 checksum sidecar", + ), + ) + try: + checksum_text = checksum_path.read_text(encoding="utf-8").strip() + except OSError as exc: + return ( + self._integrity_issue( + HistoryIntegrityIssueCode.CHECKSUM_INVALID, + checksum_path, + artifact_kind=artifact_kind, + artifact_id=artifact_id, + detail=f"checksum could not be read: {exc}", + ), + ) + if not _CHECKSUM_PATTERN.fullmatch(checksum_text): + return ( + self._integrity_issue( + HistoryIntegrityIssueCode.CHECKSUM_INVALID, + checksum_path, + artifact_kind=artifact_kind, + artifact_id=artifact_id, + detail="checksum sidecar is not canonical sha256:", + ), + ) + if checksum_text != _checksum_record(raw).strip(): + return ( + self._integrity_issue( + HistoryIntegrityIssueCode.CHECKSUM_MISMATCH, + path, + artifact_kind=artifact_kind, + artifact_id=artifact_id, + detail="artifact content does not match its checksum sidecar", + ), + ) + return () + + def _resolve_history_spec( + self, + path: Path, + ) -> tuple[_HistoryKindSpec[BaseModel], str, str | None] | None: + try: + relative = path.relative_to(self._history_root) + except ValueError: + return None + parts = relative.parts + if not parts: + return None + spec = _HISTORY_KIND_SPECS.get(parts[0]) + if spec is None: + return None + + parent_change_set_id: str | None = None + if spec is _CHANGE_SET_DECISIONS: + if len(parts) != 3 or not parts[2].endswith(".json"): + return None + try: + parent_change_set_id = _safe_artifact_id(parts[1]) + artifact_id = _safe_artifact_id(parts[2][:-5]) + except (TypeError, ValueError): + return None + else: + if len(parts) != 2 or not parts[1].endswith(".json"): + return None + try: + artifact_id = _safe_artifact_id(parts[1][:-5]) + except (TypeError, ValueError): + return None + return spec, artifact_id, parent_change_set_id + + def _integrity_issue( + self, + code: HistoryIntegrityIssueCode, + path: Path, + *, + detail: str, + artifact_kind: str | None = None, + artifact_id: str | None = None, + ) -> HistoryStorageIntegrityIssue: + if artifact_kind is None: + try: + artifact_kind = path.relative_to(self._history_root).parts[0] + except (ValueError, IndexError): + artifact_kind = "unknown" + return HistoryStorageIntegrityIssue( + code=code, + artifact_kind=artifact_kind, + artifact_id=artifact_id, + storage_reference=self._storage_reference(path), + detail=detail, + ) + + def _storage_reference(self, path: Path) -> str: + try: + return path.relative_to(self._repository_root).as_posix() + except ValueError: + return path.as_posix() + + def _path_is_within_history(self, path: Path) -> bool: + try: + resolved = path.resolve(strict=False) + return resolved.is_relative_to( + self._repository_root + ) and resolved.is_relative_to(self._history_root) + except OSError: + return False + + def _assert_path_within_history(self, path: Path) -> None: + if not self._path_is_within_history(path): + raise ValueError("history path escapes configured history root") + + +def _integrity_issue_sort_key( + issue: HistoryStorageIntegrityIssue, +) -> tuple[str, str, str, str, str]: + return ( + issue.storage_reference, + issue.code.value, + issue.artifact_kind, + issue.artifact_id or "", + issue.detail, + ) + + +def _path_name(path: Path) -> str: + return path.name def _canonical_model_json(artifact: BaseModel) -> str: @@ -680,6 +1014,32 @@ def _canonical_model_json(artifact: BaseModel) -> str: return canonical_compact_json(artifact.model_dump(mode="json", by_alias=True)) +def _checksum_path(path: Path) -> Path: + return path.with_name(f"{path.name}.sha256") + + +def _checksum_record(raw: str) -> str: + digest = hashlib.sha256(raw.encode("utf-8")).hexdigest() + return f"sha256:{digest}\n" + + +def _fsync_directory(path: Path) -> None: + """Persist directory metadata where the host supports directory fsync.""" + flags = os.O_RDONLY + if hasattr(os, "O_DIRECTORY"): + flags |= os.O_DIRECTORY + try: + fd = os.open(path, flags) + except OSError: + return + try: + os.fsync(fd) + except OSError: + pass + finally: + os.close(fd) + + def _safe_artifact_id(value: str) -> str: cleaned = _required_text(value, "artifact_id") if not _SAFE_ARTIFACT_ID.fullmatch(cleaned): diff --git a/tests/test_history_integrity.py b/tests/test_history_integrity.py new file mode 100644 index 00000000..e9f6cb46 --- /dev/null +++ b/tests/test_history_integrity.py @@ -0,0 +1,263 @@ +from __future__ import annotations + +from datetime import date +from pathlib import Path + +import pytest +from open_data_contract_standard.model import ( + OpenDataContractStandard, + SchemaObject, + SchemaProperty, +) + +from semapact.application.services.evolution import EvolutionChainService +from semapact.application.services.history_integrity import HistoryIntegrityService +from semapact.change_context import ChangeContext +from semapact.contractops import build_change_set_from_decision +from semapact.governance import GovernanceDecision, evaluate_governance_decision +from semapact.history import ( + ChangeSetDecisionLink, + HistoryConflictError, + HistoryCorruptionError, + HistoryIntegrityIssueCode, +) +from semapact.platforms.git import GitWorkingTreeHistoryRepository + + +CONTEXT = ChangeContext(effective_date=date(2026, 9, 15)) + + +def _contract(*, name: str) -> OpenDataContractStandard: + return OpenDataContractStandard( + apiVersion="v3.1.0", + kind="DataContract", + id="orders-product", + name=name, + version="1.0.0", + status="active", + schema=[ + SchemaObject( + name="orders", + properties=[ + SchemaProperty( + name="id", + logicalType="string", + physicalType="varchar(255)", + required=True, + ) + ], + ) + ], + ) + + +def _decision() -> GovernanceDecision: + return evaluate_governance_decision( + _contract(name="orders-base"), + _contract(name="orders-candidate"), + context=CONTEXT, + ) + + +def _decision_path(tmp_path: Path, decision: GovernanceDecision) -> Path: + return ( + tmp_path + / ".semapact" + / "history" + / "decisions" + / f"{decision.decision_id}.json" + ) + + +def _evolution_service(backend: GitWorkingTreeHistoryRepository) -> EvolutionChainService: + return EvolutionChainService( + revisions=backend, + change_sets=backend, + decisions=backend, + decision_links=backend, + releases=backend, + deployments=backend, + observations=backend, + runtime_reconciliations=backend, + ) + + +def test_new_history_write_is_checksum_sealed_and_clean(tmp_path: Path) -> None: + repository = GitWorkingTreeHistoryRepository(tmp_path) + decision = _decision() + + repository.put_decision(decision) + + artifact_path = _decision_path(tmp_path, decision) + checksum_path = artifact_path.with_name(f"{artifact_path.name}.sha256") + assert artifact_path.is_file() + assert checksum_path.is_file() + assert checksum_path.read_text(encoding="utf-8").startswith("sha256:") + assert repository.get_decision(decision.decision_id) == decision + assert repository.inspect_history_integrity() == () + + +def test_checksum_sealed_tamper_fails_closed_on_read_and_scan(tmp_path: Path) -> None: + repository = GitWorkingTreeHistoryRepository(tmp_path) + decision = _decision() + repository.put_decision(decision) + artifact_path = _decision_path(tmp_path, decision) + artifact_path.write_text( + artifact_path.read_text(encoding="utf-8").replace( + "orders-product", + "tampered-product", + ), + encoding="utf-8", + ) + + with pytest.raises(HistoryCorruptionError, match="checksum"): + repository.get_decision(decision.decision_id) + + issues = repository.inspect_history_integrity() + assert HistoryIntegrityIssueCode.CHECKSUM_MISMATCH in {item.code for item in issues} + + +def test_idempotent_rewrite_can_seal_pre_checksum_history(tmp_path: Path) -> None: + repository = GitWorkingTreeHistoryRepository(tmp_path) + decision = _decision() + repository.put_decision(decision) + artifact_path = _decision_path(tmp_path, decision) + checksum_path = artifact_path.with_name(f"{artifact_path.name}.sha256") + checksum_path.unlink() + + issues = repository.inspect_history_integrity() + assert [item.code for item in issues] == [HistoryIntegrityIssueCode.CHECKSUM_MISSING] + + repository.put_decision(decision) + + assert checksum_path.is_file() + assert repository.inspect_history_integrity() == () + + +def test_orphan_checksum_is_reported(tmp_path: Path) -> None: + repository = GitWorkingTreeHistoryRepository(tmp_path) + checksum_path = ( + tmp_path + / ".semapact" + / "history" + / "decisions" + / "orphan.json.sha256" + ) + checksum_path.parent.mkdir(parents=True) + checksum_path.write_text("sha256:" + "0" * 64 + "\n", encoding="utf-8") + + issues = repository.inspect_history_integrity() + + assert len(issues) == 1 + assert issues[0].code is HistoryIntegrityIssueCode.ORPHAN_CHECKSUM + + +def test_conflicting_content_remains_immutable_with_checksum_seal(tmp_path: Path) -> None: + repository = GitWorkingTreeHistoryRepository(tmp_path) + decision = _decision() + repository.put_decision(decision) + conflicting = decision.model_copy(update={"contract_id": "different-contract"}) + + with pytest.raises(HistoryConflictError, match="different content"): + repository.put_decision(conflicting) + + +def test_history_state_directory_cannot_escape_repository_root(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="repository-relative"): + GitWorkingTreeHistoryRepository(tmp_path, state_directory=tmp_path / "outside") + + with pytest.raises(ValueError, match="must not contain"): + GitWorkingTreeHistoryRepository(tmp_path, state_directory="../outside") + + with pytest.raises(ValueError, match="must not contain"): + GitWorkingTreeHistoryRepository( + tmp_path, + state_directory="safe/../.semapact/history", + ) + + +def test_symlink_path_escape_is_rejected(tmp_path: Path) -> None: + history_root = tmp_path / ".semapact" / "history" + history_root.mkdir(parents=True) + outside = tmp_path / "outside" + outside.mkdir() + decisions = history_root / "decisions" + try: + decisions.symlink_to(outside, target_is_directory=True) + except OSError: + pytest.skip("host does not permit symlink creation") + + repository = GitWorkingTreeHistoryRepository(tmp_path) + with pytest.raises(ValueError, match="escapes"): + repository.put_decision(_decision()) + + +def test_history_root_replaced_by_symlink_after_init_is_rejected(tmp_path: Path) -> None: + repository = GitWorkingTreeHistoryRepository(tmp_path) + history_root = tmp_path / ".semapact" / "history" + history_root.mkdir(parents=True) + outside = tmp_path / "outside" + outside.mkdir() + history_root.rmdir() + try: + history_root.symlink_to(outside, target_is_directory=True) + except OSError: + pytest.skip("host does not permit symlink creation") + + with pytest.raises(ValueError, match="escapes"): + repository.put_decision(_decision()) + + +def test_integrity_service_reuses_evolution_broken_reference_diagnostics( + tmp_path: Path, +) -> None: + backend = GitWorkingTreeHistoryRepository(tmp_path) + decision = _decision() + change_set = build_change_set_from_decision( + decision, + base_revision_ref="missing-base-revision", + candidate_revision_ref="missing-candidate-revision", + ) + backend.put_decision(decision) + backend.put_change_set(change_set) + backend.put_change_set_decision_link( + ChangeSetDecisionLink( + change_set_id=change_set.change_set_id, + decision_id=decision.decision_id, + ) + ) + service = HistoryIntegrityService( + storage=backend, + evolution=_evolution_service(backend), + ) + + report = service.check_contract("orders-product") + + assert report.storage_issues == () + assert report.references_checked is True + assert report.valid is False + assert {item.reference_field for item in report.broken_references} == { + "base_revision_ref", + "candidate_revision_ref", + } + + +def test_integrity_service_stops_reference_traversal_on_storage_corruption( + tmp_path: Path, +) -> None: + backend = GitWorkingTreeHistoryRepository(tmp_path) + decision = _decision() + backend.put_decision(decision) + artifact_path = _decision_path(tmp_path, decision) + artifact_path.write_text("{}", encoding="utf-8") + service = HistoryIntegrityService( + storage=backend, + evolution=_evolution_service(backend), + ) + + report = service.check_contract("orders-product") + + assert report.storage_issues + assert report.references_checked is False + assert report.broken_references == () + assert report.valid is False From 79b090200b62d7d3cf2fd77bc36ab35089897d7a Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Tue, 15 Sep 2026 16:15:33 +1000 Subject: [PATCH 24/35] test(history): add golden lifecycle scenarios (#230) * test(history): add golden lifecycle scenarios * test(history): add golden history fixture placeholder * test(history): compact golden persistence snapshot * test(history): seal golden history fixture --- .../v1/review_multi_deploy.json | 132 ++++ tests/test_history_golden_scenarios.py | 647 ++++++++++++++++++ 2 files changed, 779 insertions(+) create mode 100644 tests/fixtures/history_golden/v1/review_multi_deploy.json create mode 100644 tests/test_history_golden_scenarios.py diff --git a/tests/fixtures/history_golden/v1/review_multi_deploy.json b/tests/fixtures/history_golden/v1/review_multi_deploy.json new file mode 100644 index 00000000..f8a3a3ee --- /dev/null +++ b/tests/fixtures/history_golden/v1/review_multi_deploy.json @@ -0,0 +1,132 @@ +{ + "fixtureVersion": "history-v1", + "files": { + ".semapact/history/change_set_decisions/c2f4ee02-90e5-532a-ac8a-edd43e49f156/14295d1b-b52d-5d8b-8dad-2121529f099c.json": { + "checksum": "sha256:d77324f66e332dfdd6cb3ff081e65987a6eaa469d5b84bab2cb425d55e23dd62", + "topLevelKeys": ["change_set_id", "decision_id"] + }, + ".semapact/history/change_sets/c2f4ee02-90e5-532a-ac8a-edd43e49f156.json": { + "checksum": "sha256:66861c622ee7f42f4340a761d17c16a006bf738ad682c39359c552376ce95005", + "topLevelKeys": ["actor_reference", "base_revision_ref", "candidate_revision_ref", "change_set_id", "changes", "context", "contract_id", "source"] + }, + ".semapact/history/contract_revisions/290bd707-017f-56ca-a379-c8879b9ede13.json": { + "checksum": "sha256:e70f5404098efaf4cec40f9a021adbba17cad53ed0c57db6984126fdcb84373e", + "topLevelKeys": ["content_fingerprint", "contract", "revision_id"] + }, + ".semapact/history/contract_revisions/352d83a9-4ee6-508a-981f-4bc9819263c9.json": { + "checksum": "sha256:bf1e3e1836fff7827bc014caccfefd36a01aaf480ecd17eae41c9a8245e5eb9e", + "topLevelKeys": ["content_fingerprint", "contract", "revision_id"] + }, + ".semapact/history/contract_revisions/fb437fde-4baa-5dad-a382-c4df381ba412.json": { + "checksum": "sha256:be036b579fe23c3d47afb96978c1e05416564b5ad0e30df034803c2c86bbf9c2", + "topLevelKeys": ["content_fingerprint", "contract", "revision_id"] + }, + ".semapact/history/decisions/14295d1b-b52d-5d8b-8dad-2121529f099c.json": { + "checksum": "sha256:5c0ddb33910feb477c32252d21c318e273e031185f52f4295aa0074dd78bf485", + "topLevelKeys": ["breaking", "changes", "context", "contract_id", "decision", "decision_id", "evidence", "policy", "reasons", "required_version_bump", "validation"] + }, + ".semapact/history/deployment_authorizations/87ea9883-2583-51d7-8c25-fe4d1c643dc0.json": { + "checksum": "sha256:7a2ea0240736846da3a701253ad913f8a048066e30a20f7680d5385170a687c7", + "topLevelKeys": ["allowed", "applied_release_id", "contract_ops_authorization_id", "deployment_authorization_id", "deployment_plan_id"] + }, + ".semapact/history/deployment_authorizations/a1a210ef-25d0-5d00-a105-7e7907d63acb.json": { + "checksum": "sha256:915e95cdac4b9eac879221c1ffeef9ab95a8538ed0efbaf1c8f446afc77090cb", + "topLevelKeys": ["allowed", "applied_release_id", "contract_ops_authorization_id", "deployment_authorization_id", "deployment_plan_id"] + }, + ".semapact/history/deployment_plans/c684e277-793b-5628-ae09-0b67316306a8.json": { + "checksum": "sha256:f6c254152dbfe16ff017f1cc43f40390e5dc59452df730e9701cb81868a98deb", + "topLevelKeys": ["actions", "applied_release_id", "contract_id", "deployment_plan_id", "plan_version", "release_plan_id", "released_revision_ref", "selected_version", "target"] + }, + ".semapact/history/deployment_plans/fecc2ad3-b5fe-58a1-99a5-3fb30e28a6c3.json": { + "checksum": "sha256:446646a122bfc65c085a084f4bbb52d6f1ac9f6c94a1724061c2f19308c58f2c", + "topLevelKeys": ["actions", "applied_release_id", "contract_id", "deployment_plan_id", "plan_version", "release_plan_id", "released_revision_ref", "selected_version", "target"] + }, + ".semapact/history/deployment_previews/6eb94c8f-68ad-546a-8eb4-b1fb27ec21fa.json": { + "checksum": "sha256:8a42d5eeaacfed1c35fffd9da860bfb3b7ee6294d2887ac665ce0c6b2f7f500d", + "topLevelKeys": ["deployment_plan_id", "deployment_preview_id", "observation_fingerprint", "operations", "platform", "preview_version", "runtime_target", "source_identifier"] + }, + ".semapact/history/deployment_previews/778b10a8-6133-572e-b89b-6d76be994154.json": { + "checksum": "sha256:ca3f88358aaea04096290f9cb3d6a4e39218a63aec986dd5b19f93ca8ecd9d9b", + "topLevelKeys": ["deployment_plan_id", "deployment_preview_id", "observation_fingerprint", "operations", "platform", "preview_version", "runtime_target", "source_identifier"] + }, + ".semapact/history/deployment_records/61edafd1-bba0-5e2f-8fa1-f48ff7eeaafb.json": { + "checksum": "sha256:155b3dd66eca9cb519ccd3f5b3e0a6f3a4d4c430428d96ef0cddf5b6c3392aef", + "topLevelKeys": ["actor_reference", "completed_at", "deployment_authorization_id", "deployment_plan_id", "deployment_preview_id", "deployment_record_id", "external_reference", "platform", "release_record_id", "runtime_target", "source_reference", "started_at", "status"] + }, + ".semapact/history/deployment_records/c1cbc2d8-f340-5944-827a-c5bbc8d73835.json": { + "checksum": "sha256:17f423508bb8500f00f85c0ae4a9f9d1900e78c9cfff7a2b936e6e1f14357af1", + "topLevelKeys": ["actor_reference", "completed_at", "deployment_authorization_id", "deployment_plan_id", "deployment_preview_id", "deployment_record_id", "external_reference", "platform", "release_record_id", "runtime_target", "source_reference", "started_at", "status"] + }, + ".semapact/history/release_plans/45a429c1-9ff4-54da-897a-b02c7ee6027f.json": { + "checksum": "sha256:e1f6e06acd06a1b0ff719ddce44430f293b03d65e5586607f13907d6b617b237", + "topLevelKeys": ["change_set_id", "contract_id", "decision_id", "preconditions", "release_plan_id", "release_revision_ref", "required_version_bump"] + }, + ".semapact/history/release_records/de7c4ed0-d54a-5a7a-b82e-c626eab39d23.json": { + "checksum": "sha256:961815c36c9f21e9ca1a277bb256bdc6571bcd94187a2712f333fa2bf6834b9d", + "topLevelKeys": ["actual_version_bump", "applied_release_id", "authority_reference", "authorization_id", "change_set_id", "contract_id", "contract_version", "decision_id", "release_plan_id", "release_record_id", "released_revision_id", "required_version_bump", "review_evidence_action", "review_evidence_reference", "version_authority", "version_resolution_id"] + }, + ".semapact/history/runtime_observations/28a82950-d467-57e8-803f-2b705b201780.json": { + "checksum": "sha256:82121d967e651fc1af170a184352e8c5379db6098c5ea569754db9bcf9cca0c2", + "topLevelKeys": ["observation", "observation_record_id"] + }, + ".semapact/history/runtime_observations/6b0abefd-7cd6-5bcc-bd2d-714ae994735b.json": { + "checksum": "sha256:a7c61cb5589c804785346568eadef629477af3488709ecfc1f179684b2414999", + "topLevelKeys": ["observation", "observation_record_id"] + }, + ".semapact/history/runtime_reconciliations/0488eb0b-50c1-549a-af56-48807338098a.json": { + "checksum": "sha256:2fb8122baa8e16c488b73d782729cf9aaad5fc5d102bb88879bae8b9c0b522db", + "topLevelKeys": ["deployment_record_id", "observation_record_id", "release_record_id", "result", "runtime_reconciliation_record_id", "status"] + }, + ".semapact/history/runtime_reconciliations/b4c58a42-bea7-57da-ab1d-88594e425dfb.json": { + "checksum": "sha256:6e41715135cf33b62f19a4a82949bc471a42b6e20a89aa52b0a79854bdd33727", + "topLevelKeys": ["deployment_record_id", "observation_record_id", "release_record_id", "result", "runtime_reconciliation_record_id", "status"] + } + }, + "evolution": { + "contractId": "orders-product", + "proposals": [ + { + "changeSetId": "c2f4ee02-90e5-532a-ac8a-edd43e49f156", + "decisionId": "14295d1b-b52d-5d8b-8dad-2121529f099c", + "decision": "REVIEW", + "baseRevisionId": "fb437fde-4baa-5dad-a382-c4df381ba412", + "candidateRevisionId": "352d83a9-4ee6-508a-981f-4bc9819263c9", + "releases": [ + { + "releaseRecordId": "de7c4ed0-d54a-5a7a-b82e-c626eab39d23", + "contractVersion": "1.3.0", + "releasedRevisionId": "290bd707-017f-56ca-a379-c8879b9ede13", + "deployments": [ + { + "deploymentRecordId": "61edafd1-bba0-5e2f-8fa1-f48ff7eeaafb", + "runtimeTarget": "main.stage", + "status": "FAILED", + "runtime": [] + }, + { + "deploymentRecordId": "c1cbc2d8-f340-5944-827a-c5bbc8d73835", + "runtimeTarget": "main.prod", + "status": "SUCCEEDED", + "runtime": [ + { + "reconciliationRecordId": "0488eb0b-50c1-549a-af56-48807338098a", + "observationRecordId": "28a82950-d467-57e8-803f-2b705b201780", + "status": "DRIFT" + }, + { + "reconciliationRecordId": "b4c58a42-bea7-57da-ab1d-88594e425dfb", + "observationRecordId": "6b0abefd-7cd6-5bcc-bd2d-714ae994735b", + "status": "IN_SYNC" + } + ] + } + ] + } + ] + } + ], + "unlinkedReleaseIds": [], + "unlinkedRuntimeIds": [], + "brokenReferences": [] + } +} diff --git a/tests/test_history_golden_scenarios.py b/tests/test_history_golden_scenarios.py new file mode 100644 index 00000000..0ed8ee99 --- /dev/null +++ b/tests/test_history_golden_scenarios.py @@ -0,0 +1,647 @@ +from __future__ import annotations + +import json +from datetime import datetime, timedelta, timezone +from pathlib import Path +from types import SimpleNamespace + +import pytest +from open_data_contract_standard.model import ( + OpenDataContractStandard, + SchemaObject, + SchemaProperty, +) + +from semapact.application.services.deployment_history import DeploymentHistoryService +from semapact.application.services.evolution import EvolutionChainService +from semapact.application.services.governance import GovernanceService +from semapact.application.services.history import ProposalHistoryService +from semapact.application.services.history_integrity import HistoryIntegrityService +from semapact.application.services.release_history import ReleaseHistoryService +from semapact.application.services.runtime_history import RuntimeHistoryService +from semapact.contractops import ( + ReviewAuthorizationEvidence, + ReviewEvidenceAction, + VersionAuthorityConfig, + apply_contract_release, + authorize_contract_operation, + build_change_set, + build_release_plan, + resolve_release_version, +) +from semapact.deployment import ( + DeploymentAuthorization, + DeploymentPreview, + DeploymentTarget, + authorize_deployment, + build_deployment_plan, +) +from semapact.deployment.models import compute_deployment_preview_id +from semapact.governance import DecisionResult +from semapact.governance.gate import GovernanceOperation +from semapact.history import DeploymentStatus, HistoryConflictError +from semapact.observation import ( + ObservedAsset, + ObservedAssetIdentity, + ObservedPlatformState, + with_observed_state_fingerprint, +) +from semapact.platforms.git import GitWorkingTreeHistoryRepository +from semapact.reconciliation import ( + ReconciliationDifference, + ReconciliationDifferenceType, + ReconciliationResult, + ReconciliationSubject, + RuntimeDriftStatus, + RuntimeReasonCode, +) +from semapact.revision import build_contract_revision + + +GOLDEN_FIXTURE = ( + Path(__file__).parent + / "fixtures" + / "history_golden" + / "v1" + / "review_multi_deploy.json" +) +CONTRACT_ID = "orders-product" +CURRENT_VERSION = "1.2.3" +EFFECTIVE_DATE = "2026-09-15" + + +def _contract( + *, + name: str = "Orders", + version: str = CURRENT_VERSION, + include_note: bool = False, +) -> OpenDataContractStandard: + properties = [ + SchemaProperty( + name="id", + logicalType="string", + physicalType="varchar(255)", + required=True, + ) + ] + if include_note: + properties.append( + SchemaProperty( + name="note", + logicalType="string", + physicalType="varchar(255)", + required=False, + ) + ) + return OpenDataContractStandard( + apiVersion="v3.1.0", + kind="DataContract", + id=CONTRACT_ID, + name=name, + version=version, + status="active", + schema=[SchemaObject(name="orders", properties=properties)], + ) + + +def _proposal_history(backend: GitWorkingTreeHistoryRepository) -> ProposalHistoryService: + return ProposalHistoryService( + revisions=backend, + change_sets=backend, + decisions=backend, + decision_links=backend, + ) + + +def _release_history(backend: GitWorkingTreeHistoryRepository) -> ReleaseHistoryService: + return ReleaseHistoryService( + revisions=backend, + change_sets=backend, + decisions=backend, + decision_links=backend, + release_plans=backend, + release_records=backend, + ) + + +def _deployment_history(backend: GitWorkingTreeHistoryRepository) -> DeploymentHistoryService: + return DeploymentHistoryService( + releases=backend, + deployment_plans=backend, + deployment_previews=backend, + deployment_authorizations=backend, + deployment_records=backend, + ) + + +def _runtime_history(backend: GitWorkingTreeHistoryRepository) -> RuntimeHistoryService: + return RuntimeHistoryService( + observations=backend, + reconciliations=backend, + releases=backend, + deployments=backend, + ) + + +def _evolution(backend: GitWorkingTreeHistoryRepository) -> EvolutionChainService: + return EvolutionChainService( + revisions=backend, + change_sets=backend, + decisions=backend, + decision_links=backend, + releases=backend, + deployments=backend, + observations=backend, + runtime_reconciliations=backend, + ) + + +def _record_release( + root: Path, + *, + base: OpenDataContractStandard, + candidate: OpenDataContractStandard, + source: str, +): + backend = GitWorkingTreeHistoryRepository(root) + base_revision = build_contract_revision(base) + candidate_revision = build_contract_revision(candidate) + proposal = GovernanceService().evaluate_proposal( + base_revision.contract, + candidate_revision.contract, + effective_date=EFFECTIVE_DATE, + base_revision_ref=base_revision.revision_id, + candidate_revision_ref=candidate_revision.revision_id, + source=source, + actor_reference="actor:golden", + ) + _proposal_history(backend).record_proposal( + proposal, + base_revision=base_revision, + candidate_revision=candidate_revision, + ) + + release_plan = build_release_plan(proposal.change_set, proposal.decision) + version_resolution = resolve_release_version( + release_plan, + current_version=CURRENT_VERSION, + config=VersionAuthorityConfig(), + ) + evidence = None + if proposal.decision.decision is DecisionResult.REVIEW: + evidence = ReviewAuthorizationEvidence( + evidence_reference="review:apply:golden", + decision_id=proposal.decision.decision_id, + change_set_id=proposal.change_set.change_set_id, + release_plan_id=release_plan.release_plan_id, + version_resolution_id=version_resolution.version_resolution_id, + operation=GovernanceOperation.APPLY, + action=ReviewEvidenceAction.APPROVE, + ) + authorization = authorize_contract_operation( + proposal.decision, + proposal.change_set, + release_plan, + version_resolution, + GovernanceOperation.APPLY, + evidence=evidence, + ) + applied_release = apply_contract_release( + candidate_revision.contract, + candidate_revision_ref=candidate_revision.revision_id, + decision=proposal.decision, + change_set=proposal.change_set, + release_plan=release_plan, + version_resolution=version_resolution, + authorization=authorization, + ) + release_record = _release_history(backend).record_release( + release_plan=release_plan, + version_resolution=version_resolution, + authorization=authorization, + applied_release=applied_release, + ) + return SimpleNamespace( + backend=backend, + base_revision=base_revision, + candidate_revision=candidate_revision, + proposal=proposal, + release_plan=release_plan, + version_resolution=version_resolution, + authorization=authorization, + applied_release=applied_release, + release_record=release_record, + ) + + +def _deployment_preview(plan, *, label: str) -> DeploymentPreview: + observation_fingerprint = f"preview-observation:{label}" + preview_id = compute_deployment_preview_id( + deployment_plan_id=plan.deployment_plan_id, + platform=plan.target.platform, + runtime_target=plan.target.runtime_target, + source_identifier=plan.target.source_reference, + observation_fingerprint=observation_fingerprint, + operations=(), + ) + return DeploymentPreview( + deployment_preview_id=preview_id, + deployment_plan_id=plan.deployment_plan_id, + platform=plan.target.platform, + runtime_target=plan.target.runtime_target, + source_identifier=plan.target.source_reference, + observation_fingerprint=observation_fingerprint, + operations=(), + ) + + +def _record_deployment( + bundle, + *, + label: str, + runtime_target: str, + source_reference: str, + status: DeploymentStatus, + started_at: datetime, +): + plan = build_deployment_plan( + bundle.applied_release, + DeploymentTarget( + platform="databricks", + runtime_target=runtime_target, + source_reference=source_reference, + server_name=label, + ), + ) + deploy_evidence = ReviewAuthorizationEvidence( + evidence_reference=f"review:deploy:{label}", + decision_id=bundle.proposal.decision.decision_id, + change_set_id=bundle.proposal.change_set.change_set_id, + release_plan_id=bundle.release_plan.release_plan_id, + version_resolution_id=bundle.version_resolution.version_resolution_id, + operation=GovernanceOperation.DEPLOY, + action=ReviewEvidenceAction.APPROVE, + scope_reference=plan.deployment_plan_id, + ) + contractops_authorization = authorize_contract_operation( + bundle.proposal.decision, + bundle.proposal.change_set, + bundle.release_plan, + bundle.version_resolution, + GovernanceOperation.DEPLOY, + evidence=deploy_evidence, + ) + deployment_authorization: DeploymentAuthorization = authorize_deployment( + plan, + bundle.applied_release, + contractops_authorization, + ) + preview = _deployment_preview(plan, label=label) + record = _deployment_history(bundle.backend).record_execution( + plan=plan, + preview=preview, + authorization=deployment_authorization, + status=status, + started_at=started_at, + completed_at=started_at + timedelta(seconds=5), + actor_reference="agent:golden-deploy", + external_reference=f"run:{label}", + ) + return SimpleNamespace( + plan=plan, + preview=preview, + authorization=deployment_authorization, + record=record, + ) + + +def _observation( + *, + captured_at: datetime, + source_reference: str, + namespace: tuple[str, str], + asset_type: str, +) -> ObservedPlatformState: + return with_observed_state_fingerprint( + ObservedPlatformState( + platform="databricks", + source_identifier=source_reference, + assets=( + ObservedAsset( + identity=ObservedAssetIdentity( + platform="databricks", + namespace=namespace, + asset="orders", + ), + asset_type=asset_type, + ), + ), + captured_at=captured_at, + ) + ) + + +def _reconciliation_result( + observation: ObservedPlatformState, + *, + version: str, + drift: bool, +) -> ReconciliationResult: + differences = () + if drift: + differences = ( + ReconciliationDifference( + difference_type=ReconciliationDifferenceType.MISMATCH, + subject=ReconciliationSubject.PHYSICAL_TYPE, + reason_code=RuntimeReasonCode.RUNTIME_PHYSICAL_TYPE_CHANGED, + path="orders.id.physical_type", + asset_identity="orders", + property_identity="id", + expected="STRING", + observed="BIGINT", + ), + ) + assert observation.fingerprint is not None + return ReconciliationResult( + contract_id=CONTRACT_ID, + contract_version=version, + observation_source_identifier=observation.source_identifier, + observation_fingerprint=observation.fingerprint, + differences=differences, + ) + + +def _record_runtime(bundle, deployment, *, captured_at: datetime, drift: bool): + namespace = tuple(deployment.record.runtime_target.split(".")) + assert len(namespace) == 2 + observation = _observation( + captured_at=captured_at, + source_reference=deployment.record.source_reference, + namespace=(namespace[0], namespace[1]), + asset_type="VIEW" if drift else "TABLE", + ) + record = _runtime_history(bundle.backend).record_reconciliation( + observation, + _reconciliation_result( + observation, + version=bundle.release_record.contract_version, + drift=drift, + ), + release_record_id=bundle.release_record.release_record_id, + deployment_record_id=deployment.record.deployment_record_id, + ) + return SimpleNamespace(observation=observation, record=record) + + +def _evolution_projection(chain) -> dict[str, object]: + proposals = [] + for proposal in chain.proposals: + releases = [] + for release in proposal.releases: + deployments = [] + for deployment in release.deployments: + deployments.append( + { + "deploymentRecordId": deployment.deployment.deployment_record_id, + "runtimeTarget": deployment.deployment.runtime_target, + "status": deployment.deployment.status.value, + "runtime": [ + { + "reconciliationRecordId": item.reconciliation.runtime_reconciliation_record_id, + "observationRecordId": item.reconciliation.observation_record_id, + "status": item.reconciliation.status.value, + } + for item in deployment.runtime + ], + } + ) + releases.append( + { + "releaseRecordId": release.release.release_record_id, + "contractVersion": release.release.contract_version, + "releasedRevisionId": release.release.released_revision_id, + "deployments": deployments, + } + ) + proposals.append( + { + "changeSetId": proposal.change_set.change_set_id, + "decisionId": proposal.decision.decision_id if proposal.decision else None, + "decision": proposal.decision.decision.value if proposal.decision else None, + "baseRevisionId": ( + proposal.base_revision.revision_id if proposal.base_revision else None + ), + "candidateRevisionId": ( + proposal.candidate_revision.revision_id + if proposal.candidate_revision + else None + ), + "releases": releases, + } + ) + return { + "contractId": chain.contract_id, + "proposals": proposals, + "unlinkedReleaseIds": [ + item.release.release_record_id for item in chain.unlinked_releases + ], + "unlinkedRuntimeIds": [ + item.reconciliation.runtime_reconciliation_record_id + for item in chain.unlinked_runtime + ], + "brokenReferences": [ + { + "sourceKind": item.source_kind, + "sourceId": item.source_id, + "referenceField": item.reference_field, + "targetKind": item.target_kind, + "targetId": item.target_id, + "reason": item.reason, + } + for item in chain.broken_references + ], + } + + +def _history_files(root: Path) -> dict[str, object]: + """Snapshot persisted schema and exact canonical content through its checksum seal.""" + history_root = root / ".semapact" / "history" + files: dict[str, object] = {} + for path in sorted(history_root.rglob("*.json"), key=Path.as_posix): + relative = path.relative_to(root).as_posix() + content = json.loads(path.read_text(encoding="utf-8")) + checksum_path = path.with_name(f"{path.name}.sha256") + files[relative] = { + "checksum": checksum_path.read_text(encoding="utf-8").strip(), + "topLevelKeys": sorted(content), + } + return files + + +def _build_review_multi_deploy(root: Path): + bundle = _record_release( + root, + base=_contract(), + candidate=_contract(include_note=True), + source="golden-review", + ) + assert bundle.proposal.decision.decision is DecisionResult.REVIEW + + start = datetime(2026, 9, 15, 0, 0, tzinfo=timezone.utc) + production = _record_deployment( + bundle, + label="production", + runtime_target="main.prod", + source_reference="workspace:prod", + status=DeploymentStatus.SUCCEEDED, + started_at=start, + ) + staging = _record_deployment( + bundle, + label="staging", + runtime_target="main.stage", + source_reference="workspace:stage", + status=DeploymentStatus.FAILED, + started_at=start + timedelta(minutes=10), + ) + sync = _record_runtime( + bundle, + production, + captured_at=start + timedelta(hours=1), + drift=False, + ) + drift = _record_runtime( + bundle, + production, + captured_at=start + timedelta(hours=2), + drift=True, + ) + assert sync.record.status is RuntimeDriftStatus.IN_SYNC + assert drift.record.status is RuntimeDriftStatus.DRIFT + + reloaded = GitWorkingTreeHistoryRepository(root) + chain = _evolution(reloaded).reconstruct(CONTRACT_ID) + assert _evolution(reloaded).reconstruct(CONTRACT_ID) == chain + integrity = HistoryIntegrityService( + storage=reloaded, + evolution=_evolution(reloaded), + ).check_contract(CONTRACT_ID) + assert integrity.valid is True + + manifest = { + "fixtureVersion": "history-v1", + "files": _history_files(root), + "evolution": _evolution_projection(chain), + } + return SimpleNamespace( + manifest=manifest, + backend=reloaded, + bundle=bundle, + production=production, + staging=staging, + sync=sync, + drift=drift, + chain=chain, + ) + + +def test_review_multi_deploy_history_matches_checked_in_v1_fixture(tmp_path: Path) -> None: + first = _build_review_multi_deploy(tmp_path / "first") + second = _build_review_multi_deploy(tmp_path / "second") + + assert first.manifest == second.manifest + expected = json.loads(GOLDEN_FIXTURE.read_text(encoding="utf-8")) + if first.manifest != expected: + pytest.fail( + "history golden fixture mismatch; update only for an intentional persisted " + "schema/identity change.\nACTUAL:\n" + + json.dumps(first.manifest, indent=2, sort_keys=True) + ) + + +def test_allow_release_reconstructs_without_review_evidence(tmp_path: Path) -> None: + bundle = _record_release( + tmp_path, + base=_contract(name="Orders old"), + candidate=_contract(name="Orders new"), + source="golden-allow", + ) + + assert bundle.proposal.decision.decision is DecisionResult.ALLOW + assert bundle.authorization.evidence_reference is None + chain = _evolution(GitWorkingTreeHistoryRepository(tmp_path)).reconstruct(CONTRACT_ID) + assert len(chain.proposals) == 1 + assert chain.proposals[0].decision is not None + assert chain.proposals[0].decision.decision is DecisionResult.ALLOW + assert len(chain.proposals[0].releases) == 1 + assert chain.proposals[0].releases[0].release == bundle.release_record + + +def test_blocked_proposal_persists_without_release(tmp_path: Path) -> None: + backend = GitWorkingTreeHistoryRepository(tmp_path) + base_revision = build_contract_revision(_contract(version=CURRENT_VERSION)) + candidate_revision = build_contract_revision(_contract(version="2.0.0")) + proposal = GovernanceService().evaluate_proposal( + base_revision.contract, + candidate_revision.contract, + effective_date=EFFECTIVE_DATE, + base_revision_ref=base_revision.revision_id, + candidate_revision_ref=candidate_revision.revision_id, + source="golden-block", + actor_reference="actor:golden", + ) + assert proposal.decision.decision is DecisionResult.BLOCK + _proposal_history(backend).record_proposal( + proposal, + base_revision=base_revision, + candidate_revision=candidate_revision, + ) + + chain = _evolution(backend).reconstruct(CONTRACT_ID) + assert len(chain.proposals) == 1 + assert chain.proposals[0].decision is not None + assert chain.proposals[0].decision.decision is DecisionResult.BLOCK + assert chain.proposals[0].releases == () + assert backend.list_release_records(CONTRACT_ID) == () + + +def test_golden_history_rejects_conflicting_overwrite(tmp_path: Path) -> None: + scenario = _build_review_multi_deploy(tmp_path) + decision = scenario.bundle.proposal.decision + conflicting = decision.model_copy(update={"contract_id": "other-contract"}) + + with pytest.raises(HistoryConflictError, match="different content"): + scenario.backend.put_decision(conflicting) + + assert scenario.backend.get_decision(decision.decision_id) == decision + + +def test_broken_reference_is_explicit_and_does_not_fabricate_release(tmp_path: Path) -> None: + bundle = _record_release( + tmp_path, + base=_contract(), + candidate=_contract(include_note=True), + source="golden-broken-base", + ) + broken = build_change_set( + contract_id=CONTRACT_ID, + base_revision_ref="missing-revision", + candidate_revision_ref=bundle.candidate_revision.revision_id, + changes=bundle.proposal.change_set.changes, + context=bundle.proposal.change_set.context, + source="golden-broken", + ) + bundle.backend.put_change_set(broken) + + chain = _evolution(bundle.backend).reconstruct(CONTRACT_ID) + broken_path = next(item for item in chain.proposals if item.change_set == broken) + assert broken_path.base_revision is None + assert broken_path.decision is None + assert broken_path.releases == () + assert any( + item.source_id == broken.change_set_id + and item.reference_field == "base_revision_ref" + and item.target_id == "missing-revision" + and item.reason == "NOT_FOUND" + for item in chain.broken_references + ) From 375cf6529b63b09d25e7ec543255effeb06bf6e6 Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Tue, 15 Sep 2026 20:41:17 +1000 Subject: [PATCH 25/35] feat(observation): capture Unity Catalog governance evidence (#232) * feat(observation): model governance evidence * feat(observation): fingerprint governance evidence * feat(observation): capture Unity Catalog governance evidence * feat(observation): export governance evidence models * test(observation): enrich Unity Catalog fixture * test(observation): cover Unity Catalog governance evidence * test(observation): cover governance evidence fingerprints * fix(observation): canonicalize resolved relationship targets * test(reconciliation): accept obs-v2 golden fingerprints * test(history): accept obs-v2 runtime identities * test(observation): lock resolved relationship canonicalization * test(observation): update canonical relationship fingerprint * refactor(observation): centralize evidence classification * refactor(observation): keep models platform neutral * refactor(observation): expose shared evidence metrics * refactor(databricks): move observation adapter under platform * refactor(databricks): consume platform observation adapter * test(databricks): follow platform observation boundary * refactor(databricks): remove adapter from observation core * test(observation): standardize cross-platform evidence metrics * test(observation): separate classification from fingerprint tests * refactor(observation): define shared evidence availability * refactor(observation): centralize canonical evidence semantics * refactor(observation): enforce canonical state invariants * refactor(observation): standardize evidence coverage metrics * refactor(observation): align fingerprint canonical semantics * refactor(observation): expose shared evidence coverage API * refactor(databricks): isolate provider observation semantics * refactor(databricks): preserve observation coverage semantics * test(observation): cover shared evidence availability semantics * test(observation): lock canonical fingerprint semantics * test(databricks): cover provider evidence availability * fix(history): persist canonical observation fingerprint * test(history): enforce canonical observation persistence * test(observation): refresh intentional fingerprint goldens * test(history): refresh intentional observation identity goldens * fix(observation): model foreign keys as constraints * fix(observation): retain foreign key constraint evidence * test(observation): cover foreign key constraint evidence --- .../application/services/runtime_history.py | 34 +- semapact/observation/__init__.py | 34 ++ semapact/observation/canonical.py | 124 +++++ semapact/observation/classification.py | 116 +++++ semapact/observation/databricks.py | 192 -------- semapact/observation/evidence.py | 63 +++ semapact/observation/fingerprint.py | 54 ++- semapact/observation/models.py | 111 ++++- semapact/platforms/databricks/observation.py | 455 ++++++++++++++++++ semapact/platforms/databricks/runtime.py | 18 +- .../v1/review_multi_deploy.json | 28 +- tests/fixtures/m1_reconciliation_golden.json | 6 +- .../databricks/orders_table_info.json | 32 +- tests/test_observation_classification.py | 219 +++++++++ tests/test_observation_databricks.py | 234 ++++++++- tests/test_observation_fingerprint.py | 183 ++++++- tests/test_runtime_history.py | 74 ++- 17 files changed, 1690 insertions(+), 287 deletions(-) create mode 100644 semapact/observation/canonical.py create mode 100644 semapact/observation/classification.py delete mode 100644 semapact/observation/databricks.py create mode 100644 semapact/observation/evidence.py create mode 100644 semapact/platforms/databricks/observation.py create mode 100644 tests/test_observation_classification.py diff --git a/semapact/application/services/runtime_history.py b/semapact/application/services/runtime_history.py index 99b8038a..ce059a1d 100644 --- a/semapact/application/services/runtime_history.py +++ b/semapact/application/services/runtime_history.py @@ -16,7 +16,11 @@ compute_runtime_observation_record_id, compute_runtime_reconciliation_record_id, ) -from semapact.observation import ObservedPlatformState, fingerprint_observed_state +from semapact.observation import ( + ObservedPlatformState, + fingerprint_observed_state, + with_observed_state_fingerprint, +) from semapact.reconciliation import ( ReconciliationResult, classify_reconciliation_status, @@ -58,23 +62,24 @@ def record_reconciliation( f"result must be ReconciliationResult, got {type(result).__name__}" ) - _validate_observation_result_link(observation, result) + canonical_observation = _canonical_observation(observation) + _validate_observation_result_link(canonical_observation, result) release = _load_optional_release(self._releases, release_record_id) deployment = _load_optional_deployment( self._deployments, deployment_record_id, ) _validate_optional_history_links( - observation, + canonical_observation, result, release=release, deployment=deployment, ) - observation_record_id = compute_runtime_observation_record_id(observation) + observation_record_id = compute_runtime_observation_record_id(canonical_observation) observation_record = RuntimeObservationRecord( observation_record_id=observation_record_id, - observation=observation, + observation=canonical_observation, ) status = classify_reconciliation_status(result) @@ -109,6 +114,20 @@ def record_reconciliation( return record +def _canonical_observation(observation: ObservedPlatformState) -> ObservedPlatformState: + semantic_fingerprint = fingerprint_observed_state(observation) + if ( + observation.fingerprint is not None + and observation.fingerprint != semantic_fingerprint + ): + raise ValueError( + "ObservedPlatformState fingerprint does not match canonical semantic content" + ) + if observation.fingerprint is not None: + return observation + return with_observed_state_fingerprint(observation) + + def _validate_observation_result_link( observation: ObservedPlatformState, result: ReconciliationResult, @@ -117,10 +136,7 @@ def _validate_observation_result_link( raise ValueError( "ReconciliationResult source does not match ObservedPlatformState" ) - observation_fingerprint = observation.fingerprint or fingerprint_observed_state( - observation - ) - if observation_fingerprint != result.observation_fingerprint: + if observation.fingerprint != result.observation_fingerprint: raise ValueError( "ReconciliationResult fingerprint does not match ObservedPlatformState" ) diff --git a/semapact/observation/__init__.py b/semapact/observation/__init__.py index 1ac37915..2cc2538b 100644 --- a/semapact/observation/__init__.py +++ b/semapact/observation/__init__.py @@ -1,5 +1,18 @@ """Platform-neutral observation domain for SemaPact read-side state.""" +from semapact.observation.classification import ( + ObservedEvidenceClassCount, + ObservedEvidenceCount, + ObservedEvidenceMetrics, + classify_observed_evidence, + summarize_observed_evidence, +) +from semapact.observation.evidence import ( + ObservedEvidenceAvailability, + ObservedEvidenceAvailabilityStatus, + ObservedEvidenceClass, + ObservedEvidenceKind, +) from semapact.observation.fingerprint import ( OBSERVED_STATE_FINGERPRINT_ALGORITHM, OBSERVED_STATE_FINGERPRINT_VERSION, @@ -10,9 +23,15 @@ from semapact.observation.models import ( ObservedAsset, ObservedAssetIdentity, + ObservedConstraint, + ObservedConstraintKind, ObservedPlatformState, ObservedProperty, ObservedPropertyIdentity, + ObservedRelationship, + ObservedRelationshipDirection, + ObservedRelationshipKind, + ObservedTag, serialize_observed_state, ) from semapact.observation.providers import ( @@ -27,15 +46,30 @@ "OBSERVED_STATE_FINGERPRINT_VERSION", "ObservedAsset", "ObservedAssetIdentity", + "ObservedConstraint", + "ObservedConstraintKind", + "ObservedEvidenceAvailability", + "ObservedEvidenceAvailabilityStatus", + "ObservedEvidenceClass", + "ObservedEvidenceClassCount", + "ObservedEvidenceCount", + "ObservedEvidenceKind", + "ObservedEvidenceMetrics", "ObservedPlatformState", "ObservedProperty", "ObservedPropertyIdentity", + "ObservedRelationship", + "ObservedRelationshipDirection", + "ObservedRelationshipKind", + "ObservedTag", "RuntimeAssetBinding", "RuntimeAssetSpec", "RuntimeProvider", "RuntimeProviderRegistry", "canonical_observed_state_payload", + "classify_observed_evidence", "fingerprint_observed_state", "serialize_observed_state", + "summarize_observed_evidence", "with_observed_state_fingerprint", ] diff --git a/semapact/observation/canonical.py b/semapact/observation/canonical.py new file mode 100644 index 00000000..9094e90b --- /dev/null +++ b/semapact/observation/canonical.py @@ -0,0 +1,124 @@ +"""Canonical provider-neutral observation normalization primitives.""" + +from __future__ import annotations + +from collections.abc import Iterable + +from semapact.observation.evidence import ( + ObservedEvidenceAvailability, + ObservedEvidenceKind, + resolve_evidence_availability, +) +from semapact.observation.models import ( + ObservedConstraint, + ObservedRelationship, + ObservedTag, +) + + +def normalize_optional_text(value: str | None) -> str | None: + """Trim optional text and collapse empty strings to ``None``.""" + if value is None: + return None + cleaned = value.strip() + return cleaned or None + + +def normalize_casefold_text(value: str | None) -> str | None: + """Normalize case-insensitive semantic text.""" + cleaned = normalize_optional_text(value) + return cleaned.casefold() if cleaned is not None else None + + +def canonical_tag_payload(tag: ObservedTag) -> dict[str, object]: + return { + "key": tag.key.strip(), + "value": normalize_optional_text(tag.value), + "provenance": normalize_optional_text(tag.provenance), + } + + +def canonical_tag_key(tag: ObservedTag) -> tuple[str, str, str]: + payload = canonical_tag_payload(tag) + return ( + str(payload["key"]), + str(payload["value"] or ""), + str(payload["provenance"] or ""), + ) + + +def normalize_observed_tags(tags: Iterable[ObservedTag]) -> tuple[ObservedTag, ...]: + """Deduplicate and deterministically order canonical tag evidence.""" + unique = {canonical_tag_key(tag): tag for tag in tags} + return tuple(sorted(unique.values(), key=canonical_tag_key)) + + +def canonical_constraint_payload(constraint: ObservedConstraint) -> dict[str, object]: + return { + "kind": constraint.kind.value, + "properties": [item.casefold() for item in constraint.properties], + "name": normalize_optional_text(constraint.name), + "provenance": normalize_optional_text(constraint.provenance), + } + + +def canonical_constraint_key(constraint: ObservedConstraint) -> tuple[object, ...]: + payload = canonical_constraint_payload(constraint) + return ( + payload["kind"], + tuple(payload["properties"]), + payload["name"] or "", + payload["provenance"] or "", + ) + + +def canonical_relationship_payload( + relationship: ObservedRelationship, +) -> dict[str, object]: + target_asset = relationship.target_asset + return { + "kind": relationship.kind.value, + "source_asset": list(relationship.source_asset.canonical_key), + "source_properties": [item.casefold() for item in relationship.source_properties], + "target_asset": list(target_asset.canonical_key) if target_asset is not None else None, + "target_properties": [item.casefold() for item in relationship.target_properties], + "target_reference": ( + None + if target_asset is not None + else normalize_optional_text(relationship.target_reference) + ), + "direction": relationship.direction.value, + "name": normalize_optional_text(relationship.name), + "provenance": normalize_optional_text(relationship.provenance), + } + + +def canonical_relationship_key( + relationship: ObservedRelationship, +) -> tuple[object, ...]: + payload = canonical_relationship_payload(relationship) + target_asset = payload["target_asset"] + return ( + payload["kind"], + tuple(payload["source_asset"]), + tuple(payload["source_properties"]), + tuple(target_asset) if isinstance(target_asset, list) else (), + tuple(payload["target_properties"]), + payload["target_reference"] or "", + payload["direction"], + payload["name"] or "", + payload["provenance"] or "", + ) + + +def canonical_evidence_availability_payload( + entries: tuple[ObservedEvidenceAvailability, ...], +) -> list[dict[str, str]]: + """Return complete deterministic availability semantics for every evidence kind.""" + return [ + { + "kind": kind.value, + "status": resolve_evidence_availability(entries, kind).value, + } + for kind in ObservedEvidenceKind + ] diff --git a/semapact/observation/classification.py b/semapact/observation/classification.py new file mode 100644 index 00000000..bd36e06f --- /dev/null +++ b/semapact/observation/classification.py @@ -0,0 +1,116 @@ +"""Platform-neutral classification and metrics for observed evidence.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field + +from semapact.observation.evidence import ( + ObservedEvidenceAvailabilityStatus, + ObservedEvidenceClass, + ObservedEvidenceKind, + resolve_evidence_availability, +) +from semapact.observation.models import ObservedPlatformState + + +class ObservationMetricModel(BaseModel): + """Shared immutable base for derived observation metrics.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + +class ObservedEvidenceCount(ObservationMetricModel): + """Deterministic count and coverage state for one evidence kind.""" + + kind: ObservedEvidenceKind + evidence_class: ObservedEvidenceClass + availability: ObservedEvidenceAvailabilityStatus + count: int = Field(ge=0) + + +class ObservedEvidenceClassCount(ObservationMetricModel): + """Deterministic aggregate count for one evidence class.""" + + evidence_class: ObservedEvidenceClass + count: int = Field(ge=0) + + +class ObservedEvidenceMetrics(ObservationMetricModel): + """Standard evidence metrics derived from canonical observed state.""" + + total: int = Field(ge=0) + by_kind: tuple[ObservedEvidenceCount, ...] + by_class: tuple[ObservedEvidenceClassCount, ...] + + +_EVIDENCE_CLASSES = { + ObservedEvidenceKind.PHYSICAL_SCHEMA: ObservedEvidenceClass.STRUCTURAL, + ObservedEvidenceKind.OWNER: ObservedEvidenceClass.OPERATIONAL, + ObservedEvidenceKind.COMMENT: ObservedEvidenceClass.SEMANTIC, + ObservedEvidenceKind.TAG: ObservedEvidenceClass.SEMANTIC, + ObservedEvidenceKind.CONSTRAINT: ObservedEvidenceClass.STRUCTURAL, + ObservedEvidenceKind.RELATIONSHIP: ObservedEvidenceClass.STRUCTURAL, +} + + +def classify_observed_evidence(kind: ObservedEvidenceKind) -> ObservedEvidenceClass: + """Return the descriptive evidence class for one provider-neutral kind.""" + return _EVIDENCE_CLASSES[kind] + + +def summarize_observed_evidence(state: ObservedPlatformState) -> ObservedEvidenceMetrics: + """Derive the same evidence metrics for every provider observation. + + ``PHYSICAL_SCHEMA`` counts one item per observed asset plus one item per + observed property. Other kinds count their canonical evidence objects or + populated scalar values. Availability is reported independently from count, + so zero evidence is never confused with unsupported or unknown evidence. + """ + counts = {kind: 0 for kind in ObservedEvidenceKind} + + for asset in state.assets: + counts[ObservedEvidenceKind.PHYSICAL_SCHEMA] += 1 + len(asset.properties) + if asset.owner is not None: + counts[ObservedEvidenceKind.OWNER] += 1 + if asset.comment is not None: + counts[ObservedEvidenceKind.COMMENT] += 1 + + counts[ObservedEvidenceKind.TAG] += len(asset.tags) + counts[ObservedEvidenceKind.CONSTRAINT] += len(asset.constraints) + counts[ObservedEvidenceKind.RELATIONSHIP] += len(asset.relationships) + + for prop in asset.properties: + if prop.comment is not None: + counts[ObservedEvidenceKind.COMMENT] += 1 + counts[ObservedEvidenceKind.TAG] += len(prop.tags) + + by_kind = tuple( + ObservedEvidenceCount( + kind=kind, + evidence_class=classify_observed_evidence(kind), + availability=resolve_evidence_availability( + state.evidence_availability, + kind, + ), + count=counts[kind], + ) + for kind in ObservedEvidenceKind + ) + + class_counts = {evidence_class: 0 for evidence_class in ObservedEvidenceClass} + for metric in by_kind: + class_counts[metric.evidence_class] += metric.count + + by_class = tuple( + ObservedEvidenceClassCount( + evidence_class=evidence_class, + count=class_counts[evidence_class], + ) + for evidence_class in ObservedEvidenceClass + ) + + return ObservedEvidenceMetrics( + total=sum(counts.values()), + by_kind=by_kind, + by_class=by_class, + ) diff --git a/semapact/observation/databricks.py b/semapact/observation/databricks.py deleted file mode 100644 index 3bc936eb..00000000 --- a/semapact/observation/databricks.py +++ /dev/null @@ -1,192 +0,0 @@ -"""Databricks observation adapter backed by the official Databricks SDK. - -The adapter consumes the same ``WorkspaceClient.tables.get(...) -> TableInfo`` -boundary used by datacontract-cli, but projects that source metadata into -SemaPact's platform-neutral observation model instead of into ODCS. - -Authentication and credential resolution are caller concerns. This module -accepts an already initialized/authenticated ``WorkspaceClient`` and must not -resolve PATs, OAuth credentials, Azure identity, profiles, service principals, -or other authentication mechanisms itself. -""" - -from __future__ import annotations - -from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any, Mapping, Protocol - -from semapact.observation.fingerprint import with_observed_state_fingerprint -from semapact.observation.models import ( - ObservedAsset, - ObservedAssetIdentity, - ObservedPlatformState, - ObservedProperty, - ObservedPropertyIdentity, -) - -if TYPE_CHECKING: - from databricks.sdk import WorkspaceClient - from databricks.sdk.service.catalog import TableInfo - -DATABRICKS_PLATFORM = "databricks" - - -class _TableInfoLike(Protocol): - def as_dict(self) -> dict[str, Any]: ... - - -class _TablesApiLike(Protocol): - def get(self, full_name: str) -> _TableInfoLike: ... - - -class _WorkspaceClientLike(Protocol): - tables: _TablesApiLike - - -def observe_databricks_table( - *, - client: WorkspaceClient | _WorkspaceClientLike, - table_fqn: str, - source_identifier: str, - captured_at: datetime | None = None, -) -> ObservedPlatformState: - """Observe one Databricks table without generating or mutating an ODCS contract. - - ``client`` is expected to be an already initialized/authenticated official - ``databricks.sdk.WorkspaceClient`` in production. Authentication method and - credential resolution are intentionally outside this adapter's scope. - The client is injectable so core observation tests require no live - Databricks workspace. - """ - if not table_fqn: - raise ValueError("table_fqn is required for Databricks observation") - if not source_identifier: - raise ValueError("source_identifier is required for Databricks observation") - - observed_at = captured_at or datetime.now(timezone.utc) - _require_aware_datetime(observed_at) - - table = client.tables.get(table_fqn) - return map_databricks_table_info( - table, - source_identifier=source_identifier, - captured_at=observed_at, - table_fqn=table_fqn, - ) - - -def map_databricks_table_info( - table: TableInfo | _TableInfoLike, - *, - source_identifier: str, - captured_at: datetime, - table_fqn: str | None = None, -) -> ObservedPlatformState: - """Project an SDK ``TableInfo`` into fingerprinted platform-neutral state.""" - _require_aware_datetime(captured_at) - if not source_identifier: - raise ValueError("source_identifier is required for Databricks observation") - - metadata = table.as_dict() - if not isinstance(metadata, Mapping): - raise TypeError("Databricks TableInfo.as_dict() must return a mapping") - - identity = _asset_identity(metadata, table_fqn=table_fqn) - asset = ObservedAsset( - identity=identity, - asset_type=_text(metadata.get("table_type")), - properties=_properties(metadata.get("columns"), identity=identity), - ) - - state = ObservedPlatformState( - platform=DATABRICKS_PLATFORM, - source_identifier=source_identifier.rstrip("/"), - assets=(asset,), - captured_at=captured_at, - fingerprint=None, - ) - return with_observed_state_fingerprint(state) - - -def _asset_identity( - metadata: Mapping[str, Any], *, table_fqn: str | None -) -> ObservedAssetIdentity: - catalog = _text(metadata.get("catalog_name")) - schema = _text(metadata.get("schema_name")) - asset = _text(metadata.get("name")) - - full_name = _text(metadata.get("full_name") or table_fqn) - fallback = _split_table_fqn(full_name) if full_name else None - if fallback: - catalog = catalog or fallback[0] - schema = schema or fallback[1] - asset = asset or fallback[2] - - if not catalog or not schema or not asset: - raise ValueError("Databricks TableInfo must identify catalog, schema, and table name") - - return ObservedAssetIdentity( - platform=DATABRICKS_PLATFORM, - namespace=(catalog, schema), - asset=asset, - ) - - -def _split_table_fqn(value: str) -> tuple[str, str, str] | None: - parts = tuple(part.strip() for part in value.split(".")) - if len(parts) != 3 or not all(parts): - return None - return parts - - -def _properties( - value: Any, *, identity: ObservedAssetIdentity -) -> tuple[ObservedProperty, ...]: - if not isinstance(value, list): - return () - - observed: list[tuple[int | None, ObservedProperty]] = [] - for item in value: - if not isinstance(item, Mapping): - continue - - name = _text(item.get("name")) - if not name: - continue - - nullable = item.get("nullable") if isinstance(item.get("nullable"), bool) else None - position = item.get("position") - if isinstance(position, bool) or not isinstance(position, int): - position = None - - observed.append( - ( - position, - ObservedProperty( - identity=ObservedPropertyIdentity(asset=identity, property=name), - physical_type=_text(item.get("type_text") or item.get("type_name")), - nullable=nullable, - ), - ) - ) - - observed.sort( - key=lambda item: ( - item[0] is None, - item[0] if item[0] is not None else 0, - item[1].identity.property.casefold(), - ) - ) - return tuple(item[1] for item in observed) - - -def _text(value: Any) -> str | None: - if value is None: - return None - text = str(value).strip() - return text or None - - -def _require_aware_datetime(value: datetime) -> None: - if value.tzinfo is None or value.utcoffset() is None: - raise ValueError("captured_at must be timezone-aware") diff --git a/semapact/observation/evidence.py b/semapact/observation/evidence.py new file mode 100644 index 00000000..76f5db40 --- /dev/null +++ b/semapact/observation/evidence.py @@ -0,0 +1,63 @@ +"""Provider-neutral evidence taxonomy and observation availability semantics.""" + +from __future__ import annotations + +from enum import Enum + +from pydantic import BaseModel, ConfigDict, field_validator + + +class ObservedEvidenceClass(str, Enum): + """Descriptive class for observed evidence, never a governance verdict.""" + + STRUCTURAL = "STRUCTURAL" + SEMANTIC = "SEMANTIC" + OPERATIONAL = "OPERATIONAL" + + +class ObservedEvidenceKind(str, Enum): + """Provider-neutral kinds of evidence carried by an observation.""" + + PHYSICAL_SCHEMA = "PHYSICAL_SCHEMA" + OWNER = "OWNER" + COMMENT = "COMMENT" + TAG = "TAG" + CONSTRAINT = "CONSTRAINT" + RELATIONSHIP = "RELATIONSHIP" + + +class ObservedEvidenceAvailabilityStatus(str, Enum): + """Whether a provider observation could establish one evidence kind.""" + + AVAILABLE = "AVAILABLE" + UNAVAILABLE = "UNAVAILABLE" + UNKNOWN = "UNKNOWN" + + +class ObservedEvidenceAvailability(BaseModel): + """Availability state for one provider-neutral evidence kind.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + kind: ObservedEvidenceKind + status: ObservedEvidenceAvailabilityStatus + detail: str | None = None + + @field_validator("detail") + @classmethod + def _normalize_detail(cls, value: str | None) -> str | None: + if value is None: + return None + cleaned = value.strip() + return cleaned or None + + +def resolve_evidence_availability( + entries: tuple[ObservedEvidenceAvailability, ...], + kind: ObservedEvidenceKind, +) -> ObservedEvidenceAvailabilityStatus: + """Resolve one kind, treating unspecified availability as UNKNOWN.""" + for entry in entries: + if entry.kind is kind: + return entry.status + return ObservedEvidenceAvailabilityStatus.UNKNOWN diff --git a/semapact/observation/fingerprint.py b/semapact/observation/fingerprint.py index 0d6ecd81..14fd0f99 100644 --- a/semapact/observation/fingerprint.py +++ b/semapact/observation/fingerprint.py @@ -5,10 +5,10 @@ and ``source_identifier`` so repeated captures of the same platform state have the same fingerprint. -Version ``obs-v1`` hashes the minimal observation model introduced by M1: -platform, asset identity/type, and property identity/physical type/nullability. -Provider-specific semantic normalization remains the responsibility of the -provider adapter before it constructs the platform-neutral observation model. +Version ``obs-v2`` extends the physical observation payload with normalized +owner, comments, tags, constraints, relationships, and evidence availability. +Provider-specific normalization remains the responsibility of the provider +adapter before it constructs the platform-neutral observation model. """ from __future__ import annotations @@ -17,24 +17,31 @@ import json from typing import Any -from semapact.observation.models import ( - ObservedAsset, - ObservedPlatformState, - ObservedProperty, +from semapact.observation.canonical import ( + canonical_constraint_payload, + canonical_evidence_availability_payload, + canonical_relationship_payload, + canonical_tag_payload, + normalize_casefold_text, + normalize_optional_text, ) +from semapact.observation.models import ObservedAsset, ObservedPlatformState, ObservedProperty -OBSERVED_STATE_FINGERPRINT_VERSION = "obs-v1" +OBSERVED_STATE_FINGERPRINT_VERSION = "obs-v2" OBSERVED_STATE_FINGERPRINT_ALGORITHM = "sha256" def canonical_observed_state_payload(state: ObservedPlatformState) -> dict[str, object]: - """Return the versioned semantic payload used by the v1 fingerprint.""" + """Return the versioned semantic payload used by the current fingerprint.""" assets = [_canonical_asset(asset) for asset in state.assets] assets.sort(key=_canonical_json) return { "fingerprint_version": OBSERVED_STATE_FINGERPRINT_VERSION, - "platform": state.platform.casefold(), + "platform": state.platform.strip().casefold(), + "evidence_availability": canonical_evidence_availability_payload( + state.evidence_availability + ), "assets": assets, } @@ -57,28 +64,37 @@ def with_observed_state_fingerprint(state: ObservedPlatformState) -> ObservedPla def _canonical_asset(asset: ObservedAsset) -> dict[str, object]: properties = [_canonical_property(prop) for prop in asset.properties] properties.sort(key=_canonical_json) + tags = [canonical_tag_payload(tag) for tag in asset.tags] + tags.sort(key=_canonical_json) + constraints = [canonical_constraint_payload(item) for item in asset.constraints] + constraints.sort(key=_canonical_json) + relationships = [canonical_relationship_payload(item) for item in asset.relationships] + relationships.sort(key=_canonical_json) return { "identity": list(asset.identity.canonical_key), - "asset_type": _normalize_optional_text(asset.asset_type), + "asset_type": normalize_casefold_text(asset.asset_type), + "owner": normalize_optional_text(asset.owner), + "comment": normalize_optional_text(asset.comment), + "tags": tags, "properties": properties, + "constraints": constraints, + "relationships": relationships, } def _canonical_property(prop: ObservedProperty) -> dict[str, object]: + tags = [canonical_tag_payload(tag) for tag in prop.tags] + tags.sort(key=_canonical_json) return { "identity": list(prop.identity.canonical_key), - "physical_type": _normalize_optional_text(prop.physical_type), + "physical_type": normalize_casefold_text(prop.physical_type), "nullable": prop.nullable, + "comment": normalize_optional_text(prop.comment), + "tags": tags, } -def _normalize_optional_text(value: str | None) -> str | None: - if value is None: - return None - return value.strip() - - def _canonical_json(value: Any) -> str: return json.dumps( value, diff --git a/semapact/observation/models.py b/semapact/observation/models.py index acbd7606..1bc9d507 100644 --- a/semapact/observation/models.py +++ b/semapact/observation/models.py @@ -9,8 +9,11 @@ import json from datetime import datetime +from enum import Enum -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, model_validator + +from semapact.observation.evidence import ObservedEvidenceAvailability class ObservationModel(BaseModel): @@ -22,9 +25,8 @@ class ObservationModel(BaseModel): class ObservedAssetIdentity(ObservationModel): """Platform-local identity for one observed asset. - ``namespace`` is intentionally provider-neutral. A Databricks adapter may - populate it with ``(catalog, schema)`` while another platform may use a - different hierarchy without changing the domain model. + ``namespace`` is intentionally provider-neutral so adapters can map their + own hierarchy without changing the observation domain model. """ platform: str @@ -53,20 +55,98 @@ def canonical_key(self) -> tuple[str, ...]: return (*self.asset.canonical_key, self.property.casefold()) +class ObservedTag(ObservationModel): + """One normalized platform tag assignment.""" + + key: str + value: str | None = None + provenance: str | None = None + + +class ObservedConstraintKind(str, Enum): + """Constraint semantics that a provider can identify without inference.""" + + PRIMARY_KEY = "PRIMARY_KEY" + FOREIGN_KEY = "FOREIGN_KEY" + UNIQUE = "UNIQUE" + NAMED = "NAMED" + + +class ObservedConstraint(ObservationModel): + """Normalized constraint evidence for one asset.""" + + kind: ObservedConstraintKind + properties: tuple[str, ...] = () + name: str | None = None + provenance: str | None = None + + +class ObservedRelationshipKind(str, Enum): + """Relationship semantics that a provider reports explicitly.""" + + FOREIGN_KEY = "FOREIGN_KEY" + + +class ObservedRelationshipDirection(str, Enum): + """Direction from the asset carrying the relationship evidence.""" + + OUTBOUND = "OUTBOUND" + + +class ObservedRelationship(ObservationModel): + """Normalized explicit relationship evidence between observed assets. + + ``target_reference`` preserves provider evidence when the target cannot be + normalized into a platform-local asset identity. Empty property tuples mean + the provider did not identify those columns; they must never be inferred. + """ + + kind: ObservedRelationshipKind + source_asset: ObservedAssetIdentity + source_properties: tuple[str, ...] = () + target_asset: ObservedAssetIdentity | None = None + target_properties: tuple[str, ...] = () + target_reference: str | None = None + direction: ObservedRelationshipDirection = ObservedRelationshipDirection.OUTBOUND + name: str | None = None + provenance: str | None = None + + class ObservedProperty(ObservationModel): - """Observed physical property/column state.""" + """Observed physical and semantic property/column state.""" identity: ObservedPropertyIdentity physical_type: str | None = None nullable: bool | None = None + comment: str | None = None + tags: tuple[ObservedTag, ...] = () class ObservedAsset(ObservationModel): - """Observed physical state for one external asset.""" + """Observed physical, semantic, and operational state for one external asset.""" identity: ObservedAssetIdentity asset_type: str | None = None + owner: str | None = None + comment: str | None = None + tags: tuple[ObservedTag, ...] = () properties: tuple[ObservedProperty, ...] = () + constraints: tuple[ObservedConstraint, ...] = () + relationships: tuple[ObservedRelationship, ...] = () + + @model_validator(mode="after") + def _validate_nested_identity(self) -> ObservedAsset: + for prop in self.properties: + if prop.identity.asset.canonical_key != self.identity.canonical_key: + raise ValueError( + "Observed property asset identity must match its containing asset" + ) + for relationship in self.relationships: + if relationship.source_asset.canonical_key != self.identity.canonical_key: + raise ValueError( + "Observed relationship source identity must match its containing asset" + ) + return self class ObservedPlatformState(ObservationModel): @@ -76,14 +156,33 @@ class ObservedPlatformState(ObservationModel): observed state. Provider adapters may populate it through the shared fingerprint capability; manually constructed observations may leave it unset until fingerprinting is requested. + + ``evidence_availability`` records whether each provider-neutral evidence + kind was actually observable. Unspecified kinds are treated as ``UNKNOWN`` + by shared classification and fingerprinting logic. """ platform: str source_identifier: str assets: tuple[ObservedAsset, ...] captured_at: datetime + evidence_availability: tuple[ObservedEvidenceAvailability, ...] = () fingerprint: str | None = None + @model_validator(mode="after") + def _validate_state_identity(self) -> ObservedPlatformState: + platform = self.platform.casefold() + for asset in self.assets: + if asset.identity.platform.casefold() != platform: + raise ValueError( + "Observed asset platform must match ObservedPlatformState.platform" + ) + + kinds = [entry.kind for entry in self.evidence_availability] + if len(kinds) != len(set(kinds)): + raise ValueError("Observed evidence availability must contain each kind at most once") + return self + def serialize_observed_state(state: ObservedPlatformState) -> str: """Serialize observed state deterministically for machine-readable use.""" diff --git a/semapact/platforms/databricks/observation.py b/semapact/platforms/databricks/observation.py new file mode 100644 index 00000000..c5cc076f --- /dev/null +++ b/semapact/platforms/databricks/observation.py @@ -0,0 +1,455 @@ +"""Databricks observation adapter backed by the official Databricks SDK. + +This module contains Databricks/Unity Catalog extraction only. It maps provider +metadata into SemaPact's canonical platform-neutral observation model; shared +classification, metrics, canonicalization, and fingerprint semantics remain in +``semapact.observation``. + +Authentication and credential resolution are caller concerns. Callers supply an +already initialized/authenticated ``WorkspaceClient``. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Iterable, Mapping, Protocol, cast + +from semapact.observation.canonical import ( + canonical_constraint_key, + canonical_relationship_key, + normalize_observed_tags, +) +from semapact.observation.evidence import ( + ObservedEvidenceAvailability, + ObservedEvidenceAvailabilityStatus, + ObservedEvidenceKind, +) +from semapact.observation.fingerprint import with_observed_state_fingerprint +from semapact.observation.models import ( + ObservedAsset, + ObservedAssetIdentity, + ObservedConstraint, + ObservedConstraintKind, + ObservedPlatformState, + ObservedProperty, + ObservedPropertyIdentity, + ObservedRelationship, + ObservedRelationshipKind, + ObservedTag, +) + +if TYPE_CHECKING: + from databricks.sdk import WorkspaceClient + from databricks.sdk.service.catalog import TableInfo + +DATABRICKS_PLATFORM = "databricks" +UNITY_CATALOG_PROVENANCE = "unity_catalog" + + +class _SdkObjectLike(Protocol): + def as_dict(self) -> dict[str, Any]: ... + + +class _TablesApiLike(Protocol): + def get(self, full_name: str) -> _SdkObjectLike: ... + + +class _EntityTagAssignmentsApiLike(Protocol): + def list(self, entity_type: str, entity_name: str) -> Iterable[_SdkObjectLike]: ... + + +class _WorkspaceClientLike(Protocol): + tables: _TablesApiLike + + +def databricks_evidence_availability( + client: WorkspaceClient | _WorkspaceClientLike, +) -> tuple[ObservedEvidenceAvailability, ...]: + """Describe which canonical evidence kinds this client can observe.""" + tag_status = ( + ObservedEvidenceAvailabilityStatus.AVAILABLE + if _entity_tag_assignments_api(client) is not None + else ObservedEvidenceAvailabilityStatus.UNAVAILABLE + ) + return _table_info_evidence_availability(tag_status=tag_status) + + +def observe_databricks_table( + *, + client: WorkspaceClient | _WorkspaceClientLike, + table_fqn: str, + source_identifier: str, + captured_at: datetime | None = None, +) -> ObservedPlatformState: + """Observe one Unity Catalog table without mutating governed contract state.""" + table_fqn = table_fqn.strip() + source_identifier = source_identifier.strip() + if not table_fqn: + raise ValueError("table_fqn is required for Databricks observation") + if not source_identifier: + raise ValueError("source_identifier is required for Databricks observation") + + observed_at = captured_at or datetime.now(timezone.utc) + _require_aware_datetime(observed_at) + + table = client.tables.get(table_fqn) + metadata = _sdk_mapping(table, context="Databricks TableInfo") + identity = _asset_identity(metadata, table_fqn=table_fqn) + canonical_table_fqn = ".".join((*identity.namespace, identity.asset)) + tag_api = _entity_tag_assignments_api(client) + + table_tags = _read_entity_tags( + tag_api, + entity_type="tables", + entity_name=canonical_table_fqn, + ) + column_tags = { + column_name: _read_entity_tags( + tag_api, + entity_type="columns", + entity_name=f"{canonical_table_fqn}.{column_name}", + ) + for column_name in _column_names(metadata.get("columns")) + } + + return map_databricks_table_info( + table, + source_identifier=source_identifier, + captured_at=observed_at, + table_fqn=table_fqn, + table_tags=table_tags, + column_tags=column_tags, + evidence_availability=_table_info_evidence_availability( + tag_status=( + ObservedEvidenceAvailabilityStatus.AVAILABLE + if tag_api is not None + else ObservedEvidenceAvailabilityStatus.UNAVAILABLE + ) + ), + ) + + +def map_databricks_table_info( + table: TableInfo | _SdkObjectLike, + *, + source_identifier: str, + captured_at: datetime, + table_fqn: str | None = None, + table_tags: tuple[ObservedTag, ...] = (), + column_tags: Mapping[str, tuple[ObservedTag, ...]] | None = None, + evidence_availability: tuple[ObservedEvidenceAvailability, ...] | None = None, +) -> ObservedPlatformState: + """Project one SDK ``TableInfo`` into canonical platform-neutral state.""" + _require_aware_datetime(captured_at) + source_identifier = source_identifier.strip() + if not source_identifier: + raise ValueError("source_identifier is required for Databricks observation") + + metadata = _sdk_mapping(table, context="Databricks TableInfo") + identity = _asset_identity(metadata, table_fqn=table_fqn) + constraints, relationships = _governance_structure( + metadata.get("table_constraints"), + source_identity=identity, + ) + asset = ObservedAsset( + identity=identity, + asset_type=_text(metadata.get("table_type")), + owner=_text(metadata.get("owner")), + comment=_text(metadata.get("comment")), + tags=normalize_observed_tags(table_tags), + properties=_properties( + metadata.get("columns"), + identity=identity, + column_tags=column_tags or {}, + ), + constraints=constraints, + relationships=relationships, + ) + + state = ObservedPlatformState( + platform=DATABRICKS_PLATFORM, + source_identifier=source_identifier.rstrip("/"), + assets=(asset,), + captured_at=captured_at, + evidence_availability=( + evidence_availability + if evidence_availability is not None + else _table_info_evidence_availability( + tag_status=ObservedEvidenceAvailabilityStatus.UNKNOWN + ) + ), + fingerprint=None, + ) + return with_observed_state_fingerprint(state) + + +def _table_info_evidence_availability( + *, + tag_status: ObservedEvidenceAvailabilityStatus, +) -> tuple[ObservedEvidenceAvailability, ...]: + statuses = { + ObservedEvidenceKind.PHYSICAL_SCHEMA: ObservedEvidenceAvailabilityStatus.AVAILABLE, + ObservedEvidenceKind.OWNER: ObservedEvidenceAvailabilityStatus.AVAILABLE, + ObservedEvidenceKind.COMMENT: ObservedEvidenceAvailabilityStatus.AVAILABLE, + ObservedEvidenceKind.TAG: tag_status, + ObservedEvidenceKind.CONSTRAINT: ObservedEvidenceAvailabilityStatus.AVAILABLE, + ObservedEvidenceKind.RELATIONSHIP: ObservedEvidenceAvailabilityStatus.AVAILABLE, + } + return tuple( + ObservedEvidenceAvailability(kind=kind, status=statuses[kind]) + for kind in ObservedEvidenceKind + ) + + +def _entity_tag_assignments_api( + client: WorkspaceClient | _WorkspaceClientLike, +) -> _EntityTagAssignmentsApiLike | None: + api = getattr(client, "entity_tag_assignments", None) + if api is None: + return None + if not callable(getattr(api, "list", None)): + raise TypeError("Databricks entity_tag_assignments must expose list()") + return cast(_EntityTagAssignmentsApiLike, api) + + +def _asset_identity( + metadata: Mapping[str, Any], *, table_fqn: str | None +) -> ObservedAssetIdentity: + catalog = _text(metadata.get("catalog_name")) + schema = _text(metadata.get("schema_name")) + asset = _text(metadata.get("name")) + + full_name = _text(metadata.get("full_name") or table_fqn) + fallback = _split_table_fqn(full_name) if full_name else None + if fallback: + catalog = catalog or fallback[0] + schema = schema or fallback[1] + asset = asset or fallback[2] + + if not catalog or not schema or not asset: + raise ValueError("Databricks TableInfo must identify catalog, schema, and table name") + + return ObservedAssetIdentity( + platform=DATABRICKS_PLATFORM, + namespace=(catalog, schema), + asset=asset, + ) + + +def _split_table_fqn(value: str) -> tuple[str, str, str] | None: + parts = tuple(part.strip() for part in value.split(".")) + if len(parts) != 3 or not all(parts): + return None + return parts + + +def _properties( + value: Any, + *, + identity: ObservedAssetIdentity, + column_tags: Mapping[str, tuple[ObservedTag, ...]], +) -> tuple[ObservedProperty, ...]: + if not isinstance(value, list): + return () + + observed: list[tuple[int | None, ObservedProperty]] = [] + for item in value: + if not isinstance(item, Mapping): + continue + + name = _text(item.get("name")) + if not name: + continue + + nullable = item.get("nullable") if isinstance(item.get("nullable"), bool) else None + position = item.get("position") + if isinstance(position, bool) or not isinstance(position, int): + position = None + + observed.append( + ( + position, + ObservedProperty( + identity=ObservedPropertyIdentity(asset=identity, property=name), + physical_type=_text(item.get("type_text") or item.get("type_name")), + nullable=nullable, + comment=_text(item.get("comment")), + tags=normalize_observed_tags(column_tags.get(name, ())), + ), + ) + ) + + observed.sort(key=_positioned_property_sort_key) + return tuple(item[1] for item in observed) + + +def _positioned_property_sort_key( + item: tuple[int | None, ObservedProperty], +) -> tuple[bool, int, str]: + position, prop = item + return ( + position is None, + position if position is not None else 0, + prop.identity.property.casefold(), + ) + + +def _governance_structure( + value: Any, + *, + source_identity: ObservedAssetIdentity, +) -> tuple[tuple[ObservedConstraint, ...], tuple[ObservedRelationship, ...]]: + if not isinstance(value, list): + return (), () + + constraints: dict[tuple[object, ...], ObservedConstraint] = {} + relationships: dict[tuple[object, ...], ObservedRelationship] = {} + for item in value: + if not isinstance(item, Mapping): + continue + + primary_key = item.get("primary_key_constraint") + if isinstance(primary_key, Mapping): + constraint = ObservedConstraint( + kind=ObservedConstraintKind.PRIMARY_KEY, + properties=_string_tuple(primary_key.get("child_columns")), + name=_text(primary_key.get("name")), + provenance=UNITY_CATALOG_PROVENANCE, + ) + constraints[canonical_constraint_key(constraint)] = constraint + + unique = item.get("unique_constraint") + if isinstance(unique, Mapping): + constraint = ObservedConstraint( + kind=ObservedConstraintKind.UNIQUE, + properties=_string_tuple(unique.get("child_columns")), + name=_text(unique.get("name")), + provenance=UNITY_CATALOG_PROVENANCE, + ) + constraints[canonical_constraint_key(constraint)] = constraint + + named = item.get("named_table_constraint") + if isinstance(named, Mapping): + constraint = ObservedConstraint( + kind=ObservedConstraintKind.NAMED, + name=_text(named.get("name")), + provenance=UNITY_CATALOG_PROVENANCE, + ) + constraints[canonical_constraint_key(constraint)] = constraint + + foreign_key = item.get("foreign_key_constraint") + if isinstance(foreign_key, Mapping): + constraint = ObservedConstraint( + kind=ObservedConstraintKind.FOREIGN_KEY, + properties=_string_tuple(foreign_key.get("child_columns")), + name=_text(foreign_key.get("name")), + provenance=UNITY_CATALOG_PROVENANCE, + ) + constraints[canonical_constraint_key(constraint)] = constraint + + relationship = _foreign_key_relationship( + foreign_key, + source_identity=source_identity, + ) + relationships[canonical_relationship_key(relationship)] = relationship + + return ( + tuple(sorted(constraints.values(), key=canonical_constraint_key)), + tuple(sorted(relationships.values(), key=canonical_relationship_key)), + ) + + +def _foreign_key_relationship( + value: Mapping[str, Any], + *, + source_identity: ObservedAssetIdentity, +) -> ObservedRelationship: + target_reference = _text(value.get("parent_table")) + target_asset = None + if target_reference: + target_parts = _split_table_fqn(target_reference) + if target_parts is not None: + target_asset = ObservedAssetIdentity( + platform=DATABRICKS_PLATFORM, + namespace=(target_parts[0], target_parts[1]), + asset=target_parts[2], + ) + + return ObservedRelationship( + kind=ObservedRelationshipKind.FOREIGN_KEY, + source_asset=source_identity, + source_properties=_string_tuple(value.get("child_columns")), + target_asset=target_asset, + target_properties=_string_tuple(value.get("parent_columns")), + target_reference=target_reference, + name=_text(value.get("name")), + provenance=UNITY_CATALOG_PROVENANCE, + ) + + +def _read_entity_tags( + api: _EntityTagAssignmentsApiLike | None, + *, + entity_type: str, + entity_name: str, +) -> tuple[ObservedTag, ...]: + if api is None: + return () + + tags = [] + for assignment in api.list(entity_type=entity_type, entity_name=entity_name): + metadata = _sdk_mapping(assignment, context="Databricks EntityTagAssignment") + key = _text(metadata.get("tag_key")) + if key is None: + continue + tags.append( + ObservedTag( + key=key, + value=_text(metadata.get("tag_value")), + provenance=UNITY_CATALOG_PROVENANCE, + ) + ) + return normalize_observed_tags(tags) + + +def _column_names(value: Any) -> tuple[str, ...]: + if not isinstance(value, list): + return () + names = { + name + for item in value + if isinstance(item, Mapping) + for name in [_text(item.get("name"))] + if name is not None + } + return tuple(sorted(names, key=str.casefold)) + + +def _string_tuple(value: Any) -> tuple[str, ...]: + if not isinstance(value, list): + return () + return tuple(text for item in value if (text := _text(item)) is not None) + + +def _sdk_mapping(value: object, *, context: str) -> Mapping[str, Any]: + if isinstance(value, Mapping): + return value + as_dict = getattr(value, "as_dict", None) + if not callable(as_dict): + raise TypeError(f"{context} must expose as_dict()") + metadata = as_dict() + if not isinstance(metadata, Mapping): + raise TypeError(f"{context}.as_dict() must return a mapping") + return metadata + + +def _text(value: Any) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None + + +def _require_aware_datetime(value: datetime) -> None: + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError("captured_at must be timezone-aware") diff --git a/semapact/platforms/databricks/runtime.py b/semapact/platforms/databricks/runtime.py index c92ff784..08daba6a 100644 --- a/semapact/platforms/databricks/runtime.py +++ b/semapact/platforms/databricks/runtime.py @@ -5,10 +5,13 @@ from datetime import datetime, timezone from typing import Any, Sequence -from semapact.observation.databricks import observe_databricks_table from semapact.observation.fingerprint import with_observed_state_fingerprint -from semapact.observation.models import ObservedAssetIdentity, ObservedPlatformState +from semapact.observation.models import ObservedAsset, ObservedAssetIdentity, ObservedPlatformState from semapact.observation.providers import RuntimeAssetBinding, RuntimeAssetSpec +from semapact.platforms.databricks.observation import ( + databricks_evidence_availability, + observe_databricks_table, +) from semapact.platforms.databricks.target import parse_databricks_runtime_target @@ -42,7 +45,7 @@ def resolve_bindings( ) for asset in assets ) - return tuple(sorted(bindings, key=lambda item: item.governed_asset)) + return tuple(sorted(bindings, key=_binding_sort_key)) def observe( self, @@ -51,10 +54,10 @@ def observe( ) -> ObservedPlatformState: """Observe every bound UC asset; missing tables remain absent evidence.""" captured_at = datetime.now(timezone.utc) - assets = [] + assets: list[ObservedAsset] = [] not_found_error = _load_databricks_not_found_error() - for binding in sorted(bindings, key=lambda item: item.governed_asset): + for binding in sorted(bindings, key=_binding_sort_key): identity = binding.observed_asset if identity.platform.casefold() != self.key: raise ValueError("Databricks provider received a non-Databricks binding") @@ -79,11 +82,16 @@ def observe( source_identifier=self._source_identifier, assets=tuple(assets), captured_at=captured_at, + evidence_availability=databricks_evidence_availability(self._client), fingerprint=None, ) return with_observed_state_fingerprint(state) +def _binding_sort_key(binding: RuntimeAssetBinding) -> str: + return binding.governed_asset + + def _load_databricks_not_found_error() -> type[BaseException]: try: from databricks.sdk.errors import NotFound diff --git a/tests/fixtures/history_golden/v1/review_multi_deploy.json b/tests/fixtures/history_golden/v1/review_multi_deploy.json index f8a3a3ee..02d2e165 100644 --- a/tests/fixtures/history_golden/v1/review_multi_deploy.json +++ b/tests/fixtures/history_golden/v1/review_multi_deploy.json @@ -65,20 +65,20 @@ "checksum": "sha256:961815c36c9f21e9ca1a277bb256bdc6571bcd94187a2712f333fa2bf6834b9d", "topLevelKeys": ["actual_version_bump", "applied_release_id", "authority_reference", "authorization_id", "change_set_id", "contract_id", "contract_version", "decision_id", "release_plan_id", "release_record_id", "released_revision_id", "required_version_bump", "review_evidence_action", "review_evidence_reference", "version_authority", "version_resolution_id"] }, - ".semapact/history/runtime_observations/28a82950-d467-57e8-803f-2b705b201780.json": { - "checksum": "sha256:82121d967e651fc1af170a184352e8c5379db6098c5ea569754db9bcf9cca0c2", + ".semapact/history/runtime_observations/38695939-4fb5-583c-acfa-1a21397d89c8.json": { + "checksum": "sha256:dcad511a463004815942e9404e2d6ad5edd7efeb95afa14926f2b08b2c0bc389", "topLevelKeys": ["observation", "observation_record_id"] }, - ".semapact/history/runtime_observations/6b0abefd-7cd6-5bcc-bd2d-714ae994735b.json": { - "checksum": "sha256:a7c61cb5589c804785346568eadef629477af3488709ecfc1f179684b2414999", + ".semapact/history/runtime_observations/e259c151-972f-532a-99d1-a14dcd78b166.json": { + "checksum": "sha256:d2af366ab491fdd0bfc123940f17f90257a456264343d017218c5791d2e1a6ad", "topLevelKeys": ["observation", "observation_record_id"] }, - ".semapact/history/runtime_reconciliations/0488eb0b-50c1-549a-af56-48807338098a.json": { - "checksum": "sha256:2fb8122baa8e16c488b73d782729cf9aaad5fc5d102bb88879bae8b9c0b522db", + ".semapact/history/runtime_reconciliations/1e53fecd-750f-556e-97a8-060dc05e3cc2.json": { + "checksum": "sha256:87f4076f1428aa835565ced57aedd8371ff4d01de144ce6c34d395b2bd6d1a97", "topLevelKeys": ["deployment_record_id", "observation_record_id", "release_record_id", "result", "runtime_reconciliation_record_id", "status"] }, - ".semapact/history/runtime_reconciliations/b4c58a42-bea7-57da-ab1d-88594e425dfb.json": { - "checksum": "sha256:6e41715135cf33b62f19a4a82949bc471a42b6e20a89aa52b0a79854bdd33727", + ".semapact/history/runtime_reconciliations/eeaf42b7-6d15-5b80-b1c3-ce51da73293f.json": { + "checksum": "sha256:041909fb90b3c4ad24e093883d4dfed0c18bb445fbb17e9a6c5c1a47e6b6b42e", "topLevelKeys": ["deployment_record_id", "observation_record_id", "release_record_id", "result", "runtime_reconciliation_record_id", "status"] } }, @@ -109,14 +109,14 @@ "status": "SUCCEEDED", "runtime": [ { - "reconciliationRecordId": "0488eb0b-50c1-549a-af56-48807338098a", - "observationRecordId": "28a82950-d467-57e8-803f-2b705b201780", - "status": "DRIFT" + "reconciliationRecordId": "1e53fecd-750f-556e-97a8-060dc05e3cc2", + "observationRecordId": "e259c151-972f-532a-99d1-a14dcd78b166", + "status": "IN_SYNC" }, { - "reconciliationRecordId": "b4c58a42-bea7-57da-ab1d-88594e425dfb", - "observationRecordId": "6b0abefd-7cd6-5bcc-bd2d-714ae994735b", - "status": "IN_SYNC" + "reconciliationRecordId": "eeaf42b7-6d15-5b80-b1c3-ce51da73293f", + "observationRecordId": "38695939-4fb5-583c-acfa-1a21397d89c8", + "status": "DRIFT" } ] } diff --git a/tests/fixtures/m1_reconciliation_golden.json b/tests/fixtures/m1_reconciliation_golden.json index 5be828c0..a77b5f81 100644 --- a/tests/fixtures/m1_reconciliation_golden.json +++ b/tests/fixtures/m1_reconciliation_golden.json @@ -65,7 +65,7 @@ "subject": "asset" } ], - "observation_fingerprint": "obs-v1:sha256:f95eb513603e9b6273fe7426789e6632de4038c2255bad4d92487a77121b6e0e", + "observation_fingerprint": "obs-v2:sha256:60be8d78af2750b7f2c467a5bca05accd4ac3c8807c7fbfac338f97baf6e6fb5", "observation_source_identifier": "https://adb.example", "unverified_paths": [ "schema[orders].properties[created_at].nullability", @@ -79,7 +79,7 @@ "contract_id": "orders-contract", "contract_version": "1.2.3", "differences": [], - "observation_fingerprint": "obs-v1:sha256:59d00bd9e734dfec75ad898cc61117a0cb7a7c7664c331aa3e4185e90fd3c14d", + "observation_fingerprint": "obs-v2:sha256:4e449f416efc0fa80a087c1cae27d1779aae1231994ac29f85e6d02489501c86", "observation_source_identifier": "https://adb.example", "unverified_paths": [] }, @@ -90,7 +90,7 @@ "contract_id": "orders-contract", "contract_version": "1.2.3", "differences": [], - "observation_fingerprint": "obs-v1:sha256:6fb881ad1e9807df3162a12d2bcd77cdb87a3b39f103b14ccd441d5d969e6bdf", + "observation_fingerprint": "obs-v2:sha256:00c4be0a6fb125118be257697d907e311c94273a5af60d4996e355481fc3131f", "observation_source_identifier": "https://adb.example", "unverified_paths": [ "schema[orders].properties[id].nullability", diff --git a/tests/fixtures/observation/databricks/orders_table_info.json b/tests/fixtures/observation/databricks/orders_table_info.json index 42b613bf..9dd6cbd3 100644 --- a/tests/fixtures/observation/databricks/orders_table_info.json +++ b/tests/fixtures/observation/databricks/orders_table_info.json @@ -4,27 +4,53 @@ "schema_name": "silver", "full_name": "main.silver.orders", "table_type": "MANAGED", + "owner": "data-platform@example.com", + "comment": "Curated customer orders", "columns": [ { "name": "customer_id", "type_name": "STRING", "type_text": "string", "position": 1, - "nullable": false + "nullable": false, + "comment": "Customer business identifier" }, { "name": "note", "type_name": "STRING", "type_text": "string", "position": 2, - "nullable": true + "nullable": true, + "comment": null }, { "name": "order_id", "type_name": "BIGINT", "type_text": "bigint", "position": 0, - "nullable": false + "nullable": false, + "comment": "Order business identifier" + } + ], + "table_constraints": [ + { + "foreign_key_constraint": { + "name": "fk_orders_customer", + "child_columns": ["customer_id"], + "parent_table": "main.silver.customers", + "parent_columns": ["customer_id"] + } + }, + { + "primary_key_constraint": { + "name": "pk_orders", + "child_columns": ["order_id"] + } + }, + { + "named_table_constraint": { + "name": "named_orders_constraint" + } } ] } diff --git a/tests/test_observation_classification.py b/tests/test_observation_classification.py new file mode 100644 index 00000000..44e788fa --- /dev/null +++ b/tests/test_observation_classification.py @@ -0,0 +1,219 @@ +from __future__ import annotations + +import inspect +from datetime import datetime, timezone + +import pytest +from pydantic import ValidationError + +from semapact.observation import canonical, classification, evidence, fingerprint, models, providers +from semapact.observation import ( + ObservedAsset, + ObservedAssetIdentity, + ObservedConstraint, + ObservedConstraintKind, + ObservedEvidenceAvailability, + ObservedEvidenceAvailabilityStatus, + ObservedEvidenceClass, + ObservedEvidenceKind, + ObservedPlatformState, + ObservedProperty, + ObservedPropertyIdentity, + ObservedRelationship, + ObservedRelationshipKind, + ObservedTag, + classify_observed_evidence, + summarize_observed_evidence, +) + +CAPTURED_AT = datetime(2026, 9, 15, 0, 0, tzinfo=timezone.utc) + + +def _availability( + *, + tag_status: ObservedEvidenceAvailabilityStatus = ObservedEvidenceAvailabilityStatus.AVAILABLE, +) -> tuple[ObservedEvidenceAvailability, ...]: + return tuple( + ObservedEvidenceAvailability( + kind=kind, + status=( + tag_status + if kind is ObservedEvidenceKind.TAG + else ObservedEvidenceAvailabilityStatus.AVAILABLE + ), + ) + for kind in ObservedEvidenceKind + ) + + +def _state(platform: str) -> ObservedPlatformState: + asset_identity = ObservedAssetIdentity( + platform=platform, + namespace=("governed",), + asset="orders", + ) + target_identity = ObservedAssetIdentity( + platform=platform, + namespace=("governed",), + asset="customers", + ) + properties = ( + ObservedProperty( + identity=ObservedPropertyIdentity(asset=asset_identity, property="order_id"), + physical_type="bigint", + nullable=False, + comment="Order identifier", + tags=(ObservedTag(key="Identifier", value="true"),), + ), + ObservedProperty( + identity=ObservedPropertyIdentity(asset=asset_identity, property="customer_id"), + physical_type="string", + nullable=False, + ), + ) + asset = ObservedAsset( + identity=asset_identity, + asset_type="TABLE", + owner="data-team@example.com", + comment="Orders", + tags=( + ObservedTag(key="Domain", value="Sales"), + ObservedTag(key="Sensitivity", value="Internal"), + ), + properties=properties, + constraints=( + ObservedConstraint( + kind=ObservedConstraintKind.PRIMARY_KEY, + properties=("order_id",), + ), + ), + relationships=( + ObservedRelationship( + kind=ObservedRelationshipKind.FOREIGN_KEY, + source_asset=asset_identity, + source_properties=("customer_id",), + target_asset=target_identity, + target_properties=("customer_id",), + ), + ), + ) + return ObservedPlatformState( + platform=platform, + source_identifier="test-source", + assets=(asset,), + captured_at=CAPTURED_AT, + evidence_availability=_availability(), + ) + + +def test_classification_is_provider_neutral_and_complete() -> None: + expected = { + ObservedEvidenceKind.PHYSICAL_SCHEMA: ObservedEvidenceClass.STRUCTURAL, + ObservedEvidenceKind.OWNER: ObservedEvidenceClass.OPERATIONAL, + ObservedEvidenceKind.COMMENT: ObservedEvidenceClass.SEMANTIC, + ObservedEvidenceKind.TAG: ObservedEvidenceClass.SEMANTIC, + ObservedEvidenceKind.CONSTRAINT: ObservedEvidenceClass.STRUCTURAL, + ObservedEvidenceKind.RELATIONSHIP: ObservedEvidenceClass.STRUCTURAL, + } + + assert { + kind: classify_observed_evidence(kind) for kind in ObservedEvidenceKind + } == expected + + +def test_standard_metrics_are_derived_from_canonical_observation() -> None: + metrics = summarize_observed_evidence(_state("example-platform")) + + assert metrics.total == 11 + assert {item.kind: item.count for item in metrics.by_kind} == { + ObservedEvidenceKind.PHYSICAL_SCHEMA: 3, + ObservedEvidenceKind.OWNER: 1, + ObservedEvidenceKind.COMMENT: 2, + ObservedEvidenceKind.TAG: 3, + ObservedEvidenceKind.CONSTRAINT: 1, + ObservedEvidenceKind.RELATIONSHIP: 1, + } + assert all( + item.availability is ObservedEvidenceAvailabilityStatus.AVAILABLE + for item in metrics.by_kind + ) + assert {item.evidence_class: item.count for item in metrics.by_class} == { + ObservedEvidenceClass.STRUCTURAL: 5, + ObservedEvidenceClass.SEMANTIC: 5, + ObservedEvidenceClass.OPERATIONAL: 1, + } + + +def test_zero_count_is_distinct_from_unavailable_evidence() -> None: + identity = ObservedAssetIdentity(platform="example", asset="orders") + asset = ObservedAsset(identity=identity) + available = ObservedPlatformState( + platform="example", + source_identifier="source", + assets=(asset,), + captured_at=CAPTURED_AT, + evidence_availability=_availability(), + ) + unavailable = available.model_copy( + update={ + "evidence_availability": _availability( + tag_status=ObservedEvidenceAvailabilityStatus.UNAVAILABLE + ) + } + ) + + available_tag = next( + item + for item in summarize_observed_evidence(available).by_kind + if item.kind is ObservedEvidenceKind.TAG + ) + unavailable_tag = next( + item + for item in summarize_observed_evidence(unavailable).by_kind + if item.kind is ObservedEvidenceKind.TAG + ) + + assert available_tag.count == unavailable_tag.count == 0 + assert available_tag.availability is ObservedEvidenceAvailabilityStatus.AVAILABLE + assert unavailable_tag.availability is ObservedEvidenceAvailabilityStatus.UNAVAILABLE + + +def test_equivalent_platform_observations_receive_the_same_metrics() -> None: + first = summarize_observed_evidence(_state("platform-a")) + second = summarize_observed_evidence(_state("platform-b")) + + assert first == second + + +def test_canonical_observation_rejects_nested_identity_mismatch() -> None: + container = ObservedAssetIdentity(platform="example", asset="orders") + other = ObservedAssetIdentity(platform="example", asset="customers") + + with pytest.raises(ValidationError, match="containing asset"): + ObservedAsset( + identity=container, + properties=( + ObservedProperty( + identity=ObservedPropertyIdentity(asset=other, property="id") + ), + ), + ) + + +def test_canonical_observation_rejects_platform_mismatch() -> None: + with pytest.raises(ValidationError, match="platform must match"): + ObservedPlatformState( + platform="platform-a", + source_identifier="source", + assets=( + ObservedAsset( + identity=ObservedAssetIdentity(platform="platform-b", asset="orders") + ), + ), + captured_at=CAPTURED_AT, + ) + + +def test_observation_core_does_not_depend_on_platform_adapters() -> None: + for module in (canonical, classification, evidence, fingerprint, models, providers): + assert "semapact.platforms" not in inspect.getsource(module) diff --git a/tests/test_observation_databricks.py b/tests/test_observation_databricks.py index bddcf7ea..177e6e83 100644 --- a/tests/test_observation_databricks.py +++ b/tests/test_observation_databricks.py @@ -8,12 +8,23 @@ import pytest from pydantic import ValidationError -from semapact.observation import databricks as databricks_observation -from semapact.observation.databricks import ( +from semapact.observation import ( + ObservedEvidenceAvailabilityStatus, + ObservedEvidenceKind, + summarize_observed_evidence, +) +from semapact.observation.models import ( + ObservedConstraintKind, + ObservedPlatformState, + ObservedRelationshipDirection, + ObservedRelationshipKind, + serialize_observed_state, +) +from semapact.platforms.databricks import observation as databricks_observation +from semapact.platforms.databricks.observation import ( map_databricks_table_info, observe_databricks_table, ) -from semapact.observation.models import ObservedPlatformState, serialize_observed_state FIXTURE = ( Path(__file__).parent @@ -29,7 +40,7 @@ def _payload() -> dict[str, object]: return json.loads(FIXTURE.read_text(encoding="utf-8")) -class _FakeTableInfo: +class _FakeSdkObject: def __init__(self, payload: dict[str, object]) -> None: self._payload = payload @@ -38,23 +49,47 @@ def as_dict(self) -> dict[str, object]: class _FakeTablesApi: - def __init__(self, table: _FakeTableInfo) -> None: + def __init__(self, table: _FakeSdkObject) -> None: self._table = table self.calls: list[str] = [] - def get(self, full_name: str) -> _FakeTableInfo: + def get(self, full_name: str) -> _FakeSdkObject: self.calls.append(full_name) return self._table +class _FakeEntityTagsApi: + def __init__(self, assignments: dict[tuple[str, str], list[dict[str, object]]]) -> None: + self._assignments = assignments + self.calls: list[tuple[str, str]] = [] + + def list(self, entity_type: str, entity_name: str): + self.calls.append((entity_type, entity_name)) + return ( + _FakeSdkObject(item) + for item in self._assignments.get((entity_type, entity_name), []) + ) + + class _FakeWorkspaceClient: - def __init__(self, table: _FakeTableInfo) -> None: + def __init__( + self, + table: _FakeSdkObject, + *, + assignments: dict[tuple[str, str], list[dict[str, object]]] | None = None, + ) -> None: self.tables = _FakeTablesApi(table) + if assignments is not None: + self.entity_tag_assignments = _FakeEntityTagsApi(assignments) + + +def _availability_by_kind(state: ObservedPlatformState): + return {item.kind: item.status for item in state.evidence_availability} def test_databricks_table_info_maps_to_platform_neutral_observation() -> None: state = map_databricks_table_info( - _FakeTableInfo(_payload()), + _FakeSdkObject(_payload()), source_identifier="https://adb.example/", captured_at=CAPTURED_AT, ) @@ -64,10 +99,18 @@ def test_databricks_table_info_maps_to_platform_neutral_observation() -> None: assert state.source_identifier == "https://adb.example" assert state.captured_at == CAPTURED_AT assert state.fingerprint == ( - "obs-v1:sha256:" - "cc72fb5e20b673784b0847f6f903c05178b88c3d88ffb25f830da0a373457287" + "obs-v2:sha256:" + "2b4f9ae77d71ee7623d2cd6e152b86bccdbe14d850902631fd93c8040df72660" ) + availability = _availability_by_kind(state) + assert availability[ObservedEvidenceKind.PHYSICAL_SCHEMA] is ObservedEvidenceAvailabilityStatus.AVAILABLE + assert availability[ObservedEvidenceKind.OWNER] is ObservedEvidenceAvailabilityStatus.AVAILABLE + assert availability[ObservedEvidenceKind.COMMENT] is ObservedEvidenceAvailabilityStatus.AVAILABLE + assert availability[ObservedEvidenceKind.TAG] is ObservedEvidenceAvailabilityStatus.UNKNOWN + assert availability[ObservedEvidenceKind.CONSTRAINT] is ObservedEvidenceAvailabilityStatus.AVAILABLE + assert availability[ObservedEvidenceKind.RELATIONSHIP] is ObservedEvidenceAvailabilityStatus.AVAILABLE + assert len(state.assets) == 1 asset = state.assets[0] assert asset.identity.platform == "databricks" @@ -80,6 +123,8 @@ def test_databricks_table_info_maps_to_platform_neutral_observation() -> None: "orders", ) assert asset.asset_type == "MANAGED" + assert asset.owner == "data-platform@example.com" + assert asset.comment == "Curated customer orders" assert [item.identity.property for item in asset.properties] == [ "order_id", @@ -88,13 +133,47 @@ def test_databricks_table_info_maps_to_platform_neutral_observation() -> None: ] assert asset.properties[0].physical_type == "bigint" assert asset.properties[0].nullable is False + assert asset.properties[0].comment == "Order business identifier" assert asset.properties[1].physical_type == "string" + assert asset.properties[1].comment == "Customer business identifier" assert asset.properties[2].nullable is True + assert [item.kind for item in asset.constraints] == [ + ObservedConstraintKind.FOREIGN_KEY, + ObservedConstraintKind.NAMED, + ObservedConstraintKind.PRIMARY_KEY, + ] + primary_key = next( + item for item in asset.constraints if item.kind is ObservedConstraintKind.PRIMARY_KEY + ) + assert primary_key.name == "pk_orders" + assert primary_key.properties == ("order_id",) + assert primary_key.provenance == "unity_catalog" + foreign_key = next( + item for item in asset.constraints if item.kind is ObservedConstraintKind.FOREIGN_KEY + ) + assert foreign_key.name == "fk_orders_customer" + assert foreign_key.properties == ("customer_id",) + assert foreign_key.provenance == "unity_catalog" + + assert len(asset.relationships) == 1 + relationship = asset.relationships[0] + assert relationship.kind is ObservedRelationshipKind.FOREIGN_KEY + assert relationship.direction is ObservedRelationshipDirection.OUTBOUND + assert relationship.name == "fk_orders_customer" + assert relationship.source_asset == asset.identity + assert relationship.source_properties == ("customer_id",) + assert relationship.target_asset is not None + assert relationship.target_asset.namespace == ("main", "silver") + assert relationship.target_asset.asset == "customers" + assert relationship.target_properties == ("customer_id",) + assert relationship.target_reference == "main.silver.customers" + assert relationship.provenance == "unity_catalog" + def test_observation_model_identity_does_not_encode_databricks_namespace_names() -> None: state = map_databricks_table_info( - _FakeTableInfo(_payload()), + _FakeSdkObject(_payload()), source_identifier="workspace-a", captured_at=CAPTURED_AT, ) @@ -105,18 +184,21 @@ def test_observation_model_identity_does_not_encode_databricks_namespace_names() assert "schema" not in identity_fields -def test_serialization_is_stable_when_databricks_column_order_changes() -> None: +def test_serialization_is_stable_when_databricks_payload_order_changes() -> None: payload = _payload() reordered = dict(payload) reordered["columns"] = list(reversed(payload["columns"])) # type: ignore[index] + reordered["table_constraints"] = list( # type: ignore[index] + reversed(payload["table_constraints"]) # type: ignore[index] + ) left = map_databricks_table_info( - _FakeTableInfo(payload), + _FakeSdkObject(payload), source_identifier="https://adb.example", captured_at=CAPTURED_AT, ) right = map_databricks_table_info( - _FakeTableInfo(reordered), + _FakeSdkObject(reordered), source_identifier="https://adb.example", captured_at=CAPTURED_AT, ) @@ -126,8 +208,18 @@ def test_serialization_is_stable_when_databricks_column_order_changes() -> None: assert serialize_observed_state(left) == serialize_observed_state(right) -def test_observe_databricks_table_uses_workspace_client_tables_get() -> None: - client = _FakeWorkspaceClient(_FakeTableInfo(_payload())) +def test_observe_databricks_table_reads_and_normalizes_table_and_column_tags() -> None: + assignments = { + ("tables", "main.silver.orders"): [ + {"tag_key": "Domain", "tag_value": "Sales"}, + {"tag_key": "Sensitivity", "tag_value": "Internal"}, + {"tag_key": "Domain", "tag_value": "Sales"}, + ], + ("columns", "main.silver.orders.customer_id"): [ + {"tag_key": "PII", "tag_value": "true"}, + ], + } + client = _FakeWorkspaceClient(_FakeSdkObject(_payload()), assignments=assignments) state = observe_databricks_table( client=client, @@ -138,7 +230,108 @@ def test_observe_databricks_table_uses_workspace_client_tables_get() -> None: assert client.tables.calls == ["main.silver.orders"] assert state.assets[0].identity.asset == "orders" + assert [(item.key, item.value) for item in state.assets[0].tags] == [ + ("Domain", "Sales"), + ("Sensitivity", "Internal"), + ] + customer = next( + item for item in state.assets[0].properties if item.identity.property == "customer_id" + ) + assert [(item.key, item.value) for item in customer.tags] == [("PII", "true")] + assert all(item.provenance == "unity_catalog" for item in state.assets[0].tags) assert state.fingerprint is not None + metrics = summarize_observed_evidence(state) + tag_metric = next(item for item in metrics.by_kind if item.kind is ObservedEvidenceKind.TAG) + assert tag_metric.availability is ObservedEvidenceAvailabilityStatus.AVAILABLE + assert tag_metric.count == 3 + + tag_calls = client.entity_tag_assignments.calls + assert ("tables", "main.silver.orders") in tag_calls + assert ("columns", "main.silver.orders.order_id") in tag_calls + assert ("columns", "main.silver.orders.customer_id") in tag_calls + assert ("columns", "main.silver.orders.note") in tag_calls + + +def test_tag_api_iteration_order_does_not_change_observation() -> None: + ordered = [ + {"tag_key": "A", "tag_value": "1"}, + {"tag_key": "B", "tag_value": "2"}, + ] + left = _FakeWorkspaceClient( + _FakeSdkObject(_payload()), + assignments={("tables", "main.silver.orders"): ordered}, + ) + right = _FakeWorkspaceClient( + _FakeSdkObject(_payload()), + assignments={("tables", "main.silver.orders"): list(reversed(ordered))}, + ) + + first = observe_databricks_table( + client=left, + table_fqn="main.silver.orders", + source_identifier="workspace-a", + captured_at=CAPTURED_AT, + ) + second = observe_databricks_table( + client=right, + table_fqn="main.silver.orders", + source_identifier="workspace-a", + captured_at=CAPTURED_AT, + ) + + assert first == second + assert first.fingerprint == second.fingerprint + + +def test_observe_databricks_table_marks_missing_tag_capability_unavailable() -> None: + client = _FakeWorkspaceClient(_FakeSdkObject(_payload())) + + state = observe_databricks_table( + client=client, + table_fqn="main.silver.orders", + source_identifier="https://adb.example", + captured_at=CAPTURED_AT, + ) + + assert state.assets[0].tags == () + assert all(item.tags == () for item in state.assets[0].properties) + tag_metric = next( + item + for item in summarize_observed_evidence(state).by_kind + if item.kind is ObservedEvidenceKind.TAG + ) + assert tag_metric.count == 0 + assert tag_metric.availability is ObservedEvidenceAvailabilityStatus.UNAVAILABLE + + +def test_foreign_key_with_unresolved_parent_preserves_partial_evidence() -> None: + payload = _payload() + payload["table_constraints"] = [ + { + "foreign_key_constraint": { + "name": "fk_partial", + "child_columns": ["customer_id"], + "parent_table": "unqualified-parent", + "parent_columns": [], + } + } + ] + + state = map_databricks_table_info( + _FakeSdkObject(payload), + source_identifier="workspace-a", + captured_at=CAPTURED_AT, + ) + relationship = state.assets[0].relationships[0] + + assert state.assets[0].constraints == ( + state.assets[0].constraints[0], + ) + assert state.assets[0].constraints[0].kind is ObservedConstraintKind.FOREIGN_KEY + assert state.assets[0].constraints[0].properties == ("customer_id",) + assert relationship.target_asset is None + assert relationship.target_reference == "unqualified-parent" + assert relationship.target_properties == () def test_mapper_accepts_official_databricks_sdk_table_info_when_extra_is_installed() -> None: @@ -153,12 +346,17 @@ def test_mapper_accepts_official_databricks_sdk_table_info_when_extra_is_install assert state.assets[0].identity.namespace == ("main", "silver") assert state.assets[0].properties[0].identity.property == "order_id" + assert state.assets[0].owner == "data-platform@example.com" + assert next( + item for item in state.assets[0].constraints if item.kind is ObservedConstraintKind.FOREIGN_KEY + ).name == "fk_orders_customer" + assert state.assets[0].relationships[0].name == "fk_orders_customer" assert state.fingerprint is not None def test_observation_models_are_immutable() -> None: state = map_databricks_table_info( - _FakeTableInfo(_payload()), + _FakeSdkObject(_payload()), source_identifier="https://adb.example", captured_at=CAPTURED_AT, ) @@ -180,7 +378,7 @@ def test_databricks_observer_does_not_depend_on_contract_projection_layers() -> def test_observation_requires_timezone_aware_capture_time() -> None: with pytest.raises(ValueError, match="timezone-aware"): map_databricks_table_info( - _FakeTableInfo(_payload()), + _FakeSdkObject(_payload()), source_identifier="https://adb.example", captured_at=datetime(2026, 8, 30, 3, 0), ) diff --git a/tests/test_observation_fingerprint.py b/tests/test_observation_fingerprint.py index 11b03e20..b93025e7 100644 --- a/tests/test_observation_fingerprint.py +++ b/tests/test_observation_fingerprint.py @@ -2,6 +2,11 @@ from datetime import datetime, timezone +from semapact.observation.evidence import ( + ObservedEvidenceAvailability, + ObservedEvidenceAvailabilityStatus, + ObservedEvidenceKind, +) from semapact.observation.fingerprint import ( canonical_observed_state_payload, fingerprint_observed_state, @@ -10,9 +15,14 @@ from semapact.observation.models import ( ObservedAsset, ObservedAssetIdentity, + ObservedConstraint, + ObservedConstraintKind, ObservedPlatformState, ObservedProperty, ObservedPropertyIdentity, + ObservedRelationship, + ObservedRelationshipKind, + ObservedTag, ) CAPTURED_AT = datetime(2026, 8, 30, 3, 0, tzinfo=timezone.utc) @@ -27,6 +37,11 @@ def _orders_asset( physical_types: dict[str, str] | None = None, nullable: dict[str, bool | None] | None = None, asset_type: str = "MANAGED", + owner: str | None = None, + comment: str | None = None, + tags: tuple[ObservedTag, ...] = (), + constraints: tuple[ObservedConstraint, ...] = (), + relationships: tuple[ObservedRelationship, ...] = (), ) -> ObservedAsset: identity = ObservedAssetIdentity( platform=platform, @@ -46,6 +61,9 @@ def _orders_asset( return ObservedAsset( identity=identity, asset_type=asset_type, + owner=owner, + comment=comment, + tags=tags, properties=tuple( ObservedProperty( identity=ObservedPropertyIdentity(asset=identity, property=name), @@ -54,6 +72,8 @@ def _orders_asset( ) for name in property_order ), + constraints=constraints, + relationships=relationships, ) @@ -63,6 +83,7 @@ def _state( platform: str = "databricks", source_identifier: str = "https://adb.example", captured_at: datetime = CAPTURED_AT, + evidence_availability: tuple[ObservedEvidenceAvailability, ...] = (), fingerprint: str | None = None, ) -> ObservedPlatformState: return ObservedPlatformState( @@ -70,6 +91,7 @@ def _state( source_identifier=source_identifier, assets=assets or (_orders_asset(),), captured_at=captured_at, + evidence_availability=evidence_availability, fingerprint=fingerprint, ) @@ -78,10 +100,10 @@ def test_observed_state_fingerprint_has_stable_versioned_golden_value() -> None: state = _state() assert fingerprint_observed_state(state) == ( - "obs-v1:sha256:" - "cc72fb5e20b673784b0847f6f903c05178b88c3d88ffb25f830da0a373457287" + "obs-v2:sha256:" + "3d0e3c3c84d97eb1b1f494625a2d7fb1cd174d2ba248545fe4d54928a2d19a1b" ) - assert canonical_observed_state_payload(state)["fingerprint_version"] == "obs-v1" + assert canonical_observed_state_payload(state)["fingerprint_version"] == "obs-v2" def test_observation_envelope_fields_do_not_change_semantic_fingerprint() -> None: @@ -109,6 +131,12 @@ def test_asset_property_order_and_identity_case_do_not_change_fingerprint() -> N namespace=("MAIN", "SILVER"), asset_name="ORDERS", property_order=("note", "customer_id", "order_id"), + asset_type="managed", + physical_types={ + "order_id": "BIGINT", + "customer_id": "STRING", + "note": "STRING", + }, ) left = _state(assets=(first, second)) @@ -167,6 +195,155 @@ def test_governance_relevant_observed_changes_change_fingerprint() -> None: assert fingerprint_observed_state(changed_property_identity) != baseline +def test_governance_metadata_changes_change_fingerprint() -> None: + baseline = fingerprint_observed_state(_state()) + identity = _orders_asset().identity + customer = ObservedAssetIdentity( + platform="databricks", + namespace=("main", "silver"), + asset="customers", + ) + + variants = ( + _orders_asset(owner="data-team@example.com"), + _orders_asset(comment="Curated orders"), + _orders_asset(tags=(ObservedTag(key="Domain", value="Sales"),)), + _orders_asset( + constraints=( + ObservedConstraint( + kind=ObservedConstraintKind.PRIMARY_KEY, + properties=("order_id",), + name="pk_orders", + ), + ) + ), + _orders_asset( + relationships=( + ObservedRelationship( + kind=ObservedRelationshipKind.FOREIGN_KEY, + source_asset=identity, + source_properties=("customer_id",), + target_asset=customer, + target_properties=("customer_id",), + name="fk_orders_customer", + ), + ) + ), + ) + + assert all( + fingerprint_observed_state(_state(assets=(asset,))) != baseline + for asset in variants + ) + + +def test_evidence_availability_changes_semantic_fingerprint() -> None: + available = ( + ObservedEvidenceAvailability( + kind=ObservedEvidenceKind.TAG, + status=ObservedEvidenceAvailabilityStatus.AVAILABLE, + ), + ) + unavailable = ( + ObservedEvidenceAvailability( + kind=ObservedEvidenceKind.TAG, + status=ObservedEvidenceAvailabilityStatus.UNAVAILABLE, + ), + ) + + assert fingerprint_observed_state( + _state(evidence_availability=available) + ) != fingerprint_observed_state(_state(evidence_availability=unavailable)) + + +def test_tag_constraint_and_relationship_order_do_not_change_fingerprint() -> None: + identity = _orders_asset().identity + customer = ObservedAssetIdentity( + platform="databricks", + namespace=("main", "silver"), + asset="customers", + ) + tags = ( + ObservedTag(key="Domain", value="Sales"), + ObservedTag(key="Sensitivity", value="Internal"), + ) + constraints = ( + ObservedConstraint( + kind=ObservedConstraintKind.PRIMARY_KEY, + properties=("order_id",), + name="pk_orders", + ), + ObservedConstraint( + kind=ObservedConstraintKind.NAMED, + name="constraint_b", + ), + ) + relationships = ( + ObservedRelationship( + kind=ObservedRelationshipKind.FOREIGN_KEY, + source_asset=identity, + source_properties=("customer_id",), + target_asset=customer, + target_properties=("customer_id",), + name="fk_customer", + ), + ObservedRelationship( + kind=ObservedRelationshipKind.FOREIGN_KEY, + source_asset=identity, + source_properties=("order_id",), + target_asset=customer, + target_properties=("legacy_order_id",), + name="fk_legacy_order", + ), + ) + + left = _state( + assets=( + _orders_asset( + tags=tags, + constraints=constraints, + relationships=relationships, + ), + ) + ) + right = _state( + assets=( + _orders_asset( + tags=tuple(reversed(tags)), + constraints=tuple(reversed(constraints)), + relationships=tuple(reversed(relationships)), + ), + ) + ) + + assert fingerprint_observed_state(left) == fingerprint_observed_state(right) + + +def test_resolved_relationship_reference_spelling_does_not_change_fingerprint() -> None: + source = _orders_asset().identity + target = ObservedAssetIdentity( + platform="databricks", + namespace=("main", "silver"), + asset="customers", + ) + + def state_for(reference: str) -> ObservedPlatformState: + relationship = ObservedRelationship( + kind=ObservedRelationshipKind.FOREIGN_KEY, + source_asset=source, + source_properties=("customer_id",), + target_asset=target, + target_properties=("customer_id",), + target_reference=reference, + name="fk_orders_customer", + ) + return _state(assets=(_orders_asset(relationships=(relationship,)),)) + + assert fingerprint_observed_state( + state_for("main.silver.customers") + ) == fingerprint_observed_state(state_for("MAIN.SILVER.CUSTOMERS")) + + def test_with_observed_state_fingerprint_returns_immutable_copy() -> None: state = _state() diff --git a/tests/test_runtime_history.py b/tests/test_runtime_history.py index af9424da..8172b4fb 100644 --- a/tests/test_runtime_history.py +++ b/tests/test_runtime_history.py @@ -13,6 +13,7 @@ ObservedAsset, ObservedAssetIdentity, ObservedPlatformState, + fingerprint_observed_state, with_observed_state_fingerprint, ) from semapact.platforms.git import GitWorkingTreeHistoryRepository @@ -26,23 +27,32 @@ ) -def _observation(at: datetime, *, asset_type: str = "TABLE", source: str = "workspace:test") -> ObservedPlatformState: - return with_observed_state_fingerprint( - ObservedPlatformState( - platform="databricks", - source_identifier=source, - assets=( - ObservedAsset( - identity=ObservedAssetIdentity( - platform="databricks", - namespace=("catalog", "schema"), - asset="orders", - ), - asset_type=asset_type, +def _raw_observation( + at: datetime, + *, + asset_type: str = "TABLE", + source: str = "workspace:test", +) -> ObservedPlatformState: + return ObservedPlatformState( + platform="databricks", + source_identifier=source, + assets=( + ObservedAsset( + identity=ObservedAssetIdentity( + platform="databricks", + namespace=("catalog", "schema"), + asset="orders", ), + asset_type=asset_type, ), - captured_at=at, - ) + ), + captured_at=at, + ) + + +def _observation(at: datetime, *, asset_type: str = "TABLE", source: str = "workspace:test") -> ObservedPlatformState: + return with_observed_state_fingerprint( + _raw_observation(at, asset_type=asset_type, source=source) ) @@ -153,6 +163,40 @@ def test_round_trip_and_idempotency(tmp_path: Path) -> None: assert first.status is RuntimeDriftStatus.IN_SYNC +def test_missing_fingerprint_is_materialized_before_history_persistence(tmp_path: Path) -> None: + service, backend = _service(tmp_path) + observation = _raw_observation(datetime(2026, 9, 13, 1, tzinfo=timezone.utc)) + semantic_fingerprint = fingerprint_observed_state(observation) + result = ReconciliationResult( + contract_id="orders-product", + contract_version="1.3.0", + observation_source_identifier=observation.source_identifier, + observation_fingerprint=semantic_fingerprint, + ) + + record = service.record_reconciliation(observation, result) + persisted = backend.get_runtime_observation_record(record.observation_record_id) + + assert observation.fingerprint is None + assert persisted.observation.fingerprint == semantic_fingerprint + + +def test_stale_materialized_fingerprint_is_rejected(tmp_path: Path) -> None: + service, _ = _service(tmp_path) + observation = _raw_observation(datetime(2026, 9, 13, 1, tzinfo=timezone.utc)).model_copy( + update={"fingerprint": "stale"} + ) + result = ReconciliationResult( + contract_id="orders-product", + contract_version="1.3.0", + observation_source_identifier=observation.source_identifier, + observation_fingerprint=fingerprint_observed_state(observation), + ) + + with pytest.raises(ValueError, match="canonical semantic content"): + service.record_reconciliation(observation, result) + + def test_same_semantic_state_at_different_times_keeps_both_observations(tmp_path: Path) -> None: service, backend = _service(tmp_path) first_observation = _observation(datetime(2026, 9, 13, 1, tzinfo=timezone.utc)) From 5fa5c233c28394c76f4fbfe19c55ab7128585e7a Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Tue, 15 Sep 2026 20:54:06 +1000 Subject: [PATCH 26/35] feat(observation): capture Unity Catalog lineage evidence (#233) * feat(observation): model normalized lineage evidence * feat(databricks): normalize lineage system-table evidence * feat(observation): export lineage evidence API * refactor(import): project normalized lineage into legacy ODCS fields * test(observation): cover normalized Databricks lineage evidence * test(import): preserve lineage enrichment compatibility * feat(databricks): export lineage observation capability * test(observation): add Databricks lineage fixture * test(observation): drive lineage tests from fixture --- semapact/importers/unity_lineage.py | 164 +++++---- semapact/observation/__init__.py | 18 + semapact/observation/lineage.py | 252 ++++++++++++++ semapact/platforms/databricks/__init__.py | 2 + semapact/platforms/databricks/lineage.py | 313 ++++++++++++++++++ .../databricks/lineage_orders.json | 54 +++ tests/test_observation_databricks_lineage.py | 159 +++++++++ tests/test_unity_lineage.py | 47 ++- 8 files changed, 913 insertions(+), 96 deletions(-) create mode 100644 semapact/observation/lineage.py create mode 100644 semapact/platforms/databricks/lineage.py create mode 100644 tests/fixtures/observation/databricks/lineage_orders.json create mode 100644 tests/test_observation_databricks_lineage.py diff --git a/semapact/importers/unity_lineage.py b/semapact/importers/unity_lineage.py index b3a49206..19381092 100644 --- a/semapact/importers/unity_lineage.py +++ b/semapact/importers/unity_lineage.py @@ -1,10 +1,16 @@ from __future__ import annotations import logging -from typing import Any from open_data_contract_standard.model import OpenDataContractStandard, SchemaObject +from semapact.observation.lineage import ( + ObservedLineageEvidence, + ObservedLineageEvidenceType, + ObservedLineageResult, +) +from semapact.platforms.databricks.lineage import observe_databricks_lineage + LOGGER = logging.getLogger(__name__) @@ -16,7 +22,18 @@ def enrich_unity_lineage( token: str, sql_http_path: str | None = None, ) -> OpenDataContractStandard: - """Extract lineage and logic from Unity Catalog using system tables.""" + """Compatibility projection of normalized Databricks lineage into ODCS fields. + + Runtime lineage retrieval is owned by the read-only Databricks lineage adapter. + This legacy importer remains available for callers that explicitly want ODCS + enrichment; it no longer defines or retrieves lineage semantics itself. + """ + if not sql_http_path: + LOGGER.warning( + "Skipping lineage extraction: --sql-http-path is required when using databricks-sql-connector" + ) + return contract + try: from databricks import sql except ImportError as exc: @@ -25,13 +42,6 @@ def enrich_unity_lineage( "Please install it using: pip install databricks-sql-connector (or install with the [databricks] extra)." ) from exc - if not sql_http_path: - LOGGER.warning( - "Skipping lineage extraction: --sql-http-path is required when using databricks-sql-connector" - ) - return contract - - # Remove protocol prefix if accidentally included server_hostname = ( workspace_url.replace("https://", "").replace("http://", "").rstrip("/") ) @@ -43,8 +53,12 @@ def enrich_unity_lineage( access_token=token, ) as connection: with connection.cursor() as cursor: - _apply_lineage_to_contract(contract, table_fqn, cursor) - _apply_logic_to_contract(contract, table_fqn, cursor) + result = observe_databricks_lineage( + cursor=cursor, + table_fqn=table_fqn, + source_identifier=workspace_url, + ) + _apply_lineage_evidence(contract, table_fqn=table_fqn, result=result) except Exception as exc: LOGGER.warning( "Failed to fetch lineage or logic from Databricks system tables: %s", exc @@ -53,91 +67,73 @@ def enrich_unity_lineage( return contract -def _apply_lineage_to_contract( - contract: OpenDataContractStandard, table_fqn: str, cursor: Any +def _apply_lineage_evidence( + contract: OpenDataContractStandard, + *, + table_fqn: str, + result: ObservedLineageResult, ) -> None: schema_obj = _resolve_target_schema(contract, table_fqn=table_fqn) if schema_obj is None: return - # Query system.access.column_lineage - query = """ - SELECT source_table_full_name, source_column_name, target_column_name - FROM system.access.column_lineage - WHERE target_table_full_name = ? - """ - try: - cursor.execute(query, (table_fqn,)) - rows = cursor.fetchall() - except Exception as exc: - LOGGER.warning("Failed to query system.access.column_lineage: %s", exc) - return - - col_mapping: dict[str, list[str]] = {} - - for row in rows: - source_table = row.source_table_full_name - source_column = row.source_column_name - target_column = row.target_column_name - - if not target_column or not source_column or not source_table: - continue - - target_col_lower = target_column.lower() - if target_col_lower not in col_mapping: - col_mapping[target_col_lower] = [] - - full_source = f"{source_table}.{source_column}" - if full_source not in col_mapping[target_col_lower]: - col_mapping[target_col_lower].append(full_source) - + target_key = table_fqn.casefold() fields = { - item.name.lower(): item for item in (schema_obj.properties or []) if item.name + item.name.casefold(): item + for item in (schema_obj.properties or []) + if item.name } - for col_name, sources in col_mapping.items(): - property_obj = fields.get(col_name) - if property_obj: - existing = property_obj.transformSourceObjects or [] - for s in sources: - if s not in existing: - existing.append(s) - if existing: - property_obj.transformSourceObjects = existing + for item in result.evidence: + if item.evidence_type is not ObservedLineageEvidenceType.COLUMN: + continue + if _asset_reference(item, target=True).casefold() != target_key: + continue + if not item.target_property or not item.source_property: + continue + source_table = _asset_reference(item, target=False) + if not source_table: + continue + property_obj = fields.get(item.target_property.casefold()) + if property_obj is None: + continue + source = f"{source_table}.{item.source_property}" + existing = list(property_obj.transformSourceObjects or []) + if source not in existing: + existing.append(source) + property_obj.transformSourceObjects = existing + + # Legacy compatibility only: an explicit enrichment request may project the + # latest query text into transformLogic. Runtime observation keeps this text + # as non-authoritative QUERY evidence and never mutates the contract. + query_evidence = [ + item + for item in result.evidence + if item.evidence_type is ObservedLineageEvidenceType.QUERY + and item.statement_text + and _asset_reference(item, target=True).casefold() == target_key + ] + if query_evidence: + latest = max(query_evidence, key=_query_recency_key) + for prop in schema_obj.properties or []: + if not prop.transformLogic: + prop.transformLogic = latest.statement_text -def _apply_logic_to_contract( - contract: OpenDataContractStandard, table_fqn: str, cursor: Any -) -> None: - schema_obj = _resolve_target_schema(contract, table_fqn=table_fqn) - if schema_obj is None: - return +def _query_recency_key(item: ObservedLineageEvidence) -> tuple[str, str]: + recorded_at = item.capture_context.recorded_at + return ( + recorded_at.isoformat() if recorded_at is not None else "", + item.statement_reference or "", + ) - # We want to find the latest statement that wrote to this table - # We can join table_lineage with query_history. - # Note: Using `target_table_full_name = ?` to find writes. - query = """ - SELECT qh.statement_text - FROM system.access.table_lineage tl - JOIN system.access.query_history qh ON tl.statement_id = qh.statement_id - WHERE tl.target_table_full_name = ? - AND tl.source_table_full_name IS NOT NULL - ORDER BY tl.event_time DESC - LIMIT 1 - """ - try: - cursor.execute(query, (table_fqn,)) - row = cursor.fetchone() - except Exception as exc: - LOGGER.warning("Failed to query system.access.query_history for logic: %s", exc) - return - if row and row.statement_text: - statement = row.statement_text - # Apply the logic to all properties that don't have one - for prop in schema_obj.properties or []: - if not prop.transformLogic: - prop.transformLogic = statement +def _asset_reference(item: ObservedLineageEvidence, *, target: bool) -> str: + asset = item.target_asset if target else item.source_asset + reference = item.target_reference if target else item.source_reference + if asset is not None: + return ".".join((*asset.namespace, asset.asset)) + return reference or "" def _resolve_target_schema( diff --git a/semapact/observation/__init__.py b/semapact/observation/__init__.py index 2cc2538b..3eb2d9d8 100644 --- a/semapact/observation/__init__.py +++ b/semapact/observation/__init__.py @@ -20,6 +20,16 @@ fingerprint_observed_state, with_observed_state_fingerprint, ) +from semapact.observation.lineage import ( + ObservedLineageAvailability, + ObservedLineageCaptureContext, + ObservedLineageEvidence, + ObservedLineageEvidenceType, + ObservedLineageResult, + canonical_lineage_evidence_payload, + normalize_lineage_evidence, + serialize_observed_lineage_result, +) from semapact.observation.models import ( ObservedAsset, ObservedAssetIdentity, @@ -55,6 +65,11 @@ "ObservedEvidenceCount", "ObservedEvidenceKind", "ObservedEvidenceMetrics", + "ObservedLineageAvailability", + "ObservedLineageCaptureContext", + "ObservedLineageEvidence", + "ObservedLineageEvidenceType", + "ObservedLineageResult", "ObservedPlatformState", "ObservedProperty", "ObservedPropertyIdentity", @@ -66,9 +81,12 @@ "RuntimeAssetSpec", "RuntimeProvider", "RuntimeProviderRegistry", + "canonical_lineage_evidence_payload", "canonical_observed_state_payload", "classify_observed_evidence", "fingerprint_observed_state", + "normalize_lineage_evidence", + "serialize_observed_lineage_result", "serialize_observed_state", "summarize_observed_evidence", "with_observed_state_fingerprint", diff --git a/semapact/observation/lineage.py b/semapact/observation/lineage.py new file mode 100644 index 00000000..64b45957 --- /dev/null +++ b/semapact/observation/lineage.py @@ -0,0 +1,252 @@ +"""Platform-neutral lineage evidence for runtime governance observation. + +Lineage is event evidence spanning assets, not canonical contract identity and not +part of the point-in-time physical-state fingerprint. Providers normalize their +native lineage records into these immutable models without mutating ODCS state. +""" + +from __future__ import annotations + +import json +from datetime import datetime +from enum import Enum +from typing import Iterable + +from pydantic import BaseModel, ConfigDict, field_validator, model_validator + +from semapact.observation.evidence import ObservedEvidenceAvailabilityStatus +from semapact.observation.models import ObservedAssetIdentity + + +class ObservedLineageEvidenceType(str, Enum): + """Kinds of lineage evidence that must remain distinguishable.""" + + TABLE = "TABLE" + COLUMN = "COLUMN" + QUERY = "QUERY" + + +class ObservedLineageCaptureContext(BaseModel): + """Provider-neutral execution context attached to one lineage event.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + event_reference: str | None = None + recorded_at: datetime | None = None + actor_reference: str | None = None + execution_reference: str | None = None + direct: bool | None = None + + @field_validator("event_reference", "actor_reference", "execution_reference") + @classmethod + def _normalize_optional_text(cls, value: str | None) -> str | None: + if value is None: + return None + cleaned = value.strip() + return cleaned or None + + @field_validator("recorded_at") + @classmethod + def _require_aware_time(cls, value: datetime | None) -> datetime | None: + if value is not None and (value.tzinfo is None or value.utcoffset() is None): + raise ValueError("lineage recorded_at must be timezone-aware") + return value + + +class ObservedLineageEvidence(BaseModel): + """One normalized lineage edge or query/transformation evidence record.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + evidence_type: ObservedLineageEvidenceType + source_asset: ObservedAssetIdentity | None = None + source_reference: str | None = None + source_property: str | None = None + target_asset: ObservedAssetIdentity | None = None + target_reference: str | None = None + target_property: str | None = None + statement_reference: str | None = None + statement_text: str | None = None + statement_type: str | None = None + capture_context: ObservedLineageCaptureContext = ObservedLineageCaptureContext() + provenance: str | None = None + + @field_validator( + "source_reference", + "source_property", + "target_reference", + "target_property", + "statement_reference", + "statement_text", + "statement_type", + "provenance", + ) + @classmethod + def _normalize_optional_text(cls, value: str | None) -> str | None: + if value is None: + return None + cleaned = value.strip() + return cleaned or None + + @model_validator(mode="after") + def _validate_semantics(self) -> ObservedLineageEvidence: + if self.source_asset is None and self.source_reference is None and self.target_asset is None and self.target_reference is None: + raise ValueError("lineage evidence must identify a source or target") + if self.evidence_type is ObservedLineageEvidenceType.TABLE: + if self.source_property is not None or self.target_property is not None: + raise ValueError("table lineage must not carry property identity") + elif self.evidence_type is ObservedLineageEvidenceType.COLUMN: + if self.source_property is None and self.target_property is None: + raise ValueError("column lineage must identify a source or target property") + elif self.evidence_type is ObservedLineageEvidenceType.QUERY: + if self.statement_reference is None and self.statement_text is None: + raise ValueError("query lineage must identify a statement reference or text") + return self + + +class ObservedLineageAvailability(BaseModel): + """Availability of one lineage evidence kind for a provider read.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + evidence_type: ObservedLineageEvidenceType + status: ObservedEvidenceAvailabilityStatus + detail: str | None = None + + @field_validator("detail") + @classmethod + def _normalize_detail(cls, value: str | None) -> str | None: + if value is None: + return None + cleaned = value.strip() + return cleaned or None + + +class ObservedLineageResult(BaseModel): + """Deterministic auxiliary lineage evidence collected for one runtime asset.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + platform: str + source_identifier: str + target_reference: str + captured_at: datetime + evidence: tuple[ObservedLineageEvidence, ...] = () + availability: tuple[ObservedLineageAvailability, ...] = () + + @field_validator("platform", "source_identifier", "target_reference") + @classmethod + def _require_text(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("lineage result identity fields must not be empty") + return cleaned + + @field_validator("captured_at") + @classmethod + def _require_aware_capture_time(cls, value: datetime) -> datetime: + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError("lineage captured_at must be timezone-aware") + return value + + @model_validator(mode="after") + def _validate_platform_and_availability(self) -> ObservedLineageResult: + platform = self.platform.casefold() + for item in self.evidence: + for asset in (item.source_asset, item.target_asset): + if asset is not None and asset.platform.casefold() != platform: + raise ValueError("lineage asset platform must match result platform") + kinds = [item.evidence_type for item in self.availability] + if len(kinds) != len(set(kinds)): + raise ValueError("lineage availability must contain each evidence type at most once") + return self + + +def canonical_lineage_evidence_payload(item: ObservedLineageEvidence) -> dict[str, object]: + """Return provider-neutral canonical content for one lineage record.""" + + source_asset = item.source_asset + target_asset = item.target_asset + context = item.capture_context + return { + "evidence_type": item.evidence_type.value, + "source_asset": list(source_asset.canonical_key) if source_asset is not None else None, + "source_reference": None if source_asset is not None else _normalize_reference(item.source_reference), + "source_property": _normalize_identifier(item.source_property), + "target_asset": list(target_asset.canonical_key) if target_asset is not None else None, + "target_reference": None if target_asset is not None else _normalize_reference(item.target_reference), + "target_property": _normalize_identifier(item.target_property), + "statement_reference": _normalize_text(item.statement_reference), + "statement_text": _normalize_text(item.statement_text), + "statement_type": _normalize_identifier(item.statement_type), + "capture_context": { + "event_reference": _normalize_text(context.event_reference), + "recorded_at": context.recorded_at.isoformat() if context.recorded_at is not None else None, + "actor_reference": _normalize_text(context.actor_reference), + "execution_reference": _normalize_text(context.execution_reference), + "direct": context.direct, + }, + "provenance": _normalize_text(item.provenance), + } + + +def canonical_lineage_evidence_key(item: ObservedLineageEvidence) -> str: + return json.dumps( + canonical_lineage_evidence_payload(item), + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + + +def normalize_lineage_evidence( + evidence: Iterable[ObservedLineageEvidence], +) -> tuple[ObservedLineageEvidence, ...]: + """Deduplicate and deterministically order normalized lineage evidence.""" + + unique = {canonical_lineage_evidence_key(item): item for item in evidence} + return tuple(unique[key] for key in sorted(unique)) + + +def serialize_observed_lineage_result(result: ObservedLineageResult) -> str: + """Serialize lineage evidence deterministically without redefining state fingerprinting.""" + + evidence = [canonical_lineage_evidence_payload(item) for item in result.evidence] + evidence.sort(key=lambda item: json.dumps(item, sort_keys=True, separators=(",", ":"))) + availability = sorted( + ( + { + "evidence_type": item.evidence_type.value, + "status": item.status.value, + "detail": item.detail, + } + for item in result.availability + ), + key=lambda item: str(item["evidence_type"]), + ) + payload = { + "platform": result.platform.casefold(), + "source_identifier": result.source_identifier.rstrip("/"), + "target_reference": _normalize_reference(result.target_reference), + "captured_at": result.captured_at.isoformat(), + "evidence": evidence, + "availability": availability, + } + return json.dumps(payload, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + + +def _normalize_reference(value: str | None) -> str | None: + cleaned = _normalize_text(value) + return cleaned.casefold() if cleaned is not None else None + + +def _normalize_identifier(value: str | None) -> str | None: + cleaned = _normalize_text(value) + return cleaned.casefold() if cleaned is not None else None + + +def _normalize_text(value: str | None) -> str | None: + if value is None: + return None + cleaned = value.strip() + return cleaned or None diff --git a/semapact/platforms/databricks/__init__.py b/semapact/platforms/databricks/__init__.py index 6ae24d6b..2d8db7bb 100644 --- a/semapact/platforms/databricks/__init__.py +++ b/semapact/platforms/databricks/__init__.py @@ -3,6 +3,7 @@ from semapact.platforms.databricks.client import create_databricks_workspace_client from semapact.platforms.databricks.deployment import DatabricksDeploymentAdapter from semapact.platforms.databricks.discovery import discover_databricks_tables +from semapact.platforms.databricks.lineage import observe_databricks_lineage from semapact.platforms.databricks.runtime import DatabricksRuntimeProvider __all__ = [ @@ -10,4 +11,5 @@ "DatabricksRuntimeProvider", "create_databricks_workspace_client", "discover_databricks_tables", + "observe_databricks_lineage", ] diff --git a/semapact/platforms/databricks/lineage.py b/semapact/platforms/databricks/lineage.py new file mode 100644 index 00000000..685b07ef --- /dev/null +++ b/semapact/platforms/databricks/lineage.py @@ -0,0 +1,313 @@ +"""Read-only Databricks lineage evidence adapter. + +This module reads Unity Catalog system tables through an already-open SQL cursor +and maps provider records into the platform-neutral observation lineage domain. +Each evidence surface fails independently so missing system-table permissions do +not break ordinary runtime observation or other available lineage evidence. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Mapping, Protocol + +from semapact.observation.evidence import ObservedEvidenceAvailabilityStatus +from semapact.observation.lineage import ( + ObservedLineageAvailability, + ObservedLineageCaptureContext, + ObservedLineageEvidence, + ObservedLineageEvidenceType, + ObservedLineageResult, + normalize_lineage_evidence, +) +from semapact.observation.models import ObservedAssetIdentity + +DATABRICKS_PLATFORM = "databricks" +UNITY_LINEAGE_PROVENANCE = "unity_catalog_system_tables" + + +class _SqlCursorLike(Protocol): + def execute(self, operation: str, parameters: tuple[str, ...] | None = None) -> Any: ... + + def fetchall(self) -> Any: ... + + +_TABLE_LINEAGE_QUERY = """ +SELECT + source_table_full_name, + target_table_full_name, + statement_id, + event_time, + event_id, + record_id, + created_by, + direct_access +FROM system.access.table_lineage +WHERE source_table_full_name = ? OR target_table_full_name = ? +""" + +_COLUMN_LINEAGE_QUERY = """ +SELECT + source_table_full_name, + source_column_name, + target_table_full_name, + target_column_name, + statement_id, + event_time, + event_id, + record_id, + created_by, + direct_access +FROM system.access.column_lineage +WHERE source_table_full_name = ? OR target_table_full_name = ? +""" + +_QUERY_EVIDENCE_QUERY = """ +SELECT + tl.source_table_full_name, + tl.target_table_full_name, + tl.statement_id, + tl.event_time, + tl.event_id, + tl.record_id, + tl.created_by, + tl.direct_access, + qh.statement_text, + qh.statement_type +FROM system.access.table_lineage tl +LEFT JOIN system.query.history qh + ON tl.statement_id = qh.statement_id + AND tl.workspace_id = qh.workspace_id +WHERE (tl.source_table_full_name = ? OR tl.target_table_full_name = ?) + AND tl.statement_id IS NOT NULL +""" + + +def observe_databricks_lineage( + *, + cursor: _SqlCursorLike, + table_fqn: str, + source_identifier: str, + captured_at: datetime | None = None, +) -> ObservedLineageResult: + """Collect normalized lineage evidence for one Unity Catalog table. + + Table, column, and query evidence are queried independently. A permission or + availability failure on one system table is represented explicitly and does + not discard evidence collected from the other surfaces. + """ + + table_fqn = table_fqn.strip() + source_identifier = source_identifier.strip() + if not table_fqn: + raise ValueError("table_fqn is required for Databricks lineage observation") + if not source_identifier: + raise ValueError("source_identifier is required for Databricks lineage observation") + + observed_at = captured_at or datetime.now(timezone.utc) + if observed_at.tzinfo is None or observed_at.utcoffset() is None: + raise ValueError("captured_at must be timezone-aware") + + evidence: list[ObservedLineageEvidence] = [] + availability: list[ObservedLineageAvailability] = [] + + _collect( + cursor=cursor, + query=_TABLE_LINEAGE_QUERY, + table_fqn=table_fqn, + evidence_type=ObservedLineageEvidenceType.TABLE, + mapper=_map_table_lineage, + evidence=evidence, + availability=availability, + ) + _collect( + cursor=cursor, + query=_COLUMN_LINEAGE_QUERY, + table_fqn=table_fqn, + evidence_type=ObservedLineageEvidenceType.COLUMN, + mapper=_map_column_lineage, + evidence=evidence, + availability=availability, + ) + _collect( + cursor=cursor, + query=_QUERY_EVIDENCE_QUERY, + table_fqn=table_fqn, + evidence_type=ObservedLineageEvidenceType.QUERY, + mapper=_map_query_evidence, + evidence=evidence, + availability=availability, + ) + + return ObservedLineageResult( + platform=DATABRICKS_PLATFORM, + source_identifier=source_identifier.rstrip("/"), + target_reference=table_fqn, + captured_at=observed_at, + evidence=normalize_lineage_evidence(evidence), + availability=tuple(sorted(availability, key=lambda item: item.evidence_type.value)), + ) + + +def _collect( + *, + cursor: _SqlCursorLike, + query: str, + table_fqn: str, + evidence_type: ObservedLineageEvidenceType, + mapper: Any, + evidence: list[ObservedLineageEvidence], + availability: list[ObservedLineageAvailability], +) -> None: + try: + cursor.execute(query, (table_fqn, table_fqn)) + rows = cursor.fetchall() + except Exception as exc: + availability.append( + ObservedLineageAvailability( + evidence_type=evidence_type, + status=ObservedEvidenceAvailabilityStatus.UNAVAILABLE, + detail=f"Databricks lineage surface unavailable: {type(exc).__name__}", + ) + ) + return + + evidence.extend(item for row in rows for item in [mapper(row)] if item is not None) + availability.append( + ObservedLineageAvailability( + evidence_type=evidence_type, + status=ObservedEvidenceAvailabilityStatus.AVAILABLE, + ) + ) + + +def _map_table_lineage(row: object) -> ObservedLineageEvidence | None: + source_reference = _text(_row_value(row, "source_table_full_name")) + target_reference = _text(_row_value(row, "target_table_full_name")) + if source_reference is None and target_reference is None: + return None + return _lineage_evidence( + evidence_type=ObservedLineageEvidenceType.TABLE, + row=row, + source_reference=source_reference, + target_reference=target_reference, + ) + + +def _map_column_lineage(row: object) -> ObservedLineageEvidence | None: + source_reference = _text(_row_value(row, "source_table_full_name")) + target_reference = _text(_row_value(row, "target_table_full_name")) + source_property = _text(_row_value(row, "source_column_name")) + target_property = _text(_row_value(row, "target_column_name")) + if source_reference is None and target_reference is None: + return None + if source_property is None and target_property is None: + return None + return _lineage_evidence( + evidence_type=ObservedLineageEvidenceType.COLUMN, + row=row, + source_reference=source_reference, + source_property=source_property, + target_reference=target_reference, + target_property=target_property, + ) + + +def _map_query_evidence(row: object) -> ObservedLineageEvidence | None: + statement_reference = _text(_row_value(row, "statement_id")) + statement_text = _text(_row_value(row, "statement_text")) + if statement_reference is None and statement_text is None: + return None + return _lineage_evidence( + evidence_type=ObservedLineageEvidenceType.QUERY, + row=row, + source_reference=_text(_row_value(row, "source_table_full_name")), + target_reference=_text(_row_value(row, "target_table_full_name")), + statement_reference=statement_reference, + statement_text=statement_text, + statement_type=_text(_row_value(row, "statement_type")), + ) + + +def _lineage_evidence( + *, + evidence_type: ObservedLineageEvidenceType, + row: object, + source_reference: str | None, + target_reference: str | None, + source_property: str | None = None, + target_property: str | None = None, + statement_reference: str | None = None, + statement_text: str | None = None, + statement_type: str | None = None, +) -> ObservedLineageEvidence: + execution_reference = statement_reference or _text(_row_value(row, "statement_id")) + return ObservedLineageEvidence( + evidence_type=evidence_type, + source_asset=_asset_identity(source_reference), + source_reference=source_reference, + source_property=source_property, + target_asset=_asset_identity(target_reference), + target_reference=target_reference, + target_property=target_property, + statement_reference=statement_reference, + statement_text=statement_text, + statement_type=statement_type, + capture_context=ObservedLineageCaptureContext( + event_reference=( + _text(_row_value(row, "event_id")) + or _text(_row_value(row, "record_id")) + ), + recorded_at=_datetime(_row_value(row, "event_time")), + actor_reference=_text(_row_value(row, "created_by")), + execution_reference=execution_reference, + direct=_bool_or_none(_row_value(row, "direct_access")), + ), + provenance=UNITY_LINEAGE_PROVENANCE, + ) + + +def _asset_identity(reference: str | None) -> ObservedAssetIdentity | None: + if reference is None: + return None + parts = tuple(part.strip() for part in reference.split(".")) + if len(parts) != 3 or not all(parts): + return None + return ObservedAssetIdentity( + platform=DATABRICKS_PLATFORM, + namespace=(parts[0], parts[1]), + asset=parts[2], + ) + + +def _row_value(row: object, name: str) -> Any: + if isinstance(row, Mapping): + return row.get(name) + return getattr(row, name, None) + + +def _text(value: Any) -> str | None: + if value is None: + return None + cleaned = str(value).strip() + return cleaned or None + + +def _datetime(value: Any) -> datetime | None: + if isinstance(value, datetime): + if value.tzinfo is None or value.utcoffset() is None: + return value.replace(tzinfo=timezone.utc) + return value + if isinstance(value, str): + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo is None or parsed.utcoffset() is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed + return None + + +def _bool_or_none(value: Any) -> bool | None: + return value if isinstance(value, bool) else None diff --git a/tests/fixtures/observation/databricks/lineage_orders.json b/tests/fixtures/observation/databricks/lineage_orders.json new file mode 100644 index 00000000..43fb4819 --- /dev/null +++ b/tests/fixtures/observation/databricks/lineage_orders.json @@ -0,0 +1,54 @@ +{ + "table": [ + { + "source_table_full_name": "main.raw.orders", + "target_table_full_name": "main.silver.orders", + "statement_id": "stmt-1", + "event_time": "2026-09-15T10:00:00+00:00", + "event_id": "event-1", + "record_id": "table-record-1", + "created_by": "pipeline@example.com", + "direct_access": true + } + ], + "column": [ + { + "source_table_full_name": "main.raw.orders", + "source_column_name": "raw_id", + "target_table_full_name": "main.silver.orders", + "target_column_name": "order_id", + "statement_id": "stmt-1", + "event_time": "2026-09-15T10:00:00+00:00", + "event_id": "event-1", + "record_id": "column-record-1", + "created_by": "pipeline@example.com", + "direct_access": true + }, + { + "source_table_full_name": "main.raw.orders", + "source_column_name": "raw_customer_id", + "target_table_full_name": "main.silver.orders", + "target_column_name": "customer_id", + "statement_id": "stmt-1", + "event_time": "2026-09-15T10:00:00+00:00", + "event_id": "event-1", + "record_id": "column-record-2", + "created_by": "pipeline@example.com", + "direct_access": true + } + ], + "query": [ + { + "source_table_full_name": "main.raw.orders", + "target_table_full_name": "main.silver.orders", + "statement_id": "stmt-1", + "event_time": "2026-09-15T10:00:00+00:00", + "event_id": "event-1", + "record_id": "table-record-1", + "created_by": "pipeline@example.com", + "direct_access": true, + "statement_text": "INSERT INTO main.silver.orders SELECT * FROM main.raw.orders", + "statement_type": "INSERT" + } + ] +} diff --git a/tests/test_observation_databricks_lineage.py b/tests/test_observation_databricks_lineage.py new file mode 100644 index 00000000..7ffc28f2 --- /dev/null +++ b/tests/test_observation_databricks_lineage.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import inspect +import json +from datetime import datetime, timezone +from pathlib import Path +from types import SimpleNamespace + +from semapact.observation import ( + ObservedEvidenceAvailabilityStatus, + ObservedLineageEvidenceType, + serialize_observed_lineage_result, +) +from semapact.platforms.databricks import lineage as databricks_lineage +from semapact.platforms.databricks.lineage import observe_databricks_lineage + +CAPTURED_AT = datetime(2026, 9, 15, 10, 0, tzinfo=timezone.utc) +FIXTURE = ( + Path(__file__).parent + / "fixtures" + / "observation" + / "databricks" + / "lineage_orders.json" +) + + +def _fixture() -> dict[str, list[dict[str, object]]]: + return json.loads(FIXTURE.read_text(encoding="utf-8")) + + +class _FakeCursor: + def __init__(self, *, reverse: bool = False, fail: set[str] | None = None) -> None: + self.reverse = reverse + self.fail = fail or set() + self._rows = [] + self.queries: list[str] = [] + self.fixture = _fixture() + + def execute(self, query: str, parameters: tuple[str, ...] | None = None) -> None: + self.queries.append(query) + if "system.access.column_lineage" in query: + kind = "column" + elif "system.query.history" in query: + kind = "query" + else: + kind = "table" + if kind in self.fail: + raise PermissionError(f"no access to {kind}") + rows = [SimpleNamespace(**item) for item in self.fixture[kind]] + self._rows = list(reversed(rows)) if self.reverse else rows + + def fetchall(self): + return self._rows + + +def _availability(result): + return {item.evidence_type: item.status for item in result.availability} + + +def test_databricks_lineage_is_normalized_without_odcs_mutation() -> None: + cursor = _FakeCursor() + + result = observe_databricks_lineage( + cursor=cursor, + table_fqn="main.silver.orders", + source_identifier="https://adb.example/", + captured_at=CAPTURED_AT, + ) + + assert result.platform == "databricks" + assert result.source_identifier == "https://adb.example" + assert result.target_reference == "main.silver.orders" + assert {item.evidence_type for item in result.evidence} == { + ObservedLineageEvidenceType.TABLE, + ObservedLineageEvidenceType.COLUMN, + ObservedLineageEvidenceType.QUERY, + } + + columns = [ + item + for item in result.evidence + if item.evidence_type is ObservedLineageEvidenceType.COLUMN + ] + assert [(item.source_property, item.target_property) for item in columns] == [ + ("raw_customer_id", "customer_id"), + ("raw_id", "order_id"), + ] + assert all(item.source_asset is not None for item in columns) + assert all(item.target_asset is not None for item in columns) + + query = next( + item + for item in result.evidence + if item.evidence_type is ObservedLineageEvidenceType.QUERY + ) + assert query.statement_reference == "stmt-1" + assert query.statement_type == "INSERT" + assert query.statement_text == ( + "INSERT INTO main.silver.orders SELECT * FROM main.raw.orders" + ) + assert query.capture_context.event_reference == "event-1" + assert query.capture_context.recorded_at == CAPTURED_AT + assert query.capture_context.actor_reference == "pipeline@example.com" + + assert all( + status is ObservedEvidenceAvailabilityStatus.AVAILABLE + for status in _availability(result).values() + ) + assert any("system.query.history" in query for query in cursor.queries) + assert all("system.access.query_history" not in query for query in cursor.queries) + + +def test_lineage_order_is_deterministic() -> None: + first = observe_databricks_lineage( + cursor=_FakeCursor(), + table_fqn="main.silver.orders", + source_identifier="workspace-a", + captured_at=CAPTURED_AT, + ) + second = observe_databricks_lineage( + cursor=_FakeCursor(reverse=True), + table_fqn="main.silver.orders", + source_identifier="workspace-a", + captured_at=CAPTURED_AT, + ) + + assert first == second + assert serialize_observed_lineage_result(first) == serialize_observed_lineage_result(second) + + +def test_missing_query_history_permission_preserves_other_lineage() -> None: + result = observe_databricks_lineage( + cursor=_FakeCursor(fail={"query"}), + table_fqn="main.silver.orders", + source_identifier="workspace-a", + captured_at=CAPTURED_AT, + ) + + availability = _availability(result) + assert availability[ObservedLineageEvidenceType.TABLE] is ObservedEvidenceAvailabilityStatus.AVAILABLE + assert availability[ObservedLineageEvidenceType.COLUMN] is ObservedEvidenceAvailabilityStatus.AVAILABLE + assert availability[ObservedLineageEvidenceType.QUERY] is ObservedEvidenceAvailabilityStatus.UNAVAILABLE + assert all( + item.evidence_type is not ObservedLineageEvidenceType.QUERY + for item in result.evidence + ) + assert any( + item.evidence_type is ObservedLineageEvidenceType.COLUMN + for item in result.evidence + ) + + +def test_lineage_adapter_does_not_depend_on_contract_or_governance_layers() -> None: + source = inspect.getsource(databricks_lineage) + + assert "open_data_contract_standard" not in source + assert "semapact.importers" not in source + assert "semapact.lifecycle" not in source + assert "semapact.governance" not in source diff --git a/tests/test_unity_lineage.py b/tests/test_unity_lineage.py index 72471ada..c4dd869d 100644 --- a/tests/test_unity_lineage.py +++ b/tests/test_unity_lineage.py @@ -1,6 +1,8 @@ import builtins -import pytest +from datetime import datetime, timezone from unittest.mock import MagicMock, patch + +import pytest from open_data_contract_standard.model import ( OpenDataContractStandard, SchemaObject, @@ -9,6 +11,8 @@ from semapact.importers.unity_lineage import enrich_unity_lineage +EVENT_TIME = datetime(2026, 9, 15, 10, 0, tzinfo=timezone.utc) + def test_enrich_unity_lineage_no_http_path(): prop_id = SchemaProperty(id="id", name="id") @@ -51,23 +55,43 @@ def __init__(self, **kwargs): self.__dict__.update(kwargs) def execute_side_effect(query, params): + common = dict( + target_table_full_name="main.sales.orders", + statement_id="stmt-1", + event_time=EVENT_TIME, + event_id="event-1", + record_id="record-1", + created_by="pipeline@example.com", + direct_access=True, + ) if "system.access.column_lineage" in query: mock_cursor.fetchall.return_value = [ Row( source_table_full_name="main.sales.raw_orders", source_column_name="raw_id", target_column_name="id", + **common, ), Row( source_table_full_name="main.sales.raw_orders", source_column_name="raw_amount", target_column_name="amount", + **common, ), ] - elif "system.access.query_history" in query: - mock_cursor.fetchone.return_value = Row( - statement_text="INSERT INTO main.sales.orders SELECT raw_id as id, raw_amount as amount FROM main.sales.raw_orders" - ) + elif "system.query.history" in query: + mock_cursor.fetchall.return_value = [ + Row( + source_table_full_name="main.sales.raw_orders", + statement_text="INSERT INTO main.sales.orders SELECT raw_id as id, raw_amount as amount FROM main.sales.raw_orders", + statement_type="INSERT", + **common, + ) + ] + else: + mock_cursor.fetchall.return_value = [ + Row(source_table_full_name="main.sales.raw_orders", **common) + ] mock_cursor.execute.side_effect = execute_side_effect @@ -91,13 +115,12 @@ def execute_side_effect(query, params): "main.sales.raw_orders.raw_amount" ] - assert ( - fields["id"].transformLogic - == "INSERT INTO main.sales.orders SELECT raw_id as id, raw_amount as amount FROM main.sales.raw_orders" - ) - assert ( - fields["amount"].transformLogic - == "INSERT INTO main.sales.orders SELECT raw_id as id, raw_amount as amount FROM main.sales.raw_orders" + statement = "INSERT INTO main.sales.orders SELECT raw_id as id, raw_amount as amount FROM main.sales.raw_orders" + assert fields["id"].transformLogic == statement + assert fields["amount"].transformLogic == statement + assert any( + "system.query.history" in call.args[0] + for call in mock_cursor.execute.call_args_list ) From 6182325f40adef20334d38299e284897c11dcf7d Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Wed, 16 Sep 2026 07:46:49 +1000 Subject: [PATCH 27/35] refactor(observation): keep lineage as runtime evidence (#234) * refactor(observation): keep lineage as runtime evidence * fix(import): remove lineage enrichment dependency * fix(import): reject legacy lineage enrichment requests * test(import): enforce lineage evidence boundary --- ARCHITECTURE.md | 4 + semapact/importers/unity_importer.py | 32 ++-- semapact/importers/unity_lineage.py | 152 ----------------- tests/test_unity_import_lineage_boundary.py | 28 ++++ tests/test_unity_lineage.py | 175 -------------------- 5 files changed, 50 insertions(+), 341 deletions(-) delete mode 100644 semapact/importers/unity_lineage.py create mode 100644 tests/test_unity_import_lineage_boundary.py delete mode 100644 tests/test_unity_lineage.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 203273d6..252cd38a 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -103,6 +103,7 @@ These are application DTOs, not new governance/release/deployment authorities. ### Import/export and compatibility workflows - `semapact/importers/` projects explicitly imported external metadata into ODCS and contains no lifecycle policy; +- runtime observation evidence is not an importer source of canonical ODCS semantics; evidence-assisted bootstrap or enrichment must produce an explicit proposal/candidate for normal governance rather than mutate a governed contract; - `semapact/exporters/` and `quality/` are read-only projections; - `semapact/devops/` and parts of `core/` contain stable compatibility workflows and must not become a second canonical ContractOps implementation. @@ -192,6 +193,8 @@ Provider execution success is not convergence proof. Observation captures platform-neutral runtime evidence. It never mutates ODCS or invokes governance. +Lineage is time-varying runtime evidence, not canonical contract truth. Table/column lineage and query history may support provenance, impact analysis, reconciliation, or an explicit contract proposal, but observation must not directly project them into authoritative ODCS fields such as `transformSourceObjects` or `transformLogic`. + Reconciliation compares governed desired state with fresh observation and yields the existing status vocabulary: ```text @@ -226,3 +229,4 @@ Rules: 7. **Interfaces stay thin** — CLI/API/UI parse, delegate, and render; they do not become a second business-logic implementation. 8. **Compatibility is not ownership** — legacy import paths may re-export canonical implementations but must not accumulate new logic. 9. **One canonical contract model** — SemaPact reuses ODCS rather than maintaining a parallel contract schema. +10. **Evidence is not contract truth** — runtime evidence may inform analysis or an explicit governed proposal, but it never silently mutates canonical ODCS semantics. diff --git a/semapact/importers/unity_importer.py b/semapact/importers/unity_importer.py index e21faa26..679cabee 100644 --- a/semapact/importers/unity_importer.py +++ b/semapact/importers/unity_importer.py @@ -20,7 +20,6 @@ from semapact.importers.unity_relationships import ( enrich_unity_contract_relationships, ) -from semapact.importers.unity_lineage import enrich_unity_lineage LOGGER = logging.getLogger(__name__) @@ -64,14 +63,28 @@ def import_unity_contract( sql_http_path: str | None = None, extract_lineage: bool = False, ) -> OpenDataContractStandard: - """Import a Unity Catalog contract using datacontract-cli's unity importer. + """Import Unity Catalog metadata into ODCS using datacontract-cli. - Raises ``ValueError`` when required credentials are missing. + Runtime lineage is deliberately excluded from this importer. The legacy + lineage-related keyword arguments remain temporarily accepted so existing + callers fail with an explicit boundary error instead of a Python signature + error. They must not trigger ODCS mutation. + + Raises ``ValueError`` when credentials are missing or lineage enrichment is + requested through the import boundary. """ + del sql_http_path + + if extract_lineage: + raise ValueError( + "Unity lineage is runtime observation evidence and can no longer be " + "projected into ODCS during import" + ) + from semapact.core.config import config_manager + workspace_url = workspace_url or config_manager.get("databricks.workspace_url") token = token or config_manager.get("databricks.token") - sql_http_path = sql_http_path or config_manager.get("databricks.sql_http_path") profile = config_manager.get("databricks.profile") if not profile and (not workspace_url or not token): @@ -86,18 +99,9 @@ def import_unity_contract( source=None, unity_table_full_name=[table_fqn], ) - enriched = enrich_unity_contract_relationships( + return enrich_unity_contract_relationships( imported, table_fqn=table_fqn, workspace_url=workspace_url, token=token, ) - if extract_lineage: - enriched = enrich_unity_lineage( - enriched, - table_fqn=table_fqn, - workspace_url=workspace_url, - token=token, - sql_http_path=sql_http_path, - ) - return enriched diff --git a/semapact/importers/unity_lineage.py b/semapact/importers/unity_lineage.py deleted file mode 100644 index 19381092..00000000 --- a/semapact/importers/unity_lineage.py +++ /dev/null @@ -1,152 +0,0 @@ -from __future__ import annotations - -import logging - -from open_data_contract_standard.model import OpenDataContractStandard, SchemaObject - -from semapact.observation.lineage import ( - ObservedLineageEvidence, - ObservedLineageEvidenceType, - ObservedLineageResult, -) -from semapact.platforms.databricks.lineage import observe_databricks_lineage - -LOGGER = logging.getLogger(__name__) - - -def enrich_unity_lineage( - contract: OpenDataContractStandard, - *, - table_fqn: str, - workspace_url: str, - token: str, - sql_http_path: str | None = None, -) -> OpenDataContractStandard: - """Compatibility projection of normalized Databricks lineage into ODCS fields. - - Runtime lineage retrieval is owned by the read-only Databricks lineage adapter. - This legacy importer remains available for callers that explicitly want ODCS - enrichment; it no longer defines or retrieves lineage semantics itself. - """ - if not sql_http_path: - LOGGER.warning( - "Skipping lineage extraction: --sql-http-path is required when using databricks-sql-connector" - ) - return contract - - try: - from databricks import sql - except ImportError as exc: - raise ImportError( - "The 'databricks-sql-connector' package is required to extract lineage. " - "Please install it using: pip install databricks-sql-connector (or install with the [databricks] extra)." - ) from exc - - server_hostname = ( - workspace_url.replace("https://", "").replace("http://", "").rstrip("/") - ) - - try: - with sql.connect( - server_hostname=server_hostname, - http_path=sql_http_path, - access_token=token, - ) as connection: - with connection.cursor() as cursor: - result = observe_databricks_lineage( - cursor=cursor, - table_fqn=table_fqn, - source_identifier=workspace_url, - ) - _apply_lineage_evidence(contract, table_fqn=table_fqn, result=result) - except Exception as exc: - LOGGER.warning( - "Failed to fetch lineage or logic from Databricks system tables: %s", exc - ) - - return contract - - -def _apply_lineage_evidence( - contract: OpenDataContractStandard, - *, - table_fqn: str, - result: ObservedLineageResult, -) -> None: - schema_obj = _resolve_target_schema(contract, table_fqn=table_fqn) - if schema_obj is None: - return - - target_key = table_fqn.casefold() - fields = { - item.name.casefold(): item - for item in (schema_obj.properties or []) - if item.name - } - - for item in result.evidence: - if item.evidence_type is not ObservedLineageEvidenceType.COLUMN: - continue - if _asset_reference(item, target=True).casefold() != target_key: - continue - if not item.target_property or not item.source_property: - continue - source_table = _asset_reference(item, target=False) - if not source_table: - continue - property_obj = fields.get(item.target_property.casefold()) - if property_obj is None: - continue - source = f"{source_table}.{item.source_property}" - existing = list(property_obj.transformSourceObjects or []) - if source not in existing: - existing.append(source) - property_obj.transformSourceObjects = existing - - # Legacy compatibility only: an explicit enrichment request may project the - # latest query text into transformLogic. Runtime observation keeps this text - # as non-authoritative QUERY evidence and never mutates the contract. - query_evidence = [ - item - for item in result.evidence - if item.evidence_type is ObservedLineageEvidenceType.QUERY - and item.statement_text - and _asset_reference(item, target=True).casefold() == target_key - ] - if query_evidence: - latest = max(query_evidence, key=_query_recency_key) - for prop in schema_obj.properties or []: - if not prop.transformLogic: - prop.transformLogic = latest.statement_text - - -def _query_recency_key(item: ObservedLineageEvidence) -> tuple[str, str]: - recorded_at = item.capture_context.recorded_at - return ( - recorded_at.isoformat() if recorded_at is not None else "", - item.statement_reference or "", - ) - - -def _asset_reference(item: ObservedLineageEvidence, *, target: bool) -> str: - asset = item.target_asset if target else item.source_asset - reference = item.target_reference if target else item.source_reference - if asset is not None: - return ".".join((*asset.namespace, asset.asset)) - return reference or "" - - -def _resolve_target_schema( - contract: OpenDataContractStandard, *, table_fqn: str -) -> SchemaObject | None: - schema_items = contract.schema_ or [] - if not schema_items: - return None - short_name = table_fqn.split(".")[-1].strip().lower() - for item in schema_items: - if (item.physicalName or "").strip().lower() == short_name: - return item - for item in schema_items: - if (item.name or "").strip().lower() == short_name: - return item - return schema_items[0] diff --git a/tests/test_unity_import_lineage_boundary.py b/tests/test_unity_import_lineage_boundary.py new file mode 100644 index 00000000..2f48c229 --- /dev/null +++ b/tests/test_unity_import_lineage_boundary.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +import inspect + +import pytest + +from semapact.importers import unity_importer +from semapact.importers.unity_importer import import_unity_contract + + +def test_unity_import_rejects_lineage_projection() -> None: + with pytest.raises(ValueError, match="runtime observation evidence"): + import_unity_contract( + table_fqn="main.silver.orders", + workspace_url="https://adb.example", + token="token", + sql_http_path="/sql/1.0/warehouses/example", + extract_lineage=True, + ) + + +def test_unity_importer_has_no_lineage_mutation_dependency() -> None: + source = inspect.getsource(unity_importer) + + assert "semapact.importers.unity_lineage" not in source + assert "enrich_unity_lineage" not in source + assert "transformSourceObjects" not in source + assert "transformLogic" not in source diff --git a/tests/test_unity_lineage.py b/tests/test_unity_lineage.py deleted file mode 100644 index c4dd869d..00000000 --- a/tests/test_unity_lineage.py +++ /dev/null @@ -1,175 +0,0 @@ -import builtins -from datetime import datetime, timezone -from unittest.mock import MagicMock, patch - -import pytest -from open_data_contract_standard.model import ( - OpenDataContractStandard, - SchemaObject, - SchemaProperty, -) - -from semapact.importers.unity_lineage import enrich_unity_lineage - -EVENT_TIME = datetime(2026, 9, 15, 10, 0, tzinfo=timezone.utc) - - -def test_enrich_unity_lineage_no_http_path(): - prop_id = SchemaProperty(id="id", name="id") - schema_obj = SchemaObject( - name="orders", physicalName="orders", properties=[prop_id] - ) - contract = OpenDataContractStandard( - apiVersion="3.1.0", id="test-contract", schema=[schema_obj] - ) - - enriched = enrich_unity_lineage( - contract, - table_fqn="main.sales.orders", - workspace_url="https://adb.example", - token="token", - sql_http_path=None, - ) - - assert enriched.schema_[0].properties[0].transformSourceObjects is None - - -@patch("databricks.sql.connect") -def test_enrich_unity_lineage_success(mock_sql_connect): - prop_id = SchemaProperty(id="id", name="id") - prop_amount = SchemaProperty(id="amount", name="amount") - schema_obj = SchemaObject( - name="orders", physicalName="orders", properties=[prop_id, prop_amount] - ) - contract = OpenDataContractStandard( - apiVersion="3.1.0", id="test-contract", schema=[schema_obj] - ) - - mock_conn = MagicMock() - mock_cursor = MagicMock() - mock_sql_connect.return_value.__enter__.return_value = mock_conn - mock_conn.cursor.return_value.__enter__.return_value = mock_cursor - - class Row: - def __init__(self, **kwargs): - self.__dict__.update(kwargs) - - def execute_side_effect(query, params): - common = dict( - target_table_full_name="main.sales.orders", - statement_id="stmt-1", - event_time=EVENT_TIME, - event_id="event-1", - record_id="record-1", - created_by="pipeline@example.com", - direct_access=True, - ) - if "system.access.column_lineage" in query: - mock_cursor.fetchall.return_value = [ - Row( - source_table_full_name="main.sales.raw_orders", - source_column_name="raw_id", - target_column_name="id", - **common, - ), - Row( - source_table_full_name="main.sales.raw_orders", - source_column_name="raw_amount", - target_column_name="amount", - **common, - ), - ] - elif "system.query.history" in query: - mock_cursor.fetchall.return_value = [ - Row( - source_table_full_name="main.sales.raw_orders", - statement_text="INSERT INTO main.sales.orders SELECT raw_id as id, raw_amount as amount FROM main.sales.raw_orders", - statement_type="INSERT", - **common, - ) - ] - else: - mock_cursor.fetchall.return_value = [ - Row(source_table_full_name="main.sales.raw_orders", **common) - ] - - mock_cursor.execute.side_effect = execute_side_effect - - enriched = enrich_unity_lineage( - contract, - table_fqn="main.sales.orders", - workspace_url="https://adb.example", - token="token", - sql_http_path="/sql/1.0/endpoints/12345", - ) - - mock_sql_connect.assert_called_once_with( - server_hostname="adb.example", - http_path="/sql/1.0/endpoints/12345", - access_token="token", - ) - - fields = {item.name: item for item in enriched.schema_[0].properties if item.name} - assert fields["id"].transformSourceObjects == ["main.sales.raw_orders.raw_id"] - assert fields["amount"].transformSourceObjects == [ - "main.sales.raw_orders.raw_amount" - ] - - statement = "INSERT INTO main.sales.orders SELECT raw_id as id, raw_amount as amount FROM main.sales.raw_orders" - assert fields["id"].transformLogic == statement - assert fields["amount"].transformLogic == statement - assert any( - "system.query.history" in call.args[0] - for call in mock_cursor.execute.call_args_list - ) - - -@patch("databricks.sql.connect") -def test_enrich_unity_lineage_exception(mock_sql_connect): - prop_id = SchemaProperty(id="id", name="id") - schema_obj = SchemaObject( - name="orders", physicalName="orders", properties=[prop_id] - ) - contract = OpenDataContractStandard( - apiVersion="3.1.0", id="test-contract", schema=[schema_obj] - ) - - mock_sql_connect.side_effect = Exception("Connection Failed") - - enriched = enrich_unity_lineage( - contract, - table_fqn="main.sales.orders", - workspace_url="https://adb.example", - token="token", - sql_http_path="/sql/1.0/endpoints/12345", - ) - - assert enriched.schema_[0].properties[0].transformSourceObjects is None - assert enriched.schema_[0].properties[0].transformLogic is None - - -def test_enrich_unity_lineage_missing_dependency(): - prop_id = SchemaProperty(id="id", name="id") - schema_obj = SchemaObject( - name="orders", physicalName="orders", properties=[prop_id] - ) - contract = OpenDataContractStandard( - apiVersion="3.1.0", id="test-contract", schema=[schema_obj] - ) - - original_import = builtins.__import__ - - def mock_import(name, *args, **kwargs): - if name == "databricks" or name == "databricks.sql": - raise ImportError(f"No module named '{name}'") - return original_import(name, *args, **kwargs) - - with patch("builtins.__import__", side_effect=mock_import): - with pytest.raises(ImportError, match="databricks-sql-connector"): - enrich_unity_lineage( - contract, - table_fqn="main.sales.orders", - workspace_url="https://adb.example", - token="token", - sql_http_path="/sql/1.0/endpoints/12345", - ) From a0f5e7d540db312fad8baac0e025109085d2a4fc Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Thu, 17 Sep 2026 17:42:12 +1000 Subject: [PATCH 28/35] feat(history): record immutable governance approvals (#235) * feat(approval): add immutable approval domain * feat(approval): define approval record model * feat(approval): add deterministic approval identity * feat(approval): build canonical approval records * feat(approval): project approval into review evidence * feat(history): add typed approval persistence port * feat(history): export approval history port * feat(history): persist immutable approval records * test(history): cover immutable approval records * refactor(approval): make final record construction explicit * docs(history): document immutable approval records * feat(approval): add application recording boundary * feat(cli): add approval recording command boundary * feat(sdk): expose approval recording API * test(approval): cover public API and CLI adapter * feat(cli): expose approval history recording * test(cli): cover approval record command parsing * docs(approval): document CI and SDK intake path * refactor(approval): clarify record service semantics * refactor(approval): export approval record service * refactor(approval): use approval record service in cli * test(approval): align record service naming * docs(approval): clarify record service semantics * refactor(approval): align service module with ApprovalRecord * refactor(approval): update public service module path * refactor(approval): use approval record service module * refactor(approval): rename service module to approval_record --- docs/approval_history.md | 114 ++++++++++++ docs/governance_history.md | 75 ++++++-- semapact/__init__.py | 5 + .../application/services/approval_record.py | 78 ++++++++ semapact/approval/__init__.py | 19 ++ semapact/approval/builders.py | 46 +++++ semapact/approval/evidence.py | 32 ++++ semapact/approval/integrity.py | 45 +++++ semapact/approval/models.py | 85 +++++++++ semapact/history/__init__.py | 2 + semapact/history/repository.py | 19 ++ semapact/interfaces/cli.py | 57 ++++++ semapact/interfaces/commands/approval_cmd.py | 46 +++++ semapact/platforms/git/history_repository.py | 47 +++++ tests/test_approval_api.py | 112 +++++++++++ tests/test_approval_history.py | 174 ++++++++++++++++++ 16 files changed, 945 insertions(+), 11 deletions(-) create mode 100644 docs/approval_history.md create mode 100644 semapact/application/services/approval_record.py create mode 100644 semapact/approval/__init__.py create mode 100644 semapact/approval/builders.py create mode 100644 semapact/approval/evidence.py create mode 100644 semapact/approval/integrity.py create mode 100644 semapact/approval/models.py create mode 100644 semapact/interfaces/commands/approval_cmd.py create mode 100644 tests/test_approval_api.py create mode 100644 tests/test_approval_history.py diff --git a/docs/approval_history.md b/docs/approval_history.md new file mode 100644 index 00000000..88f3ff2d --- /dev/null +++ b/docs/approval_history.md @@ -0,0 +1,114 @@ +# Approval history intake + +SemaPact records explicit review actions as immutable `ApprovalRecord` artifacts. +Approval history is an audit and authorization-evidence primitive; it is not a workflow +engine and does not decide who should review, which reviewer wins, or whether a quorum +has been reached. + +## Intake boundary + +The canonical path is: + +```text +GitHub / GitLab / Azure DevOps review event + ↓ + trusted CI / adapter + ↓ + ApprovalRecordService + ↓ + ApprovalRecord + ↓ + ApprovalHistoryRepository + ↓ + .semapact/history/ +``` + +`ApprovalRecordService` is deliberately named after the artifact it manages. It records +and queries approval evidence; it does not itself approve a change, select a reviewer, +or grant authorization. + +The external provider owns the review interaction. SemaPact records the explicit fact +that the provider supplied: actor, action, provider timestamp, exact ContractOps scope, +and stable evidence references. + +A Git commit is not approval evidence by itself. For Git-hosted workflows, the approval +fact normally comes from the pull-request review API/event and is then recorded by CI. + +Persisting a record also does not by itself authorize an operation. One explicitly +selected `ApprovalRecord` can be projected into `ReviewAuthorizationEvidence`, after +which existing M2 ContractOps authorization validates the exact decision, ChangeSet, +ReleasePlan, VersionResolution, operation, scope, and action. + +## CLI + +CI can record an explicit provider review through the application API exposed by the +CLI: + +```bash +semapact approval record \ + --repository-root . \ + --decision-id "$SEMAPACT_DECISION_ID" \ + --change-set-id "$SEMAPACT_CHANGE_SET_ID" \ + --release-plan-id "$SEMAPACT_RELEASE_PLAN_ID" \ + --version-resolution-id "$SEMAPACT_VERSION_RESOLUTION_ID" \ + --operation PUBLISH \ + --action APPROVE \ + --actor-reference "github:user:${REVIEWER}" \ + --recorded-at "$REVIEW_SUBMITTED_AT" \ + --capability-reference "github:team:data-owners" \ + --evidence-reference "github:repo:${REPOSITORY}:pull:${PR_NUMBER}:review:${REVIEW_ID}" +``` + +`--recorded-at` is required and must be timezone-aware. SemaPact intentionally does not +substitute the local execution time because the review event timestamp is part of the +audit fact and deterministic approval identity. + +`--evidence-reference` may be repeated. References should be stable provider-local +identifiers or URIs that allow an auditor to trace the recorded event back to its +source. + +The Git working-tree backend writes the resulting immutable record under +`.semapact/history/`. Committing or otherwise publishing those generated history files +remains the responsibility of the surrounding GitOps workflow; the approval command +does not push branches or bypass repository protections. + +## Python SDK + +The same application API is available from the public Python package: + +```python +from datetime import datetime, timezone + +from semapact import ApprovalRecordService +from semapact.contractops import ReviewEvidenceAction +from semapact.governance import GovernanceOperation +from semapact.platforms.git import GitWorkingTreeHistoryRepository + +service = ApprovalRecordService(GitWorkingTreeHistoryRepository(".")) +record = service.record_review_action( + decision_id="...", + change_set_id="...", + release_plan_id="...", + version_resolution_id="...", + operation=GovernanceOperation.PUBLISH, + action=ReviewEvidenceAction.APPROVE, + actor_reference="github:user:alice", + recorded_at=datetime.now(timezone.utc), + capability_reference="github:team:data-owners", + evidence_references=("github:repo:org/repo:pull:42:review:1001",), +) +``` + +Callers using another future persistence backend can provide the same +`ApprovalHistoryRepository` capability without changing `ApprovalRecordService`. + +## Trust boundary + +`ApprovalRecord` preserves review evidence; it is not an authentication credential. +The intake adapter or CI workflow is responsible for obtaining review facts from a +trusted provider context. SemaPact does not infer approval from free-text comments, +commit messages, or an arbitrary `LGTM` string. + +Likewise, this layer does not introduce `latest approval wins`, quorum, reviewer +precedence, routing, or escalation policy. Those are separate workflow concerns and +must not be hidden inside persistence or history queries. diff --git a/docs/governance_history.md b/docs/governance_history.md index 87da80c5..86b0ea7a 100644 --- a/docs/governance_history.md +++ b/docs/governance_history.md @@ -7,15 +7,15 @@ store and retrieve those exact immutable models. The persistence boundary is capability-oriented: ```text -GovernanceDecision ChangeSet ContractRevision RevisionSource - ↓ ↓ ↓ ↓ -DecisionHistory ChangeSetHistory RevisionHistory RevisionSourceHistory -Repository Repository Repository Repository - \ | | / - \ | | / - GitWorkingTreeHistoryRepository - ↓ - .semapact/history/ +GovernanceDecision ChangeSet ApprovalRecord ContractRevision RevisionSource + ↓ ↓ ↓ ↓ ↓ +DecisionHistory ChangeSetHistory ApprovalHistory RevisionHistory RevisionSourceHistory +Repository Repository Repository Repository Repository + \ | | | / + \ | | | / + GitWorkingTreeHistoryRepository + ↓ + .semapact/history/ ``` Callers depend only on the narrow typed repository capability they need. A physical @@ -98,18 +98,71 @@ ContractRevision Therefore the same canonical contract content has the same revision ID wherever it is observed, while all known source references can still be retained independently. +## Approval history + +`ApprovalRecord` is an immutable review event owned by `semapact/approval/`. It records +what one actor explicitly did against one exact version-resolved ContractOps context: + +```text +ApprovalRecord +├── approval_id +├── decision_id +├── change_set_id +├── release_plan_id +├── version_resolution_id +├── operation +├── action +├── actor_reference +├── recorded_at +├── scope_reference? +├── capability_reference? +├── comment? +└── evidence_references[] +``` + +The approval ID is deterministic over the complete normalized event payload. The +explicit timezone-aware timestamp is part of that payload, so two actions by the same +actor against the same release context remain distinct historical events when they +occur at different times. Evidence references are normalized deterministically before +identity calculation. + +Persistence does not decide which approval should control a workflow. Multiple +reviewers, approvals, rejections, and request-changes actions coexist as independent +records. There is deliberately no implicit `latest wins`, quorum, or reviewer +precedence rule in the repository. + +A selected durable approval can be projected losslessly into the existing M2 +`ReviewAuthorizationEvidence` contract: + +```text +ApprovalRecord + ↓ explicit projection +ReviewAuthorizationEvidence + ↓ +authorize_contract_operation(...) +``` + +The approval ID becomes the evidence reference. M2 remains responsible for exact +context matching and for the semantics of APPROVE, REJECT, and REQUEST_CHANGES. +Persisting an approval by itself therefore never rewrites `GovernanceDecision` and +never grants authorization outside the normal ContractOps boundary. + +Approval history is separate from ODCS. Reviewer identity, comments, capabilities, +and evidence references are governance-history facts, not contract fields. + ## Persistence semantics For supported artifacts: - writing identical content under the same artifact ID is idempotent; -- writing different content under an existing artifact ID fails closed; +- writing different or identity-inconsistent content under an existing artifact ID + fails closed; - reads rehydrate the canonical domain model and invoke domain integrity validation before trusting persisted content; - the embedded artifact ID must match the requested/file identity; - malformed or semantically inconsistent persisted content fails closed; - missing IDs produce an explicit history not-found error; -- contract-scoped listings are deterministic. +- scoped listings are deterministic. `ChangeContext` remains part of `ChangeSet` and round-trips with it. History state is not written into canonical ODCS contracts. diff --git a/semapact/__init__.py b/semapact/__init__.py index 1af74dc0..c6a5d3a8 100644 --- a/semapact/__init__.py +++ b/semapact/__init__.py @@ -10,6 +10,11 @@ from typing import Any _EXPORTS: dict[str, tuple[str, str]] = { + "ApprovalRecord": ("semapact.approval", "ApprovalRecord"), + "ApprovalRecordService": ( + "semapact.application.services.approval_record", + "ApprovalRecordService", + ), "ContractLoader": ("semapact.core.loader", "ContractLoader"), "load_contract": ("semapact.core.loader", "load_contract"), "ContractValidator": ("semapact.core.validator", "ContractValidator"), diff --git a/semapact/application/services/approval_record.py b/semapact/application/services/approval_record.py new file mode 100644 index 00000000..0c659707 --- /dev/null +++ b/semapact/application/services/approval_record.py @@ -0,0 +1,78 @@ +"""Application boundary for recording explicit governance review actions.""" + +from __future__ import annotations + +from datetime import datetime + +from semapact.approval import ApprovalRecord, build_approval_record +from semapact.contractops import ReviewEvidenceAction +from semapact.governance import GovernanceOperation +from semapact.history import ApprovalHistoryRepository + + +class ApprovalRecordService: + """Record and query immutable ApprovalRecord facts without owning approval policy. + + External surfaces such as CLI, CI integrations, SDK callers, or a future UI + provide an explicit review event. This service normalizes that event through the + canonical ApprovalRecord builder and persists the resulting artifact through the + typed history capability. + + The service deliberately does not approve a change, discover provider approvals, + choose a winning approval, apply quorum rules, or authorize ContractOps actions. + """ + + def __init__(self, approvals: ApprovalHistoryRepository) -> None: + self._approvals = approvals + + def record_review_action( + self, + *, + decision_id: str, + change_set_id: str, + release_plan_id: str, + version_resolution_id: str, + operation: GovernanceOperation, + action: ReviewEvidenceAction, + actor_reference: str, + recorded_at: datetime, + scope_reference: str | None = None, + capability_reference: str | None = None, + comment: str | None = None, + evidence_references: tuple[str, ...] = (), + ) -> ApprovalRecord: + """Build and persist one explicit immutable review event idempotently.""" + record = build_approval_record( + decision_id=decision_id, + change_set_id=change_set_id, + release_plan_id=release_plan_id, + version_resolution_id=version_resolution_id, + operation=operation, + action=action, + actor_reference=actor_reference, + recorded_at=recorded_at, + scope_reference=scope_reference, + capability_reference=capability_reference, + comment=comment, + evidence_references=evidence_references, + ) + self._approvals.put_approval_record(record) + return record + + def list_review_actions( + self, + *, + decision_id: str, + change_set_id: str, + release_plan_id: str, + version_resolution_id: str, + operation: GovernanceOperation, + ) -> tuple[ApprovalRecord, ...]: + """Return all immutable review events for one exact ContractOps context.""" + return self._approvals.list_approval_records_for_context( + decision_id=decision_id, + change_set_id=change_set_id, + release_plan_id=release_plan_id, + version_resolution_id=version_resolution_id, + operation=operation, + ) diff --git a/semapact/approval/__init__.py b/semapact/approval/__init__.py new file mode 100644 index 00000000..db970ad5 --- /dev/null +++ b/semapact/approval/__init__.py @@ -0,0 +1,19 @@ +"""Immutable review-approval artifacts and M2 evidence projection.""" + +from semapact.approval.builders import build_approval_record +from semapact.approval.evidence import project_review_authorization_evidence +from semapact.approval.integrity import ( + SEMAPACT_APPROVAL_RECORD_NAMESPACE, + compute_approval_record_id, + validate_approval_record_identity, +) +from semapact.approval.models import ApprovalRecord + +__all__ = [ + "ApprovalRecord", + "SEMAPACT_APPROVAL_RECORD_NAMESPACE", + "build_approval_record", + "compute_approval_record_id", + "project_review_authorization_evidence", + "validate_approval_record_identity", +] diff --git a/semapact/approval/builders.py b/semapact/approval/builders.py new file mode 100644 index 00000000..b1008497 --- /dev/null +++ b/semapact/approval/builders.py @@ -0,0 +1,46 @@ +"""Canonical construction of immutable approval records.""" + +from __future__ import annotations + +from datetime import datetime + +from semapact.approval.integrity import compute_approval_record_id +from semapact.approval.models import ApprovalRecord +from semapact.contractops.models import ReviewEvidenceAction +from semapact.governance.gate import GovernanceOperation + + +def build_approval_record( + *, + decision_id: str, + change_set_id: str, + release_plan_id: str, + version_resolution_id: str, + operation: GovernanceOperation, + action: ReviewEvidenceAction, + actor_reference: str, + recorded_at: datetime, + scope_reference: str | None = None, + capability_reference: str | None = None, + comment: str | None = None, + evidence_references: tuple[str, ...] = (), +) -> ApprovalRecord: + """Build one canonical approval event and derive its deterministic identity.""" + provisional = ApprovalRecord( + approval_id="pending", + decision_id=decision_id, + change_set_id=change_set_id, + release_plan_id=release_plan_id, + version_resolution_id=version_resolution_id, + operation=operation, + action=action, + actor_reference=actor_reference, + recorded_at=recorded_at, + scope_reference=scope_reference, + capability_reference=capability_reference, + comment=comment, + evidence_references=evidence_references, + ) + approval_id = compute_approval_record_id(provisional) + payload = provisional.model_dump(exclude={"approval_id"}) + return ApprovalRecord(approval_id=approval_id, **payload) diff --git a/semapact/approval/evidence.py b/semapact/approval/evidence.py new file mode 100644 index 00000000..64e034f4 --- /dev/null +++ b/semapact/approval/evidence.py @@ -0,0 +1,32 @@ +"""Lossless projection from durable ApprovalRecord to M2 review evidence.""" + +from __future__ import annotations + +from semapact.approval.integrity import validate_approval_record_identity +from semapact.approval.models import ApprovalRecord +from semapact.contractops.models import ReviewAuthorizationEvidence + + +def project_review_authorization_evidence( + record: ApprovalRecord, +) -> ReviewAuthorizationEvidence: + """Project one selected durable approval into the existing M2 evidence contract. + + Selection/routing/quorum policy is deliberately outside this function. M2 remains + responsible for matching this evidence against the exact requested release context. + """ + if not isinstance(record, ApprovalRecord): + raise TypeError( + f"record must be ApprovalRecord, got {type(record).__name__}" + ) + validate_approval_record_identity(record) + return ReviewAuthorizationEvidence( + evidence_reference=record.approval_id, + decision_id=record.decision_id, + change_set_id=record.change_set_id, + release_plan_id=record.release_plan_id, + version_resolution_id=record.version_resolution_id, + operation=record.operation, + action=record.action, + scope_reference=record.scope_reference, + ) diff --git a/semapact/approval/integrity.py b/semapact/approval/integrity.py new file mode 100644 index 00000000..7b0f521d --- /dev/null +++ b/semapact/approval/integrity.py @@ -0,0 +1,45 @@ +"""Deterministic identity rules for immutable governance approval records.""" + +from __future__ import annotations + +import uuid + +from semapact.approval.models import ApprovalRecord +from semapact.utils.deterministic import deterministic_uuid5 + + +SEMAPACT_APPROVAL_RECORD_NAMESPACE = uuid.UUID( + "c0e1984e-8674-5705-bbfd-2784f6e90e6a" +) + + +def compute_approval_record_id(record: ApprovalRecord) -> str: + """Return the stable UUIDv5 identity for one canonical approval event.""" + if not isinstance(record, ApprovalRecord): + raise TypeError( + f"record must be ApprovalRecord, got {type(record).__name__}" + ) + return deterministic_uuid5( + SEMAPACT_APPROVAL_RECORD_NAMESPACE, + { + "decisionId": record.decision_id, + "changeSetId": record.change_set_id, + "releasePlanId": record.release_plan_id, + "versionResolutionId": record.version_resolution_id, + "operation": record.operation.value, + "action": record.action.value, + "actorReference": record.actor_reference, + "recordedAt": record.recorded_at.isoformat(), + "scopeReference": record.scope_reference, + "capabilityReference": record.capability_reference, + "comment": record.comment, + "evidenceReferences": list(record.evidence_references), + }, + ) + + +def validate_approval_record_identity(record: ApprovalRecord) -> None: + """Fail if approval_id does not match the complete canonical event payload.""" + expected = compute_approval_record_id(record) + if expected != record.approval_id: + raise ValueError("approval_id does not match deterministic approval identity") diff --git a/semapact/approval/models.py b/semapact/approval/models.py new file mode 100644 index 00000000..c65aaed2 --- /dev/null +++ b/semapact/approval/models.py @@ -0,0 +1,85 @@ +"""Immutable domain model for one explicit governance review action.""" + +from __future__ import annotations + +from datetime import datetime, timezone + +from pydantic import BaseModel, ConfigDict, field_validator + +from semapact.contractops.models import ReviewEvidenceAction +from semapact.governance.gate import GovernanceOperation + + +class ApprovalRecord(BaseModel): + """One immutable review action bound to an exact ContractOps context. + + The record is audit evidence only. It does not reinterpret GovernanceDecision + and does not authorize an operation until explicitly projected into the existing + M2 ReviewAuthorizationEvidence contract. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + approval_id: str + decision_id: str + change_set_id: str + release_plan_id: str + version_resolution_id: str + operation: GovernanceOperation + action: ReviewEvidenceAction + actor_reference: str + recorded_at: datetime + scope_reference: str | None = None + capability_reference: str | None = None + comment: str | None = None + evidence_references: tuple[str, ...] = () + + @field_validator( + "approval_id", + "decision_id", + "change_set_id", + "release_plan_id", + "version_resolution_id", + "actor_reference", + ) + @classmethod + def _require_text(cls, value: str) -> str: + return _required_text(value) + + @field_validator("scope_reference", "capability_reference", "comment") + @classmethod + def _normalize_optional_text(cls, value: str | None) -> str | None: + return _optional_text(value) + + @field_validator("recorded_at") + @classmethod + def _normalize_recorded_at(cls, value: datetime) -> datetime: + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError("recorded_at must be timezone-aware") + return value.astimezone(timezone.utc) + + @field_validator("evidence_references") + @classmethod + def _normalize_evidence_references( + cls, + value: tuple[str, ...], + ) -> tuple[str, ...]: + return tuple(sorted({_required_text(item) for item in value})) + + +def _required_text(value: str) -> str: + if not isinstance(value, str): + raise TypeError("value must be str") + cleaned = value.strip() + if not cleaned: + raise ValueError("value must not be empty") + return cleaned + + +def _optional_text(value: str | None) -> str | None: + if value is None: + return None + if not isinstance(value, str): + raise TypeError("value must be str or None") + cleaned = value.strip() + return cleaned or None diff --git a/semapact/history/__init__.py b/semapact/history/__init__.py index 16d5af14..6ed31053 100644 --- a/semapact/history/__init__.py +++ b/semapact/history/__init__.py @@ -15,6 +15,7 @@ RuntimeReconciliationRecord, ) from semapact.history.repository import ( + ApprovalHistoryRepository, ChangeSetDecisionLinkHistoryRepository, ChangeSetHistoryRepository, ContractRevisionHistoryRepository, @@ -36,6 +37,7 @@ ) __all__ = [ + "ApprovalHistoryRepository", "ChangeSetDecisionLink", "ChangeSetDecisionLinkHistoryRepository", "ChangeSetHistoryRepository", diff --git a/semapact/history/repository.py b/semapact/history/repository.py index 0ddc03a2..a17f2408 100644 --- a/semapact/history/repository.py +++ b/semapact/history/repository.py @@ -4,9 +4,11 @@ from typing import Protocol +from semapact.approval.models import ApprovalRecord from semapact.contractops import ChangeSet, ReleasePlan from semapact.deployment import DeploymentAuthorization, DeploymentPlan, DeploymentPreview from semapact.governance import GovernanceDecision +from semapact.governance.gate import GovernanceOperation from semapact.history.models import ( ChangeSetDecisionLink, DeploymentRecord, @@ -67,6 +69,23 @@ def list_change_set_decision_links( ) -> tuple[ChangeSetDecisionLink, ...]: ... +class ApprovalHistoryRepository(Protocol): + """Persistence capability for immutable ApprovalRecord history only.""" + + def put_approval_record(self, record: ApprovalRecord) -> None: ... + def get_approval_record(self, approval_id: str) -> ApprovalRecord: ... + + def list_approval_records_for_context( + self, + *, + decision_id: str, + change_set_id: str, + release_plan_id: str, + version_resolution_id: str, + operation: GovernanceOperation, + ) -> tuple[ApprovalRecord, ...]: ... + + class ContractRevisionHistoryRepository(Protocol): """Persistence capability for immutable ContractRevision history only.""" diff --git a/semapact/interfaces/cli.py b/semapact/interfaces/cli.py index 0aa135f7..276fd765 100644 --- a/semapact/interfaces/cli.py +++ b/semapact/interfaces/cli.py @@ -22,6 +22,9 @@ def _add_effective_date_argument( def _build_parser() -> argparse.ArgumentParser: + from semapact.contractops import ReviewEvidenceAction + from semapact.governance import GovernanceOperation + parser = argparse.ArgumentParser(prog="semapact") subparsers = parser.add_subparsers(dest="command", required=False) @@ -220,6 +223,51 @@ def _build_parser() -> argparse.ArgumentParser: pr_parser.add_argument("--paths", nargs="*") pr_parser.add_argument("--push", action="store_true") + approval_parser = subparsers.add_parser( + "approval", + help="Record explicit governance review evidence", + ) + approval_subparsers = approval_parser.add_subparsers( + dest="approval_command", required=True + ) + approval_record_parser = approval_subparsers.add_parser( + "record", + help="Persist one explicit external review event as immutable approval history", + ) + approval_record_parser.add_argument( + "--repository-root", + default=".", + help="Repository root containing .semapact/history (default: current directory)", + ) + approval_record_parser.add_argument("--decision-id", required=True) + approval_record_parser.add_argument("--change-set-id", required=True) + approval_record_parser.add_argument("--release-plan-id", required=True) + approval_record_parser.add_argument("--version-resolution-id", required=True) + approval_record_parser.add_argument( + "--operation", + required=True, + choices=[item.value for item in GovernanceOperation], + ) + approval_record_parser.add_argument( + "--action", + required=True, + choices=[item.value for item in ReviewEvidenceAction], + ) + approval_record_parser.add_argument("--actor-reference", required=True) + approval_record_parser.add_argument( + "--recorded-at", + required=True, + help="Timezone-aware ISO-8601 timestamp from the external review event", + ) + approval_record_parser.add_argument("--scope-reference") + approval_record_parser.add_argument("--capability-reference") + approval_record_parser.add_argument("--comment") + approval_record_parser.add_argument( + "--evidence-reference", + action="append", + help="Stable external review evidence reference; may be provided multiple times", + ) + release_parser = subparsers.add_parser( "release", help="Per-contract release workflow helpers" ) @@ -495,6 +543,15 @@ def main() -> int: print(json.dumps(payload, indent=2, sort_keys=True)) return 0 + if args.command == "approval": + from semapact.interfaces.commands.approval_cmd import run_approval_record + + if args.approval_command == "record": + payload = run_approval_record(args) + print(json.dumps(payload, indent=2, sort_keys=True)) + return 0 + parser.error(f"Unknown approval command: {args.approval_command}") + if args.command == "reconcile": from semapact.interfaces.commands.reconcile_cmd import run_reconcile from semapact.interfaces.outcomes import exit_code_from_outcome diff --git a/semapact/interfaces/commands/approval_cmd.py b/semapact/interfaces/commands/approval_cmd.py new file mode 100644 index 00000000..87b1534c --- /dev/null +++ b/semapact/interfaces/commands/approval_cmd.py @@ -0,0 +1,46 @@ +"""CLI adapters for explicit governance approval recording.""" + +from __future__ import annotations + +from argparse import Namespace +from datetime import datetime + +from semapact.application.services.approval_record import ApprovalRecordService +from semapact.contractops import ReviewEvidenceAction +from semapact.governance import GovernanceOperation +from semapact.platforms.git import GitWorkingTreeHistoryRepository + + +def run_approval_record(args: Namespace) -> dict[str, object]: + """Record one explicit review event through the application service boundary.""" + recorded_at = _parse_timestamp(args.recorded_at) + service = ApprovalRecordService( + GitWorkingTreeHistoryRepository(args.repository_root), + ) + record = service.record_review_action( + decision_id=args.decision_id, + change_set_id=args.change_set_id, + release_plan_id=args.release_plan_id, + version_resolution_id=args.version_resolution_id, + operation=GovernanceOperation(args.operation), + action=ReviewEvidenceAction(args.action), + actor_reference=args.actor_reference, + recorded_at=recorded_at, + scope_reference=args.scope_reference, + capability_reference=args.capability_reference, + comment=args.comment, + evidence_references=tuple(args.evidence_reference or ()), + ) + return record.model_dump(mode="json", by_alias=True) + + +def _parse_timestamp(value: str) -> datetime: + if not isinstance(value, str): + raise TypeError("recorded_at must be an ISO-8601 string") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise ValueError("recorded_at must be an ISO-8601 timestamp") from exc + if parsed.tzinfo is None or parsed.utcoffset() is None: + raise ValueError("recorded_at must include an explicit timezone offset") + return parsed diff --git a/semapact/platforms/git/history_repository.py b/semapact/platforms/git/history_repository.py index e0eade53..dfcdaf40 100644 --- a/semapact/platforms/git/history_repository.py +++ b/semapact/platforms/git/history_repository.py @@ -17,6 +17,8 @@ from pydantic import BaseModel, ValidationError as PydanticValidationError +from semapact.approval.integrity import validate_approval_record_identity +from semapact.approval.models import ApprovalRecord from semapact.contractops import ChangeSet, ReleasePlan from semapact.contractops.integrity import ( validate_change_set_identity, @@ -29,6 +31,7 @@ validate_deployment_preview_identity, ) from semapact.governance import GovernanceDecision +from semapact.governance.gate import GovernanceOperation from semapact.history import ( ChangeSetDecisionLink, DeploymentRecord, @@ -86,6 +89,12 @@ class _HistoryKindSpec(Generic[T]): ChangeSetDecisionLink, "decision_id", ) +_APPROVAL_RECORDS = _HistoryKindSpec( + "approval_records", + ApprovalRecord, + "approval_id", + validate_approval_record_identity, +) _CONTRACT_REVISIONS = _HistoryKindSpec( "contract_revisions", ContractRevision, @@ -153,6 +162,7 @@ class _HistoryKindSpec(Generic[T]): _DECISIONS, _CHANGE_SETS, _CHANGE_SET_DECISIONS, + _APPROVAL_RECORDS, _CONTRACT_REVISIONS, _CONTRACT_REVISION_SOURCES, _RELEASE_PLANS, @@ -246,6 +256,43 @@ def list_change_set_decision_links( ) return records + def put_approval_record(self, record: ApprovalRecord) -> None: + self._put(_APPROVAL_RECORDS, record.approval_id, record) + + def get_approval_record(self, approval_id: str) -> ApprovalRecord: + return self._get(_APPROVAL_RECORDS, approval_id) + + def list_approval_records_for_context( + self, + *, + decision_id: str, + change_set_id: str, + release_plan_id: str, + version_resolution_id: str, + operation: GovernanceOperation, + ) -> tuple[ApprovalRecord, ...]: + decision_id = _required_text(decision_id, "decision_id") + change_set_id = _required_text(change_set_id, "change_set_id") + release_plan_id = _required_text(release_plan_id, "release_plan_id") + version_resolution_id = _required_text( + version_resolution_id, + "version_resolution_id", + ) + if not isinstance(operation, GovernanceOperation): + raise TypeError( + "operation must be GovernanceOperation, " + f"got {type(operation).__name__}" + ) + return tuple( + record + for record in self._list(_APPROVAL_RECORDS) + if record.decision_id == decision_id + and record.change_set_id == change_set_id + and record.release_plan_id == release_plan_id + and record.version_resolution_id == version_resolution_id + and record.operation is operation + ) + def put_revision(self, revision: ContractRevision) -> None: self._put(_CONTRACT_REVISIONS, revision.revision_id, revision) diff --git a/tests/test_approval_api.py b/tests/test_approval_api.py new file mode 100644 index 00000000..611ea2b4 --- /dev/null +++ b/tests/test_approval_api.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +from argparse import Namespace +from datetime import datetime, timezone +from pathlib import Path + +from semapact import ApprovalRecord, ApprovalRecordService +from semapact.contractops import ReviewEvidenceAction +from semapact.governance import GovernanceOperation +from semapact.interfaces.cli import _build_parser +from semapact.interfaces.commands.approval_cmd import run_approval_record +from semapact.platforms.git import GitWorkingTreeHistoryRepository + + +def _event() -> dict[str, object]: + return { + "decision_id": "decision-1", + "change_set_id": "change-set-1", + "release_plan_id": "release-plan-1", + "version_resolution_id": "version-resolution-1", + "operation": GovernanceOperation.PUBLISH, + "action": ReviewEvidenceAction.APPROVE, + "actor_reference": "github:user:alice", + "recorded_at": datetime(2026, 9, 16, 1, 0, tzinfo=timezone.utc), + "capability_reference": "github:team:data-owners", + "evidence_references": ( + "github:repo:DaorynAI/example:pull:42:review:1001", + ), + } + + +def test_public_approval_record_service_records_through_typed_history_port( + tmp_path: Path, +) -> None: + repository = GitWorkingTreeHistoryRepository(tmp_path) + service = ApprovalRecordService(repository) + + record = service.record_review_action(**_event()) + + assert isinstance(record, ApprovalRecord) + assert repository.get_approval_record(record.approval_id) == record + assert service.list_review_actions( + decision_id=record.decision_id, + change_set_id=record.change_set_id, + release_plan_id=record.release_plan_id, + version_resolution_id=record.version_resolution_id, + operation=record.operation, + ) == (record,) + + +def test_cli_parser_exposes_approval_record_command() -> None: + args = _build_parser().parse_args( + [ + "approval", + "record", + "--decision-id", + "decision-1", + "--change-set-id", + "change-set-1", + "--release-plan-id", + "release-plan-1", + "--version-resolution-id", + "version-resolution-1", + "--operation", + "PUBLISH", + "--action", + "APPROVE", + "--actor-reference", + "github:user:alice", + "--recorded-at", + "2026-09-16T01:00:00Z", + "--evidence-reference", + "github:repo:DaorynAI/example:pull:42:review:1001", + ] + ) + + assert args.command == "approval" + assert args.approval_command == "record" + assert args.operation == "PUBLISH" + assert args.action == "APPROVE" + + +def test_cli_adapter_records_external_review_evidence_without_provider_policy( + tmp_path: Path, +) -> None: + args = Namespace( + repository_root=str(tmp_path), + decision_id="decision-1", + change_set_id="change-set-1", + release_plan_id="release-plan-1", + version_resolution_id="version-resolution-1", + operation="PUBLISH", + action="APPROVE", + actor_reference="github:user:alice", + recorded_at="2026-09-16T11:00:00+10:00", + scope_reference=None, + capability_reference="github:team:data-owners", + comment="Approved in pull request review", + evidence_reference=["github:repo:DaorynAI/example:pull:42:review:1001"], + ) + + payload = run_approval_record(args) + record = GitWorkingTreeHistoryRepository(tmp_path).get_approval_record( + str(payload["approval_id"]) + ) + + assert record.actor_reference == "github:user:alice" + assert record.recorded_at == datetime(2026, 9, 16, 1, 0, tzinfo=timezone.utc) + assert record.action is ReviewEvidenceAction.APPROVE + assert record.evidence_references == ( + "github:repo:DaorynAI/example:pull:42:review:1001", + ) diff --git a/tests/test_approval_history.py b/tests/test_approval_history.py new file mode 100644 index 00000000..dba660c4 --- /dev/null +++ b/tests/test_approval_history.py @@ -0,0 +1,174 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from semapact.approval import ( + build_approval_record, + project_review_authorization_evidence, + validate_approval_record_identity, +) +from semapact.contractops import ReviewEvidenceAction +from semapact.governance import GovernanceOperation +from semapact.history import ApprovalHistoryRepository, HistoryCorruptionError +from semapact.platforms.git import GitWorkingTreeHistoryRepository + + +CONTEXT = { + "decision_id": "decision-1", + "change_set_id": "change-set-1", + "release_plan_id": "release-plan-1", + "version_resolution_id": "version-resolution-1", + "operation": GovernanceOperation.PUBLISH, +} + + +def _approval( + *, + actor_reference: str = "user:alice", + action: ReviewEvidenceAction = ReviewEvidenceAction.APPROVE, + recorded_at: datetime = datetime(2026, 9, 16, 1, 0, tzinfo=timezone.utc), + scope_reference: str | None = None, + comment: str | None = "reviewed", + evidence_references: tuple[str, ...] = ("ticket:2", "ticket:1"), +): + return build_approval_record( + **CONTEXT, + actor_reference=actor_reference, + action=action, + recorded_at=recorded_at, + scope_reference=scope_reference, + capability_reference="role:data-governance-reviewer", + comment=comment, + evidence_references=evidence_references, + ) + + +def test_approval_builder_normalizes_event_before_identity() -> None: + brisbane = timezone(timedelta(hours=10)) + first = _approval( + recorded_at=datetime(2026, 9, 16, 11, 0, tzinfo=brisbane), + evidence_references=("ticket:2", "ticket:1", "ticket:2"), + ) + second = _approval( + recorded_at=datetime(2026, 9, 16, 1, 0, tzinfo=timezone.utc), + evidence_references=("ticket:1", "ticket:2"), + ) + + assert first == second + assert first.recorded_at == datetime(2026, 9, 16, 1, 0, tzinfo=timezone.utc) + assert first.evidence_references == ("ticket:1", "ticket:2") + validate_approval_record_identity(first) + + +def test_distinct_review_events_have_distinct_identity() -> None: + first = _approval() + later = _approval( + recorded_at=datetime(2026, 9, 16, 1, 1, tzinfo=timezone.utc), + ) + other_actor = _approval(actor_reference="user:bob") + rejected = _approval(action=ReviewEvidenceAction.REJECT) + + assert len( + { + first.approval_id, + later.approval_id, + other_actor.approval_id, + rejected.approval_id, + } + ) == 4 + + +def test_approval_requires_timezone_aware_timestamp() -> None: + with pytest.raises(ValidationError, match="timezone-aware"): + _approval(recorded_at=datetime(2026, 9, 16, 1, 0)) + + +def test_tampered_approval_identity_fails_closed() -> None: + record = _approval() + tampered = record.model_copy(update={"comment": "changed after identity"}) + + with pytest.raises(ValueError, match="deterministic approval identity"): + validate_approval_record_identity(tampered) + + +def test_projection_is_lossless_for_m2_authorization_scope() -> None: + record = _approval(scope_reference="deployment-plan:abc") + + evidence = project_review_authorization_evidence(record) + + assert evidence.evidence_reference == record.approval_id + assert evidence.decision_id == record.decision_id + assert evidence.change_set_id == record.change_set_id + assert evidence.release_plan_id == record.release_plan_id + assert evidence.version_resolution_id == record.version_resolution_id + assert evidence.operation is record.operation + assert evidence.action is record.action + assert evidence.scope_reference == record.scope_reference + + +def test_approval_records_round_trip_through_typed_history_port(tmp_path: Path) -> None: + backend = GitWorkingTreeHistoryRepository(tmp_path) + approvals: ApprovalHistoryRepository = backend + record = _approval() + + approvals.put_approval_record(record) + approvals.put_approval_record(record) + + assert approvals.get_approval_record(record.approval_id) == record + assert backend.inspect_history_integrity() == () + + +def test_invalid_content_cannot_reuse_existing_approval_identity(tmp_path: Path) -> None: + repository = GitWorkingTreeHistoryRepository(tmp_path) + record = _approval() + repository.put_approval_record(record) + tampered = record.model_copy(update={"comment": "different content"}) + + with pytest.raises(HistoryCorruptionError, match="is invalid"): + repository.put_approval_record(tampered) + + +def test_exact_context_listing_preserves_all_actions_deterministically( + tmp_path: Path, +) -> None: + repository = GitWorkingTreeHistoryRepository(tmp_path) + matching = ( + _approval(actor_reference="user:alice"), + _approval(actor_reference="user:bob"), + _approval( + actor_reference="user:carol", + action=ReviewEvidenceAction.REQUEST_CHANGES, + ), + ) + wrong_operation = build_approval_record( + **{**CONTEXT, "operation": GovernanceOperation.APPLY}, + actor_reference="user:dana", + action=ReviewEvidenceAction.APPROVE, + recorded_at=datetime(2026, 9, 16, 1, 2, tzinfo=timezone.utc), + ) + wrong_decision = build_approval_record( + **{**CONTEXT, "decision_id": "decision-2"}, + actor_reference="user:erin", + action=ReviewEvidenceAction.APPROVE, + recorded_at=datetime(2026, 9, 16, 1, 3, tzinfo=timezone.utc), + ) + + for record in reversed((*matching, wrong_operation, wrong_decision)): + repository.put_approval_record(record) + + listed = repository.list_approval_records_for_context(**CONTEXT) + + assert {item.approval_id for item in listed} == { + item.approval_id for item in matching + } + assert [item.approval_id for item in listed] == sorted( + item.approval_id for item in matching + ) + assert {item.action for item in listed} == { + ReviewEvidenceAction.APPROVE, + ReviewEvidenceAction.REQUEST_CHANGES, + } From f3fab602cb7edc4ebd508e78df4421dc051b8ea8 Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Sat, 19 Sep 2026 20:36:37 +1000 Subject: [PATCH 29/35] refactor(deployment): centralize provider-extensible schema evolution (#236) * refactor(databricks): extract schema evolution planner * refactor(databricks): delegate schema evolution planning * test(databricks): cover pure schema evolution planning * test(databricks): harden schema evolution planner coverage * fix(databricks): validate evolution observation identity * test(databricks): match observed asset validation * docs(contractops): define contract as desired-state artifact * docs(deployment): separate release artifact from runtime DDL * refactor(deployment): add semantic schema transitions * refactor(databricks): separate transition planning from SQL * refactor(databricks): compile semantic transitions to SQL * refactor(databricks): compile transitions in deployment preview * test(databricks): separate transitions from SQL compilation * test(deployment): cover provider-neutral schema transitions * refactor(databricks): reuse sqlglot native type parser * style(databricks): clean native type parser imports * refactor(schema): add shared comparison package * refactor(schema): centralize normalized schema comparison * refactor(reconciliation): reuse shared schema diff enums * refactor(reconciliation): consume shared schema comparison * refactor(deployment): derive transitions from shared schema diff * refactor(databricks): compile shared schema property state * refactor(databricks): project schemas through shared comparator * refactor(databricks): use shared schema comparison mapping * refactor(databricks): use schema mapping identifier validation * test(databricks): follow shared schema comparison boundary * test(deployment): consume shared schema differences * test(schema): lock shared comparison semantics * refactor(databricks): remove provider-local schema comparator * docs(deployment): define shared schema comparison boundary * refactor(schema): add shared schema mapping contract * refactor(schema): export shared mapping contract * refactor(databricks): implement shared schema mapper contract * refactor(reconciliation): use shared schema mapping helpers * refactor(schema): preserve observed identity failure semantics * test(schema): cover shared schema mapper contract * docs(schema): define shared schema mapping contract * refactor(schema): make mapping a whole-schema provider seam * refactor(schema): expose ODCS projection separately * refactor(reconciliation): consume whole-schema mapper seam * refactor(databricks): derive desired schema from datacontract DDL * style(schema): type observed normalization callback * test(schema): cover whole-schema mapper contract * refactor(databricks): report invalid target compiler output * test(databricks): assert datacontract target schema projection * docs(schema): delegate target schema compilation to datacontract-cli * refactor(databricks): rename schema mapping implementation * refactor(databricks): import schema mapper implementation * refactor(databricks): import schema mapper implementation * test(databricks): follow schema mapper implementation rename * refactor(databricks): remove old schema mapping module * refactor(databricks): keep schema mapper adapter-only * refactor(databricks): isolate identifier policy * refactor(databricks): separate schema planning from mapping * refactor(databricks): consume planner and identifier boundaries * refactor(databricks): consume identifier boundary * test(databricks): follow planner boundary * refactor(schema): retain opaque native column definitions * refactor(schema): centralize SQL schema mapping mechanics * refactor(schema): expose shared SQL mapper implementation * refactor(deployment): centralize schema transition orchestration * refactor(databricks): instantiate shared schema mapper * refactor(databricks): use shared schema transition workflow * test(databricks): exercise shared schema workflow * refactor(databricks): remove provider-local schema mapper implementation * refactor(databricks): remove provider-local schema planning workflow * refactor(deployment): define transition compiler contract * refactor(databricks): implement transition compiler contract * refactor(databricks): use transition compiler contract * test(databricks): use transition compiler contract * refactor(databricks): quote compiled transition identifiers * refactor(databricks): remove duplicate schema SQL compiler * fix(schema): parse only top-level target columns * refactor(schema): make mapper contract explicit ABC * refactor(deployment): make compiler contract explicit ABC * refactor(schema): centralize executable identifier validation * refactor(deployment): define platform and executor contracts * refactor(deployment): centralize generic deployment orchestration * refactor(schema): export shared SQL identifier validator * refactor(databricks): inherit compiler contract and shared identifier policy * refactor(databricks): implement generic deployment platform contract * refactor(databricks): delegate lifecycle to generic orchestrator * test(databricks): exercise platform contract over shared schema workflow * refactor(databricks): remove obsolete schema.py * refactor(databricks): remove obsolete identifiers.py * style(deployment): clean generic orchestrator imports * style(databricks): clean transition compiler imports * style(schema): normalize public exports * refactor(deployment): make adapter a complete lifecycle ABC * refactor(deployment): make generic orchestrator own observation * refactor(application): delegate deployment orchestration generically * refactor(cli): route preview through generic deployment adapter * refactor(deployment): expose generic orchestration contracts * test(deployment): lock generic orchestration flow * docs(deployment): document generic orchestration and API execution * fix(deployment): validate before runtime observation * refactor(schema): canonicalize native definitions once * refactor(deployment): share native definition precondition * refactor(databricks): keep transition compiler verb-only * refactor(databricks): instantiate compiler at platform boundary * refactor(databricks): instantiate transition compiler in platform * test(databricks): instantiate compiler implementation explicitly * fix(schema): retain only governed physical column shape * test(databricks): constrain transition compiler to governed shape * refactor(deployment): include verify in unified adapter contract * refactor(deployment): route verification through generic orchestrator * refactor(application): unify deployment verify through adapter * refactor(cli): use one deployment adapter for verify * refactor(platforms): define shared platform composition contract * refactor(databricks): implement shared platform factory * refactor(platforms): move runtime target projection behind factory * refactor(databricks): own ODCS runtime target projection * refactor(platforms): centralize platform composition dispatch * refactor(reconciliation): allow platform target-schema normalization * refactor(deployment): verify through platform schema mapping * refactor(deployment): verify against compiled target schema * test(databricks): use unified deployment preview entrypoint * test(application): lock unified deployment adapter entrypoint * fix(platforms): preserve neutral CLI runtime fallback resolution * refactor(deployment): keep transition planning interpretation-only * test(databricks): avoid secondary schema orchestration helper * test(databricks): verify through target schema normalization * test(architecture): lock deployment platform extension seams * refactor(platforms): keep deployment registry options provider-neutral * refactor(cli): pass provider execution options generically * refactor(platforms): expose platform factory contract * test(deployment): cover generic verify and component key invariants * test(platforms): prove one factory loader extends runtime and deployment * test(platforms): keep fake platform fixture provider-neutral * test(application): fake adapter implements public deployment contract * fix(schema): inspect only top-level nullability constraints * test(schema): cover nested SQL target schema boundaries * test(databricks): cover statement polling and failure * fix(reconciliation): rebind only platform-compiled desired identities * test(reconciliation): preserve logical identity under physical collisions * refactor(deployment): separate verify from mutation capability policy * style(platforms): depend on narrow composition contracts * style(databricks): keep factory imports and validation narrow * test(databricks): keep verification separate from mutation capability * ci(databricks): cover schema evolution write path * test(cli): lock verify to unified deployment adapter * fix(schema): reject extra target compiler statements * test(schema): assert fail-closed target compiler output * test(schema): assert fail-closed target compiler output * test(contractops): use unified deployment preview entrypoint * test(deployment): use exact fake runtime bindings * docs(deployment): align extensibility and unified verify flow * docs(cli): describe unified deployment verify entrypoint * refactor(deployment): make transition capability pluggable * refactor(deployment): expose transition planner extension seam * refactor(deployment): delegate transition capability to platform * refactor(databricks): configure shared additive transition planner * test(deployment): configure transition planner through platform seam * refactor(deployment): expose transition planning extension contract * test(architecture): lock transition planner extension contract * docs(deployment): document transition capability extension seam * test(deployment): prove transition policy is provider-pluggable * refactor(deployment): type provider execution configuration * refactor(platforms): require typed deployment execution config * refactor(databricks): add typed deployment execution config * refactor(databricks): consume typed deployment execution config * refactor(platforms): pass typed execution config through registry * refactor(cli): construct typed provider execution config * refactor(deployment): expose typed execution config contract * test(platforms): compose typed execution configuration * fix(cli): reject Databricks execution options on other platforms * fix(platforms): bind execution config to selected platform * test(platforms): bind typed execution config to platform * refactor(platforms): remove factory indirection from composition root * refactor(platforms): remove obsolete factories.py * refactor(platforms): remove obsolete factory.py * refactor(platforms): remove obsolete test_platform_factory.py * refactor(platforms): remove factory export * test(architecture): remove composition factory assumptions * docs(platforms): keep runtime registry as composition root * test(platforms): lock runtime registry as composition root * refactor(deployment): expose observed asset to transition policy * refactor(databricks): isolate mutation capability in transition planner * refactor(databricks): validate runtime bindings at provider boundary * refactor(deployment): remove platform container contract * refactor(deployment): compose explicit behavior seams in orchestrator * refactor(databricks): wire explicit deployment behavior seams * test(databricks): exercise mapper planner compiler without platform wrapper * refactor(deployment): remove platform container export * test(deployment): compose orchestrator with explicit behavior seams * refactor(databricks): rely on runtime provider for adapter key * test(architecture): remove deployment platform wrapper seam * test(databricks): apply managed constraint only to mutation * refactor(databricks): remove deployment platform wrapper * docs(deployment): remove deployment platform wrapper concept --- .github/workflows/ci.yml | 6 +- docs/contractops_phases.md | 31 ++ docs/deployment_plans.md | 85 ++- semapact/application/services/deployment.py | 55 +- semapact/deployment/__init__.py | 18 +- semapact/deployment/adapters.py | 32 +- semapact/deployment/compilers.py | 38 ++ semapact/deployment/orchestrator.py | 396 ++++++++++++++ semapact/deployment/providers.py | 38 ++ semapact/deployment/schema_transitions.py | 228 +++++++++ semapact/deployment/verification.py | 4 + .../interfaces/commands/deployment_cmd.py | 34 +- semapact/platforms/databricks/deployment.py | 483 +++--------------- semapact/platforms/databricks/runtime.py | 6 + .../databricks/transition_compiler.py | 115 +++++ .../databricks/transition_planner.py | 52 ++ semapact/platforms/runtime_registry.py | 115 +++-- semapact/reconciliation/engine.py | 401 +++++---------- semapact/reconciliation/models.py | 24 +- semapact/schema/__init__.py | 45 ++ semapact/schema/comparison.py | 323 ++++++++++++ semapact/schema/identifiers.py | 18 + semapact/schema/mapping.py | 402 +++++++++++++++ tests/interfaces/test_deployment_cmd.py | 73 +++ tests/test_contractops_golden_scenarios.py | 6 +- tests/test_databricks_schema_evolution.py | 296 +++++++++++ tests/test_deployment_databricks.py | 128 ++++- tests/test_deployment_extensibility.py | 37 ++ tests/test_deployment_orchestrator.py | 264 ++++++++++ tests/test_deployment_service.py | 188 +++---- tests/test_reconciliation.py | 36 ++ tests/test_runtime_location_resolution.py | 39 ++ tests/test_schema_comparison.py | 138 +++++ tests/test_schema_mapping.py | 138 +++++ tests/test_schema_transitions.py | 122 +++++ 35 files changed, 3451 insertions(+), 963 deletions(-) create mode 100644 semapact/deployment/compilers.py create mode 100644 semapact/deployment/orchestrator.py create mode 100644 semapact/deployment/providers.py create mode 100644 semapact/deployment/schema_transitions.py create mode 100644 semapact/platforms/databricks/transition_compiler.py create mode 100644 semapact/platforms/databricks/transition_planner.py create mode 100644 semapact/schema/__init__.py create mode 100644 semapact/schema/comparison.py create mode 100644 semapact/schema/identifiers.py create mode 100644 semapact/schema/mapping.py create mode 100644 tests/test_databricks_schema_evolution.py create mode 100644 tests/test_deployment_extensibility.py create mode 100644 tests/test_deployment_orchestrator.py create mode 100644 tests/test_schema_comparison.py create mode 100644 tests/test_schema_mapping.py create mode 100644 tests/test_schema_transitions.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29ff7730..b57e3d55 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -161,17 +161,19 @@ jobs: - name: Install SemaPact Databricks extra from project metadata run: uv pip install --python .venv/bin/python -e ".[databricks]" pytest - - name: Verify Databricks SDK read-side boundaries + - name: Verify Databricks SDK boundaries run: >- .venv/bin/python -c "import inspect; from databricks.sdk import WorkspaceClient; from databricks.sdk.service.catalog import TableInfo, TablesAPI; from semapact.platforms.databricks import create_databricks_workspace_client, discover_databricks_tables; client_params = inspect.signature(WorkspaceClient).parameters; list_params = inspect.signature(TablesAPI.list).parameters; assert {'host', 'token', 'profile'} <= set(client_params); assert {'catalog_name', 'schema_name'} <= set(list_params); print(WorkspaceClient.__name__, TableInfo.__name__, create_databricks_workspace_client.__name__, discover_databricks_tables.__name__)" - - name: Run Databricks read-side tests with official SDK installed + - name: Run Databricks read/write tests with official SDK installed run: >- .venv/bin/python -m pytest tests/test_databricks_client.py tests/test_databricks_discovery.py tests/test_observation_databricks.py + tests/test_databricks_schema_evolution.py + tests/test_deployment_databricks.py coverage: runs-on: ubuntu-latest diff --git a/docs/contractops_phases.md b/docs/contractops_phases.md index 667ce142..5160c859 100644 --- a/docs/contractops_phases.md +++ b/docs/contractops_phases.md @@ -29,6 +29,37 @@ DEPLOY Each phase consumes artifacts from the previous phases. Later phases do not recalculate earlier decisions. +## The contract is the desired-state artifact + +SemaPact does not introduce a canonical BUILD phase that turns a contract into a separately authoritative DDL artifact. + +The governed contract remains the model of desired state throughout the lifecycle: + +```text +candidate ODCS + ↓ ANALYZE / PLAN / AUTHORIZE / APPLY +AppliedContractRelease += immutable governed desired state + ↓ DEPLOY planning against one runtime target +provider-native operations +``` + +A SQL or provider-specific export is a **derived compilation output**, not a second source of truth and not deployment authority. It may be regenerated from the exact governed contract state whenever required. + +This creates two distinct comparisons: + +```text +Contract comparison +base contract ↔ candidate contract +→ governance changes, breaking classification, version requirements + +Runtime deployment comparison +AppliedContractRelease ↔ observed runtime state +→ CREATE / ALTER / NO_OP or an explicit unsupported transition +``` + +The first comparison explains how governed intent changed. The second explains how one concrete runtime must change to converge to that already-governed intent. The same release may therefore produce different deployment previews for different targets without changing the released contract. + ## ANALYZE ANALYZE evaluates the candidate against the governed base contract and produces the authoritative `GovernanceDecision`. diff --git a/docs/deployment_plans.md b/docs/deployment_plans.md index 82a585b3..38bb1f79 100644 --- a/docs/deployment_plans.md +++ b/docs/deployment_plans.md @@ -2,6 +2,24 @@ SemaPact treats a released ODCS contract as governed desired state, not as an executable SQL, Terraform, or platform program. +The released contract is the authoritative artifact. SemaPact does **not** require a separate DDL build artifact before deployment. SQL and other provider-native commands are derived only after an exact released desired state is compared with an exact runtime target. + +```text +AppliedContractRelease + = governed desired state + + +ObservedPlatformState + = point-in-time runtime state + ↓ +semantic runtime transition assessment + ↓ +provider-native operation compilation + ↓ +DeploymentPreview +``` + +This matters because one release may require different native operations in different environments. A missing table may require CREATE in one target, an additive ALTER in another, and NO_OP in a target that already satisfies the governed state. A pre-built DDL script cannot represent those three runtime states without becoming another mutable source of truth. + The deployment planning boundary is therefore: ```text @@ -12,13 +30,29 @@ DeploymentPlan ↓ DeploymentAuthorization ↓ -platform adapter validate / preview / execute +DeploymentService ↓ -runtime +DeploymentAdapter interface + ↓ +DeploymentOrchestrator + validate / map desired + → observe + → validate / map observed + → compare + → transition + → compile + → preview + → freshness / exact authorization + → execute + → verify ↓ -reconciliation verifies convergence +provider NativeOperationExecutor / RuntimeProvider + ↓ +runtime ``` +The orchestration above is provider-neutral. Platform packages configure or implement only the narrow `SchemaMapper`, `SchemaTransitionPlanner`, `TransitionCompiler`, `RuntimeProvider`, and `NativeOperationExecutor` seams. The initial Databricks path reuses the shared fail-closed `AdditiveSchemaTransitionPlanner`; a future platform can supply a different planner without changing orchestration. + ## What a DeploymentPlan means A `DeploymentPlan` is a deterministic, provider-neutral statement of the runtime state that an exact applied contract release intends to converge toward. @@ -41,6 +75,33 @@ Likewise, an object that exists in runtime but is absent from one contract must Concrete provider-native operations therefore begin at the platform adapter boundary, where validation and preview combine the DeploymentPlan with provider semantics and fresh runtime evidence. +Provider preview should keep **comparison facts**, **transition semantics**, and SQL rendering separate. Conceptually: + +```text +normalized desired schema + normalized observed schema + ↓ +shared schema comparator + ↓ +SchemaDifference[] + ↓ +deployment transition projection + CREATE_ASSET / ADD_PROPERTIES / NO_OP + ↓ +provider compiler + ↓ +CREATE / ALTER / NO_OP native operation +``` + +The same shared schema comparison facts are consumed by runtime reconciliation. Reconciliation projects them into drift reason codes; deployment projects them into convergence intent. Provider adapters must not implement a second desired-vs-observed comparator. + +Schema projection is also shared. `semapact.schema` defines the mapping contract that converts provider target-schema output and observed runtime state into normalized `SchemaSnapshot` values. SemaPact should not reimplement an ODCS-to-platform compiler when datacontract-cli already provides one. + +For Databricks, the desired side delegates the complete ODCS → Databricks target-schema compilation to datacontract-cli's SQL exporter, including physical property names, target types, nested types, and nullability. SemaPact parses that compiler output into its normalized comparison model, rejects any output that is not exactly one CREATE TABLE statement, and retains only the governed physical column shape currently covered by comparison semantics (identity, type, nullability). The observed side maps fresh runtime evidence into the same model. + +The exported CREATE DDL is **not** execution authority: datacontract-cli currently emits full creation-oriented DDL, while SemaPact must derive CREATE / ALTER / NO_OP from the released target schema versus fresh runtime state and compile only the exact authorized transition. + +The semantic transition layer is an internal planning boundary, not a new release artifact or authorization authority. This lets compatible execution families share transition semantics while keeping provider-specific naming, capability checks, SQL rendering, authentication, and execution in their adapters. + ## Identity and physical binding The same governed identity rule applies on both deployment and reconciliation paths: @@ -128,6 +189,8 @@ Execution requires the exact plan, exact preview, and exact `DeploymentAuthoriza Provider execution success is not convergence proof. +Complete DDL export remains a useful inspection or integration utility, especially for creating new assets, but export is not a lifecycle phase. Exported SQL is derived output; deployment planning remains responsible for comparing the exact released contract with fresh runtime evidence before any mutation is authorized. + ### Verify ```bash @@ -136,7 +199,7 @@ semapact deployment verify \ --output json ``` -Verification performs fresh runtime observation and reuses the normal reconciliation semantics: +Verification enters through the same DeploymentAdapter / DeploymentOrchestrator boundary, performs fresh runtime observation, and reuses the normal reconciliation semantics with the same platform schema mapper used by preview: | Runtime status | Exit code | | --- | ---: | @@ -144,7 +207,7 @@ Verification performs fresh runtime observation and reuses the normal reconcilia | `DRIFT` | `6` | | `INDETERMINATE` | `7` | -This keeps execution status separate from convergence evidence. +This keeps execution status separate from convergence evidence. VERIFY is assurance/read-side behavior: provider mutation restrictions such as Databricks MANAGED-only writes do not prevent SemaPact from verifying an observable non-managed asset. ## Databricks deployment capability @@ -198,12 +261,18 @@ Action ordering is canonical even when schemas appear in a different order in so `DeploymentPreview` is likewise deterministic for the same plan and observed runtime evidence, but deterministic IDs provide artifact consistency rather than cryptographic authenticity. Execution still validates exact binding and fresh runtime evidence at the side-effect boundary. -## Provider support belongs to the adapter +## Provider support belongs behind generic deployment contracts + +`semapact.platforms.runtime_registry` is the composition root. It lazily constructs the selected platform's runtime provider and deployment adapter and owns the small amount of dispatch needed for supported built-in platforms. SemaPact does not introduce a separate platform-factory hierarchy merely to construct these objects. + +Platform extensibility belongs in behavior seams—`RuntimeProvider`, `SchemaMapper`, `SchemaTransitionPlanner`, `TransitionCompiler`, and `NativeOperationExecutor`—rather than in an additional platform wrapper or composition abstraction. DeploymentPlan intentionally does not contain generic `preconditions`, `adapterKey`, or guessed platform-specific operations. -A deployment adapter is responsible for explicit provider support and execution semantics. It receives an already-built DeploymentPlan and an allowed DeploymentAuthorization; it does not construct or reinterpret governance artifacts. +The public deployment contracts define the complete orchestration boundary. `DeploymentOrchestrator` owns lifecycle ordering and generic binding invariants; platform implementations provide only runtime binding/observation, target-schema mapping configuration, transition capability policy, transition compilation, and native execution. + +For Databricks, the native side effect is executed through the Databricks SDK Statement Execution API using an exact SQL warehouse. SemaPact does not shell out to the Databricks CLI for deployment. -A provider adapter must explicitly report unsupported ODCS-to-platform mappings. It must never silently ignore unsupported governed state. +A platform implementation must explicitly report unsupported ODCS-to-platform mappings or runtime capabilities. It must never silently ignore unsupported governed state. A successful execution call is also not proof of convergence. Runtime convergence is verified separately through SemaPact reconciliation. diff --git a/semapact/application/services/deployment.py b/semapact/application/services/deployment.py index 1776997a..fa5d62c9 100644 --- a/semapact/application/services/deployment.py +++ b/semapact/application/services/deployment.py @@ -10,21 +10,13 @@ DeploymentPreview, DeploymentTarget, build_deployment_plan, - verify_deployment_convergence, ) -from semapact.deployment.models import validate_deployment_plan_identity from semapact.exceptions import ValidationError -from semapact.observation import RuntimeProvider from semapact.reconciliation import ReconciliationResult -from semapact.runtime import RuntimeAssetSpec class DeploymentService: - """Compose existing deployment domain/provider boundaries for interfaces. - - The service owns orchestration only. It does not re-run governance, approval, - versioning, deployment translation, or reconciliation rules. - """ + """Application boundary over the generic deployment lifecycle.""" def plan( self, @@ -37,22 +29,10 @@ def preview( self, plan: DeploymentPlan, *, - runtime_provider: RuntimeProvider, adapter: DeploymentAdapter, ) -> DeploymentPreview: - """Observe the exact plan scope and delegate native translation to adapter.""" - validate_deployment_plan_identity(plan) - _validate_component_key(runtime_provider.key, plan.target.platform, "runtime provider") _validate_component_key(adapter.key, plan.target.platform, "deployment adapter") - - assets = _runtime_assets_from_plan(plan) - bindings = runtime_provider.resolve_bindings( - runtime_target=plan.target.runtime_target, - assets=assets, - ) - observation = runtime_provider.observe(bindings=bindings) - _validate_source_reference(observation.source_identifier, plan.target.source_reference) - return adapter.preview(plan, observation) + return adapter.preview(plan) def execute( self, @@ -62,7 +42,6 @@ def execute( *, adapter: DeploymentAdapter, ) -> None: - """Delegate the exact authorized side effect to the provider adapter.""" _validate_component_key(adapter.key, plan.target.platform, "deployment adapter") adapter.execute(plan, preview, authorization) @@ -70,33 +49,19 @@ def verify( self, plan: DeploymentPlan, *, - runtime_provider: RuntimeProvider, + adapter: DeploymentAdapter, ) -> ReconciliationResult: - """Verify exact plan convergence through the existing M1 bridge.""" - return verify_deployment_convergence(plan, runtime_provider) - - -def _runtime_assets_from_plan(plan: DeploymentPlan) -> tuple[RuntimeAssetSpec, ...]: - return tuple( - RuntimeAssetSpec( - governed_asset=action.governed_asset, - physical_name=action.physical_name, - ) - for action in plan.actions - ) + _validate_component_key(adapter.key, plan.target.platform, "deployment adapter") + return adapter.verify(plan) -def _validate_component_key(actual: str, expected: str, component: str) -> None: +def _validate_component_key( + actual: str, + expected: str, + component: str, +) -> None: if actual.strip().casefold() != expected.strip().casefold(): raise ValidationError( f"{component.capitalize()} does not match DeploymentPlan platform: " f"{actual!r} != {expected!r}" ) - - -def _validate_source_reference(actual: str, expected: str) -> None: - if actual.strip() != expected.strip(): - raise ValidationError( - "Runtime observation source does not match DeploymentPlan source reference: " - f"{actual!r} != {expected!r}" - ) diff --git a/semapact/deployment/__init__.py b/semapact/deployment/__init__.py index ed738096..8a3a7c45 100644 --- a/semapact/deployment/__init__.py +++ b/semapact/deployment/__init__.py @@ -1,7 +1,8 @@ -"""Provider-neutral deployment planning, preview, and authorization boundary.""" +"""Provider-neutral deployment planning, orchestration and authorization boundary.""" from semapact.deployment.adapters import DeploymentAdapter from semapact.deployment.authorization import authorize_deployment +from semapact.deployment.compilers import TransitionCompiler from semapact.deployment.models import ( DeploymentAction, DeploymentActionKind, @@ -12,7 +13,16 @@ NativeOperation, NativeOperationKind, ) +from semapact.deployment.orchestrator import DeploymentOrchestrator from semapact.deployment.planner import build_deployment_plan +from semapact.deployment.providers import ( + DeploymentExecutionConfig, + NativeOperationExecutor, +) +from semapact.deployment.schema_transitions import ( + AdditiveSchemaTransitionPlanner, + SchemaTransitionPlanner, +) from semapact.deployment.verification import verify_deployment_convergence __all__ = [ @@ -20,11 +30,17 @@ "DeploymentActionKind", "DeploymentAdapter", "DeploymentAuthorization", + "DeploymentExecutionConfig", + "DeploymentOrchestrator", "DeploymentPlan", "DeploymentPreview", "DeploymentTarget", + "AdditiveSchemaTransitionPlanner", "NativeOperation", + "NativeOperationExecutor", "NativeOperationKind", + "TransitionCompiler", + "SchemaTransitionPlanner", "authorize_deployment", "build_deployment_plan", "verify_deployment_convergence", diff --git a/semapact/deployment/adapters.py b/semapact/deployment/adapters.py index 08f69687..5bfbe689 100644 --- a/semapact/deployment/adapters.py +++ b/semapact/deployment/adapters.py @@ -2,32 +2,42 @@ from __future__ import annotations -from typing import Protocol +from abc import ABC, abstractmethod from semapact.deployment.models import ( DeploymentAuthorization, DeploymentPlan, DeploymentPreview, ) -from semapact.observation.models import ObservedPlatformState +from semapact.reconciliation import ReconciliationResult -class DeploymentAdapter(Protocol): - """Translate and execute one exact DeploymentPlan for a runtime provider.""" +class DeploymentAdapter(ABC): + """Own the complete provider-neutral deployment lifecycle entrypoints.""" key: str - def validate(self, plan: DeploymentPlan) -> None: ... + @abstractmethod + def validate(self, plan: DeploymentPlan) -> None: + """Validate one exact DeploymentPlan.""" + raise NotImplementedError - def preview( - self, - plan: DeploymentPlan, - observed_state: ObservedPlatformState, - ) -> DeploymentPreview: ... + @abstractmethod + def preview(self, plan: DeploymentPlan) -> DeploymentPreview: + """Observe current runtime and derive the exact deployment preview.""" + raise NotImplementedError + + @abstractmethod + def verify(self, plan: DeploymentPlan) -> ReconciliationResult: + """Observe runtime and verify convergence for one exact plan.""" + raise NotImplementedError + @abstractmethod def execute( self, plan: DeploymentPlan, preview: DeploymentPreview, authorization: DeploymentAuthorization, - ) -> None: ... + ) -> None: + """Execute only the exact authorized preview.""" + raise NotImplementedError diff --git a/semapact/deployment/compilers.py b/semapact/deployment/compilers.py new file mode 100644 index 00000000..5fc68a6c --- /dev/null +++ b/semapact/deployment/compilers.py @@ -0,0 +1,38 @@ +"""Provider-neutral compiler contract for schema transitions.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod + +from semapact.deployment.models import NativeOperation +from semapact.deployment.schema_transitions import SchemaTransition +from semapact.exceptions import ValidationError +from semapact.schema import SchemaPropertyState + + +class TransitionCompiler(ABC): + """Compile one semantic schema transition into a provider-native operation.""" + + key: str + + @abstractmethod + def compile( + self, + *, + runtime_target: str, + transition: SchemaTransition, + ) -> NativeOperation: + """Compile one semantic transition into a provider-native operation.""" + raise NotImplementedError + + + +def require_native_definition(column: SchemaPropertyState) -> str: + """Return one canonical target-compiler definition or fail closed.""" + definition = column.native_definition + if definition is None or not definition.strip(): + raise ValidationError( + "Transition compilation requires target-compiler output for " + f"column '{column.identity}'" + ) + return definition diff --git a/semapact/deployment/orchestrator.py b/semapact/deployment/orchestrator.py new file mode 100644 index 00000000..6e95b994 --- /dev/null +++ b/semapact/deployment/orchestrator.py @@ -0,0 +1,396 @@ +"""Generic deployment orchestration over explicit provider behavior seams.""" + +from __future__ import annotations + +from open_data_contract_standard.model import SchemaObject + +from semapact.deployment.adapters import DeploymentAdapter +from semapact.deployment.compilers import TransitionCompiler +from semapact.deployment.models import ( + DeploymentActionKind, + DeploymentAuthorization, + DeploymentPlan, + DeploymentPreview, + NativeOperation, + NativeOperationKind, + compute_deployment_preview_id, + validate_deployment_authorization_identity, + validate_deployment_plan_identity, + validate_deployment_preview_identity, +) +from semapact.deployment.providers import NativeOperationExecutor +from semapact.deployment.schema_transitions import SchemaTransitionPlanner +from semapact.deployment.verification import verify_deployment_convergence +from semapact.exceptions import ContractOpsAuthorizationError, ValidationError +from semapact.observation.fingerprint import fingerprint_observed_state +from semapact.observation.models import ObservedAssetIdentity, ObservedPlatformState +from semapact.observation.providers import RuntimeAssetBinding, RuntimeProvider +from semapact.reconciliation import ReconciliationResult +from semapact.runtime import RuntimeAssetSpec +from semapact.schema import ( + SchemaAssetState, + SchemaMapper, + SchemaSnapshot, + compare_schema_snapshots, +) + + +class DeploymentOrchestrator(DeploymentAdapter): + """Provider-neutral deployment lifecycle over explicit behavior dependencies.""" + + def __init__( + self, + *, + runtime_provider: RuntimeProvider, + schema_mapper: SchemaMapper, + transition_planner: SchemaTransitionPlanner, + transition_compiler: TransitionCompiler, + executor: NativeOperationExecutor, + ) -> None: + provider_key = runtime_provider.key.strip().casefold() + if not provider_key: + raise ValueError("Runtime provider key is required") + if transition_compiler.key.strip().casefold() != provider_key: + raise ValueError( + "Runtime provider and transition compiler keys must match" + ) + if executor.key.strip().casefold() != provider_key: + raise ValueError( + "Runtime provider and native operation executor keys must match" + ) + + self._runtime_provider = runtime_provider + self._schema_mapper = schema_mapper + self._transition_planner = transition_planner + self._transition_compiler = transition_compiler + self._executor = executor + + @property + def key(self) -> str: + return self._runtime_provider.key + + def validate(self, plan: DeploymentPlan) -> None: + """Validate one exact plan without runtime observation side effects.""" + self._validate_and_map_plan(plan) + + def preview(self, plan: DeploymentPlan) -> DeploymentPreview: + """Validate, observe, compare, plan, and compile one deterministic preview.""" + desired_by_action = self._validate_and_map_plan(plan) + observed_state, bindings = self._observe_plan_scope(plan) + return self._preview_from_observation( + plan, + observed_state, + bindings=bindings, + desired_by_action=desired_by_action, + ) + + def verify(self, plan: DeploymentPlan) -> ReconciliationResult: + """Verify convergence using the same runtime provider and schema mapping.""" + return verify_deployment_convergence( + plan, + self._runtime_provider, + schema_mapper=self._schema_mapper, + ) + + def execute( + self, + plan: DeploymentPlan, + preview: DeploymentPreview, + authorization: DeploymentAuthorization, + ) -> None: + """Execute only the exact authorized preview against unchanged runtime state.""" + validate_deployment_preview_identity(preview) + validate_deployment_authorization_identity(authorization) + desired_by_action = self._validate_and_map_plan(plan) + + if not authorization.allowed: + raise ContractOpsAuthorizationError( + "DeploymentAuthorization is not allowed" + ) + if authorization.deployment_plan_id != plan.deployment_plan_id: + raise ContractOpsAuthorizationError( + "DeploymentAuthorization is not bound to this DeploymentPlan" + ) + if authorization.applied_release_id != plan.applied_release_id: + raise ContractOpsAuthorizationError( + "DeploymentAuthorization release does not match DeploymentPlan" + ) + if preview.deployment_plan_id != plan.deployment_plan_id: + raise ValidationError( + "DeploymentPreview is not bound to this DeploymentPlan" + ) + if preview.platform.casefold() != self.key.casefold(): + raise ValidationError( + "DeploymentPreview platform does not match runtime provider" + ) + if preview.runtime_target != plan.target.runtime_target: + raise ValidationError( + "DeploymentPreview target does not match DeploymentPlan" + ) + if preview.source_identifier != plan.target.source_reference: + raise ValidationError( + "DeploymentPreview runtime source does not match DeploymentPlan " + "source reference" + ) + + current, bindings = self._observe_plan_scope(plan) + if current.source_identifier != preview.source_identifier: + raise ValidationError( + "Runtime source changed since DeploymentPreview was produced" + ) + if current.fingerprint != preview.observation_fingerprint: + raise ValidationError( + "Runtime state changed since DeploymentPreview was produced" + ) + + expected = self._preview_from_observation( + plan, + current, + bindings=bindings, + desired_by_action=desired_by_action, + ) + if expected != preview: + raise ValidationError( + "DeploymentPreview no longer equals the deterministic preview for " + "the authorized plan and runtime evidence" + ) + + for operation in preview.operations: + if operation.kind is NativeOperationKind.NO_OP: + continue + self._executor.execute(operation) + + def _preview_from_observation( + self, + plan: DeploymentPlan, + observed_state: ObservedPlatformState, + *, + bindings: tuple[RuntimeAssetBinding, ...], + desired_by_action: dict[str, SchemaAssetState] | None = None, + ) -> DeploymentPreview: + if desired_by_action is None: + desired_by_action = self._validate_and_map_plan(plan) + self._validate_observation(plan, observed_state, bindings=bindings) + + observed_by_asset = { + asset.identity.asset.casefold(): asset + for asset in observed_state.assets + } + + operations: list[NativeOperation] = [] + for action in plan.actions: + desired_asset = desired_by_action[action.governed_asset] + observed = observed_by_asset.get(action.physical_name.casefold()) + observed_assets = ( + () + if observed is None + else ( + self._schema_mapper.map_observed_asset( + observed, + asset_identity=action.physical_name, + ), + ) + ) + + comparison = compare_schema_snapshots( + SchemaSnapshot(assets=(desired_asset,)), + SchemaSnapshot(assets=observed_assets), + ) + transition = self._transition_planner.plan( + governed_asset=action.governed_asset, + physical_name=action.physical_name, + desired_columns=desired_asset.properties, + comparison=comparison, + observed_asset=observed, + ) + operations.append( + self._transition_compiler.compile( + runtime_target=plan.target.runtime_target, + transition=transition, + ) + ) + + if observed_state.fingerprint is None: + raise ValidationError("Runtime observation fingerprint is required") + ordered = tuple(operations) + return DeploymentPreview( + deployment_preview_id=compute_deployment_preview_id( + deployment_plan_id=plan.deployment_plan_id, + platform=self.key, + runtime_target=plan.target.runtime_target, + source_identifier=observed_state.source_identifier, + observation_fingerprint=observed_state.fingerprint, + operations=ordered, + ), + deployment_plan_id=plan.deployment_plan_id, + platform=self.key, + runtime_target=plan.target.runtime_target, + source_identifier=observed_state.source_identifier, + observation_fingerprint=observed_state.fingerprint, + operations=ordered, + ) + + def _validate_and_map_plan( + self, + plan: DeploymentPlan, + ) -> dict[str, SchemaAssetState]: + validate_deployment_plan_identity(plan) + if plan.target.platform.casefold() != self.key.casefold(): + raise ValidationError( + f"Runtime provider '{self.key}' cannot deploy " + f"'{plan.target.platform}'" + ) + + physical_assets: set[str] = set() + desired_by_action: dict[str, SchemaAssetState] = {} + + for action in plan.actions: + if action.kind is not DeploymentActionKind.ENSURE_ASSET_STATE: + raise ValidationError( + f"Unsupported deployment action kind: {action.kind.value}" + ) + + physical_key = action.physical_name.casefold() + if physical_key in physical_assets: + raise ValidationError( + "Deployment cannot bind multiple governed assets to the same " + f"physical asset '{action.physical_name}'" + ) + physical_assets.add(physical_key) + + desired = SchemaObject.model_validate_json(action.desired_state_json) + mapped = self._schema_mapper.map_desired_asset( + desired, + asset_identity=action.physical_name, + ) + if mapped.identity.casefold() != physical_key: + raise ValidationError( + "Mapped desired asset identity does not match physical target" + ) + desired_by_action[action.governed_asset] = mapped + + # Binding resolution is pure provider-local identity resolution. Calling it + # here validates runtime-target syntax and exact plan bindings without I/O. + self._resolve_plan_bindings(plan) + return desired_by_action + + def _resolve_plan_bindings( + self, + plan: DeploymentPlan, + ) -> tuple[RuntimeAssetBinding, ...]: + assets = tuple( + RuntimeAssetSpec( + governed_asset=action.governed_asset, + physical_name=action.physical_name, + ) + for action in plan.actions + ) + bindings = self._runtime_provider.resolve_bindings( + runtime_target=plan.target.runtime_target, + assets=assets, + ) + + expected = { + action.governed_asset: action.physical_name.casefold() + for action in plan.actions + } + if len(bindings) != len(expected): + raise ValidationError( + "Runtime provider did not resolve exactly one binding per deployment action" + ) + + seen_governed: set[str] = set() + seen_observed: set[tuple[str, tuple[str, ...], str]] = set() + for binding in bindings: + if binding.governed_asset not in expected: + raise ValidationError( + "Runtime provider returned a binding outside DeploymentPlan scope" + ) + if binding.governed_asset in seen_governed: + raise ValidationError( + "Runtime provider returned duplicate governed asset bindings" + ) + seen_governed.add(binding.governed_asset) + + identity = binding.observed_asset + if identity.platform.casefold() != self.key.casefold(): + raise ValidationError( + "Runtime binding platform does not match runtime provider" + ) + if identity.asset.casefold() != expected[binding.governed_asset]: + raise ValidationError( + "Runtime binding asset does not match DeploymentPlan physical asset" + ) + + identity_key = _identity_key(identity) + if identity_key in seen_observed: + raise ValidationError( + "Runtime provider returned duplicate observed asset bindings" + ) + seen_observed.add(identity_key) + + return bindings + + def _validate_observation( + self, + plan: DeploymentPlan, + observed_state: ObservedPlatformState, + *, + bindings: tuple[RuntimeAssetBinding, ...], + ) -> None: + if observed_state.platform.casefold() != self.key.casefold(): + raise ValidationError( + "Runtime observation platform does not match runtime provider" + ) + if not observed_state.source_identifier.strip(): + raise ValidationError( + "Runtime observation source_identifier is required" + ) + if observed_state.source_identifier != plan.target.source_reference: + raise ValidationError( + "Runtime observation source does not match DeploymentPlan " + "source reference" + ) + if observed_state.fingerprint is None: + raise ValidationError("Runtime observation fingerprint is required") + if observed_state.fingerprint != fingerprint_observed_state(observed_state): + raise ValidationError( + "Runtime observation fingerprint does not match its content" + ) + + expected_identities = { + _identity_key(binding.observed_asset) + for binding in bindings + } + seen: set[tuple[str, tuple[str, ...], str]] = set() + for asset in observed_state.assets: + identity_key = _identity_key(asset.identity) + if identity_key not in expected_identities: + raise ValidationError( + "Runtime evidence contains an asset outside resolved plan scope" + ) + if identity_key in seen: + raise ValidationError( + "Runtime evidence contains duplicate asset identities" + ) + seen.add(identity_key) + + def _observe_plan_scope( + self, + plan: DeploymentPlan, + ) -> tuple[ObservedPlatformState, tuple[RuntimeAssetBinding, ...]]: + bindings = self._resolve_plan_bindings(plan) + return ( + self._runtime_provider.observe(bindings=bindings), + bindings, + ) + + +def _identity_key( + identity: ObservedAssetIdentity, +) -> tuple[str, tuple[str, ...], str]: + return ( + identity.platform.casefold(), + tuple(part.casefold() for part in identity.namespace), + identity.asset.casefold(), + ) diff --git a/semapact/deployment/providers.py b/semapact/deployment/providers.py new file mode 100644 index 00000000..a6b45d01 --- /dev/null +++ b/semapact/deployment/providers.py @@ -0,0 +1,38 @@ +"""Provider contracts for generic deployment orchestration.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod + +from pydantic import BaseModel, ConfigDict, field_validator + +from semapact.deployment.models import NativeOperation + + + +class DeploymentExecutionConfig(BaseModel): + """Typed provider execution configuration passed through composition.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + platform: str + + @field_validator("platform") + @classmethod + def _normalize_platform(cls, value: str) -> str: + cleaned = value.strip().casefold() + if not cleaned: + raise ValueError("platform must not be empty") + return cleaned + + +class NativeOperationExecutor(ABC): + """Execute one provider-native operation.""" + + key: str + + @abstractmethod + def execute(self, operation: NativeOperation) -> None: + """Execute one exact native operation.""" + raise NotImplementedError + diff --git a/semapact/deployment/schema_transitions.py b/semapact/deployment/schema_transitions.py new file mode 100644 index 00000000..19f78395 --- /dev/null +++ b/semapact/deployment/schema_transitions.py @@ -0,0 +1,228 @@ +"""Interpret raw schema differences as additive convergence intent. + +Comparison belongs to semapact.schema. This module consumes the shared comparison +result and maps facts into the initial deployment transition vocabulary. It does +not compare desired and observed schemas itself. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from enum import Enum +from typing import Sequence + +from pydantic import BaseModel, ConfigDict, field_validator, model_validator + +from semapact.exceptions import ValidationError +from semapact.observation.models import ObservedAsset +from semapact.schema import ( + SchemaComparisonResult, + SchemaDifferenceType, + SchemaPropertyState, + SchemaSubject, +) + + +class SchemaTransitionModel(BaseModel): + """Shared immutable base for internal schema-transition values.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + +class SchemaTransitionKind(str, Enum): + """Semantic runtime transitions supported by the initial additive planner.""" + + CREATE_ASSET = "CREATE_ASSET" + ADD_PROPERTIES = "ADD_PROPERTIES" + NO_OP = "NO_OP" + + +class SchemaTransition(SchemaTransitionModel): + """One semantic desired-to-runtime transition before provider compilation.""" + + kind: SchemaTransitionKind + governed_asset: str + physical_name: str + columns: tuple[SchemaPropertyState, ...] = () + + @field_validator("governed_asset", "physical_name") + @classmethod + def _require_identity_text(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("schema transition identity fields must not be empty") + return cleaned + + @model_validator(mode="after") + def _validate_shape(self) -> "SchemaTransition": + if self.kind is SchemaTransitionKind.NO_OP: + if self.columns: + raise ValueError("NO_OP schema transition must not carry columns") + return self + if not self.columns: + raise ValueError(f"{self.kind.value} schema transition requires columns") + return self + + + + + + +class SchemaTransitionPlanner(ABC): + """Interpret factual schema differences as provider-supported convergence intent.""" + + key: str + + @abstractmethod + def plan( + self, + *, + governed_asset: str, + physical_name: str, + desired_columns: Sequence[SchemaPropertyState], + comparison: SchemaComparisonResult, + observed_asset: ObservedAsset | None = None, + ) -> SchemaTransition: + raise NotImplementedError + + +class AdditiveSchemaTransitionPlanner(SchemaTransitionPlanner): + """Shared fail-closed planner for the initial additive schema subset.""" + + key = "additive" + + def plan( + self, + *, + governed_asset: str, + physical_name: str, + desired_columns: Sequence[SchemaPropertyState], + comparison: SchemaComparisonResult, + observed_asset: ObservedAsset | None = None, + ) -> SchemaTransition: + del observed_asset + return plan_additive_schema_transition( + governed_asset=governed_asset, + physical_name=physical_name, + desired_columns=desired_columns, + comparison=comparison, + ) + +def plan_additive_schema_transition( + *, + governed_asset: str, + physical_name: str, + desired_columns: Sequence[SchemaPropertyState], + comparison: SchemaComparisonResult, +) -> SchemaTransition: + """Interpret shared schema differences for the initial additive subset.""" + desired = tuple(desired_columns) + if not desired: + raise ValidationError("Schema transition requires at least one desired column") + + desired_by_name = _index_desired_columns(desired) + asset_key = physical_name.casefold() + + if comparison.unverified_paths: + raise ValidationError( + "Runtime schema evidence is incomplete for deployment transition planning" + ) + + relevant = tuple( + difference + for difference in comparison.differences + if difference.asset_identity.casefold() == asset_key + ) + + missing_asset = any( + difference.difference_type is SchemaDifferenceType.MISSING + and difference.subject is SchemaSubject.ASSET + for difference in relevant + ) + if missing_asset: + return SchemaTransition( + kind=SchemaTransitionKind.CREATE_ASSET, + governed_asset=governed_asset, + physical_name=physical_name, + columns=desired, + ) + + additions: list[SchemaPropertyState] = [] + for difference in relevant: + if ( + difference.difference_type is SchemaDifferenceType.UNEXPECTED + and difference.subject is SchemaSubject.PROPERTY + ): + # Runtime-only columns never imply DROP. + continue + + if ( + difference.difference_type is SchemaDifferenceType.MISSING + and difference.subject is SchemaSubject.PROPERTY + ): + property_identity = difference.property_identity + if property_identity is None: + raise ValidationError("Missing property difference has no property identity") + column = desired_by_name.get(property_identity.casefold()) + if column is None: + raise ValidationError( + f"Schema difference references unknown desired property '{property_identity}'" + ) + if column.nullable is not True: + raise ValidationError( + f"Cannot add required column '{column.identity}' without a safe default" + ) + additions.append(column) + continue + + if ( + difference.difference_type is SchemaDifferenceType.MISMATCH + and difference.subject is SchemaSubject.PHYSICAL_TYPE + ): + raise ValidationError( + f"Unsupported existing column type mutation for " + f"'{difference.property_identity}': " + f"{difference.observed} -> {difference.expected}" + ) + + if ( + difference.difference_type is SchemaDifferenceType.MISMATCH + and difference.subject is SchemaSubject.NULLABILITY + ): + raise ValidationError( + f"Unsupported existing column nullability mutation for " + f"'{difference.property_identity}'" + ) + + if difference.subject is SchemaSubject.ASSET: + raise ValidationError( + "Unsupported runtime asset difference for additive deployment planning" + ) + + if additions: + return SchemaTransition( + kind=SchemaTransitionKind.ADD_PROPERTIES, + governed_asset=governed_asset, + physical_name=physical_name, + columns=tuple(additions), + ) + + return SchemaTransition( + kind=SchemaTransitionKind.NO_OP, + governed_asset=governed_asset, + physical_name=physical_name, + ) + + +def _index_desired_columns( + columns: Sequence[SchemaPropertyState], +) -> dict[str, SchemaPropertyState]: + index: dict[str, SchemaPropertyState] = {} + for column in columns: + key = column.identity.casefold() + if key in index: + raise ValidationError( + f"Duplicate normalized desired column identity: '{column.identity}'" + ) + index[key] = column + return index diff --git a/semapact/deployment/verification.py b/semapact/deployment/verification.py index 93185b57..2d1ae047 100644 --- a/semapact/deployment/verification.py +++ b/semapact/deployment/verification.py @@ -12,11 +12,14 @@ from semapact.observation.providers import RuntimeProvider from semapact.reconciliation import ReconciliationResult, reconcile_governed_contract from semapact.runtime import RuntimeAssetSpec +from semapact.schema import SchemaMapper def verify_deployment_convergence( plan: DeploymentPlan, runtime_provider: RuntimeProvider, + *, + schema_mapper: SchemaMapper | None = None, ) -> ReconciliationResult: """Observe and reconcile the exact desired state embedded in a DeploymentPlan. @@ -60,6 +63,7 @@ def verify_deployment_convergence( desired_contract, observation, asset_bindings=bindings, + schema_mapper=schema_mapper, ) diff --git a/semapact/interfaces/commands/deployment_cmd.py b/semapact/interfaces/commands/deployment_cmd.py index f8adbaf3..26f14e6d 100644 --- a/semapact/interfaces/commands/deployment_cmd.py +++ b/semapact/interfaces/commands/deployment_cmd.py @@ -25,7 +25,6 @@ ProcessOutcome, outcome_from_reconciliation_status, ) -from semapact.observation import RuntimeProvider from semapact.reconciliation import classify_reconciliation_status @@ -59,14 +58,11 @@ def run_deployment_plan(args: argparse.Namespace) -> DeploymentCommandResult: def run_deployment_preview(args: argparse.Namespace) -> DeploymentCommandResult: """Observe exact runtime scope and render the adapter's canonical preview.""" plan = _load_model(args.plan, DeploymentPlan) - provider = _runtime_provider(plan) - from semapact.platforms.runtime_registry import create_deployment_adapter adapter = create_deployment_adapter(plan.target.platform) preview = DeploymentService().preview( plan, - runtime_provider=provider, adapter=adapter, ) return DeploymentCommandResult( @@ -83,9 +79,23 @@ def run_deployment_execute(args: argparse.Namespace) -> DeploymentCommandResult: from semapact.platforms.runtime_registry import create_deployment_adapter + execution_config = None + if plan.target.platform == "databricks": + from semapact.platforms.databricks.deployment import ( + DatabricksDeploymentExecutionConfig, + ) + + execution_config = DatabricksDeploymentExecutionConfig( + warehouse_id=args.warehouse_id, + ) + elif args.warehouse_id is not None: + raise ValidationError( + "--warehouse-id is only supported for Databricks deployment" + ) + adapter = create_deployment_adapter( plan.target.platform, - warehouse_id=args.warehouse_id, + execution_config=execution_config, ) DeploymentService().execute( plan, @@ -109,11 +119,14 @@ def run_deployment_execute(args: argparse.Namespace) -> DeploymentCommandResult: def run_deployment_verify(args: argparse.Namespace) -> DeploymentCommandResult: - """Verify exact DeploymentPlan convergence through the existing M1 path.""" + """Verify exact DeploymentPlan convergence through the unified deployment adapter.""" plan = _load_model(args.plan, DeploymentPlan) + from semapact.platforms.runtime_registry import create_deployment_adapter + + adapter = create_deployment_adapter(plan.target.platform) result = DeploymentService().verify( plan, - runtime_provider=_runtime_provider(plan), + adapter=adapter, ) status = classify_reconciliation_status(result) rendered = ( @@ -127,13 +140,6 @@ def run_deployment_verify(args: argparse.Namespace) -> DeploymentCommandResult: ) -def _runtime_provider(plan: DeploymentPlan) -> RuntimeProvider: - from semapact.platforms.runtime_registry import create_runtime_provider_registry - - registry = create_runtime_provider_registry(plan.target.platform) - return registry.get(plan.target.platform) - - def _load_model(path: str, model_type: type[_ModelT]) -> _ModelT: try: raw = ( diff --git a/semapact/platforms/databricks/deployment.py b/semapact/platforms/databricks/deployment.py index 58c8911c..10da032d 100644 --- a/semapact/platforms/databricks/deployment.py +++ b/semapact/platforms/databricks/deployment.py @@ -1,59 +1,50 @@ -"""Databricks write-side deployment adapter.""" +"""Databricks deployment wiring and Statement Execution API executor.""" from __future__ import annotations -import re import time -from typing import Any +from typing import Any, Literal -from open_data_contract_standard.model import SchemaObject, SchemaProperty +from pydantic import field_validator -from semapact.deployment.models import ( - DeploymentActionKind, - DeploymentAuthorization, - DeploymentPlan, - DeploymentPreview, - NativeOperation, - NativeOperationKind, - compute_deployment_preview_id, - validate_deployment_authorization_identity, - validate_deployment_plan_identity, - validate_deployment_preview_identity, +from semapact.deployment.models import NativeOperation +from semapact.deployment.orchestrator import DeploymentOrchestrator +from semapact.deployment.providers import ( + DeploymentExecutionConfig, + NativeOperationExecutor, ) -from semapact.exceptions import ContractOpsAuthorizationError, ValidationError -from semapact.observation.fingerprint import fingerprint_observed_state -from semapact.observation.models import ObservedAsset, ObservedPlatformState +from semapact.exceptions import ValidationError from semapact.observation.providers import RuntimeProvider -from semapact.platforms.databricks.target import parse_databricks_runtime_target -from semapact.runtime import RuntimeAssetSpec +from semapact.platforms.databricks.transition_compiler import ( + DatabricksTransitionCompiler, +) +from semapact.platforms.databricks.transition_planner import ( + DatabricksSchemaTransitionPlanner, +) +from semapact.schema import SqlSchemaMapper + -_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_$]*$") -_DECIMAL_RE = re.compile(r"^DECIMAL\((\d{1,2}),(\d{1,2})\)$") -_CHAR_RE = re.compile(r"^(CHAR|VARCHAR)\((\d+)\)$") -_PRIMITIVE_TYPES = { - "BIGINT", - "BINARY", - "BOOLEAN", - "BYTE", - "DATE", - "DOUBLE", - "FLOAT", - "INT", - "INTEGER", - "LONG", - "REAL", - "SHORT", - "SMALLINT", - "STRING", - "TIMESTAMP", - "TIMESTAMP_NTZ", - "TINYINT", -} _TERMINAL_STATES = {"SUCCEEDED", "FAILED", "CANCELED", "CLOSED"} -class DatabricksDeploymentAdapter: - """Translate approved desired state into guarded Unity Catalog mutations.""" + +class DatabricksDeploymentExecutionConfig(DeploymentExecutionConfig): + """Typed Databricks execution configuration.""" + + platform: Literal["databricks"] = "databricks" + warehouse_id: str | None = None + + @field_validator("warehouse_id") + @classmethod + def _normalize_warehouse_id(cls, value: str | None) -> str | None: + if value is None: + return None + cleaned = value.strip() + return cleaned or None + + +class DatabricksStatementExecutor(NativeOperationExecutor): + """Execute exact native operations through Databricks Statement Execution API.""" key = "databricks" @@ -61,7 +52,6 @@ def __init__( self, *, client: Any, - runtime_provider: RuntimeProvider, warehouse_id: str | None = None, poll_interval_seconds: float = 1.0, max_poll_attempts: int = 300, @@ -70,191 +60,28 @@ def __init__( raise ValueError("poll_interval_seconds must be non-negative") if max_poll_attempts < 1: raise ValueError("max_poll_attempts must be positive") + self._client = client - self._runtime_provider = runtime_provider - self._warehouse_id = warehouse_id.strip() if warehouse_id and warehouse_id.strip() else None + self._warehouse_id = ( + warehouse_id.strip() + if warehouse_id and warehouse_id.strip() + else None + ) self._poll_interval_seconds = poll_interval_seconds self._max_poll_attempts = max_poll_attempts - def validate(self, plan: DeploymentPlan) -> None: - """Fail closed when a released desired state cannot be safely translated.""" - validate_deployment_plan_identity(plan) - if plan.target.platform.casefold() != self.key: - raise ValidationError( - f"Databricks adapter cannot deploy platform '{plan.target.platform}'" - ) - catalog, schema_name = parse_databricks_runtime_target(plan.target.runtime_target) - _validate_identifier(catalog, "catalog") - _validate_identifier(schema_name, "schema") - - physical_assets: set[str] = set() - for action in plan.actions: - if action.kind is not DeploymentActionKind.ENSURE_ASSET_STATE: - raise ValidationError( - f"Unsupported deployment action kind: {action.kind.value}" - ) - _validate_identifier(action.physical_name, "asset") - asset_key = action.physical_name.casefold() - if asset_key in physical_assets: - raise ValidationError( - "Databricks deployment cannot bind multiple governed assets to " - f"the same physical asset '{action.physical_name}'" - ) - physical_assets.add(asset_key) - desired = SchemaObject.model_validate_json(action.desired_state_json) - _desired_columns(desired) - - def preview( - self, - plan: DeploymentPlan, - observed_state: ObservedPlatformState, - ) -> DeploymentPreview: - """Derive exact CREATE/ALTER/NO_OP operations from current runtime evidence.""" - self.validate(plan) - _validate_observation(plan, observed_state) - catalog, schema_name = parse_databricks_runtime_target(plan.target.runtime_target) - observed_by_asset = { - asset.identity.asset.casefold(): asset for asset in observed_state.assets - } - - operations: list[NativeOperation] = [] - for action in plan.actions: - desired = SchemaObject.model_validate_json(action.desired_state_json) - observed = observed_by_asset.get(action.physical_name.casefold()) - if observed is None: - statement = _create_table_statement( - catalog=catalog, - schema_name=schema_name, - table_name=action.physical_name, - desired=desired, - ) - operations.append( - NativeOperation( - kind=NativeOperationKind.CREATE, - governed_asset=action.governed_asset, - statement=statement, - ) - ) - continue - - additions = _required_additions(desired, observed) - if additions: - statement = _add_columns_statement( - catalog=catalog, - schema_name=schema_name, - table_name=action.physical_name, - additions=additions, - ) - operations.append( - NativeOperation( - kind=NativeOperationKind.ALTER, - governed_asset=action.governed_asset, - statement=statement, - ) - ) - else: - operations.append( - NativeOperation( - kind=NativeOperationKind.NO_OP, - governed_asset=action.governed_asset, - ) - ) - - ordered = tuple(operations) - assert observed_state.fingerprint is not None - preview_id = compute_deployment_preview_id( - deployment_plan_id=plan.deployment_plan_id, - platform=self.key, - runtime_target=plan.target.runtime_target, - source_identifier=observed_state.source_identifier, - observation_fingerprint=observed_state.fingerprint, - operations=ordered, - ) - return DeploymentPreview( - deployment_preview_id=preview_id, - deployment_plan_id=plan.deployment_plan_id, - platform=self.key, - runtime_target=plan.target.runtime_target, - source_identifier=observed_state.source_identifier, - observation_fingerprint=observed_state.fingerprint, - operations=ordered, - ) - - def execute( - self, - plan: DeploymentPlan, - preview: DeploymentPreview, - authorization: DeploymentAuthorization, - ) -> None: - """Execute only the exact preview while its runtime preconditions still hold.""" - validate_deployment_plan_identity(plan) - validate_deployment_preview_identity(preview) - validate_deployment_authorization_identity(authorization) - self.validate(plan) - - if not authorization.allowed: - raise ContractOpsAuthorizationError("DeploymentAuthorization is not allowed") - if authorization.deployment_plan_id != plan.deployment_plan_id: - raise ContractOpsAuthorizationError( - "DeploymentAuthorization is not bound to this DeploymentPlan" - ) - if authorization.applied_release_id != plan.applied_release_id: - raise ContractOpsAuthorizationError( - "DeploymentAuthorization release does not match DeploymentPlan" - ) - if preview.deployment_plan_id != plan.deployment_plan_id: - raise ValidationError("DeploymentPreview is not bound to this DeploymentPlan") - if preview.platform.casefold() != self.key: - raise ValidationError("DeploymentPreview platform does not match adapter") - if preview.runtime_target != plan.target.runtime_target: - raise ValidationError("DeploymentPreview target does not match DeploymentPlan") - if preview.source_identifier != plan.target.source_reference: - raise ValidationError( - "DeploymentPreview runtime source does not match DeploymentPlan source reference" - ) - - current = self._observe_plan_scope(plan) - if current.source_identifier != preview.source_identifier: - raise ValidationError( - "Runtime source changed since DeploymentPreview was produced" - ) - if current.fingerprint != preview.observation_fingerprint: - raise ValidationError( - "Runtime state changed since DeploymentPreview was produced" - ) - - expected = self.preview(plan, current) - if expected != preview: + def execute(self, operation: NativeOperation) -> None: + """Execute one exact compiled operation using the Databricks SDK API.""" + statement = operation.statement + if statement is None or not statement.strip(): raise ValidationError( - "DeploymentPreview no longer equals the deterministic preview for " - "the authorized plan and runtime evidence" + "Databricks executable operation requires a SQL statement" ) - - for operation in preview.operations: - if operation.kind is NativeOperationKind.NO_OP: - continue - assert operation.statement is not None - self._execute_statement(operation.statement) - - def _observe_plan_scope(self, plan: DeploymentPlan) -> ObservedPlatformState: - assets = tuple( - RuntimeAssetSpec( - governed_asset=action.governed_asset, - physical_name=action.physical_name, - ) - for action in plan.actions - ) - bindings = self._runtime_provider.resolve_bindings( - runtime_target=plan.target.runtime_target, - assets=assets, - ) - return self._runtime_provider.observe(bindings=bindings) - - def _execute_statement(self, statement: str) -> None: if self._warehouse_id is None: raise ValidationError( "Databricks deployment execution requires a SQL warehouse_id" ) + response = self._client.statement_execution.execute_statement( statement=statement, warehouse_id=self._warehouse_id, @@ -262,6 +89,7 @@ def _execute_statement(self, statement: str) -> None: ) state = _statement_state(response) attempts = 0 + while state not in _TERMINAL_STATES: statement_id = getattr(response, "statement_id", None) if not statement_id: @@ -272,8 +100,10 @@ def _execute_statement(self, statement: str) -> None: raise RuntimeError( f"Databricks statement did not reach terminal state: {statement_id}" ) + if self._poll_interval_seconds: time.sleep(self._poll_interval_seconds) + response = self._client.statement_execution.get_statement(statement_id) state = _statement_state(response) attempts += 1 @@ -286,197 +116,42 @@ def _execute_statement(self, statement: str) -> None: ) -def _validate_observation( - plan: DeploymentPlan, - observed_state: ObservedPlatformState, -) -> None: - if observed_state.platform.casefold() != "databricks": - raise ValidationError("Databricks preview requires Databricks runtime evidence") - if not observed_state.source_identifier.strip(): - raise ValidationError("Runtime observation source_identifier is required") - if observed_state.source_identifier != plan.target.source_reference: - raise ValidationError( - "Runtime observation source does not match DeploymentPlan source reference" - ) - if observed_state.fingerprint is None: - raise ValidationError("Runtime observation fingerprint is required") - if observed_state.fingerprint != fingerprint_observed_state(observed_state): - raise ValidationError("Runtime observation fingerprint does not match its content") - - namespace = parse_databricks_runtime_target(plan.target.runtime_target) - expected_assets = {action.physical_name.casefold() for action in plan.actions} - seen: set[str] = set() - for asset in observed_state.assets: - identity = asset.identity - if identity.platform.casefold() != "databricks": - raise ValidationError("Observed asset platform does not match Databricks") - if tuple(part.casefold() for part in identity.namespace) != tuple( - part.casefold() for part in namespace - ): - raise ValidationError("Observed asset is outside DeploymentPlan runtime target") - asset_key = identity.asset.casefold() - if asset_key not in expected_assets: - raise ValidationError("Runtime evidence contains an asset outside plan scope") - if asset_key in seen: - raise ValidationError("Runtime evidence contains duplicate asset identities") - seen.add(asset_key) - - -def _desired_columns( - desired: SchemaObject, -) -> tuple[tuple[str, str, bool], ...]: - columns: list[tuple[str, str, bool]] = [] - seen: set[str] = set() - for prop in desired.properties or []: - physical_name = _property_physical_name(prop) - _validate_identifier(physical_name, "column") - key = physical_name.casefold() - if key in seen: - raise ValidationError( - f"Duplicate physical column binding in desired schema: '{physical_name}'" - ) - seen.add(key) - physical_type = getattr(prop, "physicalType", None) - if physical_type is None or not str(physical_type).strip(): - raise ValidationError( - f"Databricks deployment requires physicalType for column '{physical_name}'" - ) - rendered_type = _render_type(str(physical_type)) - columns.append((physical_name, rendered_type, bool(getattr(prop, "required", False)))) - if not columns: - raise ValidationError("Databricks deployment requires at least one schema property") - return tuple(columns) - - -def _required_additions( - desired: SchemaObject, - observed: ObservedAsset, -) -> tuple[tuple[str, str, bool], ...]: - if (observed.asset_type or "").strip().casefold() != "managed": - raise ValidationError( - "Existing Databricks asset must be a MANAGED table for deployment mutation" - ) - - observed_columns = { - prop.identity.property.casefold(): prop for prop in observed.properties - } - additions: list[tuple[str, str, bool]] = [] - for name, desired_type, required in _desired_columns(desired): - current = observed_columns.get(name.casefold()) - if current is None: - if required: - raise ValidationError( - f"Cannot add required column '{name}' without a safe default" - ) - additions.append((name, desired_type, required)) - continue - if current.physical_type is None: - raise ValidationError(f"Observed type is unknown for column '{name}'") - current_type = _render_type(current.physical_type) - if current_type != desired_type: - raise ValidationError( - f"Unsupported existing column type mutation for '{name}': " - f"{current_type} -> {desired_type}" - ) - if current.nullable is None: - raise ValidationError(f"Observed nullability is unknown for column '{name}'") - desired_nullable = not required - if current.nullable is not desired_nullable: - raise ValidationError( - f"Unsupported existing column nullability mutation for '{name}'" - ) - return tuple(additions) - - -def _property_physical_name(prop: SchemaProperty) -> str: - physical = getattr(prop, "physicalName", None) - if physical is not None and str(physical).strip(): - return str(physical).strip() - name = getattr(prop, "name", None) - if name is None or not str(name).strip(): - raise ValidationError("Schema property name is required for deployment binding") - return str(name).strip() - +class DatabricksDeploymentAdapter(DeploymentOrchestrator): + """Databricks wiring over the generic deployment orchestrator.""" -def _create_table_statement( - *, - catalog: str, - schema_name: str, - table_name: str, - desired: SchemaObject, -) -> str: - columns = [] - for name, physical_type, required in _desired_columns(desired): - suffix = " NOT NULL" if required else "" - columns.append(f"{_quote_identifier(name)} {physical_type}{suffix}") - column_sql = ", ".join(columns) - return ( - f"CREATE TABLE {_qualified_name(catalog, schema_name, table_name)} " - f"({column_sql}) USING DELTA" - ) - - -def _add_columns_statement( - *, - catalog: str, - schema_name: str, - table_name: str, - additions: tuple[tuple[str, str, bool], ...], -) -> str: - columns = ", ".join( - f"{_quote_identifier(name)} {physical_type}" - for name, physical_type, _required in additions - ) - return ( - f"ALTER TABLE {_qualified_name(catalog, schema_name, table_name)} " - f"ADD COLUMNS ({columns})" - ) - - -def _qualified_name(catalog: str, schema_name: str, table_name: str) -> str: - return ".".join( - _quote_identifier(part) for part in (catalog, schema_name, table_name) - ) - - -def _quote_identifier(value: str) -> str: - _validate_identifier(value, "identifier") - return f"`{value}`" - - -def _validate_identifier(value: str, role: str) -> None: - if not _IDENTIFIER_RE.fullmatch(value): - raise ValidationError( - f"Unsupported Databricks {role} identifier for M1 deployment: '{value}'" + def __init__( + self, + *, + client: Any, + runtime_provider: RuntimeProvider, + warehouse_id: str | None = None, + poll_interval_seconds: float = 1.0, + max_poll_attempts: int = 300, + ) -> None: + super().__init__( + runtime_provider=runtime_provider, + schema_mapper=SqlSchemaMapper( + key="databricks", + server_type="databricks", + dialect="databricks", + ), + transition_planner=DatabricksSchemaTransitionPlanner(), + transition_compiler=DatabricksTransitionCompiler(), + executor=DatabricksStatementExecutor( + client=client, + warehouse_id=warehouse_id, + poll_interval_seconds=poll_interval_seconds, + max_poll_attempts=max_poll_attempts, + ), ) -def _render_type(value: str) -> str: - normalized = re.sub(r"\s+", "", value.strip().upper()) - if normalized in _PRIMITIVE_TYPES: - return normalized - - decimal = _DECIMAL_RE.fullmatch(normalized) - if decimal: - precision = int(decimal.group(1)) - scale = int(decimal.group(2)) - if 1 <= precision <= 38 and 0 <= scale <= precision: - return f"DECIMAL({precision},{scale})" - raise ValidationError(f"Unsupported Databricks DECIMAL type: '{value}'") - - char_type = _CHAR_RE.fullmatch(normalized) - if char_type: - length = int(char_type.group(2)) - if length > 0: - return f"{char_type.group(1)}({length})" - - raise ValidationError(f"Unsupported Databricks physicalType: '{value}'") - - def _statement_state(response: Any) -> str: status = getattr(response, "status", None) state = getattr(status, "state", None) if state is None: - raise RuntimeError("Databricks statement response did not contain status.state") + raise RuntimeError( + "Databricks statement response did not contain status.state" + ) value = getattr(state, "value", state) return str(value).upper() diff --git a/semapact/platforms/databricks/runtime.py b/semapact/platforms/databricks/runtime.py index 08daba6a..b830d282 100644 --- a/semapact/platforms/databricks/runtime.py +++ b/semapact/platforms/databricks/runtime.py @@ -13,6 +13,7 @@ observe_databricks_table, ) from semapact.platforms.databricks.target import parse_databricks_runtime_target +from semapact.schema import validate_simple_sql_identifier class DatabricksRuntimeProvider: @@ -34,6 +35,11 @@ def resolve_bindings( ) -> tuple[RuntimeAssetBinding, ...]: """Resolve ``catalog.schema`` plus asset physical names into UC identities.""" namespace = parse_databricks_runtime_target(runtime_target) + catalog, schema_name = namespace + validate_simple_sql_identifier(catalog, "catalog") + validate_simple_sql_identifier(schema_name, "schema") + for asset in assets: + validate_simple_sql_identifier(asset.physical_name, "asset") bindings = tuple( RuntimeAssetBinding( governed_asset=asset.governed_asset, diff --git a/semapact/platforms/databricks/transition_compiler.py b/semapact/platforms/databricks/transition_compiler.py new file mode 100644 index 00000000..1ba36730 --- /dev/null +++ b/semapact/platforms/databricks/transition_compiler.py @@ -0,0 +1,115 @@ +"""Databricks implementation of the provider-neutral transition compiler.""" + +from __future__ import annotations + +from semapact.deployment.compilers import ( + TransitionCompiler, + require_native_definition, +) +from semapact.deployment.models import NativeOperation, NativeOperationKind +from semapact.deployment.schema_transitions import ( + SchemaTransition, + SchemaTransitionKind, +) +from semapact.exceptions import ValidationError +from semapact.platforms.databricks.target import parse_databricks_runtime_target +from semapact.schema import SchemaPropertyState, validate_simple_sql_identifier + + +class DatabricksTransitionCompiler(TransitionCompiler): + """Render Databricks-specific transition verbs around compiled target schema.""" + + key = "databricks" + + def compile( + self, + *, + runtime_target: str, + transition: SchemaTransition, + ) -> NativeOperation: + catalog, schema_name = parse_databricks_runtime_target(runtime_target) + validate_simple_sql_identifier(catalog, "catalog") + validate_simple_sql_identifier(schema_name, "schema") + validate_simple_sql_identifier(transition.physical_name, "asset") + + if transition.kind is SchemaTransitionKind.NO_OP: + return NativeOperation( + kind=NativeOperationKind.NO_OP, + governed_asset=transition.governed_asset, + ) + + if transition.kind is SchemaTransitionKind.CREATE_ASSET: + statement = _create_table_statement( + catalog=catalog, + schema_name=schema_name, + table_name=transition.physical_name, + columns=transition.columns, + ) + return NativeOperation( + kind=NativeOperationKind.CREATE, + governed_asset=transition.governed_asset, + statement=statement, + ) + + if transition.kind is SchemaTransitionKind.ADD_PROPERTIES: + statement = _add_columns_statement( + catalog=catalog, + schema_name=schema_name, + table_name=transition.physical_name, + columns=transition.columns, + ) + return NativeOperation( + kind=NativeOperationKind.ALTER, + governed_asset=transition.governed_asset, + statement=statement, + ) + + raise ValidationError( + f"Unsupported Databricks schema transition: {transition.kind.value}" + ) + + +def _create_table_statement( + *, + catalog: str, + schema_name: str, + table_name: str, + columns: tuple[SchemaPropertyState, ...], +) -> str: + rendered = ", ".join( + require_native_definition(column) + for column in columns + ) + return ( + f"CREATE TABLE {_qualified_name(catalog, schema_name, table_name)} " + f"({rendered}) USING DELTA" + ) + + +def _add_columns_statement( + *, + catalog: str, + schema_name: str, + table_name: str, + columns: tuple[SchemaPropertyState, ...], +) -> str: + rendered = ", ".join( + require_native_definition(column) + for column in columns + ) + return ( + f"ALTER TABLE {_qualified_name(catalog, schema_name, table_name)} " + f"ADD COLUMNS ({rendered})" + ) + + +def _qualified_name(catalog: str, schema_name: str, table_name: str) -> str: + return ".".join( + _quote_identifier(part) + for part in (catalog, schema_name, table_name) + ) + + +def _quote_identifier(value: str) -> str: + validate_simple_sql_identifier(value, "identifier") + return f"`{value}`" diff --git a/semapact/platforms/databricks/transition_planner.py b/semapact/platforms/databricks/transition_planner.py new file mode 100644 index 00000000..8ea730e0 --- /dev/null +++ b/semapact/platforms/databricks/transition_planner.py @@ -0,0 +1,52 @@ +"""Databricks schema-transition capability policy.""" + +from __future__ import annotations + +from collections.abc import Sequence + +from semapact.deployment.schema_transitions import ( + AdditiveSchemaTransitionPlanner, + SchemaTransition, + SchemaTransitionKind, + SchemaTransitionPlanner, +) +from semapact.exceptions import ValidationError +from semapact.observation.models import ObservedAsset +from semapact.schema import SchemaComparisonResult, SchemaPropertyState + + +class DatabricksSchemaTransitionPlanner(SchemaTransitionPlanner): + """Apply Databricks mutation capability rules around shared additive planning.""" + + key = "databricks" + + def __init__(self) -> None: + self._additive = AdditiveSchemaTransitionPlanner() + + def plan( + self, + *, + governed_asset: str, + physical_name: str, + desired_columns: Sequence[SchemaPropertyState], + comparison: SchemaComparisonResult, + observed_asset: ObservedAsset | None = None, + ) -> SchemaTransition: + transition = self._additive.plan( + governed_asset=governed_asset, + physical_name=physical_name, + desired_columns=desired_columns, + comparison=comparison, + observed_asset=observed_asset, + ) + + if ( + observed_asset is not None + and transition.kind is not SchemaTransitionKind.NO_OP + and (observed_asset.asset_type or "").strip().casefold() != "managed" + ): + raise ValidationError( + "Existing Databricks asset must be a MANAGED table for deployment mutation" + ) + + return transition diff --git a/semapact/platforms/runtime_registry.py b/semapact/platforms/runtime_registry.py index a88e7328..113d7332 100644 --- a/semapact/platforms/runtime_registry.py +++ b/semapact/platforms/runtime_registry.py @@ -8,6 +8,7 @@ from open_data_contract_standard.model import OpenDataContractStandard, Server from semapact.deployment.adapters import DeploymentAdapter +from semapact.deployment.providers import DeploymentExecutionConfig from semapact.exceptions import ValidationError from semapact.observation import RuntimeProvider, RuntimeProviderRegistry @@ -34,7 +35,10 @@ def resolve_runtime_location( servers = tuple(contract.servers or ()) if servers: selected = _select_contract_server(servers, server_name) - platform = _required(selected.type, "Selected contract server must define a runtime type") + platform = _required( + selected.type, + "Selected contract server must define a runtime type", + ).casefold() return ResolvedRuntimeLocation( platform=platform, runtime_target=_runtime_target_from_server(platform, selected), @@ -51,7 +55,7 @@ def resolve_runtime_location( "--server cannot be used because the contract defines no servers" ) - platform = _clean(fallback_platform) + platform = _clean(fallback_platform, casefold=True) runtime_target = _clean(fallback_runtime_target) if not platform or not runtime_target: raise ValidationError( @@ -84,23 +88,73 @@ def create_runtime_provider_registry( def create_deployment_adapter( platform: str, *, - warehouse_id: str | None = None, contract_server: Server | None = None, + execution_config: DeploymentExecutionConfig | None = None, ) -> DeploymentAdapter: - """Compose the selected write adapter and provider clients lazily. - - A SQL warehouse is execution configuration, not a prerequisite for read-only - validation or preview. Databricks execution fails closed if mutation is attempted - without a warehouse ID. - """ + """Compose the selected write adapter and provider clients lazily.""" normalized = platform.strip().casefold() if normalized != "databricks": raise ValidationError( f"Unsupported deployment adapter '{platform}'. Supported adapters: databricks" ) - from semapact.platforms.databricks import ( + from semapact.platforms.databricks.deployment import ( DatabricksDeploymentAdapter, + DatabricksDeploymentExecutionConfig, + ) + + config = ( + DatabricksDeploymentExecutionConfig() + if execution_config is None + else execution_config + ) + if not isinstance(config, DatabricksDeploymentExecutionConfig): + raise ValidationError( + "Databricks deployment requires DatabricksDeploymentExecutionConfig" + ) + + client, runtime_provider = _create_databricks_client_and_provider( + contract_server=contract_server + ) + return DatabricksDeploymentAdapter( + client=client, + runtime_provider=runtime_provider, + warehouse_id=config.warehouse_id, + ) + + +def _runtime_target_from_server(platform: str, server: Server) -> str: + """Project one ODCS server into the selected provider's runtime target.""" + if platform == "databricks": + catalog = _required( + server.catalog, + "Databricks contract server must define catalog", + ) + schema_name = _required( + server.schema_, + "Databricks contract server must define schema", + ) + return f"{catalog}.{schema_name}" + raise ValidationError( + f"Unsupported runtime provider '{platform}'. Supported providers: databricks" + ) + + +def _create_databricks_provider( + *, + contract_server: Server | None = None, +) -> RuntimeProvider: + _, provider = _create_databricks_client_and_provider( + contract_server=contract_server + ) + return provider + + +def _create_databricks_client_and_provider( + *, + contract_server: Server | None = None, +): + from semapact.platforms.databricks import ( DatabricksRuntimeProvider, create_databricks_workspace_client, ) @@ -111,15 +165,12 @@ def create_deployment_adapter( source_identifier = getattr(getattr(client, "config", None), "host", None) if not isinstance(source_identifier, str) or not source_identifier.strip(): raise RuntimeError("Databricks SDK did not resolve a workspace host") - runtime_provider = DatabricksRuntimeProvider( + + provider = DatabricksRuntimeProvider( client=client, source_identifier=source_identifier, ) - return DatabricksDeploymentAdapter( - client=client, - runtime_provider=runtime_provider, - warehouse_id=warehouse_id, - ) + return client, provider def _select_contract_server( @@ -153,38 +204,6 @@ def _select_contract_server( ) -def _runtime_target_from_server(platform: str, server: Server) -> str: - """Project one ODCS server into the selected provider's runtime target.""" - if platform.strip().casefold() == "databricks": - catalog = _required(server.catalog, "Databricks contract server must define catalog") - schema = _required(server.schema_, "Databricks contract server must define schema") - return f"{catalog}.{schema}" - raise ValidationError( - f"Unsupported runtime provider '{platform}'. Supported providers: databricks" - ) - - -def _create_databricks_provider( - *, - contract_server: Server | None = None, -) -> RuntimeProvider: - from semapact.platforms.databricks import ( - DatabricksRuntimeProvider, - create_databricks_workspace_client, - ) - - client = create_databricks_workspace_client( - workspace_url=_clean(contract_server.host) if contract_server else None - ) - source_identifier = getattr(getattr(client, "config", None), "host", None) - if not isinstance(source_identifier, str) or not source_identifier.strip(): - raise RuntimeError("Databricks SDK did not resolve a workspace host") - return DatabricksRuntimeProvider( - client=client, - source_identifier=source_identifier, - ) - - def _available_server_names(servers: tuple[Server, ...]) -> str: names = sorted(name for server in servers if (name := _clean(server.server))) return ", ".join(names) or "none" diff --git a/semapact/reconciliation/engine.py b/semapact/reconciliation/engine.py index 551eb079..c12a4dc6 100644 --- a/semapact/reconciliation/engine.py +++ b/semapact/reconciliation/engine.py @@ -4,17 +4,12 @@ from collections.abc import Sequence -from open_data_contract_standard.model import OpenDataContractStandard, SchemaProperty +from open_data_contract_standard.model import OpenDataContractStandard, SchemaObject from semapact.exceptions import ValidationError -from semapact.lifecycle.identity import ( - PropertyIdentity, - build_property_index, - build_schema_index, - normalize_identity_name, -) +from semapact.lifecycle.identity import build_schema_index, normalize_identity_name from semapact.observation.fingerprint import fingerprint_observed_state -from semapact.observation.models import ObservedAsset, ObservedPlatformState, ObservedProperty +from semapact.observation.models import ObservedAsset, ObservedPlatformState from semapact.observation.providers import RuntimeAssetBinding from semapact.reconciliation.models import ( ReconciliationDifference, @@ -23,13 +18,16 @@ ReconciliationSubject, RuntimeReasonCode, ) +from semapact.schema import ( + PassThroughSchemaMapper, + SchemaDifference, + SchemaMapper, + SchemaSnapshot, + build_physical_property_bindings, + compare_schema_snapshots, +) -_SUBJECT_ORDER = { - ReconciliationSubject.ASSET: 0, - ReconciliationSubject.PROPERTY: 1, - ReconciliationSubject.PHYSICAL_TYPE: 2, - ReconciliationSubject.NULLABILITY: 3, -} +_PASSTHROUGH_SCHEMA_MAPPER = PassThroughSchemaMapper() _REASON_CODE_BY_RAW_DIFFERENCE: dict[ tuple[ReconciliationDifferenceType, ReconciliationSubject], RuntimeReasonCode @@ -48,15 +46,17 @@ def reconcile_governed_contract( observation: ObservedPlatformState, *, asset_bindings: Sequence[RuntimeAssetBinding] | None = None, + schema_mapper: SchemaMapper | None = None, ) -> ReconciliationResult: """Compare governed ODCS desired state with platform-neutral observed state. - ``asset_bindings`` explicitly maps governed logical schema identity to - provider-local observed asset identity. Operational runtime-product flows - should provide bindings so physical names never redefine governed identity. - The legacy name-matching path remains available for existing library callers. + Shared schema mapping projects both source models into normalized snapshots. + The shared comparator finds raw schema facts exactly once. Reconciliation + projects those facts into stable runtime reason codes. """ governed_assets = build_schema_index(contract) + mapper = schema_mapper or _PASSTHROUGH_SCHEMA_MAPPER + desired_uses_physical_identity = schema_mapper is not None if asset_bindings is None: observed_assets = _build_observed_asset_index(observation) else: @@ -66,169 +66,121 @@ def reconcile_governed_contract( bindings=asset_bindings, ) - differences: list[ReconciliationDifference] = [] - unverified_paths: list[str] = [] - governed_keys = set(governed_assets) - observed_keys = set(observed_assets) - - for asset_key in sorted(governed_keys - observed_keys): - differences.append( - _difference( - difference_type=ReconciliationDifferenceType.MISSING, - subject=ReconciliationSubject.ASSET, - asset_identity=asset_key, - ) - ) - - for asset_key in sorted(observed_keys - governed_keys): - differences.append( - _difference( - difference_type=ReconciliationDifferenceType.UNEXPECTED, - subject=ReconciliationSubject.ASSET, - asset_identity=asset_key, - ) - ) - - for asset_key in sorted(governed_keys & observed_keys): - governed_schema = governed_assets[asset_key] - observed_asset = observed_assets[asset_key] - property_differences, property_unverified_paths = _reconcile_properties( - asset_key=asset_key, - governed_properties=list(governed_schema.properties or []), - observed_asset=observed_asset, - ) - differences.extend(property_differences) - unverified_paths.extend(property_unverified_paths) + comparison = compare_schema_snapshots( + _governed_snapshot( + governed_assets, + mapper=mapper, + rebind_physical_identity=desired_uses_physical_identity, + ), + _observed_snapshot( + governed_assets=governed_assets, + observed_assets=observed_assets, + mapper=mapper, + ), + ) - ordered = tuple(sorted(differences, key=_difference_sort_key)) return ReconciliationResult( contract_id=_required_contract_text(getattr(contract, "id", None), field="id"), contract_version=_required_contract_text(getattr(contract, "version", None), field="version"), observation_source_identifier=observation.source_identifier, observation_fingerprint=observation.fingerprint or fingerprint_observed_state(observation), - differences=ordered, - unverified_paths=tuple(sorted(unverified_paths)), + differences=tuple( + _reconciliation_difference(item) + for item in comparison.differences + ), + unverified_paths=comparison.unverified_paths, ) -def _reconcile_properties( +def _governed_snapshot( + governed_assets: dict[str, SchemaObject], *, - asset_key: str, - governed_properties: list[SchemaProperty], - observed_asset: ObservedAsset, -) -> tuple[list[ReconciliationDifference], list[str]]: - governed = build_property_index(asset_key, governed_properties) - observed = _build_observed_property_index( - asset_key, - observed_asset, - governed_properties=governed_properties, - ) - differences: list[ReconciliationDifference] = [] - unverified_paths: list[str] = [] - governed_keys = set(governed) - observed_keys = set(observed) - - for prop_key in sorted(governed_keys - observed_keys): - differences.append( - _difference( - difference_type=ReconciliationDifferenceType.MISSING, - subject=ReconciliationSubject.PROPERTY, - asset_identity=asset_key, - property_identity=prop_key[1], - ) + mapper: SchemaMapper, + rebind_physical_identity: bool, +) -> SchemaSnapshot: + assets = [] + for asset_key, governed_schema in governed_assets.items(): + mapped = mapper.map_desired_asset( + governed_schema, + asset_identity=asset_key, ) - - for prop_key in sorted(observed_keys - governed_keys): - differences.append( - _difference( - difference_type=ReconciliationDifferenceType.UNEXPECTED, - subject=ReconciliationSubject.PROPERTY, - asset_identity=asset_key, - property_identity=prop_key[1], + if rebind_physical_identity: + property_bindings = build_physical_property_bindings( + governed_schema.properties or [] ) - ) - - for prop_key in sorted(governed_keys & observed_keys): - property_differences, property_unverified_paths = _reconcile_matching_property( - asset_key=asset_key, - property_key=prop_key, - governed=governed[prop_key], - observed=observed[prop_key], - ) - differences.extend(property_differences) - unverified_paths.extend(property_unverified_paths) - - return differences, unverified_paths + mapped = mapped.model_copy( + update={ + "properties": tuple( + prop.model_copy( + update={ + "identity": property_bindings.get( + prop.identity.casefold(), + prop.identity, + ) + } + ) + for prop in mapped.properties + ) + } + ) + assets.append(mapped) + return SchemaSnapshot(assets=tuple(assets)) -def _reconcile_matching_property( +def _observed_snapshot( *, - asset_key: str, - property_key: PropertyIdentity, - governed: SchemaProperty, - observed: ObservedProperty, -) -> tuple[list[ReconciliationDifference], list[str]]: - differences: list[ReconciliationDifference] = [] - unverified_paths: list[str] = [] - property_identity = property_key[1] - - expected_physical = _optional_text(getattr(governed, "physicalType", None)) - observed_physical = _optional_text(observed.physical_type) - if expected_physical is not None: - if observed_physical is None: - unverified_paths.append( - _difference_path( - subject=ReconciliationSubject.PHYSICAL_TYPE, - asset_identity=asset_key, - property_identity=property_identity, - ) + governed_assets: dict[str, SchemaObject], + observed_assets: dict[str, ObservedAsset], + mapper: SchemaMapper, +) -> SchemaSnapshot: + assets = [] + for asset_key, observed_asset in observed_assets.items(): + governed_schema = governed_assets.get(asset_key) + property_bindings = ( + None + if governed_schema is None + else build_physical_property_bindings( + governed_schema.properties or [] ) - elif _normalize_comparable_text(expected_physical) != _normalize_comparable_text(observed_physical): - differences.append( - _difference( - difference_type=ReconciliationDifferenceType.MISMATCH, - subject=ReconciliationSubject.PHYSICAL_TYPE, - asset_identity=asset_key, - property_identity=property_identity, - expected=expected_physical, - observed=observed_physical, - ) + ) + assets.append( + mapper.map_observed_asset( + observed_asset, + asset_identity=asset_key, + property_bindings=property_bindings, ) + ) + return SchemaSnapshot(assets=tuple(assets)) - required = getattr(governed, "required", None) - nullable = observed.nullable - if isinstance(required, bool): - if not isinstance(nullable, bool): - unverified_paths.append( - _difference_path( - subject=ReconciliationSubject.NULLABILITY, - asset_identity=asset_key, - property_identity=property_identity, - ) - ) - else: - expected_nullable = not required - if expected_nullable != nullable: - differences.append( - _difference( - difference_type=ReconciliationDifferenceType.MISMATCH, - subject=ReconciliationSubject.NULLABILITY, - asset_identity=asset_key, - property_identity=property_identity, - expected=expected_nullable, - observed=nullable, - ) - ) - return differences, unverified_paths +def _reconciliation_difference( + difference: SchemaDifference, +) -> ReconciliationDifference: + return ReconciliationDifference( + difference_type=difference.difference_type, + subject=difference.subject, + reason_code=_runtime_reason_code( + difference_type=difference.difference_type, + subject=difference.subject, + ), + path=difference.path, + asset_identity=difference.asset_identity, + property_identity=difference.property_identity, + expected=difference.expected, + observed=difference.observed, + ) -def _build_observed_asset_index(observation: ObservedPlatformState) -> dict[str, ObservedAsset]: +def _build_observed_asset_index( + observation: ObservedPlatformState, +) -> dict[str, ObservedAsset]: index: dict[str, ObservedAsset] = {} for asset in observation.assets: key = normalize_identity_name(asset.identity.asset, "Observed asset") if key in index: - raise ValidationError(f"Duplicate canonical observed asset identity found: '{key}'") + raise ValidationError( + f"Duplicate canonical observed asset identity found: '{key}'" + ) index[key] = asset return index @@ -243,14 +195,23 @@ def _build_bound_observed_asset_index( bound_runtime_keys: set[tuple[str, ...]] = set() for binding in bindings: - governed_key = normalize_identity_name(binding.governed_asset, "Runtime binding governed asset") + governed_key = normalize_identity_name( + binding.governed_asset, + "Runtime binding governed asset", + ) if governed_key in binding_by_governed: - raise ValidationError(f"Duplicate runtime binding for governed asset: '{governed_key}'") + raise ValidationError( + f"Duplicate runtime binding for governed asset: '{governed_key}'" + ) if binding.observed_asset.platform.casefold() != observation.platform.casefold(): - raise ValidationError("Runtime binding platform must match observed platform state") + raise ValidationError( + "Runtime binding platform must match observed platform state" + ) runtime_key = binding.observed_asset.canonical_key if runtime_key in bound_runtime_keys: - raise ValidationError("Multiple governed assets cannot bind to one runtime asset") + raise ValidationError( + "Multiple governed assets cannot bind to one runtime asset" + ) bound_runtime_keys.add(runtime_key) binding_by_governed[governed_key] = binding @@ -287,79 +248,6 @@ def _build_bound_observed_asset_index( return index -def _build_observed_property_index( - asset_key: str, - asset: ObservedAsset, - *, - governed_properties: Sequence[SchemaProperty], -) -> dict[PropertyIdentity, ObservedProperty]: - physical_to_governed = _property_binding_index(governed_properties) - index: dict[PropertyIdentity, ObservedProperty] = {} - for prop in asset.properties: - if prop.identity.asset != asset.identity: - raise ValidationError("Observed property asset identity must match its containing asset") - observed_name = normalize_identity_name(prop.identity.property, "Observed property") - governed_name = physical_to_governed.get(observed_name, observed_name) - key: PropertyIdentity = (asset_key, governed_name) - if key in index: - raise ValidationError( - f"Duplicate canonical observed property identity found: '{governed_name}'" - f" in asset '{asset_key}'" - ) - index[key] = prop - return index - - -def _property_binding_index( - governed_properties: Sequence[SchemaProperty], -) -> dict[str, str]: - physical_to_governed: dict[str, str] = {} - for prop in governed_properties: - logical_raw = getattr(prop, "name", None) - if logical_raw is None: - raise ValidationError("Governed property name is required for runtime binding") - governed_name = normalize_identity_name(str(logical_raw), "Property") - physical_raw = getattr(prop, "physicalName", None) - physical_text = str(physical_raw).strip() if physical_raw is not None else "" - physical_name = normalize_identity_name( - physical_text or str(logical_raw), - "Property physical binding", - ) - existing = physical_to_governed.get(physical_name) - if existing is not None and existing != governed_name: - raise ValidationError( - "Multiple governed properties cannot bind to one physical runtime property: " - f"'{physical_name}'" - ) - physical_to_governed[physical_name] = governed_name - return physical_to_governed - - -def _difference( - *, - difference_type: ReconciliationDifferenceType, - subject: ReconciliationSubject, - asset_identity: str, - property_identity: str | None = None, - expected: str | bool | None = None, - observed: str | bool | None = None, -) -> ReconciliationDifference: - return ReconciliationDifference( - difference_type=difference_type, - subject=subject, - reason_code=_runtime_reason_code(difference_type=difference_type, subject=subject), - path=_difference_path( - subject=subject, - asset_identity=asset_identity, - property_identity=property_identity, - ), - asset_identity=asset_identity, - property_identity=property_identity, - expected=expected, - observed=observed, - ) - - def _runtime_reason_code( *, difference_type: ReconciliationDifferenceType, @@ -374,49 +262,14 @@ def _runtime_reason_code( return reason_code -def _difference_path( - *, - subject: ReconciliationSubject, - asset_identity: str, - property_identity: str | None, -) -> str: - asset_path = f"schema[{asset_identity}]" - if subject is ReconciliationSubject.ASSET: - return asset_path - if property_identity is None: - raise ValueError(f"property_identity is required for {subject.value}") - property_path = f"{asset_path}.properties[{property_identity}]" - if subject is ReconciliationSubject.PROPERTY: - return property_path - if subject is ReconciliationSubject.PHYSICAL_TYPE: - return f"{property_path}.physicalType" - return f"{property_path}.nullability" - - -def _difference_sort_key(difference: ReconciliationDifference) -> tuple[str, str, int, str]: - return ( - difference.asset_identity, - difference.property_identity or "", - _SUBJECT_ORDER[difference.subject], - difference.difference_type.value, - ) - - def _required_contract_text(value: object, *, field: str) -> str: - text = _optional_text(value) - if text is None: + if value is None: raise ValidationError( f"Governed desired-state contract {field} is required for reconciliation" ) - return text - - -def _optional_text(value: object | None) -> str | None: - if value is None: - return None text = str(value).strip() - return text or None - - -def _normalize_comparable_text(value: str) -> str: - return value.strip().casefold() + if not text: + raise ValidationError( + f"Governed desired-state contract {field} is required for reconciliation" + ) + return text diff --git a/semapact/reconciliation/models.py b/semapact/reconciliation/models.py index b03c4d3b..3546e8e0 100644 --- a/semapact/reconciliation/models.py +++ b/semapact/reconciliation/models.py @@ -12,6 +12,8 @@ from pydantic import BaseModel, ConfigDict +from semapact.schema import SchemaDifferenceType, SchemaSubject + class ReconciliationModel(BaseModel): """Shared immutable base for reconciliation models.""" @@ -19,21 +21,9 @@ class ReconciliationModel(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") -class ReconciliationDifferenceType(str, Enum): - """Generic raw comparison operation, not a public reason-code taxonomy.""" - - MISSING = "missing" - UNEXPECTED = "unexpected" - MISMATCH = "mismatch" - - -class ReconciliationSubject(str, Enum): - """Comparable subject represented by a raw reconciliation difference.""" - - ASSET = "asset" - PROPERTY = "property" - PHYSICAL_TYPE = "physical_type" - NULLABILITY = "nullability" +# Backward-compatible public names projected from the shared schema comparator. +ReconciliationDifferenceType = SchemaDifferenceType +ReconciliationSubject = SchemaSubject class RuntimeReasonCode(str, Enum): @@ -50,8 +40,8 @@ class RuntimeReasonCode(str, Enum): class ReconciliationDifference(ReconciliationModel): """One deterministic difference between governed and observed state.""" - difference_type: ReconciliationDifferenceType - subject: ReconciliationSubject + difference_type: SchemaDifferenceType + subject: SchemaSubject reason_code: RuntimeReasonCode path: str asset_identity: str diff --git a/semapact/schema/__init__.py b/semapact/schema/__init__.py new file mode 100644 index 00000000..5553376b --- /dev/null +++ b/semapact/schema/__init__.py @@ -0,0 +1,45 @@ +"""Provider-neutral schema state, mapping and comparison primitives.""" + +from semapact.schema.comparison import ( + SchemaAssetState, + SchemaComparisonResult, + SchemaDifference, + SchemaDifferenceType, + SchemaPropertyState, + SchemaSnapshot, + SchemaSubject, + compare_schema_snapshots, +) +from semapact.schema.identifiers import validate_simple_sql_identifier +from semapact.schema.mapping import ( + PassThroughSchemaMapper, + SchemaMapper, + SqlSchemaMapper, + build_physical_property_bindings, + map_odcs_schema_asset, + map_observed_schema_asset, + normalize_sql_type, + parse_sql_target_asset, + property_identity, +) + +__all__ = [ + "SchemaAssetState", + "SchemaComparisonResult", + "SchemaDifference", + "SchemaDifferenceType", + "SchemaPropertyState", + "SchemaSnapshot", + "SchemaSubject", + "compare_schema_snapshots", + "PassThroughSchemaMapper", + "SchemaMapper", + "SqlSchemaMapper", + "build_physical_property_bindings", + "map_odcs_schema_asset", + "map_observed_schema_asset", + "property_identity", + "parse_sql_target_asset", + "normalize_sql_type", + "validate_simple_sql_identifier", +] diff --git a/semapact/schema/comparison.py b/semapact/schema/comparison.py new file mode 100644 index 00000000..304272f0 --- /dev/null +++ b/semapact/schema/comparison.py @@ -0,0 +1,323 @@ +"""Provider-neutral comparison of normalized schema state. + +The comparator knows only expected and observed schema snapshots. ODCS, +runtime-provider models, governance policy, deployment capability, SQL rendering, +and authorization belong to projection or interpretation layers. +""" + +from __future__ import annotations + +from enum import Enum + +from pydantic import BaseModel, ConfigDict, field_validator + +from semapact.exceptions import ValidationError + + +class SchemaComparisonModel(BaseModel): + """Shared immutable base for schema comparison values.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + +class SchemaDifferenceType(str, Enum): + """Difference direction relative to expected desired state.""" + + MISSING = "missing" + UNEXPECTED = "unexpected" + MISMATCH = "mismatch" + + +class SchemaSubject(str, Enum): + """Comparable schema subject.""" + + ASSET = "asset" + PROPERTY = "property" + PHYSICAL_TYPE = "physical_type" + NULLABILITY = "nullability" + + +class SchemaPropertyState(SchemaComparisonModel): + """Normalized comparable state for one property. + + native_definition is opaque provider output retained for later transition + rendering. It is deliberately excluded from schema comparison semantics. + """ + + identity: str + physical_type: str | None = None + nullable: bool | None = None + native_definition: str | None = None + + @field_validator("identity") + @classmethod + def _require_identity(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("schema property identity must not be empty") + return cleaned + + @field_validator("physical_type", "native_definition") + @classmethod + def _normalize_optional_type(cls, value: str | None) -> str | None: + if value is None: + return None + cleaned = value.strip() + return cleaned or None + + +class SchemaAssetState(SchemaComparisonModel): + """Normalized comparable state for one asset.""" + + identity: str + properties: tuple[SchemaPropertyState, ...] = () + + @field_validator("identity") + @classmethod + def _require_identity(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("schema asset identity must not be empty") + return cleaned + + +class SchemaSnapshot(SchemaComparisonModel): + """Normalized schema state independent of its source object model.""" + + assets: tuple[SchemaAssetState, ...] = () + + +class SchemaDifference(SchemaComparisonModel): + """One raw schema fact; no policy or deployment meaning is attached.""" + + difference_type: SchemaDifferenceType + subject: SchemaSubject + path: str + asset_identity: str + property_identity: str | None = None + expected: str | bool | None = None + observed: str | bool | None = None + + +class SchemaComparisonResult(SchemaComparisonModel): + """Deterministic schema differences plus evidence gaps.""" + + differences: tuple[SchemaDifference, ...] = () + unverified_paths: tuple[str, ...] = () + + +_SUBJECT_ORDER = { + SchemaSubject.ASSET: 0, + SchemaSubject.PROPERTY: 1, + SchemaSubject.PHYSICAL_TYPE: 2, + SchemaSubject.NULLABILITY: 3, +} + + +def compare_schema_snapshots( + expected: SchemaSnapshot, + observed: SchemaSnapshot, +) -> SchemaComparisonResult: + """Compare normalized expected and observed schema state exactly once.""" + expected_assets = _asset_index(expected, role="expected") + observed_assets = _asset_index(observed, role="observed") + + differences: list[SchemaDifference] = [] + unverified_paths: list[str] = [] + + expected_keys = set(expected_assets) + observed_keys = set(observed_assets) + + for asset_key in sorted(expected_keys - observed_keys): + differences.append( + _difference( + difference_type=SchemaDifferenceType.MISSING, + subject=SchemaSubject.ASSET, + asset_identity=asset_key, + ) + ) + + for asset_key in sorted(observed_keys - expected_keys): + differences.append( + _difference( + difference_type=SchemaDifferenceType.UNEXPECTED, + subject=SchemaSubject.ASSET, + asset_identity=asset_key, + ) + ) + + for asset_key in sorted(expected_keys & observed_keys): + asset_differences, asset_unverified = _compare_asset( + expected=expected_assets[asset_key], + observed=observed_assets[asset_key], + ) + differences.extend(asset_differences) + unverified_paths.extend(asset_unverified) + + return SchemaComparisonResult( + differences=tuple(sorted(differences, key=_difference_sort_key)), + unverified_paths=tuple(sorted(unverified_paths)), + ) + + +def _compare_asset( + *, + expected: SchemaAssetState, + observed: SchemaAssetState, +) -> tuple[list[SchemaDifference], list[str]]: + expected_properties = _property_index(expected, role="expected") + observed_properties = _property_index(observed, role="observed") + + differences: list[SchemaDifference] = [] + unverified_paths: list[str] = [] + + expected_keys = set(expected_properties) + observed_keys = set(observed_properties) + + for property_key in sorted(expected_keys - observed_keys): + differences.append( + _difference( + difference_type=SchemaDifferenceType.MISSING, + subject=SchemaSubject.PROPERTY, + asset_identity=expected.identity, + property_identity=property_key, + ) + ) + + for property_key in sorted(observed_keys - expected_keys): + differences.append( + _difference( + difference_type=SchemaDifferenceType.UNEXPECTED, + subject=SchemaSubject.PROPERTY, + asset_identity=expected.identity, + property_identity=property_key, + ) + ) + + for property_key in sorted(expected_keys & observed_keys): + desired = expected_properties[property_key] + actual = observed_properties[property_key] + + if desired.physical_type is not None: + path = _difference_path( + subject=SchemaSubject.PHYSICAL_TYPE, + asset_identity=expected.identity, + property_identity=property_key, + ) + if actual.physical_type is None: + unverified_paths.append(path) + elif desired.physical_type.casefold() != actual.physical_type.casefold(): + differences.append( + _difference( + difference_type=SchemaDifferenceType.MISMATCH, + subject=SchemaSubject.PHYSICAL_TYPE, + asset_identity=expected.identity, + property_identity=property_key, + expected=desired.physical_type, + observed=actual.physical_type, + ) + ) + + if desired.nullable is not None: + path = _difference_path( + subject=SchemaSubject.NULLABILITY, + asset_identity=expected.identity, + property_identity=property_key, + ) + if actual.nullable is None: + unverified_paths.append(path) + elif desired.nullable is not actual.nullable: + differences.append( + _difference( + difference_type=SchemaDifferenceType.MISMATCH, + subject=SchemaSubject.NULLABILITY, + asset_identity=expected.identity, + property_identity=property_key, + expected=desired.nullable, + observed=actual.nullable, + ) + ) + + return differences, unverified_paths + + +def _asset_index(snapshot: SchemaSnapshot, *, role: str) -> dict[str, SchemaAssetState]: + index: dict[str, SchemaAssetState] = {} + for asset in snapshot.assets: + key = asset.identity.casefold() + if key in index: + raise ValidationError( + f"Duplicate normalized {role} asset identity: '{asset.identity}'" + ) + index[key] = asset.model_copy(update={"identity": key}) + return index + + +def _property_index( + asset: SchemaAssetState, + *, + role: str, +) -> dict[str, SchemaPropertyState]: + index: dict[str, SchemaPropertyState] = {} + for prop in asset.properties: + key = prop.identity.casefold() + if key in index: + raise ValidationError( + f"Duplicate normalized {role} property identity: '{prop.identity}'" + ) + index[key] = prop.model_copy(update={"identity": key}) + return index + + +def _difference( + *, + difference_type: SchemaDifferenceType, + subject: SchemaSubject, + asset_identity: str, + property_identity: str | None = None, + expected: str | bool | None = None, + observed: str | bool | None = None, +) -> SchemaDifference: + return SchemaDifference( + difference_type=difference_type, + subject=subject, + path=_difference_path( + subject=subject, + asset_identity=asset_identity, + property_identity=property_identity, + ), + asset_identity=asset_identity, + property_identity=property_identity, + expected=expected, + observed=observed, + ) + + +def _difference_path( + *, + subject: SchemaSubject, + asset_identity: str, + property_identity: str | None, +) -> str: + asset_path = f"schema[{asset_identity}]" + if subject is SchemaSubject.ASSET: + return asset_path + if property_identity is None: + raise ValueError(f"property_identity is required for {subject.value}") + property_path = f"{asset_path}.properties[{property_identity}]" + if subject is SchemaSubject.PROPERTY: + return property_path + if subject is SchemaSubject.PHYSICAL_TYPE: + return f"{property_path}.physicalType" + return f"{property_path}.nullability" + + +def _difference_sort_key( + difference: SchemaDifference, +) -> tuple[str, str, int, str]: + return ( + difference.asset_identity, + difference.property_identity or "", + _SUBJECT_ORDER[difference.subject], + difference.difference_type.value, + ) diff --git a/semapact/schema/identifiers.py b/semapact/schema/identifiers.py new file mode 100644 index 00000000..4c2d77f7 --- /dev/null +++ b/semapact/schema/identifiers.py @@ -0,0 +1,18 @@ +"""Shared executable identifier validation helpers.""" + +from __future__ import annotations + +import re + +from semapact.exceptions import ValidationError + + +_SIMPLE_SQL_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_$]*$") + + +def validate_simple_sql_identifier(value: str, role: str) -> None: + """Validate the conservative SQL identifier subset used by safe renderers.""" + if not _SIMPLE_SQL_IDENTIFIER_RE.fullmatch(value): + raise ValidationError( + f"Unsupported {role} identifier for executable SQL: '{value}'" + ) diff --git a/semapact/schema/mapping.py b/semapact/schema/mapping.py new file mode 100644 index 00000000..edd9060b --- /dev/null +++ b/semapact/schema/mapping.py @@ -0,0 +1,402 @@ +"""Provider-neutral schema mapping contracts and common implementations. + +The shared layer owns generic mapping mechanics. Platform modules only +instantiate a mapper with target compiler/dialect parameters or add genuinely +platform-specific validation/capability rules. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import Mapping, Sequence +from typing import Callable + +import sqlglot +from datacontract.export.sql_exporter import to_sql_ddl +from open_data_contract_standard.model import ( + OpenDataContractStandard, + SchemaObject, + SchemaProperty, +) +from sqlglot import exp + +from semapact.exceptions import ValidationError +from semapact.observation.models import ObservedAsset +from semapact.schema.comparison import SchemaAssetState, SchemaPropertyState + + +_UNRESOLVED_SQL_TYPES = { + exp.DataType.Type.UNKNOWN, + exp.DataType.Type.USERDEFINED, + exp.DataType.Type.NULL, +} + + +class SchemaMapper(ABC): + """Map desired and observed schema state into one comparable model.""" + + key: str + + @abstractmethod + def map_desired_asset( + self, + schema: SchemaObject, + *, + asset_identity: str, + ) -> SchemaAssetState: + """Map desired source state into normalized comparable state.""" + raise NotImplementedError + + @abstractmethod + def map_observed_asset( + self, + observed: ObservedAsset, + *, + asset_identity: str, + property_bindings: Mapping[str, str] | None = None, + ) -> SchemaAssetState: + """Map observed runtime state into normalized comparable state.""" + raise NotImplementedError + + +class SqlSchemaMapper(SchemaMapper): + """Common SQL target-schema mapper backed by datacontract-cli + sqlglot.""" + + def __init__( + self, + *, + key: str, + server_type: str, + dialect: str, + ) -> None: + self.key = _required_text(key, "schema mapper key") + self.server_type = _required_text(server_type, "SQL server type") + self.dialect = _required_text(dialect, "SQL dialect") + + def map_desired_asset( + self, + schema: SchemaObject, + *, + asset_identity: str, + ) -> SchemaAssetState: + """Compile ODCS with datacontract-cli and adapt the target DDL.""" + desired = schema.model_copy(update={"name": asset_identity}) + contract = OpenDataContractStandard.model_construct( + id="semapact:deployment-target", + version="0.0.0", + schema_=[desired], + servers=[], + ) + ddl = to_sql_ddl(contract, server_type=self.server_type) + return parse_sql_target_asset( + ddl, + asset_identity=asset_identity, + dialect=self.dialect, + ) + + def map_observed_asset( + self, + observed: ObservedAsset, + *, + asset_identity: str, + property_bindings: Mapping[str, str] | None = None, + ) -> SchemaAssetState: + return map_observed_schema_asset( + observed, + asset_identity=asset_identity, + property_bindings=property_bindings, + normalize_physical_type=lambda value: normalize_sql_type( + value, + dialect=self.dialect, + ), + ) + + +class PassThroughSchemaMapper(SchemaMapper): + """Provider-neutral mapper preserving ODCS/runtime physical type text.""" + + key = "generic" + + def map_desired_asset( + self, + schema: SchemaObject, + *, + asset_identity: str, + ) -> SchemaAssetState: + return map_odcs_schema_asset( + schema, + asset_identity=asset_identity, + use_physical_property_names=False, + ) + + def map_observed_asset( + self, + observed: ObservedAsset, + *, + asset_identity: str, + property_bindings: Mapping[str, str] | None = None, + ) -> SchemaAssetState: + return map_observed_schema_asset( + observed, + asset_identity=asset_identity, + property_bindings=property_bindings, + ) + + +def parse_sql_target_asset( + ddl: str, + *, + asset_identity: str, + dialect: str, +) -> SchemaAssetState: + """Parse one compiler-emitted CREATE TABLE into normalized schema state.""" + try: + statements = [ + statement + for statement in sqlglot.parse(ddl, read=dialect) + if statement is not None + ] + except Exception as exc: + raise ValidationError( + f"Target schema compiler emitted {dialect} DDL that could not be parsed" + ) from exc + + if ( + len(statements) != 1 + or not isinstance(statements[0], exp.Create) + or (statements[0].kind or "").upper() != "TABLE" + ): + raise ValidationError( + "Target schema compilation must emit exactly one CREATE TABLE " + "and no additional statements" + ) + + table_schema = statements[0].this + if not isinstance(table_schema, exp.Schema): + raise ValidationError( + "Target schema compiler emitted CREATE TABLE without a column schema" + ) + + properties: list[SchemaPropertyState] = [] + seen: set[str] = set() + + for column in ( + expression + for expression in table_schema.expressions + if isinstance(expression, exp.ColumnDef) + ): + name = column.name.strip() + if not name: + raise ValidationError("Target schema contains an empty column identity") + + key = name.casefold() + if key in seen: + raise ValidationError( + f"Duplicate physical column binding in target schema: '{name}'" + ) + seen.add(key) + + data_type = column.args.get("kind") + if ( + not isinstance(data_type, exp.DataType) + or data_type.this in _UNRESOLVED_SQL_TYPES + ): + raise ValidationError( + f"Unsupported target physicalType for column '{name}'" + ) + + constraints = column.args.get("constraints") or [] + nullable = not any( + isinstance(constraint, exp.ColumnConstraint) + and isinstance( + constraint.args.get("kind"), + exp.NotNullColumnConstraint, + ) + for constraint in constraints + ) + physical_type = data_type.sql(dialect=dialect) + quoted_name = exp.to_identifier(name, quoted=True).sql(dialect=dialect) + native_definition = f"{quoted_name} {physical_type}" + if not nullable: + native_definition += " NOT NULL" + + properties.append( + SchemaPropertyState( + identity=name, + physical_type=physical_type, + nullable=nullable, + native_definition=native_definition, + ) + ) + + if not properties: + raise ValidationError("Target schema compilation emitted no properties") + + return SchemaAssetState( + identity=asset_identity, + properties=tuple(properties), + ) + + +def normalize_sql_type( + value: str | None, + *, + dialect: str, +) -> str | None: + """Canonicalize one observed native type using a SQL dialect parser.""" + if value is None: + return None + + try: + parsed = sqlglot.parse_one( + value.strip(), + into=exp.DataType, + dialect=dialect, + ) + except Exception as exc: + raise ValidationError( + f"Unsupported {dialect} physicalType: '{value}'" + ) from exc + + if ( + not isinstance(parsed, exp.DataType) + or parsed.this in _UNRESOLVED_SQL_TYPES + ): + raise ValidationError(f"Unsupported {dialect} physicalType: '{value}'") + + return parsed.sql(dialect=dialect) + + +def map_odcs_schema_asset( + schema: SchemaObject, + *, + asset_identity: str, + use_physical_property_names: bool, +) -> SchemaAssetState: + """Direct ODCS projection for provider-neutral reconciliation callers.""" + properties: list[SchemaPropertyState] = [] + seen: set[str] = set() + + for prop in schema.properties or []: + identity = property_identity( + prop, + use_physical_name=use_physical_property_names, + ) + key = identity.casefold() + if key in seen: + raise ValidationError( + f"Duplicate normalized desired property identity: '{identity}'" + ) + seen.add(key) + + required = getattr(prop, "required", None) + properties.append( + SchemaPropertyState( + identity=identity, + physical_type=_optional_text(getattr(prop, "physicalType", None)), + nullable=(not required) if isinstance(required, bool) else None, + ) + ) + + return SchemaAssetState( + identity=asset_identity, + properties=tuple(properties), + ) + + +def map_observed_schema_asset( + observed: ObservedAsset, + *, + asset_identity: str, + property_bindings: Mapping[str, str] | None = None, + normalize_physical_type: Callable[[str | None], str | None] | None = None, +) -> SchemaAssetState: + """Project one observed runtime asset into normalized comparable state.""" + bindings = { + key.casefold(): value + for key, value in (property_bindings or {}).items() + } + properties: list[SchemaPropertyState] = [] + seen: set[str] = set() + + for prop in observed.properties: + observed_name = prop.identity.property.strip() + if not observed_name: + raise ValidationError("Observed property identity must not be empty") + + identity = bindings.get(observed_name.casefold(), observed_name) + key = identity.casefold() + if key in seen: + raise ValidationError( + f"Duplicate canonical observed property identity found: '{key}'" + ) + seen.add(key) + + physical_type = prop.physical_type + if normalize_physical_type is not None: + physical_type = normalize_physical_type(physical_type) + + properties.append( + SchemaPropertyState( + identity=identity, + physical_type=_optional_text(physical_type), + nullable=prop.nullable, + ) + ) + + return SchemaAssetState( + identity=asset_identity, + properties=tuple(properties), + ) + + +def build_physical_property_bindings( + governed_properties: Sequence[SchemaProperty], +) -> dict[str, str]: + """Map physical runtime property names to governed logical identities.""" + bindings: dict[str, str] = {} + for prop in governed_properties: + logical = property_identity(prop, use_physical_name=False) + physical = property_identity(prop, use_physical_name=True) + physical_key = physical.casefold() + + existing = bindings.get(physical_key) + if existing is not None and existing.casefold() != logical.casefold(): + raise ValidationError( + "Multiple governed properties cannot bind to one physical runtime " + f"property: '{physical}'" + ) + bindings[physical_key] = logical + + return bindings + + +def property_identity( + prop: SchemaProperty, + *, + use_physical_name: bool, +) -> str: + """Resolve one logical or physical property identity deterministically.""" + logical = _optional_text(getattr(prop, "name", None)) + if logical is None: + raise ValidationError("Schema property name is required for schema mapping") + + if not use_physical_name: + return logical + + physical = _optional_text(getattr(prop, "physicalName", None)) + return physical or logical + + +def _required_text(value: str, field: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError(f"{field} is required") + return cleaned + + +def _optional_text(value: object | None) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None diff --git a/tests/interfaces/test_deployment_cmd.py b/tests/interfaces/test_deployment_cmd.py index 0ed96f05..7074c7be 100644 --- a/tests/interfaces/test_deployment_cmd.py +++ b/tests/interfaces/test_deployment_cmd.py @@ -7,12 +7,18 @@ import pytest from semapact.contractops import AppliedContractRelease +from semapact.deployment import ( + DeploymentAdapter, + DeploymentTarget, + build_deployment_plan, +) from semapact.contractops.integrity import compute_applied_release_id from semapact.exceptions import ValidationError from semapact.interfaces import cli from semapact.interfaces.commands import deployment_cmd from semapact.interfaces.commands.deployment_cmd import DeploymentCommandResult from semapact.interfaces.outcomes import ProcessOutcome +from semapact.reconciliation import ReconciliationResult SOURCE_REFERENCE = "https://workspace.example" @@ -180,3 +186,70 @@ def test_main_preserves_verification_outcome_semantics( assert cli.main() == expected_exit assert capsys.readouterr().out.strip() == "verification" + + + +def test_verify_command_uses_unified_deployment_adapter( + tmp_path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + plan = build_deployment_plan( + _release(), + DeploymentTarget( + platform="databricks", + runtime_target="main.silver", + source_reference=SOURCE_REFERENCE, + ), + ) + plan_path = tmp_path / "plan.json" + plan_path.write_text(plan.model_dump_json(), encoding="utf-8") + + class _Adapter(DeploymentAdapter): + key = "databricks" + + def __init__(self) -> None: + self.verify_calls = 0 + + def validate(self, plan): + raise AssertionError("CLI should call the service entrypoint") + + def preview(self, plan): + raise AssertionError("preview should not be called") + + def execute(self, plan, preview, authorization): + raise AssertionError("execute should not be called") + + def verify(self, plan): + self.verify_calls += 1 + return ReconciliationResult( + contract_id=plan.contract_id, + contract_version=plan.selected_version, + observation_source_identifier=SOURCE_REFERENCE, + observation_fingerprint="obs-v2:sha256:test", + ) + + adapter = _Adapter() + + import semapact.platforms.runtime_registry as runtime_registry + + monkeypatch.setattr( + runtime_registry, + "create_deployment_adapter", + lambda platform: adapter, + ) + + args = cli._build_parser().parse_args( + [ + "deployment", + "verify", + "--plan", + str(plan_path), + "--output", + "json", + ] + ) + result = deployment_cmd.run_deployment_verify(args) + + assert result.outcome is ProcessOutcome.SUCCESS + assert adapter.verify_calls == 1 + assert json.loads(result.output)["status"] == "IN_SYNC" diff --git a/tests/test_contractops_golden_scenarios.py b/tests/test_contractops_golden_scenarios.py index 1527b947..b694cc2b 100644 --- a/tests/test_contractops_golden_scenarios.py +++ b/tests/test_contractops_golden_scenarios.py @@ -561,8 +561,8 @@ def test_execute_success_is_separate_from_runtime_convergence() -> None: provider = _RuntimeProvider(missing_state) adapter, client = _adapter(provider) - first_preview = adapter.preview(chain.deployment_plan, missing_state) - second_preview = adapter.preview(chain.deployment_plan, missing_state) + first_preview = adapter.preview(chain.deployment_plan) + second_preview = adapter.preview(chain.deployment_plan) assert first_preview == second_preview assert client.statement_execution.calls == [] @@ -601,7 +601,7 @@ def test_stale_preview_fails_before_native_mutation() -> None: missing_state = _observed_state(present=False) provider = _RuntimeProvider(missing_state) adapter, client = _adapter(provider) - preview = adapter.preview(chain.deployment_plan, missing_state) + preview = adapter.preview(chain.deployment_plan) provider.state = _observed_state(present=True) with pytest.raises(ValidationError, match="no longer equals|Runtime state changed"): diff --git a/tests/test_databricks_schema_evolution.py b/tests/test_databricks_schema_evolution.py new file mode 100644 index 00000000..f54bac53 --- /dev/null +++ b/tests/test_databricks_schema_evolution.py @@ -0,0 +1,296 @@ +from __future__ import annotations + +import pytest +from open_data_contract_standard.model import SchemaObject, SchemaProperty + +from semapact.deployment.models import NativeOperationKind +from semapact.deployment.schema_transitions import ( + SchemaTransition, + SchemaTransitionKind, +) +from semapact.exceptions import ValidationError +from semapact.observation.models import ( + ObservedAsset, + ObservedAssetIdentity, + ObservedProperty, + ObservedPropertyIdentity, +) +from semapact.platforms.databricks.transition_compiler import ( + DatabricksTransitionCompiler, +) +from semapact.platforms.databricks.transition_planner import ( + DatabricksSchemaTransitionPlanner, +) +from semapact.schema import ( + SchemaSnapshot, + SqlSchemaMapper, + compare_schema_snapshots, +) + + +_MAPPER = SqlSchemaMapper( + key="databricks", + server_type="databricks", + dialect="databricks", +) +_PLANNER = DatabricksSchemaTransitionPlanner() + + +def _property( + name: str, + physical_type: str, + *, + required: bool = False, +) -> SchemaProperty: + return SchemaProperty( + name=name, + physicalName=name, + logicalType="string", + physicalType=physical_type, + required=required, + ) + + +def _schema(*properties: SchemaProperty) -> SchemaObject: + return SchemaObject( + name="orders", + physicalName="orders", + physicalType="table", + properties=list(properties), + ) + + +def _observed( + *columns: tuple[str, str, bool], + asset_type: str = "MANAGED", +) -> ObservedAsset: + identity = ObservedAssetIdentity( + platform="databricks", + namespace=("main", "silver"), + asset="orders", + ) + return ObservedAsset( + identity=identity, + asset_type=asset_type, + properties=tuple( + ObservedProperty( + identity=ObservedPropertyIdentity( + asset=identity, + property=name, + ), + physical_type=physical_type, + nullable=nullable, + ) + for name, physical_type, nullable in columns + ), + ) + + +def _plan( + desired: SchemaObject, + observed: ObservedAsset | None, +) -> SchemaTransition: + mapped = _MAPPER.map_desired_asset( + desired, + asset_identity="orders", + ) + observed_assets = ( + () + if observed is None + else ( + _MAPPER.map_observed_asset( + observed, + asset_identity="orders", + ), + ) + ) + comparison = compare_schema_snapshots( + SchemaSnapshot(assets=(mapped,)), + SchemaSnapshot(assets=observed_assets), + ) + return _PLANNER.plan( + governed_asset="orders", + physical_name="orders", + desired_columns=mapped.properties, + comparison=comparison, + observed_asset=observed, + ) + + +def _compile(transition: SchemaTransition): + return DatabricksTransitionCompiler().compile( + runtime_target="main.silver", + transition=transition, + ) + + +def test_planner_and_compiler_create_missing_managed_delta_table() -> None: + transition = _plan( + _schema(_property("id", "BIGINT", required=True)), + None, + ) + + assert transition.kind is SchemaTransitionKind.CREATE_ASSET + assert transition.columns[0].identity == "id" + assert transition.columns[0].physical_type == "BIGINT" + assert transition.columns[0].nullable is False + + operation = _compile(transition) + assert operation.kind is NativeOperationKind.CREATE + assert operation.statement == ( + "CREATE TABLE `main`.`silver`.`orders` " + "(`id` BIGINT NOT NULL) USING DELTA" + ) + + +def test_planner_and_compiler_add_only_missing_nullable_columns() -> None: + transition = _plan( + _schema( + _property("id", "BIGINT", required=True), + _property("note", "STRING"), + ), + _observed(("id", "bigint", False)), + ) + + assert transition.kind is SchemaTransitionKind.ADD_PROPERTIES + assert [column.identity for column in transition.columns] == ["note"] + + operation = _compile(transition) + assert operation.kind is NativeOperationKind.ALTER + assert operation.statement == ( + "ALTER TABLE `main`.`silver`.`orders` " + "ADD COLUMNS (`note` STRING)" + ) + + +def test_planner_no_ops_when_governed_shape_is_satisfied() -> None: + transition = _plan( + _schema(_property("id", "BIGINT", required=True)), + _observed( + ("id", "bigint", False), + ("runtime_only", "string", True), + ), + ) + + assert transition.kind is SchemaTransitionKind.NO_OP + assert transition.columns == () + + operation = _compile(transition) + assert operation.kind is NativeOperationKind.NO_OP + assert operation.statement is None + + +def test_desired_type_normalization_reuses_datacontract_mapping() -> None: + transition = _plan( + _schema(_property("id", "integer", required=True)), + None, + ) + assert transition.columns[0].physical_type == "INT" + + +def test_desired_target_schema_comes_from_datacontract_physical_projection() -> None: + prop = SchemaProperty( + name="logical_id", + physicalName="physical_id", + logicalType="integer", + physicalType="integer", + required=True, + ) + + transition = _plan(_schema(prop), None) + + assert transition.columns[0].identity == "physical_id" + assert transition.columns[0].physical_type == "INT" + assert transition.columns[0].nullable is False + + +def test_transition_compiler_executes_only_governed_physical_shape() -> None: + prop = SchemaProperty( + name="id", + physicalName="id", + logicalType="integer", + physicalType="BIGINT", + required=True, + primaryKey=True, + description="business identifier", + ) + + transition = _plan(_schema(prop), None) + + assert transition.columns[0].native_definition == "`id` BIGINT NOT NULL" + operation = _compile(transition) + assert operation.statement == ( + "CREATE TABLE `main`.`silver`.`orders` " + "(`id` BIGINT NOT NULL) USING DELTA" + ) + assert "PRIMARY KEY" not in operation.statement + assert "COMMENT" not in operation.statement + + +@pytest.mark.parametrize( + ("desired", "observed", "message"), + [ + ( + _schema(_property("id", "STRING", required=True)), + _observed(("id", "bigint", False)), + "type mutation", + ), + ( + _schema(_property("id", "BIGINT", required=True)), + _observed(("id", "bigint", True)), + "nullability mutation", + ), + ( + _schema( + _property("id", "BIGINT", required=True), + _property("required_new", "STRING", required=True), + ), + _observed(("id", "bigint", False)), + "safe default", + ), + ], +) +def test_planner_fails_closed_on_unsafe_existing_mutation( + desired: SchemaObject, + observed: ObservedAsset, + message: str, +) -> None: + with pytest.raises(ValidationError, match=message): + _plan(desired, observed) + + +def test_planner_rejects_non_managed_asset_only_when_mutation_is_required() -> None: + with pytest.raises(ValidationError, match="MANAGED"): + _plan( + _schema( + _property("id", "BIGINT", required=True), + _property("note", "STRING"), + ), + _observed( + ("id", "bigint", False), + asset_type="EXTERNAL", + ), + ) + + +def test_planner_allows_no_op_assurance_for_non_managed_asset() -> None: + transition = _plan( + _schema(_property("id", "BIGINT", required=True)), + _observed( + ("id", "bigint", False), + asset_type="EXTERNAL", + ), + ) + + assert transition.kind is SchemaTransitionKind.NO_OP + + +def test_desired_schema_validation_rejects_unsafe_physical_type() -> None: + with pytest.raises( + ValidationError, + match="physicalType|could not be parsed|exactly one CREATE TABLE", + ): + _MAPPER.map_desired_asset( + _schema(_property("id", "STRING);DROP")), + asset_identity="orders", + ) diff --git a/tests/test_deployment_databricks.py b/tests/test_deployment_databricks.py index e72c0b07..8e0f5662 100644 --- a/tests/test_deployment_databricks.py +++ b/tests/test_deployment_databricks.py @@ -29,7 +29,10 @@ ObservedPropertyIdentity, ) from semapact.observation.providers import RuntimeAssetBinding -from semapact.platforms.databricks.deployment import DatabricksDeploymentAdapter +from semapact.platforms.databricks.deployment import ( + DatabricksDeploymentAdapter, + DatabricksStatementExecutor, +) CAPTURED_AT = datetime(2026, 9, 10, 5, 0, tzinfo=timezone.utc) SOURCE_REFERENCE = "workspace-a" @@ -200,7 +203,7 @@ def test_preview_create_alter_and_no_op() -> None: create_plan = _plan(_property("id", "BIGINT", required=True)) missing = _state(present=False) adapter, _, _ = _adapter(missing) - create_preview = adapter.preview(create_plan, missing) + create_preview = adapter.preview(create_plan) assert create_preview.operations[0].kind is NativeOperationKind.CREATE alter_plan = _plan( @@ -209,11 +212,11 @@ def test_preview_create_alter_and_no_op() -> None: ) current = _state(("id", "bigint", False)) adapter, _, _ = _adapter(current) - alter_preview = adapter.preview(alter_plan, current) + alter_preview = adapter.preview(alter_plan) assert alter_preview.operations[0].kind is NativeOperationKind.ALTER assert "ADD COLUMNS (`note` STRING)" in alter_preview.operations[0].statement - no_op = adapter.preview(create_plan, current) + no_op = adapter.preview(create_plan) assert no_op.operations == ( NativeOperation(kind=NativeOperationKind.NO_OP, governed_asset="orders"), ) @@ -223,7 +226,7 @@ def test_preview_never_drops_extra_runtime_columns() -> None: plan = _plan(_property("id", "BIGINT", required=True)) current = _state(("id", "bigint", False), ("extra", "string", True)) adapter, _, _ = _adapter(current) - assert adapter.preview(plan, current).operations[0].kind is NativeOperationKind.NO_OP + assert adapter.preview(plan).operations[0].kind is NativeOperationKind.NO_OP @pytest.mark.parametrize( @@ -252,19 +255,22 @@ def test_preview_never_drops_extra_runtime_columns() -> None: def test_unsafe_existing_mutations_fail_closed(plan, state, message) -> None: adapter, _, _ = _adapter(state) with pytest.raises(ValidationError, match=message): - adapter.preview(plan, state) + adapter.preview(plan) -def test_non_managed_asset_and_unsafe_type_fail_closed() -> None: - plan = _plan(_property("id", "BIGINT", required=True)) +def test_non_managed_mutation_and_unsafe_type_fail_closed() -> None: + plan = _plan( + _property("id", "BIGINT", required=True), + _property("note", "STRING"), + ) external = _state(("id", "bigint", False), asset_type="EXTERNAL") adapter, _, _ = _adapter(external) with pytest.raises(ValidationError, match="MANAGED"): - adapter.preview(plan, external) + adapter.preview(plan) malicious_type = "STRING);DROP" malicious = _plan(_property("id", malicious_type)) - with pytest.raises(ValidationError, match="physicalType"): + with pytest.raises(ValidationError, match="physicalType|exactly one CREATE TABLE"): adapter.validate(malicious) @@ -274,14 +280,14 @@ def test_preview_rejects_cross_source_runtime_evidence() -> None: adapter, _, _ = _adapter(other_workspace) with pytest.raises(ValidationError, match="source reference"): - adapter.preview(plan, other_workspace) + adapter.preview(plan) def test_execute_fails_closed_for_denied_stale_and_cross_source() -> None: plan = _plan(_property("id", "BIGINT", required=True)) before = _state(("id", "bigint", False)) adapter, provider, _ = _adapter(before) - preview = adapter.preview(plan, before) + preview = adapter.preview(plan) with pytest.raises(ContractOpsAuthorizationError, match="not allowed"): adapter.execute(plan, preview, _authorization(plan, False)) @@ -302,7 +308,7 @@ def test_execute_rejects_tampered_plan_and_forged_native_command() -> None: plan = _plan(_property("id", "BIGINT", required=True)) current = _state(("id", "bigint", False)) adapter, _, client = _adapter(current) - preview = adapter.preview(plan, current) + preview = adapter.preview(plan) tampered = plan.model_copy(update={"selected_version": "9.9.9"}) with pytest.raises(ValueError, match="DeploymentPlan deterministic identity"): @@ -338,7 +344,7 @@ def test_execute_runs_exact_preview_statement() -> None: ) current = _state(("id", "bigint", False)) adapter, _, client = _adapter(current) - preview = adapter.preview(plan, current) + preview = adapter.preview(plan) adapter.execute(plan, preview, _authorization(plan)) @@ -351,7 +357,7 @@ def test_no_op_execute_does_not_require_warehouse() -> None: plan = _plan(_property("id", "BIGINT", required=True)) current = _state(("id", "bigint", False)) adapter, _, client = _adapter(current, warehouse_id=None) - preview = adapter.preview(plan, current) + preview = adapter.preview(plan) assert preview.operations[0].kind is NativeOperationKind.NO_OP adapter.execute(plan, preview, _authorization(plan)) @@ -365,15 +371,105 @@ def test_mutation_execute_without_warehouse_fails_closed() -> None: ) current = _state(("id", "bigint", False)) adapter, _, client = _adapter(current, warehouse_id=None) - preview = adapter.preview(plan, current) + preview = adapter.preview(plan) with pytest.raises(ValidationError, match="warehouse_id"): adapter.execute(plan, preview, _authorization(plan)) assert client.statement_execution.calls == [] +def test_verify_can_assure_non_managed_runtime_without_claiming_mutation_support() -> None: + plan = _plan(_property("id", "BIGINT", required=True)) + external = _state( + ("id", "BIGINT", False), + asset_type="EXTERNAL", + ) + adapter, _, _ = _adapter(external) + + result = adapter.verify(plan) + + assert result.differences == () + assert result.unverified_paths == () + + +def test_verify_uses_same_databricks_target_mapping_as_preview() -> None: + plan = _plan(_property("id", "integer", required=True)) + current = _state(("id", "INT", False)) + adapter, _, _ = _adapter(current) + + result = adapter.verify(plan) + + assert result.differences == () + assert result.unverified_paths == () + + def test_runtime_source_participates_in_plan_identity() -> None: plan_a = _plan(_property("id", "BIGINT", required=True), source_reference="workspace-a") plan_b = _plan(_property("id", "BIGINT", required=True), source_reference="workspace-b") assert plan_a.deployment_plan_id != plan_b.deployment_plan_id + + + +class _PollingStatements: + def __init__(self, terminal_state: str = "SUCCEEDED") -> None: + self.terminal_state = terminal_state + self.get_calls = 0 + + def execute_statement(self, *, statement, warehouse_id, wait_timeout): + return SimpleNamespace( + statement_id="s-2", + status=SimpleNamespace(state="PENDING", error=None), + ) + + def get_statement(self, statement_id): + self.get_calls += 1 + state = "RUNNING" if self.get_calls == 1 else self.terminal_state + return SimpleNamespace( + statement_id=statement_id, + status=SimpleNamespace( + state=state, + error="boom" if state == "FAILED" else None, + ), + ) + + +def test_statement_executor_polls_until_success() -> None: + client = _Client() + client.statement_execution = _PollingStatements() + executor = DatabricksStatementExecutor( + client=client, + warehouse_id="warehouse-1", + poll_interval_seconds=0, + max_poll_attempts=3, + ) + + executor.execute( + NativeOperation( + kind=NativeOperationKind.ALTER, + governed_asset="orders", + statement="ALTER TABLE x ADD COLUMNS (y STRING)", + ) + ) + + assert client.statement_execution.get_calls == 2 + + +def test_statement_executor_surfaces_failed_terminal_state() -> None: + client = _Client() + client.statement_execution = _PollingStatements(terminal_state="FAILED") + executor = DatabricksStatementExecutor( + client=client, + warehouse_id="warehouse-1", + poll_interval_seconds=0, + max_poll_attempts=3, + ) + + with pytest.raises(RuntimeError, match="FAILED"): + executor.execute( + NativeOperation( + kind=NativeOperationKind.ALTER, + governed_asset="orders", + statement="ALTER TABLE x ADD COLUMNS (y STRING)", + ) + ) diff --git a/tests/test_deployment_extensibility.py b/tests/test_deployment_extensibility.py new file mode 100644 index 00000000..86688fcd --- /dev/null +++ b/tests/test_deployment_extensibility.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from semapact.deployment import ( + AdditiveSchemaTransitionPlanner, + DeploymentAdapter, + DeploymentOrchestrator, + NativeOperationExecutor, + SchemaTransitionPlanner, + TransitionCompiler, +) +from semapact.platforms.databricks.deployment import ( + DatabricksDeploymentAdapter, + DatabricksStatementExecutor, +) +from semapact.platforms.databricks.transition_compiler import ( + DatabricksTransitionCompiler, +) +from semapact.platforms.databricks.transition_planner import ( + DatabricksSchemaTransitionPlanner, +) +from semapact.schema import SchemaMapper, SqlSchemaMapper + + +def test_shared_deployment_contracts_own_behavior_seams() -> None: + assert issubclass(SqlSchemaMapper, SchemaMapper) + assert issubclass(AdditiveSchemaTransitionPlanner, SchemaTransitionPlanner) + assert issubclass(DatabricksSchemaTransitionPlanner, SchemaTransitionPlanner) + assert issubclass(DatabricksTransitionCompiler, TransitionCompiler) + assert issubclass(DatabricksStatementExecutor, NativeOperationExecutor) + + +def test_databricks_adapter_is_only_generic_orchestrator_wiring() -> None: + assert issubclass(DeploymentOrchestrator, DeploymentAdapter) + assert issubclass(DatabricksDeploymentAdapter, DeploymentOrchestrator) + assert "preview" not in DatabricksDeploymentAdapter.__dict__ + assert "verify" not in DatabricksDeploymentAdapter.__dict__ + assert "execute" not in DatabricksDeploymentAdapter.__dict__ diff --git a/tests/test_deployment_orchestrator.py b/tests/test_deployment_orchestrator.py new file mode 100644 index 00000000..20b5de40 --- /dev/null +++ b/tests/test_deployment_orchestrator.py @@ -0,0 +1,264 @@ +from __future__ import annotations + +from datetime import datetime, timezone + +import pytest + +from open_data_contract_standard.model import SchemaObject, SchemaProperty + +from semapact.deployment.compilers import TransitionCompiler +from semapact.deployment.models import ( + DeploymentAction, + DeploymentActionKind, + DeploymentAuthorization, + DeploymentPlan, + DeploymentTarget, + NativeOperation, + NativeOperationKind, + compute_deployment_authorization_id, + compute_deployment_plan_id, +) +from semapact.deployment.orchestrator import DeploymentOrchestrator +from semapact.deployment.providers import NativeOperationExecutor +from semapact.deployment.schema_transitions import ( + AdditiveSchemaTransitionPlanner, + SchemaTransition, + SchemaTransitionKind, + SchemaTransitionPlanner, +) +from semapact.observation.fingerprint import with_observed_state_fingerprint +from semapact.observation.models import ( + ObservedAsset, + ObservedAssetIdentity, + ObservedPlatformState, +) +from semapact.observation.providers import RuntimeAssetBinding +from semapact.reconciliation import RuntimeDriftStatus, classify_reconciliation_status +from semapact.schema import PassThroughSchemaMapper + + +class _RuntimeProvider: + key = "fake" + + def __init__(self, state: ObservedPlatformState) -> None: + self.state = state + self.observe_calls = 0 + + def resolve_bindings(self, *, runtime_target, assets): + assert runtime_target == "main" + assert tuple(asset.physical_name for asset in assets) == ("orders",) + return ( + RuntimeAssetBinding( + governed_asset="orders", + observed_asset=ObservedAssetIdentity( + platform="fake", + namespace=("main",), + asset="orders", + ), + ), + ) + + def observe(self, *, bindings): + assert len(tuple(bindings)) == 1 + self.observe_calls += 1 + return self.state + + +class _AlwaysNoOpPlanner(SchemaTransitionPlanner): + key = "always-no-op" + + def plan( + self, + *, + governed_asset, + physical_name, + desired_columns, + comparison, + observed_asset=None, + ): + del observed_asset + return SchemaTransition( + kind=SchemaTransitionKind.NO_OP, + governed_asset=governed_asset, + physical_name=physical_name, + ) + + +class _Compiler(TransitionCompiler): + key = "fake" + + def compile( + self, + *, + runtime_target: str, + transition: SchemaTransition, + ) -> NativeOperation: + assert runtime_target == "main" + if transition.kind is SchemaTransitionKind.NO_OP: + return NativeOperation( + kind=NativeOperationKind.NO_OP, + governed_asset=transition.governed_asset, + ) + return NativeOperation( + kind=( + NativeOperationKind.CREATE + if transition.kind is SchemaTransitionKind.CREATE_ASSET + else NativeOperationKind.ALTER + ), + governed_asset=transition.governed_asset, + statement=f"{transition.kind.value} {transition.physical_name}", + ) + + +class _Executor(NativeOperationExecutor): + key = "fake" + + def __init__(self) -> None: + self.operations: list[NativeOperation] = [] + + def execute(self, operation: NativeOperation) -> None: + self.operations.append(operation) + + +def _plan() -> DeploymentPlan: + schema = SchemaObject( + name="orders", + physicalName="orders", + physicalType="table", + properties=[ + SchemaProperty( + name="id", + logicalType="integer", + physicalType="BIGINT", + required=False, + ) + ], + ) + action = DeploymentAction( + kind=DeploymentActionKind.ENSURE_ASSET_STATE, + governed_asset="orders", + physical_name="orders", + desired_state_json=schema.model_dump_json(by_alias=True), + ) + target = DeploymentTarget( + platform="fake", + runtime_target="main", + source_reference="source-1", + ) + kwargs = dict( + applied_release_id="release-1", + contract_id="contract-1", + release_plan_id="release-plan-1", + released_revision_ref="revision-1", + selected_version="1.0.0", + target=target, + actions=(action,), + ) + return DeploymentPlan( + deployment_plan_id=compute_deployment_plan_id(**kwargs), + **kwargs, + ) + + +def _observation() -> ObservedPlatformState: + return with_observed_state_fingerprint( + ObservedPlatformState( + platform="fake", + source_identifier="source-1", + assets=(), + captured_at=datetime(2026, 9, 19, tzinfo=timezone.utc), + ) + ) + + +def test_generic_orchestrator_owns_observe_preview_freshness_and_execute() -> None: + plan = _plan() + runtime_provider = _RuntimeProvider(_observation()) + executor = _Executor() + orchestrator = DeploymentOrchestrator( + runtime_provider=runtime_provider, + schema_mapper=PassThroughSchemaMapper(), + transition_planner=AdditiveSchemaTransitionPlanner(), + transition_compiler=_Compiler(), + executor=executor, + ) + + preview = orchestrator.preview(plan) + + assert runtime_provider.observe_calls == 1 + assert len(preview.operations) == 1 + assert preview.operations[0].kind is NativeOperationKind.CREATE + assert preview.operations[0].statement == "CREATE_ASSET orders" + + authorization_id = compute_deployment_authorization_id( + contract_ops_authorization_id="contract-auth-1", + deployment_plan_id=plan.deployment_plan_id, + applied_release_id=plan.applied_release_id, + allowed=True, + ) + authorization = DeploymentAuthorization( + deployment_authorization_id=authorization_id, + contract_ops_authorization_id="contract-auth-1", + deployment_plan_id=plan.deployment_plan_id, + applied_release_id=plan.applied_release_id, + allowed=True, + ) + + orchestrator.execute(plan, preview, authorization) + + assert runtime_provider.observe_calls == 2 + assert executor.operations == [preview.operations[0]] + + +def test_generic_orchestrator_delegates_transition_policy_to_platform() -> None: + plan = _plan() + runtime_provider = _RuntimeProvider(_observation()) + orchestrator = DeploymentOrchestrator( + runtime_provider=runtime_provider, + schema_mapper=PassThroughSchemaMapper(), + transition_planner=_AlwaysNoOpPlanner(), + transition_compiler=_Compiler(), + executor=_Executor(), + ) + + preview = orchestrator.preview(plan) + + assert preview.operations == ( + NativeOperation( + kind=NativeOperationKind.NO_OP, + governed_asset="orders", + ), + ) + + +def test_generic_orchestrator_owns_verification_entrypoint() -> None: + plan = _plan() + runtime_provider = _RuntimeProvider(_observation()) + orchestrator = DeploymentOrchestrator( + runtime_provider=runtime_provider, + schema_mapper=PassThroughSchemaMapper(), + transition_planner=AdditiveSchemaTransitionPlanner(), + transition_compiler=_Compiler(), + executor=_Executor(), + ) + + result = orchestrator.verify(plan) + + assert runtime_provider.observe_calls == 1 + assert classify_reconciliation_status(result) is RuntimeDriftStatus.DRIFT + assert result.differences[0].asset_identity == "orders" + + +def test_generic_orchestrator_rejects_component_key_mismatch() -> None: + runtime_provider = _RuntimeProvider(_observation()) + executor = _Executor() + executor.key = "other" + + with pytest.raises(ValueError, match="executor keys must match"): + DeploymentOrchestrator( + runtime_provider=runtime_provider, + schema_mapper=PassThroughSchemaMapper(), + transition_planner=AdditiveSchemaTransitionPlanner(), + transition_compiler=_Compiler(), + executor=executor, + ) diff --git a/tests/test_deployment_service.py b/tests/test_deployment_service.py index 7b904a97..6cb90a3b 100644 --- a/tests/test_deployment_service.py +++ b/tests/test_deployment_service.py @@ -7,6 +7,7 @@ from semapact.deployment import ( DeploymentAction, + DeploymentAdapter, DeploymentActionKind, DeploymentAuthorization, DeploymentPlan, @@ -21,70 +22,40 @@ compute_deployment_preview_id, ) from semapact.exceptions import ValidationError -from semapact.observation import ( - ObservedAsset, - ObservedAssetIdentity, - ObservedPlatformState, - ObservedProperty, - ObservedPropertyIdentity, - RuntimeAssetBinding, - with_observed_state_fingerprint, -) -from semapact.reconciliation import RuntimeDriftStatus, classify_reconciliation_status +from semapact.observation import ObservedPlatformState, with_observed_state_fingerprint +from semapact.reconciliation import ReconciliationResult from semapact.services.deployment_service import DeploymentService CAPTURED_AT = datetime(2026, 9, 10, 10, 0, tzinfo=timezone.utc) SOURCE_REFERENCE = "https://adb.example" -class FakeRuntimeProvider: +class FakeDeploymentAdapter(DeploymentAdapter): key = "databricks" - def __init__(self, observation: ObservedPlatformState) -> None: - self.observation = observation - self.resolve_calls = 0 - self.observe_calls = 0 - self.bindings: tuple[RuntimeAssetBinding, ...] = () - - def resolve_bindings(self, *, runtime_target: str, assets): - assert runtime_target == "main.silver" - self.resolve_calls += 1 - self.bindings = tuple( - RuntimeAssetBinding( - governed_asset=asset.governed_asset, - observed_asset=ObservedAssetIdentity( - platform="databricks", - namespace=("main", "silver"), - asset=asset.physical_name, - ), - ) - for asset in assets - ) - return self.bindings - - def observe(self, *, bindings): - assert tuple(bindings) == self.bindings - self.observe_calls += 1 - return self.observation - - -class FakeDeploymentAdapter: - key = "databricks" - - def __init__(self, preview: DeploymentPreview) -> None: + def __init__( + self, + preview: DeploymentPreview, + verification: ReconciliationResult, + ) -> None: self.preview_result = preview + self.verification_result = verification self.preview_calls = 0 + self.verify_calls = 0 self.execute_calls = 0 self.executed = None def validate(self, plan: DeploymentPlan) -> None: pass - def preview(self, plan: DeploymentPlan, observed_state: ObservedPlatformState): + def preview(self, plan: DeploymentPlan) -> DeploymentPreview: self.preview_calls += 1 - assert observed_state.fingerprint == self.preview_result.observation_fingerprint return self.preview_result + def verify(self, plan: DeploymentPlan) -> ReconciliationResult: + self.verify_calls += 1 + return self.verification_result + def execute(self, plan, preview, authorization) -> None: self.execute_calls += 1 self.executed = (plan, preview, authorization) @@ -136,39 +107,20 @@ def _plan() -> DeploymentPlan: ) -def _observation(*, source: str = SOURCE_REFERENCE) -> ObservedPlatformState: - asset_identity = ObservedAssetIdentity( - platform="databricks", - namespace=("main", "silver"), - asset="orders_v2", - ) +def _observation() -> ObservedPlatformState: return with_observed_state_fingerprint( ObservedPlatformState( platform="databricks", - source_identifier=source, + source_identifier=SOURCE_REFERENCE, captured_at=CAPTURED_AT, - assets=( - ObservedAsset( - identity=asset_identity, - asset_type="MANAGED", - properties=( - ObservedProperty( - identity=ObservedPropertyIdentity( - asset=asset_identity, - property="order_pk", - ), - physical_type="BIGINT", - nullable=False, - ), - ), - ), - ), + assets=(), fingerprint=None, ) ) -def _preview(plan: DeploymentPlan, observation: ObservedPlatformState) -> DeploymentPreview: +def _preview(plan: DeploymentPlan) -> DeploymentPreview: + observation = _observation() operation = NativeOperation( kind=NativeOperationKind.NO_OP, governed_asset="orders", @@ -193,6 +145,24 @@ def _preview(plan: DeploymentPlan, observation: ObservedPlatformState) -> Deploy ) +def _verification(plan: DeploymentPlan) -> ReconciliationResult: + observation = _observation() + assert observation.fingerprint is not None + return ReconciliationResult( + contract_id=plan.contract_id, + contract_version=plan.selected_version, + observation_source_identifier=observation.source_identifier, + observation_fingerprint=observation.fingerprint, + ) + + +def _adapter(plan: DeploymentPlan) -> FakeDeploymentAdapter: + return FakeDeploymentAdapter( + _preview(plan), + _verification(plan), + ) + + def _authorization(plan: DeploymentPlan) -> DeploymentAuthorization: authorization_id = compute_deployment_authorization_id( contract_ops_authorization_id="contractops-auth-1", @@ -209,48 +179,22 @@ def _authorization(plan: DeploymentPlan) -> DeploymentAuthorization: ) -def test_preview_orchestrates_observation_without_execution() -> None: +def test_preview_delegates_to_unified_adapter_entrypoint() -> None: plan = _plan() - observation = _observation() - provider = FakeRuntimeProvider(observation) - adapter = FakeDeploymentAdapter(_preview(plan, observation)) + adapter = _adapter(plan) - result = DeploymentService().preview( - plan, - runtime_provider=provider, - adapter=adapter, - ) + result = DeploymentService().preview(plan, adapter=adapter) assert result == adapter.preview_result - assert provider.resolve_calls == 1 - assert provider.observe_calls == 1 assert adapter.preview_calls == 1 assert adapter.execute_calls == 0 -def test_preview_rejects_runtime_source_mismatch() -> None: - plan = _plan() - observation = _observation(source="https://other-workspace.example") - provider = FakeRuntimeProvider(observation) - adapter = FakeDeploymentAdapter(_preview(plan, observation)) - - with pytest.raises(ValidationError, match="source reference"): - DeploymentService().preview( - plan, - runtime_provider=provider, - adapter=adapter, - ) - - assert provider.observe_calls == 1 - assert adapter.preview_calls == 0 - - def test_execute_delegates_exact_canonical_artifacts() -> None: plan = _plan() - observation = _observation() - preview = _preview(plan, observation) + preview = _preview(plan) authorization = _authorization(plan) - adapter = FakeDeploymentAdapter(preview) + adapter = _adapter(plan) DeploymentService().execute( plan, @@ -263,31 +207,35 @@ def test_execute_delegates_exact_canonical_artifacts() -> None: assert adapter.executed == (plan, preview, authorization) -def test_verify_reuses_existing_m1_reconciliation() -> None: +def test_verify_delegates_to_same_adapter_entrypoint() -> None: plan = _plan() - provider = FakeRuntimeProvider(_observation()) + adapter = _adapter(plan) - result = DeploymentService().verify(plan, runtime_provider=provider) + result = DeploymentService().verify(plan, adapter=adapter) - assert classify_reconciliation_status(result) is RuntimeDriftStatus.IN_SYNC - assert result.contract_id == plan.contract_id - assert result.contract_version == plan.selected_version + assert result == adapter.verification_result + assert adapter.verify_calls == 1 -def test_preview_provider_mismatch_fails_before_observation() -> None: +@pytest.mark.parametrize("operation", ["preview", "verify", "execute"]) +def test_service_rejects_adapter_platform_mismatch(operation: str) -> None: plan = _plan() - observation = _observation() - provider = FakeRuntimeProvider(observation) - provider.key = "snowflake" - adapter = FakeDeploymentAdapter(_preview(plan, observation)) - - with pytest.raises(ValidationError, match="Runtime provider does not match"): - DeploymentService().preview( - plan, - runtime_provider=provider, - adapter=adapter, - ) + adapter = _adapter(plan) + adapter.key = "snowflake" + + with pytest.raises(ValidationError, match="Deployment adapter does not match"): + if operation == "preview": + DeploymentService().preview(plan, adapter=adapter) + elif operation == "verify": + DeploymentService().verify(plan, adapter=adapter) + else: + DeploymentService().execute( + plan, + _preview(plan), + _authorization(plan), + adapter=adapter, + ) - assert provider.resolve_calls == 0 - assert provider.observe_calls == 0 assert adapter.preview_calls == 0 + assert adapter.verify_calls == 0 + assert adapter.execute_calls == 0 diff --git a/tests/test_reconciliation.py b/tests/test_reconciliation.py index dfc88d00..a9b76e56 100644 --- a/tests/test_reconciliation.py +++ b/tests/test_reconciliation.py @@ -290,6 +290,42 @@ def test_duplicate_canonical_observed_property_identity_fails_closed() -> None: reconcile_governed_contract(contract, observation) +def test_default_reconciliation_does_not_rebind_logical_names_as_physical_names() -> None: + contract = _contract( + SchemaObject( + name="orders", + properties=[ + SchemaProperty( + name="logical_a", + physicalName="logical_b", + type="integer", + physicalType="BIGINT", + required=True, + ), + SchemaProperty( + name="logical_b", + physicalName="physical_b", + type="string", + physicalType="STRING", + required=False, + ), + ], + ) + ) + observation = _observation( + _asset( + "orders", + ("logical_b", "BIGINT", False), + ("physical_b", "STRING", True), + ) + ) + + result = reconcile_governed_contract(contract, observation) + + assert result.differences == () + assert result.unverified_paths == () + + def test_difference_order_and_serialization_are_deterministic() -> None: orders = SchemaObject( name="orders", diff --git a/tests/test_runtime_location_resolution.py b/tests/test_runtime_location_resolution.py index 79e69d3e..58f0887c 100644 --- a/tests/test_runtime_location_resolution.py +++ b/tests/test_runtime_location_resolution.py @@ -1,9 +1,12 @@ from __future__ import annotations +from types import SimpleNamespace + import pytest from open_data_contract_standard.model import OpenDataContractStandard, Server from semapact.exceptions import ValidationError +from semapact.platforms import runtime_registry from semapact.platforms.runtime_registry import resolve_runtime_location @@ -99,3 +102,39 @@ def test_contract_without_servers_requires_both_fallback_values( def test_server_selector_is_invalid_when_contract_has_no_servers() -> None: with pytest.raises(ValidationError, match="contract defines no servers"): resolve_runtime_location(_contract(), server_name="production") + + + +class _FakeRuntimeProvider: + key = "databricks" + + def resolve_bindings(self, *, runtime_target, assets): + return () + + def observe(self, *, bindings): + raise AssertionError("observation is not needed for composition test") + + +def test_runtime_registry_is_single_read_write_composition_root( + monkeypatch: pytest.MonkeyPatch, +) -> None: + provider = _FakeRuntimeProvider() + client = SimpleNamespace() + calls: list[object | None] = [] + + def _compose(*, contract_server=None): + calls.append(contract_server) + return client, provider + + monkeypatch.setattr( + runtime_registry, + "_create_databricks_client_and_provider", + _compose, + ) + + registry = runtime_registry.create_runtime_provider_registry("databricks") + adapter = runtime_registry.create_deployment_adapter("databricks") + + assert registry.get("databricks") is provider + assert adapter.key == "databricks" + assert calls == [None, None] diff --git a/tests/test_schema_comparison.py b/tests/test_schema_comparison.py new file mode 100644 index 00000000..56b592d3 --- /dev/null +++ b/tests/test_schema_comparison.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +from semapact.schema import ( + SchemaAssetState, + SchemaDifferenceType, + SchemaPropertyState, + SchemaSnapshot, + SchemaSubject, + compare_schema_snapshots, +) + + +def _property( + name: str, + physical_type: str | None, + nullable: bool | None, +) -> SchemaPropertyState: + return SchemaPropertyState( + identity=name, + physical_type=physical_type, + nullable=nullable, + ) + + +def test_shared_schema_comparator_reports_structural_and_value_facts() -> None: + expected = SchemaSnapshot( + assets=( + SchemaAssetState( + identity="orders", + properties=( + _property("id", "BIGINT", False), + _property("amount", "DECIMAL(18,2)", True), + ), + ), + SchemaAssetState(identity="customers"), + ) + ) + observed = SchemaSnapshot( + assets=( + SchemaAssetState( + identity="orders", + properties=( + _property("id", "STRING", True), + _property("runtime_only", "STRING", True), + ), + ), + SchemaAssetState(identity="payments"), + ) + ) + + result = compare_schema_snapshots(expected, observed) + + assert [ + (item.difference_type, item.subject, item.path) + for item in result.differences + ] == [ + ( + SchemaDifferenceType.MISSING, + SchemaSubject.ASSET, + "schema[customers]", + ), + ( + SchemaDifferenceType.MISSING, + SchemaSubject.PROPERTY, + "schema[orders].properties[amount]", + ), + ( + SchemaDifferenceType.MISMATCH, + SchemaSubject.PHYSICAL_TYPE, + "schema[orders].properties[id].physicalType", + ), + ( + SchemaDifferenceType.MISMATCH, + SchemaSubject.NULLABILITY, + "schema[orders].properties[id].nullability", + ), + ( + SchemaDifferenceType.UNEXPECTED, + SchemaSubject.PROPERTY, + "schema[orders].properties[runtime_only]", + ), + ( + SchemaDifferenceType.UNEXPECTED, + SchemaSubject.ASSET, + "schema[payments]", + ), + ] + + +def test_shared_schema_comparator_reports_evidence_gaps_without_policy() -> None: + expected = SchemaSnapshot( + assets=( + SchemaAssetState( + identity="orders", + properties=(_property("id", "BIGINT", False),), + ), + ) + ) + observed = SchemaSnapshot( + assets=( + SchemaAssetState( + identity="orders", + properties=(_property("id", None, None),), + ), + ) + ) + + result = compare_schema_snapshots(expected, observed) + + assert result.differences == () + assert result.unverified_paths == ( + "schema[orders].properties[id].nullability", + "schema[orders].properties[id].physicalType", + ) + + +def test_shared_schema_comparator_is_case_insensitive_for_identity_and_type() -> None: + expected = SchemaSnapshot( + assets=( + SchemaAssetState( + identity="Orders", + properties=(_property("ID", "BIGINT", False),), + ), + ) + ) + observed = SchemaSnapshot( + assets=( + SchemaAssetState( + identity="orders", + properties=(_property("id", "bigint", False),), + ), + ) + ) + + result = compare_schema_snapshots(expected, observed) + + assert result.differences == () + assert result.unverified_paths == () diff --git a/tests/test_schema_mapping.py b/tests/test_schema_mapping.py new file mode 100644 index 00000000..e3e205cb --- /dev/null +++ b/tests/test_schema_mapping.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +from open_data_contract_standard.model import SchemaObject, SchemaProperty + +from semapact.observation.models import ( + ObservedAsset, + ObservedAssetIdentity, + ObservedProperty, + ObservedPropertyIdentity, +) +from semapact.schema import ( + PassThroughSchemaMapper, + SchemaMapper, + build_physical_property_bindings, + parse_sql_target_asset, +) + + +def _schema() -> SchemaObject: + return SchemaObject( + name="orders", + properties=[ + SchemaProperty( + name="customer_id", + physicalName="customerId", + physicalType="BIGINT", + required=True, + ), + SchemaProperty( + name="note", + physicalType="STRING", + required=False, + ), + ], + ) + + +def _observed() -> ObservedAsset: + identity = ObservedAssetIdentity( + platform="example", + namespace=("main",), + asset="orders", + ) + return ObservedAsset( + identity=identity, + properties=( + ObservedProperty( + identity=ObservedPropertyIdentity( + asset=identity, + property="customerId", + ), + physical_type="bigint", + nullable=False, + ), + ObservedProperty( + identity=ObservedPropertyIdentity( + asset=identity, + property="note", + ), + physical_type="string", + nullable=True, + ), + ), + ) + + +def test_pass_through_mapper_projects_governed_logical_identity() -> None: + mapper = PassThroughSchemaMapper() + + desired = mapper.map_desired_asset( + _schema(), + asset_identity="orders", + ) + + assert [prop.identity for prop in desired.properties] == [ + "customer_id", + "note", + ] + assert desired.properties[0].physical_type == "BIGINT" + assert desired.properties[0].nullable is False + assert desired.properties[1].nullable is True + + +def test_pass_through_mapper_projects_runtime_physical_names_back_to_governed_identity() -> None: + schema = _schema() + mapper = PassThroughSchemaMapper() + + observed = mapper.map_observed_asset( + _observed(), + asset_identity="orders", + property_bindings=build_physical_property_bindings( + schema.properties or [] + ), + ) + + assert [prop.identity for prop in observed.properties] == [ + "customer_id", + "note", + ] + + +def test_schema_mapper_contract_operates_on_whole_assets() -> None: + mapper: SchemaMapper = PassThroughSchemaMapper() + + desired = mapper.map_desired_asset( + _schema(), + asset_identity="orders", + ) + observed = mapper.map_observed_asset( + _observed(), + asset_identity="orders", + ) + + assert desired.identity == "orders" + assert observed.identity == "orders" + + + +def test_sql_target_parser_keeps_only_top_level_nested_columns() -> None: + asset = parse_sql_target_asset( + ( + "CREATE TABLE orders (" + "payload STRUCT, " + "attrs MAP, " + "values ARRAY" + ")" + ), + asset_identity="orders", + dialect="databricks", + ) + + assert [prop.identity for prop in asset.properties] == [ + "payload", + "attrs", + "values", + ] + assert len(asset.properties) == 3 + assert asset.properties[0].native_definition.startswith("`payload` STRUCT") diff --git a/tests/test_schema_transitions.py b/tests/test_schema_transitions.py new file mode 100644 index 00000000..aea14701 --- /dev/null +++ b/tests/test_schema_transitions.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +import pytest + +from semapact.deployment.schema_transitions import ( + SchemaTransitionKind, + plan_additive_schema_transition, +) +from semapact.exceptions import ValidationError +from semapact.schema import ( + SchemaAssetState, + SchemaPropertyState, + SchemaSnapshot, + compare_schema_snapshots, +) + + +def _column(name: str, physical_type: str, *, nullable: bool) -> SchemaPropertyState: + return SchemaPropertyState( + identity=name, + physical_type=physical_type, + nullable=nullable, + ) + + +def _compare( + desired: tuple[SchemaPropertyState, ...], + observed: tuple[SchemaPropertyState, ...] | None, +): + expected = SchemaSnapshot( + assets=(SchemaAssetState(identity="orders", properties=desired),) + ) + actual = SchemaSnapshot( + assets=() + if observed is None + else (SchemaAssetState(identity="orders", properties=observed),) + ) + return compare_schema_snapshots(expected, actual) + + +def test_transition_planner_consumes_shared_missing_property_difference() -> None: + desired = ( + _column("id", "BIGINT", nullable=False), + _column("note", "STRING", nullable=True), + ) + + transition = plan_additive_schema_transition( + governed_asset="orders", + physical_name="orders", + desired_columns=desired, + comparison=_compare( + desired, + (_column("id", "BIGINT", nullable=False),), + ), + ) + + assert transition.kind is SchemaTransitionKind.ADD_PROPERTIES + assert transition.columns == (desired[1],) + + +def test_transition_planner_never_interprets_runtime_only_property_as_drop() -> None: + desired = (_column("id", "BIGINT", nullable=False),) + + transition = plan_additive_schema_transition( + governed_asset="orders", + physical_name="orders", + desired_columns=desired, + comparison=_compare( + desired, + ( + _column("id", "BIGINT", nullable=False), + _column("runtime_only", "STRING", nullable=True), + ), + ), + ) + + assert transition.kind is SchemaTransitionKind.NO_OP + + +def test_transition_planner_interprets_missing_asset_as_create() -> None: + desired = (_column("id", "BIGINT", nullable=False),) + + transition = plan_additive_schema_transition( + governed_asset="orders", + physical_name="orders", + desired_columns=desired, + comparison=_compare(desired, None), + ) + + assert transition.kind is SchemaTransitionKind.CREATE_ASSET + assert transition.columns == desired + + +def test_transition_planner_fails_closed_on_required_addition() -> None: + desired = ( + _column("id", "BIGINT", nullable=False), + _column("required_new", "STRING", nullable=False), + ) + with pytest.raises(ValidationError, match="safe default"): + plan_additive_schema_transition( + governed_asset="orders", + physical_name="orders", + desired_columns=desired, + comparison=_compare( + desired, + (_column("id", "BIGINT", nullable=False),), + ), + ) + + +def test_transition_planner_fails_closed_on_type_mismatch_fact() -> None: + desired = (_column("id", "BIGINT", nullable=False),) + with pytest.raises(ValidationError, match="type mutation"): + plan_additive_schema_transition( + governed_asset="orders", + physical_name="orders", + desired_columns=desired, + comparison=_compare( + desired, + (_column("id", "STRING", nullable=False),), + ), + ) From 6c2376426e24be2a661ea8b5540accdec44f4a99 Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Sun, 20 Sep 2026 07:32:24 +1000 Subject: [PATCH 30/35] test(databricks): prove contract-driven convergence end to end (#237) * test(databricks): add convergence golden workflows * test(databricks): use public ContractOps versioning API * ci(databricks): protect convergence golden workflow * test(databricks): lock numeric type mutation fail-closed cases --- .github/workflows/ci.yml | 1 + tests/test_databricks_convergence_e2e.py | 654 ++++++++++++++++++++++ tests/test_databricks_schema_evolution.py | 21 + 3 files changed, 676 insertions(+) create mode 100644 tests/test_databricks_convergence_e2e.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b57e3d55..7d9e575d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -174,6 +174,7 @@ jobs: tests/test_observation_databricks.py tests/test_databricks_schema_evolution.py tests/test_deployment_databricks.py + tests/test_databricks_convergence_e2e.py coverage: runs-on: ubuntu-latest diff --git a/tests/test_databricks_convergence_e2e.py b/tests/test_databricks_convergence_e2e.py new file mode 100644 index 00000000..29f80bf6 --- /dev/null +++ b/tests/test_databricks_convergence_e2e.py @@ -0,0 +1,654 @@ +from __future__ import annotations + +import argparse +import json +import re +from datetime import date, datetime, timedelta, timezone +from pathlib import Path +from types import SimpleNamespace + +import pytest +from open_data_contract_standard.model import ( + OpenDataContractStandard, + SchemaObject, + SchemaProperty, +) + +from semapact.application.services.deployment import DeploymentService +from semapact.change_context import ChangeContext +from semapact.contractops import ( + apply_contract_release, + authorize_contract_operation, + build_change_set_from_decision, + build_release_plan, + resolve_release_version, + VersionAuthorityConfig, +) +from semapact.deployment import ( + DeploymentPlan, + DeploymentPreview, + DeploymentTarget, + authorize_deployment, +) +from semapact.deployment.models import ( + NativeOperationKind, + compute_deployment_authorization_id, +) +from semapact.exceptions import ContractOpsAuthorizationError, ValidationError +from semapact.governance import DecisionResult, evaluate_governance_decision +from semapact.governance.gate import GovernanceOperation +from semapact.interfaces.commands import deployment_cmd +from semapact.observation.fingerprint import with_observed_state_fingerprint +from semapact.observation.models import ( + ObservedAsset, + ObservedAssetIdentity, + ObservedPlatformState, + ObservedProperty, + ObservedPropertyIdentity, +) +from semapact.observation.providers import RuntimeAssetBinding +from semapact.platforms.databricks.deployment import DatabricksDeploymentAdapter +from semapact.reconciliation import RuntimeDriftStatus, classify_reconciliation_status + + +_CONTEXT = ChangeContext(effective_date=date(2026, 9, 19)) +_SOURCE = "workspace:golden" +_RUNTIME_TARGET = "main.silver" +_CAPTURED_AT = datetime(2026, 9, 19, 10, 0, tzinfo=timezone.utc) + + +def _property( + name: str, + physical_type: str, + *, + required: bool = False, +) -> SchemaProperty: + logical_type = "integer" if physical_type.casefold() == "integer" else "string" + return SchemaProperty( + name=name, + physicalName=name, + logicalType=logical_type, + physicalType=physical_type, + required=required, + ) + + +def _contract( + *properties: SchemaProperty, + name: str, +) -> OpenDataContractStandard: + return OpenDataContractStandard( + apiVersion="v3.1.0", + kind="DataContract", + id="orders-product", + name=name, + version="1.0.0", + status="active", + schema=[ + SchemaObject( + name="orders", + physicalName="orders", + physicalType="table", + properties=list(properties), + ) + ], + ) + + +def _release_context(*properties: SchemaProperty): + base = _contract(*properties, name="orders-old") + candidate = _contract(*properties, name="orders-new") + decision = evaluate_governance_decision(base, candidate, context=_CONTEXT) + assert decision.decision is DecisionResult.ALLOW + + change_set = build_change_set_from_decision( + decision, + base_revision_ref="git:base", + candidate_revision_ref="git:candidate", + source="databricks-convergence-golden", + actor_reference="service:ci", + ) + release_plan = build_release_plan(change_set, decision) + version_resolution = resolve_release_version( + release_plan, + current_version="1.0.0", + config=VersionAuthorityConfig(), + ) + apply_authorization = authorize_contract_operation( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.APPLY, + ) + release = apply_contract_release( + candidate, + candidate_revision_ref="git:candidate", + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + authorization=apply_authorization, + ) + deploy_authorization = authorize_contract_operation( + decision, + change_set, + release_plan, + version_resolution, + GovernanceOperation.DEPLOY, + ) + return SimpleNamespace( + release=release, + deploy_authorization=deploy_authorization, + ) + + +def _target(*, source_reference: str = _SOURCE) -> DeploymentTarget: + return DeploymentTarget( + platform="databricks", + runtime_target=_RUNTIME_TARGET, + source_reference=source_reference, + server_name="production", + ) + + +class _StatefulWorkspace: + def __init__( + self, + *, + present: bool, + columns: tuple[tuple[str, str, bool], ...] = (), + asset_type: str = "MANAGED", + source_identifier: str = _SOURCE, + ) -> None: + self.present = present + self.columns = list(columns) + self.asset_type = asset_type + self.source_identifier = source_identifier + self.capture_count = 0 + self.statements: list[str] = [] + + def observe(self) -> ObservedPlatformState: + identity = ObservedAssetIdentity( + platform="databricks", + namespace=("main", "silver"), + asset="orders", + ) + assets = () + if self.present: + assets = ( + ObservedAsset( + identity=identity, + asset_type=self.asset_type, + properties=tuple( + ObservedProperty( + identity=ObservedPropertyIdentity( + asset=identity, + property=name, + ), + physical_type=physical_type, + nullable=nullable, + ) + for name, physical_type, nullable in self.columns + ), + ), + ) + + captured_at = _CAPTURED_AT + timedelta(seconds=self.capture_count) + self.capture_count += 1 + return with_observed_state_fingerprint( + ObservedPlatformState( + platform="databricks", + source_identifier=self.source_identifier, + assets=assets, + captured_at=captured_at, + ) + ) + + def apply(self, statement: str) -> None: + self.statements.append(statement) + + create_match = re.fullmatch( + r"CREATE TABLE `main`.`silver`.`orders` " + r"\((?P.*)\) USING DELTA", + statement, + ) + if create_match is not None: + self.present = True + self.asset_type = "MANAGED" + self.columns = list(_parse_columns(create_match.group("columns"))) + return + + alter_match = re.fullmatch( + r"ALTER TABLE `main`.`silver`.`orders` " + r"ADD COLUMNS \((?P.*)\)", + statement, + ) + if alter_match is not None: + if not self.present: + raise AssertionError("ALTER cannot mutate a missing mock asset") + self.columns.extend(_parse_columns(alter_match.group("columns"))) + return + + raise AssertionError(f"unexpected Databricks statement: {statement}") + + +def _parse_columns(value: str) -> tuple[tuple[str, str, bool], ...]: + # Golden scenarios intentionally use scalar types so a simple harness parser + # can model the runtime side effect without becoming another SQL compiler. + parsed: list[tuple[str, str, bool]] = [] + for raw in value.split(","): + item = raw.strip() + match = re.fullmatch( + r"`(?P[^`]+)`\s+(?P.+?)(?P\s+NOT NULL)?", + item, + ) + if match is None: + raise AssertionError(f"unexpected mock column definition: {item}") + physical_type = match.group("type").strip() + parsed.append( + ( + match.group("name"), + physical_type, + match.group("not_null") is None, + ) + ) + return tuple(parsed) + + +class _StatefulRuntimeProvider: + key = "databricks" + + def __init__(self, workspace: _StatefulWorkspace) -> None: + self.workspace = workspace + + def resolve_bindings(self, *, runtime_target, assets): + assert runtime_target == _RUNTIME_TARGET + return tuple( + RuntimeAssetBinding( + governed_asset=asset.governed_asset, + observed_asset=ObservedAssetIdentity( + platform="databricks", + namespace=("main", "silver"), + asset=asset.physical_name, + ), + ) + for asset in assets + ) + + def observe(self, *, bindings): + assert tuple(bindings) + return self.workspace.observe() + + +class _StatefulStatements: + def __init__(self, workspace: _StatefulWorkspace) -> None: + self.workspace = workspace + + def execute_statement(self, *, statement, warehouse_id, wait_timeout): + assert warehouse_id == "warehouse-golden" + assert wait_timeout == "10s" + self.workspace.apply(statement) + return SimpleNamespace( + statement_id="statement-golden", + status=SimpleNamespace(state="SUCCEEDED", error=None), + ) + + def get_statement(self, statement_id): + raise AssertionError(f"unexpected statement polling: {statement_id}") + + +class _Client: + def __init__(self, workspace: _StatefulWorkspace) -> None: + self.statement_execution = _StatefulStatements(workspace) + + +def _adapter(workspace: _StatefulWorkspace) -> DatabricksDeploymentAdapter: + return DatabricksDeploymentAdapter( + client=_Client(workspace), + runtime_provider=_StatefulRuntimeProvider(workspace), + warehouse_id="warehouse-golden", + poll_interval_seconds=0, + ) + + +def _workflow( + workspace: _StatefulWorkspace, + *properties: SchemaProperty, +): + release_context = _release_context(*properties) + service = DeploymentService() + plan = service.plan(release_context.release, _target()) + authorization = authorize_deployment( + plan, + release_context.release, + release_context.deploy_authorization, + ) + return SimpleNamespace( + service=service, + plan=plan, + authorization=authorization, + adapter=_adapter(workspace), + release_context=release_context, + ) + + +def _assert_in_sync(workflow) -> None: + result = workflow.service.verify( + workflow.plan, + adapter=workflow.adapter, + ) + assert classify_reconciliation_status(result) is RuntimeDriftStatus.IN_SYNC + assert result.differences == () + assert result.unverified_paths == () + + +def test_missing_table_create_execute_and_fresh_verify_converge() -> None: + workspace = _StatefulWorkspace(present=False) + workflow = _workflow( + workspace, + _property("id", "integer", required=True), + ) + + preview = workflow.service.preview( + workflow.plan, + adapter=workflow.adapter, + ) + assert preview.operations[0].kind is NativeOperationKind.CREATE + + workflow.service.execute( + workflow.plan, + preview, + workflow.authorization, + adapter=workflow.adapter, + ) + + assert workspace.columns == [("id", "INT", False)] + assert workspace.statements == [ + "CREATE TABLE `main`.`silver`.`orders` " + "(`id` INT NOT NULL) USING DELTA" + ] + _assert_in_sync(workflow) + + +def test_missing_nullable_column_alter_execute_and_fresh_verify_converge() -> None: + workspace = _StatefulWorkspace( + present=True, + columns=(("id", "INT", False),), + ) + workflow = _workflow( + workspace, + _property("id", "integer", required=True), + _property("note", "STRING"), + ) + + preview = workflow.service.preview( + workflow.plan, + adapter=workflow.adapter, + ) + assert preview.operations[0].kind is NativeOperationKind.ALTER + + workflow.service.execute( + workflow.plan, + preview, + workflow.authorization, + adapter=workflow.adapter, + ) + + assert workspace.columns == [ + ("id", "INT", False), + ("note", "STRING", True), + ] + assert workspace.statements == [ + "ALTER TABLE `main`.`silver`.`orders` " + "ADD COLUMNS (`note` STRING)" + ] + _assert_in_sync(workflow) + + +def test_compliant_table_no_op_execute_and_verify_converge() -> None: + workspace = _StatefulWorkspace( + present=True, + columns=(("id", "INT", False),), + ) + workflow = _workflow( + workspace, + _property("id", "integer", required=True), + ) + + preview = workflow.service.preview( + workflow.plan, + adapter=workflow.adapter, + ) + assert preview.operations[0].kind is NativeOperationKind.NO_OP + + workflow.service.execute( + workflow.plan, + preview, + workflow.authorization, + adapter=workflow.adapter, + ) + + assert workspace.statements == [] + _assert_in_sync(workflow) + + +def test_stale_runtime_between_preview_and_execute_fails_closed() -> None: + workspace = _StatefulWorkspace(present=False) + workflow = _workflow( + workspace, + _property("id", "integer", required=True), + ) + preview = workflow.service.preview( + workflow.plan, + adapter=workflow.adapter, + ) + + workspace.present = True + workspace.columns = [("id", "INT", False)] + + with pytest.raises(ValidationError, match="Runtime state changed"): + workflow.service.execute( + workflow.plan, + preview, + workflow.authorization, + adapter=workflow.adapter, + ) + + assert workspace.statements == [] + + +def test_external_asset_requiring_mutation_fails_closed() -> None: + workspace = _StatefulWorkspace( + present=True, + columns=(("id", "INT", False),), + asset_type="EXTERNAL", + ) + workflow = _workflow( + workspace, + _property("id", "integer", required=True), + _property("note", "STRING"), + ) + + with pytest.raises(ValidationError, match="MANAGED"): + workflow.service.preview( + workflow.plan, + adapter=workflow.adapter, + ) + + assert workspace.statements == [] + + +def test_wrong_workspace_source_fails_preview_and_execute_closed() -> None: + wrong_workspace = _StatefulWorkspace( + present=False, + source_identifier="workspace:other", + ) + wrong_workflow = _workflow( + wrong_workspace, + _property("id", "integer", required=True), + ) + + with pytest.raises(ValidationError, match="source reference"): + wrong_workflow.service.preview( + wrong_workflow.plan, + adapter=wrong_workflow.adapter, + ) + + workspace = _StatefulWorkspace(present=False) + workflow = _workflow( + workspace, + _property("id", "integer", required=True), + ) + preview = workflow.service.preview( + workflow.plan, + adapter=workflow.adapter, + ) + workspace.source_identifier = "workspace:other" + + with pytest.raises(ValidationError, match="Runtime source changed"): + workflow.service.execute( + workflow.plan, + preview, + workflow.authorization, + adapter=workflow.adapter, + ) + + assert workspace.statements == [] + + +def test_denied_deployment_authorization_fails_before_native_mutation() -> None: + workspace = _StatefulWorkspace(present=False) + workflow = _workflow( + workspace, + _property("id", "integer", required=True), + ) + preview = workflow.service.preview( + workflow.plan, + adapter=workflow.adapter, + ) + denied = workflow.authorization.model_copy( + update={ + "allowed": False, + "deployment_authorization_id": compute_deployment_authorization_id( + contract_ops_authorization_id=( + workflow.authorization.contract_ops_authorization_id + ), + deployment_plan_id=workflow.plan.deployment_plan_id, + applied_release_id=workflow.plan.applied_release_id, + allowed=False, + ), + } + ) + + with pytest.raises(ContractOpsAuthorizationError, match="not allowed"): + workflow.service.execute( + workflow.plan, + preview, + denied, + adapter=workflow.adapter, + ) + + assert workspace.statements == [] + + +def test_databricks_integer_target_alias_does_not_create_false_drift() -> None: + workspace = _StatefulWorkspace( + present=True, + columns=(("id", "INT", False),), + ) + workflow = _workflow( + workspace, + _property("id", "integer", required=True), + ) + + preview = workflow.service.preview( + workflow.plan, + adapter=workflow.adapter, + ) + assert preview.operations[0].kind is NativeOperationKind.NO_OP + + _assert_in_sync(workflow) + + +def test_cli_round_trip_matches_application_service_and_converges( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + release_context = _release_context( + _property("id", "integer", required=True), + ) + workspace = _StatefulWorkspace(present=False) + adapter = _adapter(workspace) + service = DeploymentService() + + expected_plan = service.plan(release_context.release, _target()) + expected_preview = service.preview(expected_plan, adapter=adapter) + authorization = authorize_deployment( + expected_plan, + release_context.release, + release_context.deploy_authorization, + ) + + release_path = tmp_path / "release.json" + plan_path = tmp_path / "plan.json" + preview_path = tmp_path / "preview.json" + authorization_path = tmp_path / "authorization.json" + release_path.write_text( + release_context.release.model_dump_json(), + encoding="utf-8", + ) + + plan_result = deployment_cmd.run_deployment_plan( + argparse.Namespace( + release=str(release_path), + platform="databricks", + runtime=_RUNTIME_TARGET, + source_reference=_SOURCE, + server="production", + ) + ) + cli_plan = DeploymentPlan.model_validate_json(plan_result.output) + assert cli_plan == expected_plan + plan_path.write_text(cli_plan.model_dump_json(), encoding="utf-8") + + import semapact.platforms.runtime_registry as runtime_registry + + monkeypatch.setattr( + runtime_registry, + "create_deployment_adapter", + lambda platform, **kwargs: adapter, + ) + + preview_result = deployment_cmd.run_deployment_preview( + argparse.Namespace(plan=str(plan_path)) + ) + cli_preview = DeploymentPreview.model_validate_json(preview_result.output) + assert cli_preview == expected_preview + preview_path.write_text(cli_preview.model_dump_json(), encoding="utf-8") + authorization_path.write_text( + authorization.model_dump_json(), + encoding="utf-8", + ) + + execute_result = deployment_cmd.run_deployment_execute( + argparse.Namespace( + plan=str(plan_path), + preview=str(preview_path), + authorization=str(authorization_path), + warehouse_id="warehouse-golden", + ) + ) + assert json.loads(execute_result.output)["providerExecution"] == "SUCCEEDED" + + verify_result = deployment_cmd.run_deployment_verify( + argparse.Namespace( + plan=str(plan_path), + output="json", + ) + ) + assert json.loads(verify_result.output)["status"] == "IN_SYNC" + assert workspace.statements == [ + "CREATE TABLE `main`.`silver`.`orders` " + "(`id` INT NOT NULL) USING DELTA" + ] diff --git a/tests/test_databricks_schema_evolution.py b/tests/test_databricks_schema_evolution.py index f54bac53..cf98b38d 100644 --- a/tests/test_databricks_schema_evolution.py +++ b/tests/test_databricks_schema_evolution.py @@ -259,6 +259,27 @@ def test_planner_fails_closed_on_unsafe_existing_mutation( _plan(desired, observed) +@pytest.mark.parametrize( + ("desired_type", "observed_type"), + [ + ("INT", "DECIMAL(18,2)"), + ("DECIMAL(10,2)", "DECIMAL(18,2)"), + ("DECIMAL(18,2)", "DECIMAL(18,4)"), + ("DECIMAL(18,2)", "DECIMAL(10,2)"), + ("BIGINT", "INT"), + ], +) +def test_existing_numeric_type_changes_fail_closed( + desired_type: str, + observed_type: str, +) -> None: + with pytest.raises(ValidationError, match="type mutation"): + _plan( + _schema(_property("amount", desired_type)), + _observed(("amount", observed_type, True)), + ) + + def test_planner_rejects_non_managed_asset_only_when_mutation_is_required() -> None: with pytest.raises(ValidationError, match="MANAGED"): _plan( From 6c567c661912f85358d2ef1e4924ce26760619ff Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Sun, 20 Sep 2026 07:43:18 +1000 Subject: [PATCH 31/35] feat(readiness): add Databricks deployment doctor (#238) * feat(readiness): add diagnostic foundations * feat(readiness): add diagnostic foundations * feat(readiness): add diagnostic foundations * feat(readiness): add diagnostic foundations * feat(readiness): add diagnostic foundations * feat(readiness): compose runtime probes * feat(cli): expose doctor diagnostics * test(readiness): cover doctor prerequisites * test(readiness): cover doctor prerequisites * test(readiness): cover doctor prerequisites --- semapact/application/models/readiness.py | 56 ++++++ semapact/application/services/readiness.py | 58 ++++++ semapact/interfaces/cli.py | 45 +++++ semapact/interfaces/commands/doctor_cmd.py | 101 ++++++++++ semapact/platforms/databricks/readiness.py | 222 +++++++++++++++++++++ semapact/platforms/git/readiness.py | 135 +++++++++++++ semapact/platforms/runtime_registry.py | 44 +++- tests/interfaces/test_doctor_cmd.py | 156 +++++++++++++++ tests/test_databricks_readiness.py | 161 +++++++++++++++ tests/test_readiness.py | 94 +++++++++ 10 files changed, 1071 insertions(+), 1 deletion(-) create mode 100644 semapact/application/models/readiness.py create mode 100644 semapact/application/services/readiness.py create mode 100644 semapact/interfaces/commands/doctor_cmd.py create mode 100644 semapact/platforms/databricks/readiness.py create mode 100644 semapact/platforms/git/readiness.py create mode 100644 tests/interfaces/test_doctor_cmd.py create mode 100644 tests/test_databricks_readiness.py create mode 100644 tests/test_readiness.py diff --git a/semapact/application/models/readiness.py b/semapact/application/models/readiness.py new file mode 100644 index 00000000..6c3e4262 --- /dev/null +++ b/semapact/application/models/readiness.py @@ -0,0 +1,56 @@ +"""Application read model for production-readiness diagnostics.""" + +from __future__ import annotations + +from enum import Enum + +from pydantic import BaseModel, ConfigDict, computed_field, field_validator + + +class ReadinessStatus(str, Enum): + """Stable status vocabulary for one readiness check.""" + + PASS = "PASS" + FAIL = "FAIL" + WARN = "WARN" + SKIP = "SKIP" + + +class ReadinessCheck(BaseModel): + """One secret-safe prerequisite check.""" + + model_config = ConfigDict(frozen=True) + + check_id: str + status: ReadinessStatus + required: bool + summary: str + remediation: str | None = None + error_type: str | None = None + + @field_validator("check_id", "summary") + @classmethod + def _require_text(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("readiness check text fields must not be empty") + return cleaned + + +class ReadinessReport(BaseModel): + """Provider-neutral readiness report for one runtime target.""" + + model_config = ConfigDict(frozen=True) + + platform: str + runtime_target: str + checks: tuple[ReadinessCheck, ...] + + @computed_field + @property + def ready(self) -> bool: + """Required checks must pass; optional warnings do not block readiness.""" + return all( + (not check.required) or check.status is ReadinessStatus.PASS + for check in self.checks + ) diff --git a/semapact/application/services/readiness.py b/semapact/application/services/readiness.py new file mode 100644 index 00000000..536a3d52 --- /dev/null +++ b/semapact/application/services/readiness.py @@ -0,0 +1,58 @@ +"""Application orchestration for production-readiness diagnostics.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Protocol + +from semapact.application.models.readiness import ( + ReadinessCheck, + ReadinessReport, + ReadinessStatus, +) + + +class ReadinessProbe(Protocol): + """Narrow infrastructure probe consumed by the application service.""" + + key: str + + def run(self) -> tuple[ReadinessCheck, ...]: ... + + +class ReadinessService: + """Aggregate provider/storage probes without owning infrastructure behavior.""" + + def __init__(self, probes: Iterable[ReadinessProbe]) -> None: + self._probes = tuple(probes) + + def check(self, *, platform: str, runtime_target: str) -> ReadinessReport: + checks: list[ReadinessCheck] = [] + for probe in self._probes: + try: + checks.extend(probe.run()) + except Exception as exc: + checks.append( + ReadinessCheck( + check_id=f"{probe.key}.probe", + status=ReadinessStatus.FAIL, + required=True, + summary="Readiness probe failed unexpectedly.", + remediation="Inspect the failing integration and retry the diagnostic.", + error_type=type(exc).__name__, + ) + ) + return ReadinessReport( + platform=_required_text(platform, "platform"), + runtime_target=_required_text(runtime_target, "runtime_target"), + checks=tuple(checks), + ) + + +def _required_text(value: str, field_name: str) -> str: + if not isinstance(value, str): + raise TypeError(f"{field_name} must be str") + cleaned = value.strip() + if not cleaned: + raise ValueError(f"{field_name} must not be empty") + return cleaned diff --git a/semapact/interfaces/cli.py b/semapact/interfaces/cli.py index 276fd765..abc20f60 100644 --- a/semapact/interfaces/cli.py +++ b/semapact/interfaces/cli.py @@ -379,6 +379,42 @@ def _build_parser() -> argparse.ArgumentParser: release_prs_parser.add_argument("--pat-token") release_prs_parser.add_argument("--push", action="store_true") + + doctor_parser = subparsers.add_parser( + "doctor", + help="Check production prerequisites for the contract-selected runtime", + ) + doctor_parser.add_argument( + "--contract", required=True, help="Path or URL to the governed ODCS contract" + ) + doctor_parser.add_argument( + "--server", + help="Contract server identifier when the contract defines multiple servers", + ) + doctor_parser.add_argument( + "--platform", + help="Fallback runtime provider when the contract defines no servers", + ) + doctor_parser.add_argument( + "--runtime", + help="Fallback provider-local runtime target when the contract defines no servers", + ) + doctor_parser.add_argument( + "--warehouse-id", + help="Databricks SQL warehouse used for read-only Statement Execution readiness checks", + ) + doctor_parser.add_argument( + "--repository-root", + default=".", + help="Repository root containing governance history (default: current directory)", + ) + doctor_parser.add_argument( + "--output", + choices=["text", "json"], + default="text", + help="Output format (default: text)", + ) + reconcile_parser = subparsers.add_parser( "reconcile", help="Compare a governed data product with its runtime implementation", @@ -552,6 +588,15 @@ def main() -> int: return 0 parser.error(f"Unknown approval command: {args.approval_command}") + + if args.command == "doctor": + from semapact.interfaces.commands.doctor_cmd import run_doctor + from semapact.interfaces.outcomes import exit_code_from_outcome + + result = run_doctor(args) + print(result.output) + return int(exit_code_from_outcome(result.outcome)) + if args.command == "reconcile": from semapact.interfaces.commands.reconcile_cmd import run_reconcile from semapact.interfaces.outcomes import exit_code_from_outcome diff --git a/semapact/interfaces/commands/doctor_cmd.py b/semapact/interfaces/commands/doctor_cmd.py new file mode 100644 index 00000000..1ad86042 --- /dev/null +++ b/semapact/interfaces/commands/doctor_cmd.py @@ -0,0 +1,101 @@ +"""CLI adapter for production-readiness diagnostics.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +import json + +from semapact.application.models.readiness import ReadinessReport +from semapact.application.services.readiness import ReadinessService +from semapact.core.loader import load_contract +from semapact.interfaces.outcomes import ProcessOutcome +from semapact.platforms.git.readiness import GitWorkingTreeReadinessProbe + + +@dataclass(frozen=True) +class DoctorCommandResult: + """Rendered readiness output plus its process outcome.""" + + output: str + outcome: ProcessOutcome + + +def run_doctor(args: argparse.Namespace) -> DoctorCommandResult: + """Check the deployed prerequisites for the contract-selected runtime.""" + from semapact.platforms.runtime_registry import ( + create_runtime_readiness_probe, + resolve_runtime_location, + ) + + contract = load_contract(args.contract) + location = resolve_runtime_location( + contract, + server_name=args.server, + fallback_platform=args.platform, + fallback_runtime_target=args.runtime, + ) + + execution_config = None + if location.platform == "databricks": + from semapact.platforms.databricks.deployment import ( + DatabricksDeploymentExecutionConfig, + ) + + execution_config = DatabricksDeploymentExecutionConfig( + warehouse_id=args.warehouse_id, + ) + + runtime_probe = create_runtime_readiness_probe( + location.platform, + runtime_target=location.runtime_target, + contract_server=location.contract_server, + execution_config=execution_config, + ) + report = ReadinessService( + ( + runtime_probe, + GitWorkingTreeReadinessProbe(args.repository_root), + ) + ).check( + platform=location.platform, + runtime_target=location.runtime_target, + ) + + rendered = _json_output(report) if args.output == "json" else _text_output(report) + return DoctorCommandResult( + output=rendered, + outcome=( + ProcessOutcome.SUCCESS + if report.ready + else ProcessOutcome.VALIDATION_FAILED + ), + ) + + +def _json_output(report: ReadinessReport) -> str: + payload = report.model_dump(mode="json") + payload["ready"] = report.ready + return json.dumps( + payload, + indent=2, + ensure_ascii=False, + sort_keys=True, + ) + + +def _text_output(report: ReadinessReport) -> str: + lines = [ + f"Readiness: {'READY' if report.ready else 'NOT_READY'}", + f"Platform: {report.platform}", + f"Runtime target: {report.runtime_target}", + "Checks:", + ] + for check in report.checks: + requirement = "required" if check.required else "optional" + lines.append( + f" [{check.status.value}] {check.check_id} ({requirement}) - {check.summary}" + ) + if check.remediation: + lines.append(f" remediation: {check.remediation}") + return "\n".join(lines) diff --git a/semapact/platforms/databricks/readiness.py b/semapact/platforms/databricks/readiness.py new file mode 100644 index 00000000..76642921 --- /dev/null +++ b/semapact/platforms/databricks/readiness.py @@ -0,0 +1,222 @@ +"""Secret-safe read-only readiness checks for Databricks deployment.""" + +from __future__ import annotations + +import time +from typing import Any + +from semapact.application.models.readiness import ReadinessCheck, ReadinessStatus +from semapact.platforms.databricks.client import create_databricks_workspace_client + + +_TERMINAL_STATES = {"SUCCEEDED", "FAILED", "CANCELED", "CLOSED"} + + +class DatabricksReadinessProbe: + """Check the exact prerequisites used by SemaPact's Databricks deployment path.""" + + key = "databricks" + + def __init__( + self, + *, + runtime_target: str, + workspace_url: str | None = None, + warehouse_id: str | None = None, + poll_interval_seconds: float = 0.2, + max_poll_attempts: int = 10, + ) -> None: + self._runtime_target = runtime_target.strip() + self._workspace_url = _clean(workspace_url) + self._warehouse_id = _clean(warehouse_id) + self._poll_interval_seconds = poll_interval_seconds + self._max_poll_attempts = max_poll_attempts + + def run(self) -> tuple[ReadinessCheck, ...]: + try: + client = create_databricks_workspace_client( + workspace_url=self._workspace_url, + ) + except Exception as exc: + return ( + _failure( + "databricks.configuration", + "Databricks SDK client could not be initialized.", + exc, + remediation='Install the Databricks extra and configure Databricks unified authentication.', + ), + _skipped( + "databricks.identity", + "Workspace identity was not checked because client initialization failed.", + ), + _skipped( + "databricks.unity_catalog", + "Unity Catalog access was not checked because client initialization failed.", + ), + _skipped( + "databricks.statement_execution", + "Statement Execution access was not checked because client initialization failed.", + ), + ) + + host = getattr(getattr(client, "config", None), "host", None) + if not isinstance(host, str) or not host.strip(): + return ( + ReadinessCheck( + check_id="databricks.configuration", + status=ReadinessStatus.FAIL, + required=True, + summary="Databricks SDK did not resolve a workspace host.", + remediation="Configure a workspace host through the contract server or Databricks unified authentication.", + ), + _skipped( + "databricks.identity", + "Workspace identity was not checked because no workspace host was resolved.", + ), + _skipped( + "databricks.unity_catalog", + "Unity Catalog access was not checked because no workspace host was resolved.", + ), + _skipped( + "databricks.statement_execution", + "Statement Execution access was not checked because no workspace host was resolved.", + ), + ) + + checks = [ + ReadinessCheck( + check_id="databricks.configuration", + status=ReadinessStatus.PASS, + required=True, + summary="Databricks SDK resolved workspace configuration.", + ), + self._check_identity(client), + self._check_unity_catalog(client), + self._check_statement_execution(client), + ] + return tuple(checks) + + def _check_identity(self, client: Any) -> ReadinessCheck: + try: + client.current_user.me() + except Exception as exc: + return _failure( + "databricks.identity", + "Current Databricks identity could not be resolved.", + exc, + remediation="Verify workspace connectivity and the configured Databricks credentials.", + ) + return ReadinessCheck( + check_id="databricks.identity", + status=ReadinessStatus.PASS, + required=True, + summary="Databricks workspace identity is authenticated.", + ) + + def _check_unity_catalog(self, client: Any) -> ReadinessCheck: + try: + client.schemas.get(full_name=self._runtime_target) + except Exception as exc: + return _failure( + "databricks.unity_catalog", + "Target Unity Catalog schema is not readable by the current identity.", + exc, + remediation="Grant the deployment identity metadata access to the governed catalog and schema.", + ) + return ReadinessCheck( + check_id="databricks.unity_catalog", + status=ReadinessStatus.PASS, + required=True, + summary="Target Unity Catalog schema is readable.", + ) + + def _check_statement_execution(self, client: Any) -> ReadinessCheck: + if self._warehouse_id is None: + return ReadinessCheck( + check_id="databricks.statement_execution", + status=ReadinessStatus.FAIL, + required=True, + summary="No Databricks SQL warehouse was configured for deployment execution.", + remediation="Provide --warehouse-id for the SQL warehouse used by governed deployment mutations.", + ) + + try: + response = client.statement_execution.execute_statement( + statement="SELECT 1", + warehouse_id=self._warehouse_id, + wait_timeout="10s", + ) + state = _statement_state(response) + attempts = 0 + while state not in _TERMINAL_STATES and attempts < self._max_poll_attempts: + statement_id = getattr(response, "statement_id", None) + if not statement_id: + raise RuntimeError( + "Databricks readiness statement returned no statement_id" + ) + if self._poll_interval_seconds: + time.sleep(self._poll_interval_seconds) + response = client.statement_execution.get_statement(statement_id) + state = _statement_state(response) + attempts += 1 + + if state != "SUCCEEDED": + raise RuntimeError( + "Databricks readiness statement did not complete successfully" + ) + except Exception as exc: + return _failure( + "databricks.statement_execution", + "Databricks Statement Execution is not usable with the configured warehouse.", + exc, + remediation="Verify warehouse availability and CAN USE / statement-execution permissions for the deployment identity.", + ) + + return ReadinessCheck( + check_id="databricks.statement_execution", + status=ReadinessStatus.PASS, + required=True, + summary="Read-only Statement Execution succeeded on the configured SQL warehouse.", + ) + + +def _failure( + check_id: str, + summary: str, + exc: Exception, + *, + remediation: str, +) -> ReadinessCheck: + return ReadinessCheck( + check_id=check_id, + status=ReadinessStatus.FAIL, + required=True, + summary=summary, + remediation=remediation, + error_type=type(exc).__name__, + ) + + +def _skipped(check_id: str, summary: str) -> ReadinessCheck: + return ReadinessCheck( + check_id=check_id, + status=ReadinessStatus.SKIP, + required=True, + summary=summary, + ) + + +def _statement_state(response: Any) -> str: + status = getattr(response, "status", None) + state = getattr(status, "state", None) + if state is None: + raise RuntimeError("Databricks statement response did not contain status.state") + value = getattr(state, "value", state) + return str(value).upper() + + +def _clean(value: object) -> str | None: + if value is None: + return None + cleaned = str(value).strip() + return cleaned or None diff --git a/semapact/platforms/git/readiness.py b/semapact/platforms/git/readiness.py new file mode 100644 index 00000000..ed080d74 --- /dev/null +++ b/semapact/platforms/git/readiness.py @@ -0,0 +1,135 @@ +"""Read-only readiness checks for Git working-tree governance storage.""" + +from __future__ import annotations + +import os +from pathlib import Path + +from semapact.application.models.readiness import ReadinessCheck, ReadinessStatus + + +class GitWorkingTreeReadinessProbe: + """Validate governance-history storage prerequisites without writing files.""" + + key = "history" + + def __init__( + self, + repository_root: str | Path, + *, + state_directory: str | Path = ".semapact/history", + ) -> None: + self._repository_root = Path(repository_root) + self._state_directory = Path(state_directory) + + def run(self) -> tuple[ReadinessCheck, ...]: + root = self._repository_root.resolve(strict=False) + root_check = self._check_repository_root(root) + if root_check.status is not ReadinessStatus.PASS: + return ( + root_check, + ReadinessCheck( + check_id="history.storage", + status=ReadinessStatus.SKIP, + required=True, + summary="Governance history storage was not checked because the repository root is invalid.", + ), + ReadinessCheck( + check_id="git.worktree", + status=ReadinessStatus.WARN, + required=False, + summary="Git worktree integration could not be checked.", + ), + ) + + return ( + root_check, + self._check_history_storage(root), + self._check_git_worktree(root), + ) + + def _check_repository_root(self, root: Path) -> ReadinessCheck: + if not root.exists() or not root.is_dir(): + return ReadinessCheck( + check_id="history.repository_root", + status=ReadinessStatus.FAIL, + required=True, + summary="Repository root does not exist or is not a directory.", + remediation="Point --repository-root at the Git working tree used for SemaPact governance history.", + ) + return ReadinessCheck( + check_id="history.repository_root", + status=ReadinessStatus.PASS, + required=True, + summary="Repository root is available.", + ) + + def _check_history_storage(self, root: Path) -> ReadinessCheck: + state = self._state_directory + if state.is_absolute() or ".." in state.parts: + return ReadinessCheck( + check_id="history.storage", + status=ReadinessStatus.FAIL, + required=True, + summary="Governance history state directory is not repository-relative.", + remediation="Use a repository-relative history directory inside the governance working tree.", + ) + + target = (root / state).resolve(strict=False) + if not target.is_relative_to(root): + return ReadinessCheck( + check_id="history.storage", + status=ReadinessStatus.FAIL, + required=True, + summary="Governance history storage resolves outside the repository root.", + remediation="Keep governance history inside the configured repository root.", + ) + + if target.exists() and not target.is_dir(): + return ReadinessCheck( + check_id="history.storage", + status=ReadinessStatus.FAIL, + required=True, + summary="Governance history path exists but is not a directory.", + remediation="Replace the conflicting path with a writable history directory.", + ) + + probe_path = target if target.exists() else _nearest_existing_parent(target, root) + if not os.access(probe_path, os.R_OK | os.W_OK | os.X_OK): + return ReadinessCheck( + check_id="history.storage", + status=ReadinessStatus.FAIL, + required=True, + summary="Governance history storage is not readable and writable by the current process.", + remediation="Grant the SemaPact process read/write access to the governance working tree.", + ) + + return ReadinessCheck( + check_id="history.storage", + status=ReadinessStatus.PASS, + required=True, + summary="Governance history storage is accessible.", + ) + + def _check_git_worktree(self, root: Path) -> ReadinessCheck: + if (root / ".git").exists(): + return ReadinessCheck( + check_id="git.worktree", + status=ReadinessStatus.PASS, + required=False, + summary="Git worktree metadata is present.", + ) + return ReadinessCheck( + check_id="git.worktree", + status=ReadinessStatus.WARN, + required=False, + summary="Git worktree metadata was not found.", + remediation="Use a Git working tree when governance history should be versioned through GitOps.", + ) + + +def _nearest_existing_parent(target: Path, root: Path) -> Path: + current = target + while not current.exists() and current != root: + current = current.parent + return current diff --git a/semapact/platforms/runtime_registry.py b/semapact/platforms/runtime_registry.py index 113d7332..03f199c7 100644 --- a/semapact/platforms/runtime_registry.py +++ b/semapact/platforms/runtime_registry.py @@ -3,7 +3,7 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Literal +from typing import TYPE_CHECKING, Literal from open_data_contract_standard.model import OpenDataContractStandard, Server @@ -12,6 +12,9 @@ from semapact.exceptions import ValidationError from semapact.observation import RuntimeProvider, RuntimeProviderRegistry +if TYPE_CHECKING: + from semapact.application.services.readiness import ReadinessProbe + @dataclass(frozen=True) class ResolvedRuntimeLocation: @@ -85,6 +88,45 @@ def create_runtime_provider_registry( ) +def create_runtime_readiness_probe( + platform: str, + *, + runtime_target: str, + contract_server: Server | None = None, + execution_config: DeploymentExecutionConfig | None = None, +) -> ReadinessProbe: + """Compose the selected provider readiness probe without performing checks yet.""" + normalized = platform.strip().casefold() + if normalized != "databricks": + raise ValidationError( + f"Unsupported readiness provider '{platform}'. Supported providers: databricks" + ) + + from semapact.platforms.databricks.deployment import ( + DatabricksDeploymentExecutionConfig, + ) + from semapact.platforms.databricks.readiness import DatabricksReadinessProbe + + config = ( + DatabricksDeploymentExecutionConfig() + if execution_config is None + else execution_config + ) + if not isinstance(config, DatabricksDeploymentExecutionConfig): + raise ValidationError( + "Databricks readiness requires DatabricksDeploymentExecutionConfig" + ) + + return DatabricksReadinessProbe( + runtime_target=_required( + runtime_target, + "Databricks readiness requires a runtime target", + ), + workspace_url=_clean(contract_server.host) if contract_server else None, + warehouse_id=config.warehouse_id, + ) + + def create_deployment_adapter( platform: str, *, diff --git a/tests/interfaces/test_doctor_cmd.py b/tests/interfaces/test_doctor_cmd.py new file mode 100644 index 00000000..94b76ba5 --- /dev/null +++ b/tests/interfaces/test_doctor_cmd.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +import json +import sys + +from open_data_contract_standard.model import OpenDataContractStandard, Server +import pytest + +from semapact.application.models.readiness import ( + ReadinessCheck, + ReadinessStatus, +) +from semapact.interfaces import cli +from semapact.interfaces.commands import doctor_cmd +from semapact.interfaces.commands.doctor_cmd import DoctorCommandResult +from semapact.interfaces.outcomes import ProcessOutcome + + +def _contract() -> OpenDataContractStandard: + server = Server.model_validate( + { + "server": "production", + "type": "databricks", + "host": "https://workspace.example", + "catalog": "main", + "schema": "sales", + } + ) + return OpenDataContractStandard.model_construct( + id="sales-product", + version="1.0.0", + servers=[server], + schema_=[], + ) + + +class _PassingProbe: + key = "databricks" + + def run(self) -> tuple[ReadinessCheck, ...]: + return ( + ReadinessCheck( + check_id="databricks.configuration", + status=ReadinessStatus.PASS, + required=True, + summary="Databricks configuration is ready.", + ), + ) + + +def test_doctor_parser_is_contract_first_and_does_not_accept_credentials() -> None: + args = cli._build_parser().parse_args( + [ + "doctor", + "--contract", + "contracts/sales.yaml", + "--warehouse-id", + "warehouse-123", + "--output", + "json", + ] + ) + + assert args.command == "doctor" + assert args.contract == "contracts/sales.yaml" + assert args.server is None + assert args.platform is None + assert args.runtime is None + assert args.warehouse_id == "warehouse-123" + assert args.output == "json" + assert not hasattr(args, "token") + assert not hasattr(args, "workspace_url") + + +def test_doctor_uses_contract_runtime_and_returns_machine_readable_report( + tmp_path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + import semapact.platforms.runtime_registry as runtime_registry + + captured: dict[str, object] = {} + + def _create_probe( + platform: str, + *, + runtime_target: str, + contract_server=None, + execution_config=None, + ): + captured.update( + platform=platform, + runtime_target=runtime_target, + contract_server=contract_server, + execution_config=execution_config, + ) + return _PassingProbe() + + monkeypatch.setattr(doctor_cmd, "load_contract", lambda path: _contract()) + monkeypatch.setattr( + runtime_registry, + "create_runtime_readiness_probe", + _create_probe, + ) + + args = cli._build_parser().parse_args( + [ + "doctor", + "--contract", + "contracts/sales.yaml", + "--warehouse-id", + "warehouse-123", + "--repository-root", + str(tmp_path), + "--output", + "json", + ] + ) + result = doctor_cmd.run_doctor(args) + payload = json.loads(result.output) + + assert result.outcome is ProcessOutcome.SUCCESS + assert payload["ready"] is True + assert payload["platform"] == "databricks" + assert payload["runtime_target"] == "main.sales" + assert captured["platform"] == "databricks" + assert captured["runtime_target"] == "main.sales" + assert getattr(captured["contract_server"], "server") == "production" + assert getattr(captured["execution_config"], "warehouse_id") == "warehouse-123" + assert any( + check["check_id"] == "git.worktree" + and check["status"] == "WARN" + and check["required"] is False + for check in payload["checks"] + ) + + +def test_main_maps_not_ready_to_validation_exit_code( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.setattr( + doctor_cmd, + "run_doctor", + lambda args: DoctorCommandResult( + output="Readiness: NOT_READY", + outcome=ProcessOutcome.VALIDATION_FAILED, + ), + ) + monkeypatch.setattr( + sys, + "argv", + ["semapact", "doctor", "--contract", "contracts/sales.yaml"], + ) + + assert cli.main() == 2 + assert capsys.readouterr().out.strip() == "Readiness: NOT_READY" diff --git a/tests/test_databricks_readiness.py b/tests/test_databricks_readiness.py new file mode 100644 index 00000000..5defb975 --- /dev/null +++ b/tests/test_databricks_readiness.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import json +from types import SimpleNamespace + +import pytest + +from semapact.application.models.readiness import ReadinessStatus +from semapact.platforms.databricks import readiness +from semapact.platforms.databricks.readiness import DatabricksReadinessProbe + + +class _CurrentUser: + def __init__(self) -> None: + self.calls = 0 + + def me(self): + self.calls += 1 + return SimpleNamespace(user_name="svc-semapact@example.com") + + +class _Schemas: + def __init__(self) -> None: + self.full_names: list[str] = [] + + def get(self, *, full_name: str): + self.full_names.append(full_name) + return SimpleNamespace(full_name=full_name) + + +class _StatementExecution: + def __init__(self) -> None: + self.calls: list[tuple[str, str, str]] = [] + + def execute_statement(self, *, statement: str, warehouse_id: str, wait_timeout: str): + self.calls.append((statement, warehouse_id, wait_timeout)) + return SimpleNamespace( + statement_id="stmt-1", + status=SimpleNamespace(state="SUCCEEDED"), + ) + + def get_statement(self, statement_id: str): + raise AssertionError(f"unexpected poll for {statement_id}") + + +class _Client: + def __init__(self) -> None: + self.config = SimpleNamespace(host="https://workspace.example") + self.current_user = _CurrentUser() + self.schemas = _Schemas() + self.statement_execution = _StatementExecution() + + +def test_databricks_probe_checks_identity_uc_and_read_only_statement_execution( + monkeypatch: pytest.MonkeyPatch, +) -> None: + client = _Client() + monkeypatch.setattr( + readiness, + "create_databricks_workspace_client", + lambda **kwargs: client, + ) + + checks = DatabricksReadinessProbe( + runtime_target="main.sales", + workspace_url="https://workspace.example", + warehouse_id="warehouse-123", + poll_interval_seconds=0, + ).run() + + assert [check.status for check in checks] == [ + ReadinessStatus.PASS, + ReadinessStatus.PASS, + ReadinessStatus.PASS, + ReadinessStatus.PASS, + ] + assert client.current_user.calls == 1 + assert client.schemas.full_names == ["main.sales"] + assert client.statement_execution.calls == [ + ("SELECT 1", "warehouse-123", "10s") + ] + + +def test_missing_warehouse_blocks_deployment_readiness( + monkeypatch: pytest.MonkeyPatch, +) -> None: + client = _Client() + monkeypatch.setattr( + readiness, + "create_databricks_workspace_client", + lambda **kwargs: client, + ) + + checks = DatabricksReadinessProbe( + runtime_target="main.sales", + warehouse_id=None, + ).run() + + statement_check = next( + check + for check in checks + if check.check_id == "databricks.statement_execution" + ) + assert statement_check.status is ReadinessStatus.FAIL + assert statement_check.required is True + assert client.statement_execution.calls == [] + + +def test_client_initialization_failure_does_not_leak_exception_message( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def _raise(**kwargs): + raise RuntimeError("token=secret-token") + + monkeypatch.setattr( + readiness, + "create_databricks_workspace_client", + _raise, + ) + + checks = DatabricksReadinessProbe( + runtime_target="main.sales", + warehouse_id="warehouse-123", + ).run() + rendered = json.dumps( + [check.model_dump(mode="json") for check in checks], + sort_keys=True, + ) + + assert checks[0].status is ReadinessStatus.FAIL + assert checks[0].error_type == "RuntimeError" + assert "secret-token" not in rendered + + +def test_uc_permission_failure_is_reported_without_exposing_provider_message( + monkeypatch: pytest.MonkeyPatch, +) -> None: + client = _Client() + + def _deny(*, full_name: str): + raise PermissionError("bearer secret-token") + + client.schemas.get = _deny + monkeypatch.setattr( + readiness, + "create_databricks_workspace_client", + lambda **kwargs: client, + ) + + checks = DatabricksReadinessProbe( + runtime_target="main.sales", + warehouse_id="warehouse-123", + poll_interval_seconds=0, + ).run() + uc_check = next( + check for check in checks if check.check_id == "databricks.unity_catalog" + ) + + assert uc_check.status is ReadinessStatus.FAIL + assert uc_check.error_type == "PermissionError" + assert "secret-token" not in json.dumps(uc_check.model_dump(mode="json")) diff --git a/tests/test_readiness.py b/tests/test_readiness.py new file mode 100644 index 00000000..9e976bbe --- /dev/null +++ b/tests/test_readiness.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +from semapact.application.models.readiness import ( + ReadinessCheck, + ReadinessStatus, +) +from semapact.application.services.readiness import ReadinessService +from semapact.platforms.git.readiness import GitWorkingTreeReadinessProbe + + +class _Probe: + key = "test" + + def __init__(self, *checks: ReadinessCheck) -> None: + self._checks = checks + + def run(self) -> tuple[ReadinessCheck, ...]: + return self._checks + + +def test_optional_warning_does_not_block_readiness() -> None: + report = ReadinessService( + ( + _Probe( + ReadinessCheck( + check_id="test.required", + status=ReadinessStatus.PASS, + required=True, + summary="Required prerequisite passed.", + ), + ReadinessCheck( + check_id="test.optional", + status=ReadinessStatus.WARN, + required=False, + summary="Optional integration is unavailable.", + ), + ), + ) + ).check(platform="databricks", runtime_target="main.sales") + + assert report.ready is True + + +def test_required_failure_blocks_readiness() -> None: + report = ReadinessService( + ( + _Probe( + ReadinessCheck( + check_id="test.required", + status=ReadinessStatus.FAIL, + required=True, + summary="Required prerequisite failed.", + ) + ), + ) + ).check(platform="databricks", runtime_target="main.sales") + + assert report.ready is False + + +def test_unexpected_probe_failure_is_secret_safe() -> None: + class _ExplodingProbe: + key = "explode" + + def run(self) -> tuple[ReadinessCheck, ...]: + raise RuntimeError("credential=secret-token") + + report = ReadinessService((_ExplodingProbe(),)).check( + platform="databricks", + runtime_target="main.sales", + ) + + check = report.checks[0] + assert check.status is ReadinessStatus.FAIL + assert check.error_type == "RuntimeError" + assert "secret-token" not in check.summary + assert "secret-token" not in (check.remediation or "") + + +def test_git_history_probe_is_read_only_and_missing_git_is_optional(tmp_path) -> None: + probe = GitWorkingTreeReadinessProbe(tmp_path) + + report = ReadinessService((probe,)).check( + platform="databricks", + runtime_target="main.sales", + ) + checks = {check.check_id: check for check in report.checks} + + assert checks["history.repository_root"].status is ReadinessStatus.PASS + assert checks["history.storage"].status is ReadinessStatus.PASS + assert checks["git.worktree"].status is ReadinessStatus.WARN + assert checks["git.worktree"].required is False + assert report.ready is True + assert not (tmp_path / ".semapact").exists() From ffada6062b2b3ee21f02fcfa1f474667afbf09b3 Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Mon, 21 Sep 2026 21:15:07 +1000 Subject: [PATCH 32/35] feat(deployment): add immutable CI deployment bundle handoff (#239) * refactor(history): remove obsolete release deployment runtime records * refactor(history): keep governance ledger repositories only * refactor(history): expose only canonical ledger and operational sink * refactor(history): keep Git ledger governance-only * refactor(history): model governance evolution only * refactor(history): reconstruct ContractRelease governance chain * refactor(history): remove obsolete git operational history * refactor(history): remove obsolete git operational history * refactor(history): remove obsolete git operational history * refactor(history): remove obsolete git operational history * refactor(history): remove obsolete git operational history * refactor(history): remove obsolete git operational history * refactor(history): remove obsolete git operational history * fix(contractops): restore canonical version validation * test(contractops): cover canonical release snapshot only * test(contractops): rehydrate canonical release artifacts only * test(architecture): assert canonical owners only * test(contractops): keep golden scenarios canonical * test(history): remove obsolete deployment history golden * test(history): remove obsolete deployment history golden * refactor(cli): remove obsolete release compatibility commands * refactor(cli): expose canonical release commands only * docs(contractops): remove obsolete compatibility lifecycle * docs(deployment): document canonical-only artifact model * docs(examples): remove obsolete legacy wording * docs(config): remove obsolete legacy wording * docs(config): refresh canonical schema description * test(cli): describe canonical deployment surface * test(cli): assert canonical release commands only * refactor(release): isolate repository classification * refactor(release): remove legacy batch release commands * refactor(cli): remove legacy batch release surface * refactor: remove obsolete compatibility workflow * refactor: remove obsolete compatibility workflow * refactor: remove obsolete compatibility workflow * refactor: remove obsolete compatibility workflow * refactor: remove obsolete compatibility workflow * refactor: remove obsolete compatibility workflow * refactor: remove obsolete compatibility workflow * refactor: remove obsolete compatibility workflow * refactor: remove obsolete compatibility workflow * refactor: remove obsolete compatibility workflow * refactor: remove obsolete release workflow artifact * refactor: remove obsolete release workflow artifact * refactor: remove obsolete release workflow artifact * refactor: remove obsolete release workflow artifact * refactor(devops): remove obsolete release workflow exports * refactor(examples): remove obsolete release promotion workflow * test(architecture): remove services compatibility assertions * test(examples): keep only canonical release workflows * docs(examples): remove obsolete release-promotion examples * docs: describe greenfield canonical architecture * refactor(lifecycle): remove root legacy status fallback * refactor(lifecycle): remove deprecated compatibility helpers * refactor(editor): remove lifecycle compatibility alias * fix(lifecycle): reference canonical release workflow * refactor(lifecycle): remove deprecated public helper * test(lifecycle): enforce canonical root lifecycle status * test(lifecycle): remove compatibility lifecycle paths * test(lifecycle): remove proposed status alias * test(editor): use canonical editor semantics directly * refactor(editor): remove compatibility facade * docs(lifecycle): document only canonical lifecycle semantics * docs(governance): remove obsolete manifest boundary * refactor(api): remove obsolete release workflow exports * refactor(core): remove deleted release compatibility exports * refactor(governance): import version type from canonical owner * test(governance): use canonical change classification owner * test(governance): use canonical version type owner * test(identity): use canonical change classification owner * test(governance): use canonical application service * test(governance): remove services compatibility dependency * test(reconciliation): use canonical application service * test(release): use canonical application services * test(versioning): use canonical application service * test(lifecycle): remove obsolete release compatibility scenarios * refactor(deployment): start canonical plan schema at v1 * refactor(deployment): start deployment bundle schema at v1 * refactor(history): start operational event schema at v1 * test(deployment): assert greenfield v1 plan schema * test(history): assert greenfield v1 operational schema * test(deployment): assert greenfield plan version * docs(deployment): describe greenfield v1 artifact schema * refactor(application): remove deleted evolution exports * test(cli): import version type from canonical owner * test(delta): use canonical governance service * test(deployment): use canonical deployment service * test(governance): remove obsolete release compatibility paths * test(governance): remove final obsolete release API cases * refactor(import): use canonical governance service * test(reconcile): use canonical application model * test(history): align evolution chain with governance ledger * refactor(lifecycle): use canonical governance service * refactor(deployment): emit greenfield v1 plans * refactor(import): use canonical governance service * test(reconcile): use canonical reconciliation model * test(cli): remove deleted release prepare subprocess cases * test(history): use canonical governance evolution boundary * test(cli): remove obsolete release promotion surfaces * test(history): remove obsolete runtime evolution chain * test(history): align integrity evolution with governance ledger * test(lifecycle): remove noncanonical contract lifecycle fallback * test(contractops): assert fail-closed tampered version identity * test(governance): refresh canonical release guidance fixture * test(cli): remove obsolete release manifest regression * test(cicd): execute canonical release and deployment artifact handoff * refactor(cicd): emit collision-resistant contract artifact keys * fix(cicd): use collision-resistant central-repo artifact keys * test(cicd): execute central-repo release and deployment fan-out * test(cicd): lock collision-resistant central artifact identity * fix(cicd): repair regression fixture contract factory * fix(cicd): normalize regression fixture signature * refactor(governance): remove execution date from decision * refactor(governance): make decision identity date independent * refactor(contractops): remove date from changeset * refactor(contractops): make changeset date independent * refactor(contractops): remove date from changeset identity * fix(governance): remove final date-dependent decision field * refactor(governance): separate mutation context from evaluation * refactor(release): remove business date from planning * refactor(release): make assessment date independent * refactor(release): make repository classification date independent * refactor(deployment): remove business date from assessment * refactor(cli): remove release effective date plumbing * refactor(cli): remove deployment assessment effective date * refactor(cli): keep effective date only on mutation commands * refactor(cicd): remove execution date from assessment * refactor(cicd): remove execution date from assessment * test(cicd): use date-independent assess commands * test(deployment): remove assessment date surface * test(cli): remove date from release analysis commands * refactor(governance): remove effective date from public decision protocol * refactor(governance): stop exporting lifecycle mutation context * refactor(contractops): remove date context coupling * refactor(lifecycle): scope effective date to mutations * refactor(lifecycle): keep date only for mutation * refactor(pipeline): separate merge date from governance identity * refactor(plan): keep date only in in-memory merge * refactor(history): remove date coupling from proposal links * test(release): remove assessment date * test(deployment): make workflow assessment date independent * test(release): make planning date independent * test(release): make formal assessment date independent * test(release): remove effective date from public release CLI * test(governance): separate mutation date from decision * test(governance): proposal evaluation is date independent * test(release): remove date context from proposal artifacts * test(contractops): make changeset identity date independent * test(history): make proposal history date independent * test(governance): make decisions independent of effective date * test(governance): remove effective date from public decision protocol * test(contractops): remove date from release execution inputs * test(contractops): remove date from golden release scenarios * test(governance): make golden decisions date independent * test(history): remove date from changeset integrity * test(governance): remove date from gate decisions * test(pipeline): separate merge date from decision payload * test(governance): evaluation is independent of merge date * test(governance): remove date context from change evaluation * test(governance): make reason codes date independent * test(contractops): remove date from authorization inputs * test(contractops): remove date from artifact rehydration * test(lifecycle): import mutation context from lifecycle boundary * test(lifecycle): import mutation context directly * test(lifecycle): import mutation context from canonical module * test(lifecycle): import mutation context from canonical module * test(lifecycle): import mutation context from canonical module * test(lifecycle): import mutation context from canonical module * test(lifecycle): import mutation context from canonical module * test(lifecycle): import mutation context from canonical module * test(databricks): remove date from deployment assessment * test(context): scope effective date to lifecycle mutation * test(pipeline): separate mutation context from governance * test(databricks): make bundle assessment date independent * fix(lifecycle): preserve mutation date without coupling governance * fix(lifecycle): preserve mutation date without coupling governance * fix(release): remove stale effective date from plan command * test(cli): remove decision date context * test(architecture): remove date from release and governance identity * test(import): keep effective date only on merge mutation * test(governance): remove date from read-only evaluation * test(devops): remove date from governance decision fixtures * test(history): remove date from persisted governance artifacts * test(lifecycle): keep date on mutation, not evaluation * test(merge): separate mutation date from governance evaluation * test(retired): remove date from read-only governance paths * test(governance): remove date from pure analysis paths * test(retired): keep effective date only on mutation commands * test(cli): remove effective date from release classify subprocess cases * test(lifecycle): remove effective context from pure governance evaluation * test(governance): replace legacy decision snapshots with semantic golden checks * test(governance): remove obsolete public decision snapshots * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(governance): remove obsolete dated decision snapshot * test(lifecycle): decouple evaluation from mutation context * test(cicd): remove stale release effective date * docs(deployment): remove effective date from assess workflows * docs(readme): remove execution date from release and deployment assess * test(lifecycle): restore mutation context in fail-closed cases * feat(cli): accept approval artifact at release finalization * feat(release): prefer exact approval artifact over history lookup * test(cli): finalize release from exact approval artifact * docs(examples): hand off release approval as exact artifact * docs(examples): hand off approvals without Git transport * refactor(approval): separate history conflict check from artifact resolution * fix(release): keep explicit approval handoff fail closed on ledger conflicts * test(approval): fail closed on exact release review conflict * docs(readme): show approval artifact handoff * docs(deployment): make approval artifact the canonical handoff * docs(examples): document approval artifact handoff * refactor(approval): avoid duplicate ledger reads * refactor(deployment): align planner docs with canonical source model * test(cicd): exercise exact release approval artifact handoff * docs(approval): align history with publish-only release approval * docs(contractops): scope authorization to formal publication * docs(architecture): make release and deployment boundaries canonical * refactor(deployment): name provenance validation by responsibility * refactor(deployment): import canonical provenance validator * refactor(deployment): remove obsolete authorization module name * refactor(deployment): remove obsolete authorization terminology * refactor(contractops): remove deployment approval example * docs(deployment): remove obsolete authorization wording * fix(deployment): bind candidate bundle to exact change set * test(deployment): reject mixed candidate governance artifacts --- .semapact.yaml | 1 + ARCHITECTURE.md | 198 ++++-- README.md | 186 ++++- docs/approval_history.md | 33 + docs/configuration.md | 59 ++ docs/contractops_authorization.md | 79 +-- docs/contractops_phases.md | 112 ++- docs/deployment_plans.md | 324 +++++++-- docs/governance_analysis_boundary.md | 93 +-- docs/lifecycle_semantics.md | 192 ++---- examples/README.md | 56 +- examples/azure-devops/semapact-release.yml | 42 -- examples/ci/release.example.sh | 31 - .../github/central-contract-repo-ci-cd.yml | 327 +++++++++ examples/github/data-product-ci-cd.yml | 235 +++++++ examples/github/semapact-release.yml | 59 -- .../release/release-manifest.example.json | 18 - schemas/semapact-config.schema.json | 101 +++ semapact/__init__.py | 41 -- semapact/application/models/__init__.py | 14 +- .../application/models/deployment_workflow.py | 199 ++++++ semapact/application/models/evolution.py | 38 +- semapact/application/models/release.py | 145 +++- semapact/application/services/deployment.py | 15 +- .../services/deployment_history.py | 191 ------ .../services/deployment_workflow.py | 249 +++++++ semapact/application/services/evolution.py | 276 +------- semapact/application/services/governance.py | 9 +- semapact/application/services/history.py | 2 - .../application/services/release_approval.py | 51 ++ .../application/services/release_history.py | 214 ------ .../application/services/release_planning.py | 4 - .../application/services/release_workflow.py | 139 ++++ .../services/repository_classification.py | 147 ++++ .../application/services/runtime_history.py | 207 ------ semapact/change_context.py | 12 +- semapact/contractops/__init__.py | 26 +- semapact/contractops/changeset.py | 5 - semapact/contractops/context.py | 4 - semapact/contractops/execution.py | 165 +---- semapact/contractops/execution_models.py | 71 +- semapact/contractops/integrity.py | 117 ++-- semapact/contractops/models.py | 6 +- semapact/core/__init__.py | 18 - semapact/core/config_schema.py | 101 +++ semapact/core/editor_contract.py | 74 -- semapact/core/editor_semantics.py | 4 - semapact/core/lifecycle_cli.py | 3 +- semapact/core/release.py | 166 ----- semapact/deployment/__init__.py | 36 +- semapact/deployment/adapters.py | 31 +- semapact/deployment/authorization.py | 120 ---- semapact/deployment/models.py | 120 +--- semapact/deployment/orchestrator.py | 156 +++-- semapact/deployment/planner.py | 82 ++- semapact/deployment/provenance.py | 75 ++ semapact/deployment/source.py | 181 +++++ semapact/deployment/verification.py | 2 +- semapact/devops/__init__.py | 24 - semapact/devops/release_workflow.py | 465 ------------- semapact/governance/__init__.py | 4 - semapact/governance/evaluator.py | 6 - semapact/governance/models.py | 2 - semapact/governance/public.py | 30 +- semapact/history/__init__.py | 49 +- semapact/history/integrity.py | 222 ------ semapact/history/models.py | 195 ------ semapact/history/operational.py | 194 ++++++ semapact/history/operational_registry.py | 34 + semapact/history/repository.py | 107 +-- semapact/interfaces/cli.py | 292 ++++---- semapact/interfaces/commands/approval_cmd.py | 15 +- .../interfaces/commands/deployment_cmd.py | 374 +++++++--- semapact/interfaces/commands/import_cmd.py | 2 +- semapact/interfaces/commands/lifecycle_cmd.py | 2 +- semapact/interfaces/commands/plan_cmd.py | 1 - semapact/interfaces/commands/release_cmd.py | 239 ++++--- semapact/interfaces/parsing.py | 18 + semapact/lifecycle/__init__.py | 6 +- semapact/lifecycle/helpers.py | 29 - semapact/lifecycle/policy.py | 2 +- semapact/lifecycle/status.py | 21 +- semapact/orchestrator/pipeline.py | 6 +- semapact/platforms/databricks/deployment.py | 62 +- semapact/platforms/delta/__init__.py | 5 + .../platforms/delta/operational_history.py | 48 ++ semapact/platforms/git/history_repository.py | 261 ++----- semapact/platforms/sqlite/__init__.py | 5 + .../platforms/sqlite/operational_history.py | 70 ++ semapact/services/__init__.py | 29 - semapact/services/deployment_service.py | 5 - semapact/services/governance_service.py | 6 - semapact/services/reconciliation_service.py | 6 - semapact/services/release_models.py | 5 - semapact/services/release_planning_service.py | 5 - .../services/version_authority_service.py | 5 - .../allow_clean_decision.json | 39 -- .../block_retired_decision.json | 72 -- .../block_validation_decision.json | 55 -- .../breaking_review_decision.json | 74 -- .../review_deprecate_decision.json | 56 -- .../expected.json | 71 -- .../contract_id_change/expected.json | 73 -- .../decimal_precision_reduction/expected.json | 74 -- .../decimal_scale_reduction/expected.json | 74 -- .../decimal_widening/expected.json | 56 -- .../deprecated_entity_change/expected.json | 88 --- .../descriptive_metadata_only/expected.json | 106 --- .../draft_entity_change/expected.json | 88 --- .../enum_reduction/expected.json | 39 -- .../logical_type_change/expected.json | 90 --- .../manual_version_change/expected.json | 73 -- .../merge_conflict/expected.json | 51 -- .../no_change/expected.json | 39 -- .../expected.json | 71 -- .../physical_type_narrowing/expected.json | 74 -- .../property_addition/expected.json | 71 -- .../property_removal/expected.json | 82 --- .../relationship_removal/expected.json | 77 --- .../required_tightening/expected.json | 74 -- .../retired_contract_mutation/expected.json | 75 -- .../schema_addition/expected.json | 78 --- .../schema_removal/expected.json | 87 --- .../validation_failure/expected.json | 55 -- .../v1/review_multi_deploy.json | 132 ---- tests/interfaces/test_cli_outcomes.py | 4 +- .../test_cli_subprocess_outcomes.py | 79 --- tests/interfaces/test_deployment_cmd.py | 372 +++++----- tests/interfaces/test_reconcile_cmd.py | 2 +- tests/interfaces/test_release_plan_cmd.py | 101 ++- tests/interfaces/test_release_workflow_cmd.py | 101 +++ tests/test_application_architecture.py | 27 +- tests/test_architecture_consolidation.py | 227 ++---- tests/test_change_context.py | 40 +- tests/test_change_set.py | 29 +- tests/test_changeset_history.py | 24 +- tests/test_cicd_workflow_regression.py | 516 ++++++++++++++ .../test_contractops_artifact_rehydration.py | 72 +- tests/test_contractops_authorization.py | 5 +- tests/test_contractops_execution.py | 331 ++------- tests/test_contractops_golden_scenarios.py | 559 ++------------- tests/test_databricks_convergence_e2e.py | 308 +++------ tests/test_delta_import_integration.py | 4 +- tests/test_deployment_databricks.py | 103 +-- tests/test_deployment_history.py | 285 -------- tests/test_deployment_orchestrator.py | 40 +- tests/test_deployment_plan.py | 177 +++-- tests/test_deployment_service.py | 63 +- tests/test_deployment_verification.py | 16 +- tests/test_deployment_workflow_service.py | 461 +++++++++++++ tests/test_evolution_chain.py | 297 -------- ...test_fail_closed_governance_integration.py | 187 +---- tests/test_governance_analysis_boundary.py | 10 +- tests/test_governance_change_integration.py | 13 +- tests/test_governance_decision.py | 15 +- tests/test_governance_gate.py | 3 - tests/test_governance_golden_scenarios.py | 69 +- tests/test_governance_proposal.py | 7 +- tests/test_governance_reason_codes.py | 6 +- tests/test_governance_service.py | 14 +- tests/test_history_golden_scenarios.py | 647 ------------------ tests/test_history_integrity.py | 7 - tests/test_history_repository.py | 6 +- tests/test_identity_integration.py | 4 +- .../test_lifecycle_effective_scope_matrix.py | 11 - .../test_lifecycle_fail_closed_governance.py | 34 +- tests/test_lifecycle_status_resolver.py | 13 +- tests/test_operational_history.py | 64 ++ tests/test_operational_history_config.py | 79 +++ tests/test_pipeline_manifest.py | 5 +- tests/test_public_governance_decision.py | 86 +-- tests/test_reconciliation_service.py | 2 +- tests/test_release_history.py | 302 -------- tests/test_release_plan.py | 9 +- tests/test_release_planning_service.py | 6 +- tests/test_release_workflow_service.py | 170 +++++ tests/test_retired_immutability.py | 147 +--- tests/test_runtime_history.py | 287 -------- tests/test_semapact_cli_readable.py | 297 -------- .../test_semapact_devops_examples_readable.py | 82 ++- tests/test_semapact_devops_readable.py | 5 - ...mapact_devops_release_workflow_readable.py | 331 --------- .../test_semapact_editor_contract_readable.py | 10 +- tests/test_semapact_lifecycle_cli.py | 2 +- ...emapact_lifecycle_merge_engine_readable.py | 2 +- tests/test_semapact_merge_engine.py | 3 +- ...semapact_orchestrator_pipeline_readable.py | 4 +- ...test_semapact_release_workflow_readable.py | 131 ---- tests/test_version_authority_service.py | 2 +- 189 files changed, 6692 insertions(+), 11230 deletions(-) create mode 100644 docs/configuration.md delete mode 100644 examples/azure-devops/semapact-release.yml delete mode 100644 examples/ci/release.example.sh create mode 100644 examples/github/central-contract-repo-ci-cd.yml create mode 100644 examples/github/data-product-ci-cd.yml delete mode 100644 examples/github/semapact-release.yml delete mode 100644 examples/release/release-manifest.example.json create mode 100644 schemas/semapact-config.schema.json create mode 100644 semapact/application/models/deployment_workflow.py delete mode 100644 semapact/application/services/deployment_history.py create mode 100644 semapact/application/services/deployment_workflow.py create mode 100644 semapact/application/services/release_approval.py delete mode 100644 semapact/application/services/release_history.py create mode 100644 semapact/application/services/release_workflow.py create mode 100644 semapact/application/services/repository_classification.py delete mode 100644 semapact/application/services/runtime_history.py create mode 100644 semapact/core/config_schema.py delete mode 100644 semapact/core/editor_contract.py delete mode 100644 semapact/core/release.py delete mode 100644 semapact/deployment/authorization.py create mode 100644 semapact/deployment/provenance.py create mode 100644 semapact/deployment/source.py delete mode 100644 semapact/devops/release_workflow.py delete mode 100644 semapact/history/integrity.py create mode 100644 semapact/history/operational.py create mode 100644 semapact/history/operational_registry.py create mode 100644 semapact/interfaces/parsing.py create mode 100644 semapact/platforms/delta/__init__.py create mode 100644 semapact/platforms/delta/operational_history.py create mode 100644 semapact/platforms/sqlite/__init__.py create mode 100644 semapact/platforms/sqlite/operational_history.py delete mode 100644 semapact/services/__init__.py delete mode 100644 semapact/services/deployment_service.py delete mode 100644 semapact/services/governance_service.py delete mode 100644 semapact/services/reconciliation_service.py delete mode 100644 semapact/services/release_models.py delete mode 100644 semapact/services/release_planning_service.py delete mode 100644 semapact/services/version_authority_service.py delete mode 100644 tests/fixtures/governance_decisions/allow_clean_decision.json delete mode 100644 tests/fixtures/governance_decisions/block_retired_decision.json delete mode 100644 tests/fixtures/governance_decisions/block_validation_decision.json delete mode 100644 tests/fixtures/governance_decisions/breaking_review_decision.json delete mode 100644 tests/fixtures/governance_decisions/review_deprecate_decision.json delete mode 100644 tests/fixtures/governance_scenarios/active_to_retired_transition/expected.json delete mode 100644 tests/fixtures/governance_scenarios/contract_id_change/expected.json delete mode 100644 tests/fixtures/governance_scenarios/decimal_precision_reduction/expected.json delete mode 100644 tests/fixtures/governance_scenarios/decimal_scale_reduction/expected.json delete mode 100644 tests/fixtures/governance_scenarios/decimal_widening/expected.json delete mode 100644 tests/fixtures/governance_scenarios/deprecated_entity_change/expected.json delete mode 100644 tests/fixtures/governance_scenarios/descriptive_metadata_only/expected.json delete mode 100644 tests/fixtures/governance_scenarios/draft_entity_change/expected.json delete mode 100644 tests/fixtures/governance_scenarios/enum_reduction/expected.json delete mode 100644 tests/fixtures/governance_scenarios/logical_type_change/expected.json delete mode 100644 tests/fixtures/governance_scenarios/manual_version_change/expected.json delete mode 100644 tests/fixtures/governance_scenarios/merge_conflict/expected.json delete mode 100644 tests/fixtures/governance_scenarios/no_change/expected.json delete mode 100644 tests/fixtures/governance_scenarios/physical_name_identity_stability/expected.json delete mode 100644 tests/fixtures/governance_scenarios/physical_type_narrowing/expected.json delete mode 100644 tests/fixtures/governance_scenarios/property_addition/expected.json delete mode 100644 tests/fixtures/governance_scenarios/property_removal/expected.json delete mode 100644 tests/fixtures/governance_scenarios/relationship_removal/expected.json delete mode 100644 tests/fixtures/governance_scenarios/required_tightening/expected.json delete mode 100644 tests/fixtures/governance_scenarios/retired_contract_mutation/expected.json delete mode 100644 tests/fixtures/governance_scenarios/schema_addition/expected.json delete mode 100644 tests/fixtures/governance_scenarios/schema_removal/expected.json delete mode 100644 tests/fixtures/governance_scenarios/validation_failure/expected.json delete mode 100644 tests/fixtures/history_golden/v1/review_multi_deploy.json create mode 100644 tests/interfaces/test_release_workflow_cmd.py create mode 100644 tests/test_cicd_workflow_regression.py delete mode 100644 tests/test_deployment_history.py create mode 100644 tests/test_deployment_workflow_service.py delete mode 100644 tests/test_evolution_chain.py delete mode 100644 tests/test_history_golden_scenarios.py create mode 100644 tests/test_operational_history.py create mode 100644 tests/test_operational_history_config.py delete mode 100644 tests/test_release_history.py create mode 100644 tests/test_release_workflow_service.py delete mode 100644 tests/test_runtime_history.py delete mode 100644 tests/test_semapact_devops_release_workflow_readable.py delete mode 100644 tests/test_semapact_release_workflow_readable.py diff --git a/.semapact.yaml b/.semapact.yaml index b1f82569..f1e9f0b6 100644 --- a/.semapact.yaml +++ b/.semapact.yaml @@ -17,3 +17,4 @@ llm: model_name: gpt-4-turbo api_key: '' base_url: '' + diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 252cd38a..6155d253 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -2,9 +2,9 @@ ## Purpose -SemaPact is an ODCS-first, change-driven contract governance and production-assurance control plane. It separates governed contract semantics, release authorization, runtime mutation, and runtime verification so each concern has one authoritative owner. +SemaPact is an ODCS-first, change-driven contract governance and production-assurance control plane. It separates governed contract semantics, formal release publication, runtime deployment, and runtime verification so each concern has one authoritative owner. -The canonical product flow is: +The canonical product flow has two paths after governance: ```text ODCS base + candidate @@ -12,23 +12,49 @@ ODCS base + candidate lifecycle / governance ↓ GovernanceDecision - ↓ -ChangeSet → ReleasePlan → VersionResolution - ↓ -ContractOpsAuthorization - ↓ -AppliedContractRelease - ↓ -DeploymentPlan → DeploymentAuthorization - ↓ -DeploymentAdapter - ↓ -runtime - ↓ -observation / reconciliation + ├──────────────────────────────────────────────┐ + │ candidate deployment │ formal release + │ │ + ↓ ↓ +DeploymentSourceSnapshot(candidate) ChangeSet + ↓ ↓ +DeploymentPlan ReleasePlan + ↓ ↓ +CI-time DeploymentPreview VersionResolution + ↓ ↓ +DeploymentBundle ReleaseSnapshot + │ ↓ + │ ReleaseBundle + │ ↓ + │ PUBLISH approval + │ when REVIEW + │ ↓ + │ ContractRelease + │ ↓ + │ DeploymentSourceSnapshot(contract_release) + │ ↓ + │ DeploymentPlan + │ ↓ + │ CI-time DeploymentPreview + │ ↓ + └───────────────────────┬──────────────DeploymentBundle + ↓ + protected CD execution context + ↓ + fresh runtime observation + ↓ + fresh DeploymentPreview + ↓ + apply + ↓ + observation / reconciliation + ↓ + IN_SYNC / DRIFT / INDETERMINATE ``` -Later phases consume exact artifacts from earlier phases. They do not re-run governance, lifecycle classification, version authority, or approval semantics. +Later phases consume exact immutable artifacts from earlier phases. They do not re-run governance, version selection, approval semantics, or release identity. + +Formal release approval protects PUBLISH. Runtime deployment permission belongs to the surrounding protected CI/CD execution context. A `ContractRelease` is release provenance, not DEPLOY authorization. ## Dependency Direction @@ -55,13 +81,15 @@ Domain models live with the rules that give them meaning: - `semapact/lifecycle/` — canonical identity, lifecycle policy, merge/change semantics; - `semapact/governance/` — `GovernanceDecision`, reason codes, centralized gate; - `semapact/revision/` — immutable governed contract content identity and source-provenance links; -- `semapact/contractops/` — `ChangeSet`, `ReleasePlan`, `VersionResolution`, ContractOps authorization, APPLY/PUBLISH artifacts; -- `semapact/deployment/` — provider-neutral `DeploymentPlan`, `DeploymentPreview`, deployment authorization and adapter contract; +- `semapact/approval/` — immutable structured review evidence; +- `semapact/contractops/` — `ChangeSet`, `ReleasePlan`, `VersionResolution`, `ReleaseSnapshot`, publication authorization, and `ContractRelease`; +- `semapact/deployment/` — `DeploymentSourceSnapshot`, provider-neutral `DeploymentPlan`, `DeploymentPreview`, provenance validation, and adapter contracts; - `semapact/runtime/` — provider-neutral governed runtime asset projection; - `semapact/observation/` — provider-neutral point-in-time runtime state; -- `semapact/reconciliation/` — desired-vs-observed comparison and `RuntimeDriftStatus`. +- `semapact/reconciliation/` — desired-vs-observed comparison and `RuntimeDriftStatus`; +- `semapact/history/` — storage-neutral typed governance and operational-history ports/models. -A domain artifact does not move into the application or persistence layer merely because an application service returns it or a history backend stores it. +A domain artifact does not move into the application or persistence layer merely because an application service returns it or a backend stores it. ### Application layer @@ -73,39 +101,40 @@ semapact/application/ └── services/ ``` -`application/services/` owns thin, interface-independent use-case orchestration. It may resolve workflow context/configuration and compose existing domain functions or ports, but it must not reimplement domain policy. +`application/services/` owns thin, interface-independent use-case orchestration. It may resolve configuration and compose existing domain functions or ports, but it must not reimplement domain policy. `application/models/` owns typed use-case results that aggregate canonical domain artifacts. Examples include: - `GovernanceAnalysis`; - `GovernanceProposal`; - `RuntimeReconciliation`; -- `ReleasePlanningResult`. - -These are application DTOs, not new governance/release/deployment authorities. - -### Compatibility package +- `ReleasePlanningResult`; +- deployment workflow result DTOs. -`semapact/services/` is a backward-compatible import surface for the former package layout. It contains re-exports only and owns no models or business logic. New code must import from `semapact.application`. +These are application DTOs, not new governance, release, deployment, or reconciliation authorities. ### Interfaces -`semapact/interfaces/` owns parsing, loading input artifacts at the interface edge, rendering, and process-outcome mapping. Interfaces delegate to application/domain boundaries and must not independently calculate governance, version, deployment, or reconciliation results. +`semapact/interfaces/` owns argument parsing, loading input artifacts at the interface edge, rendering, and process-outcome mapping. Interfaces delegate to application/domain boundaries and must not independently calculate governance, versions, deployment plans, or reconciliation results. ### History persistence -`semapact/history/` owns storage-neutral typed persistence/query ports and persistence errors. It stores canonical artifacts from their owning domains but does not redefine their models, identity formulas, or business semantics. +Git is the low-frequency governance ledger for facts such as `ApprovalRecord` and `ContractRelease`. It is not the CI/CD artifact transport and not an operational telemetry database. + +Operational deployment telemetry is disabled by default and can be configured through typed `history.operational` settings with SQLite or Delta backends. High-frequency deployment/reconciliation events are never written to Git by the canonical workflow. ### Platform adapters -`semapact/platforms/` owns provider SDK/client integration and physical-platform translation. Databricks DDL generation/execution is provider behavior; it does not belong in ContractOps or application DTOs. Storage backend layout and physical persistence mechanics likewise belong to the corresponding platform adapter. +`semapact/platforms/` owns provider SDK/client integration and physical-platform translation. Databricks DDL generation/execution and Unity Catalog tag projection are provider behavior; they do not belong in ContractOps or application DTOs. -### Import/export and compatibility workflows +### Import/export and integration workflows - `semapact/importers/` projects explicitly imported external metadata into ODCS and contains no lifecycle policy; - runtime observation evidence is not an importer source of canonical ODCS semantics; evidence-assisted bootstrap or enrichment must produce an explicit proposal/candidate for normal governance rather than mutate a governed contract; - `semapact/exporters/` and `quality/` are read-only projections; -- `semapact/devops/` and parts of `core/` contain stable compatibility workflows and must not become a second canonical ContractOps implementation. +- `semapact/devops/` contains Git/CI integration helpers only and must not become a second canonical release implementation. + +SemaPact is treated as a greenfield architecture. Superseded package aliases, artifact schemas, and compatibility workflows are removed rather than retained as parallel public surfaces. ## Model Placement Rule @@ -115,10 +144,12 @@ Do not create a generic root `schema/`, `models/`, or `data_models/` directory t | --- | --- | | governed ODCS contract | ODCS model | | governed contract revision identity/provenance | `semapact/revision/` | -| governance/release/deployment/reconciliation artifact | owning domain package | +| structured review evidence | `semapact/approval/` | +| release artifact | `semapact/contractops/` or release application DTO owner | +| deployment source/plan/preview artifact | `semapact/deployment/` | | application/use-case aggregate result | `application/models/` | | application orchestration | `application/services/` | -| storage-neutral history persistence/query capability | `semapact/history/` | +| storage-neutral persistence/query capability | `semapact/history/` | | provider/SDK/physical representation | `platforms//` | | presentation-only rendering state | `interfaces/` | @@ -128,9 +159,7 @@ The fact that every object is “data” is not a useful architectural boundary. SemaPact reuses `OpenDataContractStandard` as the canonical logical contract model. It must not create a second contract representation merely to support governance, revision history, persistence, or deployment. -Domain artifacts may reference or wrap the canonical ODCS model while adding only semantics owned by that domain. For example, `ContractRevision` adds content identity around an exact `OpenDataContractStandard`; it does not duplicate `contract.id`, `contract.version`, schema fields, or a serialized contract copy as parallel logical fields. - -Canonical JSON may be derived transiently for deterministic hashing, signatures, persistence, or transport. That serialization is not a second logical contract model. +Domain artifacts may reference or serialize the canonical ODCS model while adding only semantics owned by that domain. Canonical JSON may be derived for deterministic hashing, persistence, or transport; that serialization is not a second logical contract model. ## Governed Identity @@ -145,34 +174,60 @@ property identity = lowercase(schema.name) + lowercase(property.name) ## Lifecycle and Governance -Lifecycle/governance is the sole authority for change meaning. Active entities participate in governance; draft/deprecated entities are excluded where policy specifies; retired state is immutable. Interfaces, application services, adapters, and exporters must not independently reinterpret these rules. +Lifecycle/governance is the sole authority for change meaning. Interfaces, application services, adapters, and exporters must not independently reinterpret those rules. + +A business effective date is mutation context only, for operations that materialize dated lifecycle state such as deprecation. Pure governance evaluation, release assessment, repository classification, and deployment assessment are date-independent. Governance produces one immutable `GovernanceDecision`. Downstream phases consume that decision and its projected artifacts rather than diffing again. -## ContractOps Release Boundary +## Formal Release Boundary -Canonical planning is: +Canonical formal release planning is: ```text base + candidate + exact revision refs ↓ -GovernanceDecision + ChangeSet +GovernanceDecision + ↓ +ChangeSet ↓ ReleasePlan ↓ VersionResolution + ↓ +ReleaseSnapshot + ↓ +ReleaseBundle ``` Version selection is separate from governance classification. SemaPact-managed and Git-managed authority both resolve through the canonical version-authority boundary. -APPLY materializes the exact released ODCS snapshot only after matching authorization. PUBLISH publishes a released artifact and is distinct from DEPLOY. +For `REVIEW`, an exact `ApprovalRecord` must bind the PUBLISH operation, exact `ReleaseSnapshot`, and exact `ReleaseBundle` digest. The preferred CI/CD handoff is the approval artifact itself; Git retains the same fact for audit/conflict detection and fallback lookup. + +Finalization creates one target-neutral `ContractRelease` and materializes the selected semantic version into the released ODCS snapshot. The same `ContractRelease` may subsequently be deployed to multiple targets without another version calculation. + +Candidate deployment bypasses this formal release path entirely. It does not calculate a release version, create release approval, or write release history. ## Deployment Boundary -Deployment planning starts from an exact `AppliedContractRelease`: +Deployment planning starts from one exact `DeploymentSourceSnapshot`: ```text -AppliedContractRelease + DeploymentTarget +candidate contract + governance decision + ↓ +DeploymentSourceSnapshot(source_kind=candidate) + +or + +ContractRelease + ↓ +DeploymentSourceSnapshot(source_kind=contract_release) +``` + +The source snapshot plus one exact target produces the deployment artifacts: + +```text +DeploymentSourceSnapshot + DeploymentTarget ↓ DeploymentPlan ↓ @@ -180,12 +235,34 @@ fresh runtime observation + adapter preview ↓ DeploymentPreview ↓ -exact plan + preview + DeploymentAuthorization +DeploymentBundle +``` + +The CI-time preview is review evidence only; CD never blindly replays it. At execution time: + +```text +protected CI/CD execution context + + +exact DeploymentBundle + ↓ +validate source / plan / bundle integrity + ↓ +fresh runtime observation + ↓ +fresh DeploymentPreview ↓ -DeploymentAdapter.execute(...) +apply supported operations + ↓ +fresh verification + ↓ +IN_SYNC / DRIFT / INDETERMINATE ``` -`DeploymentPlan` stays provider-neutral. Provider-native CREATE/ALTER/NO_OP operations begin at the adapter boundary. Runtime mutation must fail closed when capability or evidence is insufficient. +SemaPact does not create a `DeploymentAuthorization` artifact. Runtime execution permission is delegated to the surrounding protected environment. The deployment domain owns provenance validation and runtime freshness, not IAM approval policy. + +Candidate `BLOCK` decisions fail closed. Candidate `REVIEW` may still be deployed for validation/test because no formal publication occurs. + +For Databricks, a finalized formal release that reaches `IN_SYNC` also projects reserved SemaPact release provenance tags to governed Unity Catalog tables. Candidate deployments do not publish formal release/version tags. Provider execution success is not convergence proof. @@ -193,9 +270,9 @@ Provider execution success is not convergence proof. Observation captures platform-neutral runtime evidence. It never mutates ODCS or invokes governance. -Lineage is time-varying runtime evidence, not canonical contract truth. Table/column lineage and query history may support provenance, impact analysis, reconciliation, or an explicit contract proposal, but observation must not directly project them into authoritative ODCS fields such as `transformSourceObjects` or `transformLogic`. +Lineage is time-varying runtime evidence, not canonical contract truth. Table/column lineage and query history may support provenance, impact analysis, reconciliation, or an explicit contract proposal, but observation must not directly project them into authoritative ODCS fields. -Reconciliation compares governed desired state with fresh observation and yields the existing status vocabulary: +Reconciliation compares governed desired state with fresh observation and yields: ```text IN_SYNC @@ -207,7 +284,7 @@ Deployment verification reuses this same reconciliation authority rather than in ## Application Services -Application services exist only when a use case genuinely coordinates multiple domain/port calls. Current examples include governance context construction, configured version authority, canonical release planning, runtime reconciliation orchestration, and deployment orchestration. +Application services exist only when a use case genuinely coordinates multiple domain/port calls. Rules: @@ -220,13 +297,14 @@ Rules: ## Public Architecture Invariants -1. **Change-driven, not CRUD** — governed state evolves through explicit analysis/planning/authorization boundaries. -2. **One authority per rule** — lifecycle, governance, revision identity, version selection, deployment translation, and reconciliation each have one canonical owner. +1. **Change-driven, not CRUD** — governed state evolves through explicit analysis, release, deployment, and reconciliation boundaries. +2. **One authority per rule** — lifecycle, governance, version selection, release publication, deployment translation, and reconciliation each have one canonical owner. 3. **Exact artifacts cross boundaries** — side effects consume exact immutable artifacts; mutable current state is not silently substituted. -4. **Operation-scoped authorization** — APPLY, PUBLISH, and DEPLOY are distinct operations; authorization for one cannot authorize another. -5. **Logical identity is stable** — `physicalName` binds runtime state but does not redefine governed identity. -6. **Execution is not convergence** — runtime state must be observed and reconciled independently. -7. **Interfaces stay thin** — CLI/API/UI parse, delegate, and render; they do not become a second business-logic implementation. -8. **Compatibility is not ownership** — legacy import paths may re-export canonical implementations but must not accumulate new logic. -9. **One canonical contract model** — SemaPact reuses ODCS rather than maintaining a parallel contract schema. -10. **Evidence is not contract truth** — runtime evidence may inform analysis or an explicit governed proposal, but it never silently mutates canonical ODCS semantics. +4. **PUBLISH approval is not DEPLOY authority** — release approval protects formal publication; runtime mutation permission comes from the protected execution environment. +5. **Candidate deployment is not release** — it performs no release version calculation, approval persistence, or release-history write. +6. **Logical identity is stable** — `physicalName` binds runtime state but does not redefine governed identity. +7. **Execution is not convergence** — runtime state must be observed and reconciled independently. +8. **Interfaces stay thin** — CLI/API/UI parse, delegate, and render; they do not become a second business-logic implementation. +9. **Greenfield means one surface** — superseded aliases, legacy artifacts, and compatibility workflows are removed rather than maintained beside canonical paths. +10. **One canonical contract model** — SemaPact reuses ODCS rather than maintaining a parallel contract schema. +11. **Evidence is not contract truth** — runtime evidence may inform analysis or an explicit governed proposal, but it never silently mutates canonical ODCS semantics. diff --git a/README.md b/README.md index aff85e2c..c9b850e0 100644 --- a/README.md +++ b/README.md @@ -67,20 +67,32 @@ GovernanceDecision Governance Gate ``` -Canonical ContractOps then carries the exact governed decision forward: +The exact governed decision then feeds either candidate deployment or an explicit formal release: ```text GovernanceDecision → ChangeSet -→ ReleasePlan -→ VersionResolution -→ ContractOpsAuthorization -→ AppliedContractRelease -→ DeploymentPlan -→ DeploymentAuthorization -→ runtime mutation + ├─ candidate deployment + │ → DeploymentSourceSnapshot(source_kind=candidate) + │ → DeploymentPlan + │ → DeploymentBundle + │ + └─ formal release + → ReleasePlan + → VersionResolution + → ReleaseSnapshot + → ReleaseBundle + → ContractRelease + ↓ + + target + ↓ + DeploymentSourceSnapshot(source_kind=contract_release) + → DeploymentPlan + → DeploymentBundle ``` +Release approval protects formal publication. Runtime deployment permission belongs to the surrounding protected execution context; ContractRelease is provenance, not DEPLOY authorization. + For production assurance: ```text @@ -126,9 +138,9 @@ DRAFT → ACTIVE → DEPRECATED → RETIRED Lifecycle status does not itself mean that a revision has been authorized for release. -### Side effects are operation-scoped +### Side effects have explicit trust boundaries -SemaPact distinguishes analysis and planning from side effects. APPLY, PUBLISH, and DEPLOY are separate protected operations. A publication authorization cannot be reused as runtime deployment authority. +SemaPact distinguishes pure planning/materialization from external side effects. Formal release PUBLISH approval is recorded explicitly when governance requires review. Runtime DEPLOY permission belongs to the surrounding protected execution boundary (for example a GitHub or Azure DevOps Environment), not to the ContractRelease artifact. Release state therefore cannot authorize runtime mutation by itself. ### Runtime-aware without becoming platform-owned @@ -155,7 +167,7 @@ SemaPact currently supports deterministic change analysis and lifecycle-aware po - relationship change handling; - version-policy classification; - deterministic `GovernanceDecision` artifacts; -- centralized governance gates for ANALYZE / PROPOSE / APPLY / PUBLISH / DEPLOY / CI operations. +- centralized governance and release-approval boundaries, with runtime deployment permission delegated to the protected execution environment. ### Canonical ContractOps release planning @@ -177,33 +189,67 @@ SemaPact supports two version-authority modes: See [`docs/contractops_phases.md`](docs/contractops_phases.md) and [`docs/version_authority.md`](docs/version_authority.md). -### Governed runtime deployment +### Contract release and runtime deployment + +SemaPact treats formal contract release and runtime deployment as separate lifecycles. -`semapact deployment` exposes a canonical CLI/CI surface: +A formal release is target-neutral: ```text -plan -→ DeploymentPlan +release assess +→ GovernanceDecision +→ ChangeSet +→ ReleasePlan +→ VersionResolution +→ ReleaseSnapshot +→ ReleaseBundle + +REVIEW → exact release approval + ↓ +release finalize +→ ContractRelease +→ materialize the selected version back to ODCS +``` + +The selected version is calculated once. The resulting `ContractRelease` can then be deployed to any number of runtime targets without another version bump: + +```text +orders@1.4.0 +├── dev +├── test +└── prod +``` -preview -→ fresh runtime observation -→ DeploymentPreview +Deployment is target-specific: -execute -→ exact plan + preview + DeploymentAuthorization -→ provider execution +```text +deployment assess --release ./artifacts/contract-release.json +→ DeploymentSourceSnapshot +→ DeploymentPlan +→ fresh runtime preview +→ DeploymentBundle -verify -→ fresh observation + reconciliation +deployment deploy +→ fresh preview → execute → fresh verify → IN_SYNC / DRIFT / INDETERMINATE ``` -Planning and preview are read-only. Runtime mutation occurs only through `deployment execute`, and provider execution success is not treated as convergence proof. +Candidate/non-release deployment remains available directly from base + candidate and does not create a new contract version or release history. -The first Databricks write capability is intentionally narrow: create a missing managed Delta table, add missing nullable governed columns to an existing managed table, or perform NO_OP when the governed shape is already satisfied. Rename, existing-column type/nullability mutation, required-column addition without a safe migration strategy, DROP, and existing external/non-managed asset mutation fail closed. +Deployment telemetry is disabled by default. High-frequency execution history can be enabled once in typed `.semapact.yaml` configuration with a SQLite or Delta backend; it is not written into Git governance history. The `--operational-history` CLI option is only an override. -See [`docs/deployment_plans.md`](docs/deployment_plans.md). +For Databricks, once deployment of a finalized formal release verifies `IN_SYNC`, SemaPact projects the finalized release provenance to governed Unity Catalog tables using reserved tags: +```text +semapact_contract_id +semapact_contract_version +semapact_release_id +semapact_source_revision +``` + +Candidate deployments do not publish formal version/release tags. Business classifications or ABAC tags are not automatically mapped. + +See [`docs/deployment_plans.md`](docs/deployment_plans.md). ### Databricks discovery and observation With the `databricks` extra, SemaPact provides a thin read-side integration using the official Databricks SDK: @@ -329,15 +375,82 @@ pip install "semapact[databricks]" The Databricks SDK owns authentication-provider selection. SemaPact forwards supported connection hints rather than implementing a separate credential system. -After an exact `AppliedContractRelease` and deployment authorization have been produced, the runtime path is exposed through: +Assess a candidate deployment without creating a new contract version: + +```bash +semapact deployment assess \ + --base ./contracts/orders.yaml \ + --candidate ./contracts/orders.candidate.yaml \ + --base-revision-ref git:abc123 \ + --candidate-revision-ref git:def456 \ + --server development \ + --bundle-out ./artifacts/orders-dev.bundle.json +``` + +Build a target-neutral formal release separately: + +```bash +semapact release assess \ + --base ./contracts/orders.yaml \ + --candidate ./contracts/orders.candidate.yaml \ + --base-revision-ref git:abc123 \ + --candidate-revision-ref git:def456 \ + --bundle-out ./artifacts/orders.release.bundle.json +``` + +For REVIEW releases, record the exact external approval and then finalize the release: ```bash -semapact deployment plan --help -semapact deployment preview --help -semapact deployment execute --help -semapact deployment verify --help +semapact release approve \ + --bundle ./artifacts/orders.release.bundle.json \ + --actor-reference github-environment:contract-release \ + --recorded-at 2026-09-20T10:00:00+10:00 \ + --approval-out ./artifacts/orders.approval.json + +semapact release finalize \ + --bundle ./artifacts/orders.release.bundle.json \ + --approval ./artifacts/orders.approval.json \ + --output-contract ./contracts/orders.yaml \ + --release-out ./artifacts/orders.contract-release.json ``` +Finalization writes the selected semantic version back to the ODCS contract and records the immutable `ContractRelease` in the Git governance ledger. The release records the source revision from which it was derived; the finalized release identity is the immutable released-contract snapshot, not a claim that the source revision already contained the materialized version bump. + +Deployment then consumes the finalized release artifact directly: + +```bash +semapact deployment assess \ + --release ./artifacts/orders.contract-release.json \ + --server production \ + --bundle-out ./artifacts/orders-prod.deployment.bundle.json + +semapact deployment deploy \ + --bundle ./artifacts/orders-prod.deployment.bundle.json \ + --warehouse-id +``` + +Candidate deployments do not require release approval. + +Operational deployment history is configured project-wide rather than repeated on every deploy: + +```yaml +history: + operational: + backend: sqlite + path: .semapact/operational.db +``` + +For a shared Delta sink: + +```yaml +history: + operational: + backend: delta + table_uri: s3://governance/semapact/operational-history +``` + +The config is fail-closed against the typed `SemaPactConfigSchema`. `--operational-history` remains available only as a per-invocation override. If neither config nor override is present, operational persistence stays disabled. See [`docs/configuration.md`](docs/configuration.md) for precedence and schema details. + ## Optional Dependencies | Extra | Purpose | @@ -359,16 +472,15 @@ Optional extras are intentionally separate from the base distribution. If an int ```text semapact/ - core/ # loading, validation, compatibility workflow boundaries + core/ # loading, configuration and validation lifecycle/ # canonical identity, lifecycle and change policy governance/ # GovernanceDecision and centralized gate - contractops/ # deterministic release planning / authorization / apply / publish domain - deployment/ # provider-neutral deployment plans, authorization, preview contracts + contractops/ # deterministic release planning, approval and release artifacts + deployment/ # provider-neutral deployment source, plan and preview contracts runtime/ # provider-neutral governed runtime asset projection application/ # interface-independent use-case models + orchestration models/ # application result DTOs; no domain authority services/ # thin orchestration over canonical domain rules/ports - services/ # backward-compatible imports only observation/ # platform-neutral observed state + fingerprint reconciliation/ # governed desired vs observed comparison platforms/ # provider adapters such as Databricks @@ -376,12 +488,12 @@ semapact/ exporters/ # SQL / graph and other outputs quality/ # quality intent adapters interfaces/ # CLI and user-facing boundaries - devops/ # Git / CI compatibility helpers + devops/ # Git / CI integration helpers ``` A central architectural rule is: -> **Interfaces parse and render. Application services orchestrate. Domain packages own business meaning. Platform adapters own provider-specific effects. Compatibility packages do not become new owners.** +> **Interfaces parse and render. Application services orchestrate. Domain packages own business meaning. Platform adapters own provider-specific effects. There is one canonical workflow per lifecycle.** See [`ARCHITECTURE.md`](ARCHITECTURE.md) for package/model placement rules. diff --git a/docs/approval_history.md b/docs/approval_history.md index 88f3ff2d..db001917 100644 --- a/docs/approval_history.md +++ b/docs/approval_history.md @@ -102,6 +102,39 @@ record = service.record_review_action( Callers using another future persistence backend can provide the same `ApprovalHistoryRepository` capability without changing `ApprovalRecordService`. +## Release approval handoff and resolution + +Formal REVIEW approval belongs to the PUBLISH boundary, not to runtime deployment. + +The canonical CI/CD handoff is an exact `ApprovalRecord` artifact: + +```text +release assess +→ ReleaseBundle + +protected release environment +→ release approve --approval-out release.approval.json + +release finalize --approval release.approval.json +→ ContractRelease +``` + +The approval must match all of the following: + +- exact `decisionId`; +- exact `changeSetId`; +- exact `releasePlanId`; +- exact `versionResolutionId`; +- operation `PUBLISH`; +- exact `ReleaseSnapshot` as `scopeReference`; +- exact `ReleaseBundle` digest in `evidenceReferences`. + +`release approve` also persists the same approval in the Git governance ledger. That ledger is durable audit/conflict evidence, not the preferred in-pipeline transport. + +As a convenience fallback, `release finalize` may resolve the same exact PUBLISH approval from the Git ledger when `--approval` is omitted. If the exact scope contains conflicting review evidence such as `REQUEST_CHANGES` or `REJECT`, finalization fails closed. SemaPact does not apply a hidden "latest review wins" rule. + +Runtime deployment has no SemaPact deployment-approval artifact. Whether `semapact deployment deploy` may run is controlled by the surrounding protected CI/CD execution context. Candidate deployment therefore does not create or resolve release approval records, and a finalized `ContractRelease` is provenance rather than DEPLOY authority. + ## Trust boundary `ApprovalRecord` preserves review evidence; it is not an authentication credential. diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 00000000..9c1b0d79 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,59 @@ +# Configuration + +SemaPact loads configuration in this order: + +1. explicit CLI override for the current operation; +2. local project config at `.semapact.yaml`; +3. global config at `~/.config/semapact/config.yaml`; +4. feature default. + +Operational deployment history is disabled when no backend is configured. + +## Operational history + +SQLite: + +```yaml +history: + operational: + backend: sqlite + path: .semapact/operational.db +``` + +Delta: + +```yaml +history: + operational: + backend: delta + table_uri: s3://governance/semapact/operational-history +``` + +The configuration is validated fail closed. Supported backends require exactly their backend-specific fields: + +- `sqlite`: `backend` + `path`; +- `delta`: `backend` + `table_uri`. + +Unknown backends, missing required fields, and extra fields under `history.operational` are rejected. + +The canonical schema is defined by `SemaPactConfigSchema` and published for editor/tooling use at: + +```text +schemas/semapact-config.schema.json +``` + +## CLI override + +`--operational-history` is a per-invocation override, not the normal configuration path. + +For example: + +```bash +semapact deployment deploy \ + --bundle ./artifacts/orders.bundle.json \ + --operational-history sqlite:///./tmp/one-run.db +``` + +If the flag is omitted, SemaPact resolves `history.operational` from project/global config. If neither exists, operational history persistence remains disabled. + +Operational telemetry is separate from the Git governance ledger. Formal release/approval facts may be stored in Git, while high-frequency deployment events are written only to an explicitly configured SQLite or Delta backend. diff --git a/docs/contractops_authorization.md b/docs/contractops_authorization.md index e47f4b73..da5662b4 100644 --- a/docs/contractops_authorization.md +++ b/docs/contractops_authorization.md @@ -1,102 +1,91 @@ # ContractOps review authorization -SemaPact keeps governance classification and review authorization as separate immutable facts. +SemaPact keeps governance classification and formal-release approval as separate immutable facts. + +The current canonical use of ContractOps authorization is the PUBLISH boundary for a formal release: ```text GovernanceDecision(REVIEW) + -exact version-resolved release context +ChangeSet + + +ReleasePlan + + +VersionResolution + -matching explicit approval evidence +matching explicit PUBLISH approval evidence ↓ ContractOpsAuthorization(allowed=true) + ↓ +ContractRelease ``` The original `GovernanceDecision` remains `REVIEW`. Approval never rewrites it to `ALLOW`. ## Why authorization happens after version resolution -A review must authorize the release that will actually be executed, not an earlier approximation of it. +A reviewer must approve the release that will actually be published, including the selected semantic version. ```text GovernanceDecision → ChangeSet → ReleasePlan → VersionResolution -→ ContractOpsAuthorization -→ APPLY / PUBLISH / DEPLOY +→ ReleaseSnapshot +→ ReleaseBundle +→ PUBLISH approval when REVIEW +→ ContractRelease ``` -Review evidence is scoped to all of: +Review evidence is scoped to the exact release context: ```text decisionId changeSetId releasePlanId versionResolutionId -operation -scopeReference? # optional downstream scope +operation = PUBLISH +scopeReference = releaseSnapshotId +evidenceReference = releaseBundleDigest ``` -Changing the proposal, release plan, selected version, requested operation, or an explicitly bound downstream scope invalidates the evidence for that new action. +Changing the proposal, release plan, selected version, release snapshot, or bundle invalidates the approval for the new release. ## Decision behavior -| Governance gate result | Review evidence | Authorization | +| Governance gate result | Review evidence | Publication authorization | | --- | --- | --- | | `allowed` | not required | allowed by governance | -| `review_required` | missing | denied: authorization required | +| `review_required` | missing | denied: approval required | | `review_required` | exact `APPROVE` | allowed by review | | `review_required` | `REJECT` / `REQUEST_CHANGES` | denied | | `review_required` | stale or mismatched | denied | | `blocked` | any | denied; review cannot override BLOCK | -`GovernanceGateResult` remains authoritative for whether the operation is already allowed, requires review, or is blocked. ContractOps authorization only satisfies `review_required`; it does not re-run policy. +`GovernanceGateResult` remains authoritative for whether publication is already allowed, requires review, or is blocked. ContractOps authorization satisfies the review requirement; it does not re-run policy. ## Explicit evidence, not comment parsing -ContractOps consumes structured evidence: - -```text -ReviewAuthorizationEvidence -├── evidenceReference -├── decisionId -├── changeSetId -├── releasePlanId -├── versionResolutionId -├── operation -├── scopeReference? -└── action -``` - -`evidenceReference` is opaque. `scopeReference`, when present, is also opaque to ContractOps and may be used by a downstream boundary to bind approval to an exact target-specific artifact such as a deployment plan. - -This layer does not store approvals or infer approval from human comments, PR text, Slack messages, or similar free-form content. Durable review history and reviewer workflow are separate persistence/application concerns and can project structured evidence into this boundary without changing its authorization semantics. +ContractOps consumes structured `ReviewAuthorizationEvidence`. The evidence reference and scope reference are opaque identifiers whose exact values are produced by the release workflow. -## Operation scope +This layer does not parse human comments, PR text, Slack messages, or similar free-form content. `ApprovalRecord` is the durable structured fact; Git history is an audit/conflict ledger and the exact approval artifact is the preferred CI/CD handoff. -Approval is not a generic bypass token. +## Runtime deployment is a separate trust boundary -```text -approval for APPLY -≠ approval for PUBLISH -≠ approval for DEPLOY -``` +A `ContractRelease` proves what was formally released. It does not grant permission to mutate a runtime. -Likewise, approval for one `VersionResolution` does not authorize a different selected version. +SemaPact does not create a `DeploymentAuthorization` artifact. Runtime execution permission belongs to the surrounding protected CI/CD environment, such as a GitHub or Azure DevOps Environment. The deployment domain instead validates exact source/plan provenance, re-observes runtime state, derives a fresh preview, applies only supported transitions, and verifies convergence. -Runtime deployment adds another scope boundary: a review-required deployment must bind approval to the exact `DeploymentPlan`, so approval for one runtime target cannot be rebound to another target. +Candidate deployment likewise does not create release approval or release history. `BLOCK` fails closed; `REVIEW` may still be used for non-release validation/test deployment because no formal publication occurs. -## Invalid context vs denied authorization +## Invalid context vs denied publication -SemaPact distinguishes malformed artifact composition from a valid release that simply lacks approval. +SemaPact distinguishes malformed release composition from a valid release that lacks required approval. If `GovernanceDecision`, `ChangeSet`, `ReleasePlan`, and `VersionResolution` do not refer to the same immutable release context, authorization fails closed with a validation error. -If the release context is valid but evidence is missing, rejected, stale, or mismatched, SemaPact returns a deterministic `ContractOpsAuthorization` with `allowed=false` and a machine-readable reason. - -This lets APPLY, PUBLISH, and DEPLOY boundaries consume one authoritative release-context authorization result without implementing their own approval rules. +If the release context is valid but required PUBLISH evidence is missing, rejected, stale, or mismatched, SemaPact returns a deterministic `ContractOpsAuthorization` with `allowed=false` and a machine-readable reason. ## Non-goals -This boundary does not implement approval persistence, reviewer routing, quorum, IAM/SSO, approval UI, GitHub review parsing, or BLOCK overrides. +This boundary does not implement approval persistence mechanics, reviewer routing, quorum, IAM/SSO, approval UI, GitHub review parsing, runtime deployment permission, or BLOCK overrides. diff --git a/docs/contractops_phases.md b/docs/contractops_phases.md index 5160c859..3c55a509 100644 --- a/docs/contractops_phases.md +++ b/docs/contractops_phases.md @@ -4,31 +4,39 @@ ContractOps separates reasoning from side effects so CI, agents, APIs, and user ## Canonical contract release flow +Contract release is explicit and distinct from ordinary runtime deployment. + +Candidate deployment does not enter the version-resolution path: + +```text +ANALYZE +→ GovernanceDecision +→ ChangeSet +→ candidate DeploymentSourceSnapshot +→ DeploymentPlan +→ runtime deployment +``` + +A formal release enters ContractOps release planning exactly once: + ```text ANALYZE → GovernanceDecision -PLAN +PLAN RELEASE → ChangeSet → ReleasePlan → VersionResolution +→ ReleaseSnapshot -AUTHORIZE -→ ContractOpsAuthorization - -APPLY -→ AppliedContractRelease - -PUBLISH -→ PublicationResult +FORMAL RELEASE +→ immutable ContractRelease DEPLOY -→ DeploymentPlan + DeploymentAuthorization -→ runtime mutation through a platform adapter +→ one or more target-specific DeploymentPlan / deployment occurrences ``` -Each phase consumes artifacts from the previous phases. Later phases do not recalculate earlier decisions. - +The same formal contract version may be deployed to dev, test, and prod without another version bump. Runtime promotion is not a new contract release. ## The contract is the desired-state artifact SemaPact does not introduce a canonical BUILD phase that turns a contract into a separately authoritative DDL artifact. @@ -37,11 +45,13 @@ The governed contract remains the model of desired state throughout the lifecycl ```text candidate ODCS - ↓ ANALYZE / PLAN / AUTHORIZE / APPLY -AppliedContractRelease -= immutable governed desired state - ↓ DEPLOY planning against one runtime target -provider-native operations + ↓ ANALYZE / PLAN +ReleaseSnapshot += immutable selected governed desired state + ↓ target-specific DeploymentPlan +fresh runtime observation + ↓ +provider-native preview operations ``` A SQL or provider-specific export is a **derived compilation output**, not a second source of truth and not deployment authority. It may be regenerated from the exact governed contract state whenever required. @@ -54,7 +64,7 @@ base contract ↔ candidate contract → governance changes, breaking classification, version requirements Runtime deployment comparison -AppliedContractRelease ↔ observed runtime state +ReleaseSnapshot / DeploymentPlan ↔ observed runtime state → CREATE / ALTER / NO_OP or an explicit unsupported transition ``` @@ -138,71 +148,57 @@ operation scopeReference? ``` -`GovernanceDecision(REVIEW)` remains `REVIEW` after approval. Matching explicit review evidence produces an allowed `ContractOpsAuthorization`; it does not rewrite governance history. +`GovernanceDecision(REVIEW)` remains `REVIEW` after approval. Formal-release approval is represented by an immutable `ApprovalRecord` bound to the exact `ReleaseSnapshot` and `ReleaseBundle` digest. `ReleaseFinalizer` projects that evidence into the PUBLISH authorization check without rewriting governance history. -APPLY, PUBLISH, and DEPLOY are distinct operation scopes. Runtime DEPLOY additionally binds authorization to the exact `DeploymentPlan` before mutation. +Runtime deployment is a separate boundary. SemaPact does not create a deployment-authorization artifact; the surrounding protected CI/CD environment controls whether runtime mutation may be invoked, while SemaPact validates exact source/plan binding and runtime freshness. -## APPLY +## RELEASE SNAPSHOT -APPLY materializes the exact released ODCS state from the planned candidate and `VersionResolution.selectedVersion`. +Release snapshot construction is pure. It materializes the exact selected ODCS state from the planned candidate and `VersionResolution.selectedVersion` without crossing an external side-effect boundary. -The canonical APPLY path: +`build_release_snapshot(...)`: -- requires an allowed `ContractOpsAuthorization(operation=APPLY)`; - requires the supplied candidate revision reference to match the planned revision; - validates contract identity and the expected current version; - copies the candidate and synchronizes only the selected release version; - does not mutate the input candidate; -- does not rerun diffing, lifecycle policy, breaking-change classification, or version authority. - -The output is an immutable `AppliedContractRelease` containing provenance IDs and a canonical JSON snapshot of the released ODCS state. Consumers can materialize a fresh ODCS model from that snapshot. +- does not rerun governance or version authority; +- produces a deterministic `ReleaseSnapshot` with no authorization ID. -This also covers metadata-only governed releases: `requiredVersionBump=none` may have been resolved by SemaPact version authority to an actual patch release, and APPLY uses that already-selected version directly. +This pure snapshot is used only for formal `--release` bundles. Candidate deployment freezes the candidate directly in a non-release `DeploymentSourceSnapshot` and does not calculate a new semantic version. ## PUBLISH -PUBLISH publishes an applied contract release or release artifact. It is distinct from runtime deployment. +PUBLISH is the formal-release authorization boundary. For a REVIEW decision, the approval must bind the exact `ReleaseSnapshot` and `ReleaseBundle` digest. For ALLOW, no explicit approval record is required. -It requires an allowed `ContractOpsAuthorization(operation=PUBLISH)` matching the exact applied release context before the publisher adapter is invoked. An APPLY authorization cannot authorize PUBLISH. +After that check, `ReleaseFinalizer` constructs the immutable, target-neutral `ContractRelease`. The CLI persists that release fact to the Git governance ledger and may also emit the `ContractRelease` JSON artifact for CI/CD handoff. -ContractOps defines only a narrow publisher port: - -```text -AppliedContractRelease - ↓ -ContractReleasePublisher.publish(...) - ↓ -opaque publication reference - ↓ -PublicationResult -``` - -Git, storage, and other release-artifact publication behavior belongs in adapters rather than the ContractOps domain. +PUBLISH does not authorize runtime mutation. A `ContractRelease` records what was formally released; deployment remains a separate protected execution boundary. ## DEPLOY -DEPLOY mutates a runtime toward a provider-neutral `DeploymentPlan` and is separate from release publication. +DEPLOY converges one runtime toward an exact target-specific `DeploymentPlan`. The desired state comes from one immutable `DeploymentSourceSnapshot`: -A release-context `ContractOpsAuthorization(operation=DEPLOY)` is not enough on its own. Runtime execution also requires a `DeploymentAuthorization` bound to the exact deployment plan, including its target. A review approval scoped to one deployment plan therefore cannot be rebound to another target. +- `source_kind=candidate` for validation/test deployment without a formal release; +- `source_kind=contract_release` for deployment of an already-finalized `ContractRelease`. -Platform-specific execution belongs behind a deployment adapter. The adapter must not recompute governance, version authority, release planning, or approval semantics. +Candidate `BLOCK` decisions fail closed. Candidate `REVIEW` may still be deployed for non-release validation/test because this path creates no formal release fact or release approval. -See [`deployment_plans.md`](deployment_plans.md) for the deployment CLI, Databricks capability boundary, preview integrity checks, and convergence verification semantics. +Formal REVIEW approval belongs to the earlier PUBLISH boundary. A `ContractRelease` proves what was released; it does not grant permission to mutate a runtime. Whether `semapact deployment deploy` may run is controlled by the surrounding protected execution context, such as a GitHub or Azure DevOps Environment. -## Failure semantics +The deployment adapter validates exact plan/source binding, re-observes runtime, derives a fresh deterministic preview, checks runtime freshness, applies safe operations, then verifies convergence. For Databricks, a finalized release that reaches `IN_SYNC` also projects SemaPact-owned release/version provenance into Unity Catalog tags. -ContractOps distinguishes invalid context from denied authorization: +Runtime deployment history is optional operational telemetry and is not written to Git by default. -- mismatched revision/artifact/operation context → `ReleaseValidationError`; -- a valid context whose explicit authorization is denied → `ContractOpsAuthorizationError`; -- external publishers or runtime adapters are never invoked when authorization validation fails. +See [`deployment_plans.md`](deployment_plans.md) for the complete candidate/release CLI flow, operational-history backends, Databricks tags, and convergence semantics. -Unexpected publisher/runtime failures are not converted into governance decisions; they propagate as execution failures. +## Failure semantics -## Compatibility helpers +ContractOps distinguishes invalid release context from denied publication authorization: -`semapact release prepare` and `semapact release create-pr` remain compatibility workflows for existing Git-based release processes. They are not the canonical ContractOps PLAN/APPLY/PUBLISH path and should not be treated as equivalent to `release plan` plus explicit authorization. +- mismatched release revision/artifact context → `ReleaseValidationError`; +- a valid formal release whose required PUBLISH approval is denied/missing → `ContractOpsAuthorizationError`; +- runtime deployment fails closed on invalid source/plan provenance, stale runtime evidence, or unsupported provider transitions. -`semapact.core.release.prepare_release_candidate()` likewise remains a backward-compatible helper and is not the canonical ContractOps APPLY path because it may classify changes itself. +Unexpected persistence/runtime failures are not converted into governance decisions; they propagate as execution failures. -New ContractOps flows consume the existing authoritative `GovernanceDecision`, `ChangeSet`, `ReleasePlan`, and `VersionResolution` instead of recomputing them. diff --git a/docs/deployment_plans.md b/docs/deployment_plans.md index 38bb1f79..36c8ff3f 100644 --- a/docs/deployment_plans.md +++ b/docs/deployment_plans.md @@ -5,8 +5,8 @@ SemaPact treats a released ODCS contract as governed desired state, not as an ex The released contract is the authoritative artifact. SemaPact does **not** require a separate DDL build artifact before deployment. SQL and other provider-native commands are derived only after an exact released desired state is compared with an exact runtime target. ```text -AppliedContractRelease - = governed desired state +Candidate contract or finalized ContractRelease + = exact governed desired state + ObservedPlatformState = point-in-time runtime state @@ -23,13 +23,11 @@ This matters because one release may require different native operations in diff The deployment planning boundary is therefore: ```text -AppliedContractRelease +DeploymentSourceSnapshot + DeploymentTarget ↓ DeploymentPlan ↓ -DeploymentAuthorization - ↓ DeploymentService ↓ DeploymentAdapter interface @@ -42,8 +40,8 @@ DeploymentOrchestrator → transition → compile → preview - → freshness / exact authorization - → execute + → freshness validation + → apply → verify ↓ provider NativeOperationExecutor / RuntimeProvider @@ -55,9 +53,9 @@ The orchestration above is provider-neutral. Platform packages configure or impl ## What a DeploymentPlan means -A `DeploymentPlan` is a deterministic, provider-neutral statement of the runtime state that an exact applied contract release intends to converge toward. +A `DeploymentPlan` is a deterministic, provider-neutral statement of the runtime state that one exact deployment source intends to converge toward. -It is built only from `AppliedContractRelease`; drafts and raw candidate contracts are not deployment authority. +Canonical plans use schema version 1 and bind only the immutable `DeploymentSourceSnapshot` plus target-specific convergence actions. Release provenance and candidate-versus-release mode live only in the source snapshot; they are not duplicated in `DeploymentPlan`. The initial action vocabulary deliberately contains only: @@ -140,124 +138,316 @@ DeploymentTarget `platform` is the downstream adapter dispatch key. `runtimeTarget` is an opaque provider-local product target. `sourceReference` identifies the exact runtime source/end point against which the plan is authorized; for Databricks this is the workspace host used by runtime observation. It is an identity reference, never a credential. -Preview, execution, and verification fail closed when fresh runtime evidence comes from a different source than the plan's `sourceReference`. This prevents an authorization for the same catalog/schema name from being reused against another workspace. +Preview, execution, and verification fail closed when fresh runtime evidence comes from a different source than the plan's `sourceReference`. This prevents a plan or bundle prepared for one catalog/schema binding from being reused against another workspace. The plan does not contain credentials, workspace clients, SQL connections, or provider sessions. +## ReleaseBundle and DeploymentBundle boundaries + +Formal contract release and runtime deployment are separate lifecycles with separate immutable handoff artifacts. + +A formal release is target-neutral: + +```text +base + candidate + ↓ +GovernanceDecision +→ ChangeSet +→ ReleasePlan +→ VersionResolution +→ ReleaseSnapshot +→ ReleaseBundle +``` + +`ReleaseBundle` contains no runtime target, workspace, catalog, schema, warehouse, or deployment preview. It freezes exactly what is intended to become a released contract. + +For REVIEW decisions, approval binds: + +```text +PUBLISH ++ exact ReleaseSnapshot ID ++ exact ReleaseBundle digest +``` + +Finalization then creates the formal release fact and materializes the selected semantic version back into ODCS: + +```text +ReleaseBundle ++ exact approval when REVIEW + ↓ +release finalize + ↓ +ContractRelease ++ versioned ODCS contract +``` + +A finalized release is environment-neutral. One release can subsequently fan out to multiple runtime targets without another version bump: + +```text +ContractRelease orders@1.4.0 +├── dev +├── test +└── prod +``` + +Deployment begins only after the release exists, or directly from an unreleased candidate for validation/test use. + +Candidate deployment: + +```text +base + candidate +→ GovernanceDecision + ChangeSet +→ DeploymentSourceSnapshot(source_kind=candidate) +→ target-specific DeploymentPlan +→ fresh DeploymentPreview +→ DeploymentBundle +``` + +Finalized-release deployment: + +```text +ContractRelease +→ DeploymentSourceSnapshot(source_kind=contract_release) +→ target-specific DeploymentPlan +→ fresh DeploymentPreview +→ DeploymentBundle +``` + +A release-mode `DeploymentBundle` therefore carries the exact finalized `ContractRelease`, not `ReleasePlan`, `VersionResolution`, or `ReleaseSnapshot`. Deployment never creates or versions a contract release. + ## CLI workflow -The deployment CLI consumes and emits canonical JSON artifacts. Planning and preview are read-only; `execute` is the runtime mutation boundary. +### Candidate deployment -### Plan +Candidate deployment remains a direct `assess → deploy` path: ```bash -semapact deployment plan \ - --release ./artifacts/applied-release.json \ - --platform databricks \ - --runtime main.sales \ - --source-reference https://dbc-example.cloud.databricks.com +semapact deployment assess \ + --base ./contracts/orders.yaml \ + --candidate ./contracts/orders.candidate.yaml \ + --base-revision-ref git:abc123 \ + --candidate-revision-ref git:def456 \ + --server development \ + --bundle-out ./artifacts/orders-dev.deployment.bundle.json + +semapact deployment deploy \ + --bundle ./artifacts/orders-dev.deployment.bundle.json \ + --warehouse-id ``` -`--source-reference` must match the stable source identity reported by the runtime provider. Optional `--server` preserves the selected contract-server reference as target provenance. +This does not calculate another semantic version, create release approval, or write release history. REVIEW may proceed for candidate runtime validation; BLOCK always fails closed. -The output is the canonical `DeploymentPlan` JSON. +### Formal release -### Preview +Build the target-neutral release artifact first: ```bash -semapact deployment preview \ - --plan ./artifacts/deployment-plan.json +semapact release assess \ + --base ./contracts/orders.yaml \ + --candidate ./contracts/orders.candidate.yaml \ + --base-revision-ref git:abc123 \ + --candidate-revision-ref git:def456 \ + --bundle-out ./artifacts/orders.release.bundle.json ``` -Preview observes the exact target scope and derives a canonical `DeploymentPreview`. It does not mutate runtime and does not require a Databricks SQL warehouse merely to inspect provider-native operations. +For a REVIEW release, an external workflow may persist exact approval with the convenience command: -### Execute +```bash +semapact release approve \ + --bundle ./artifacts/orders.release.bundle.json \ + --actor-reference github-environment:contract-release \ + --recorded-at 2026-09-20T10:00:00+10:00 \ + --approval-out ./artifacts/orders.approval.json +``` + +The approval is `PUBLISH`-scoped to the exact `ReleaseSnapshot` and `ReleaseBundle` digest. The ApprovalRecord artifact is the CI/CD handoff; Git remains the durable governance ledger and is consulted only for conflict detection or fallback resolution. ALLOW releases do not require an ApprovalRecord. -For a preview containing CREATE or ALTER operations, provide the SQL warehouse used for mutation: +Finalize exactly once: ```bash -semapact deployment execute \ - --plan ./artifacts/deployment-plan.json \ - --preview ./artifacts/deployment-preview.json \ - --authorization ./artifacts/deployment-authorization.json \ - --warehouse-id +semapact release finalize \ + --bundle ./artifacts/orders.release.bundle.json \ + --approval ./artifacts/orders.approval.json \ + --output-contract ./contracts/orders.yaml \ + --release-out ./artifacts/orders.contract-release.json ``` -For an all-`NO_OP` preview, `--warehouse-id` may be omitted because no native mutation is executed. Any CREATE/ALTER attempt without a warehouse fails closed. +Finalization: -Execution requires the exact plan, exact preview, and exact `DeploymentAuthorization`. The adapter re-observes the target, validates the authorized runtime source and observation fingerprint, re-derives the expected preview for integrity/freshness validation, and executes only the supplied operations when the artifacts still match. +- authorizes the exact formal release; +- writes one immutable `ContractRelease` to the Git governance ledger; +- writes the selected version back to the ODCS contract file. -Provider execution success is not convergence proof. +After finalization, hand the immutable ContractRelease artifact directly to target-specific deployment: -Complete DDL export remains a useful inspection or integration utility, especially for creating new assets, but export is not a lifecycle phase. Exported SQL is derived output; deployment planning remains responsible for comparing the exact released contract with fresh runtime evidence before any mutation is authorized. +```bash +semapact deployment assess \ + --release ./artifacts/orders.contract-release.json \ + --server production \ + --bundle-out ./artifacts/orders-prod.deployment.bundle.json +``` -### Verify +The same ContractRelease artifact may be assessed/deployed against dev, test, prod, or another runtime target. `--release-id` remains a convenience fallback for resolving the same artifact from the Git governance ledger; Git history is not the canonical CI/CD transport. -```bash -semapact deployment verify \ - --plan ./artifacts/deployment-plan.json \ - --output json +### CI/CD handoff rules + +Two immutable boundaries are now explicit: + +```text +CI release planning +→ ReleaseBundle +→ approval/finalize +→ ContractRelease + +runtime planning +ContractRelease + target +→ DeploymentBundle +→ fresh CD execution ``` -Verification enters through the same DeploymentAdapter / DeploymentOrchestrator boundary, performs fresh runtime observation, and reuses the normal reconciliation semantics with the same platform schema mapper used by preview: +CD must not rebuild either artifact from mutable repository state after its trust boundary. Deployment always re-observes runtime before mutation, so the CI-time preview remains review evidence only and is never replayed blindly. -| Runtime status | Exit code | -| --- | ---: | -| `IN_SYNC` | `0` | -| `DRIFT` | `6` | -| `INDETERMINATE` | `7` | +At execution time: -This keeps execution status separate from convergence evidence. VERIFY is assurance/read-side behavior: provider mutation restrictions such as Databricks MANAGED-only writes do not prevent SemaPact from verifying an observable non-managed asset. +```text +protected CI/CD execution boundary +→ exact DeploymentBundle +→ validate candidate/finalized-release provenance +→ fresh runtime observation +→ fresh DeploymentPreview +→ apply +→ fresh reconciliation +→ IN_SYNC / DRIFT / INDETERMINATE +``` + +## Governance ledger vs operational history + +SemaPact deliberately separates low-frequency governance facts from high-frequency deployment telemetry. + +### Git governance ledger + +Git-backed history is appropriate for durable reviewable facts such as: + +```text +ApprovalRecord +ContractRelease +``` + +`semapact release finalize` creates one immutable, target-neutral `ContractRelease` containing the released contract version, the exact source revision from which the release was derived, and release-artifact identities. The released contract snapshot itself is canonical; the source revision is provenance and need not already contain the materialized version bump. Target-specific deployment bundles remain separate. Candidate deployment creates no release history. + +The Git adapter writes deterministic history files under `.semapact/history/`. The surrounding GitOps workflow remains responsible for committing/publishing those files; SemaPact does not silently push repository branches. + +The formal release fact is independent of runtime convergence. Once `ContractRelease` is created, a later runtime deployment failure does not undo or renumber the release. + +### Operational history — opt in + +Deployment executions can be much more frequent than releases, so SemaPact does **not** write deployment/reconciliation telemetry to Git by default. + +Without configuration: + +```text +deployment execute + verify +→ return DeploymentExecutionResult +→ no operational history persistence +``` + +Configure operational telemetry once in `.semapact.yaml`: + +```yaml +history: + operational: + backend: sqlite + path: .semapact/operational.db +``` + +For a shared/higher-volume Delta sink: + +```yaml +history: + operational: + backend: delta + table_uri: s3://governance/semapact/operational-history +``` + +The `history.operational` subsection is validated fail-closed by the typed `SemaPactConfigSchema`. Unknown backends, missing backend-specific fields, or extra fields are rejected. + +Resolution precedence is: + +```text +--operational-history CLI override + ↓ +history.operational project/global config + ↓ +disabled +``` + +The CLI URI remains available for one-off/custom pipeline overrides, but ordinary CI/CD does not repeat the history destination on every deployment. + +Supported operational history backends in this slice are SQLite and Delta. The Delta backend is lazy and requires the `delta` optional extra. Operational event payloads carry an explicit schema version so the telemetry contract can evolve independently of governance-history models. + +The Git ledger stores low-frequency governance facts only. Deployment/runtime telemetry is written only when an operational SQLite or Delta backend is configured. + +Operational events record concrete deployment occurrence facts including success/failure, exact bundle/plan/source provenance, target, timestamps, and reconciliation status when available. ## Databricks deployment capability -The first Databricks write slice is intentionally narrow and fail-closed. +The first Databricks write slice remains intentionally narrow and fail-closed for schema mutation: | Observed state | Supported behavior | | --- | --- | | Governed table is missing | `CREATE TABLE ... USING DELTA` as a managed table | | Existing `MANAGED` table is missing a governed nullable column | `ALTER TABLE ... ADD COLUMNS (...)` | | Existing `MANAGED` table already satisfies the governed shape | `NO_OP` | -| Runtime contains extra columns not governed by this contract | Leave them untouched; no inferred `DROP` | +| Runtime contains extra columns not governed by this contract | Leave them untouched | + +Rename, existing-column type/nullability mutation, unsafe required-column addition, DROP, non-managed mutation, and ambiguous mappings fail closed. + +### Unity Catalog release provenance tags -For this slice, `ALTER` means **only additive nullable-column change**. The adapter does not interpret `ALTER` as generic schema evolution. +After deployment of a **finalized formal release** reaches schema reconciliation status `IN_SYNC`, the Databricks adapter projects SemaPact-owned release provenance onto every governed Unity Catalog table: -The following are rejected rather than guessed or silently converted: +```text +semapact_contract_id +semapact_contract_version +semapact_release_id +semapact_source_revision +``` -- column rename; -- existing-column physical type change; -- existing-column nullability change; -- adding a required/non-null column without an explicit safe migration/default strategy; -- `DROP` or other destructive reconciliation; -- mutation of existing external/non-managed assets; -- unsupported or ambiguous provider mappings. +For example, a released `orders@1.4.0` table is tagged with version `1.4.0` and the exact `ContractRelease` identity. Candidate/non-release deployments never publish formal release/version tags. -Existing external/non-managed assets remain observable through the runtime read side, but this deployment adapter does not claim mutation authority over them. +This projection is deliberately limited to SemaPact provenance. ODCS business tags, classifications, PII labels, or governed ABAC tags are **not** automatically mapped to Unity Catalog tags; those require a separate explicit mapping policy. -The adapter is also not a general Databricks infrastructure engine. Workspace, catalog, schema, SQL warehouse, credentials, external locations, storage configuration, grants, jobs, and clusters are outside this deployment boundary and must be provisioned separately. +The Databricks principal running CD must have the Unity Catalog privileges needed to apply table tags, including `APPLY TAG` on the target object plus `USE CATALOG` and `USE SCHEMA` on its parents. SemaPact uses `ALTER TABLE ... SET TAGS`, supported by current Unity Catalog SQL surfaces. -## Authorization scope +Tag mutation is a provider side effect and therefore requires a Databricks SQL warehouse even when the schema deployment itself resolves to `NO_OP`. A tag-write failure fails the release deployment command rather than silently claiming the runtime is fully projected. -Runtime deployment is a separate protected operation from publishing a contract release artifact. +## Execution trust boundary -A `ContractOpsAuthorization(operation=DEPLOY)` establishes release-context authorization. Before runtime mutation, it must be bound to the exact `DeploymentPlan` as a `DeploymentAuthorization`. +Candidate and finalized-release deployments use different provenance, but neither turns release state into runtime execution authority: -For review-required changes, structured review evidence may carry an opaque `scopeReference`. Deployment requires that scope to match the exact `deploymentPlanId`, so an approval for one exact platform/runtime/source target cannot be reused for another target. +```text +candidate deployment +→ exact GovernanceDecision + candidate source snapshot +→ BLOCK fails closed + +formal release deployment +→ exact finalized ContractRelease provenance +``` -A PUBLISH authorization cannot authorize DEPLOY. +Whether runtime mutation may execute is decided by the surrounding protected CD context, such as a GitHub Environment or Azure DevOps Environment. SemaPact validates the exact bundle, source/plan linkage, fresh runtime observation, deterministic preview, and convergence; it does not manufacture DEPLOY authority from a ContractRelease. ## Determinism `deploymentPlanId` is UUID5-derived from the full stable plan record: -- exact `AppliedContractRelease` identity and provenance; +- exact deployment source snapshot (candidate or release) and revision/version provenance; - exact deployment target, including runtime source reference; - canonical actions ordered by governed asset identity; - plan schema version. -The same exact applied release and target therefore produce the same DeploymentPlan. +The same exact deployment source and target therefore produce the same DeploymentPlan. -Action ordering is canonical even when schemas appear in a different order in source ODCS. However, DeploymentPlan does not redefine release identity: two distinct `AppliedContractRelease` artifacts remain distinct authorities even if their projected actions happen to be equivalent. +Action ordering is canonical even when schemas appear in a different order in source ODCS. However, DeploymentPlan does not redefine release identity: two distinct exact release artifacts remain distinct plan inputs even if their projected actions happen to be equivalent. `DeploymentPreview` is likewise deterministic for the same plan and observed runtime evidence, but deterministic IDs provide artifact consistency rather than cryptographic authenticity. Execution still validates exact binding and fresh runtime evidence at the side-effect boundary. diff --git a/docs/governance_analysis_boundary.md b/docs/governance_analysis_boundary.md index 7bbb0a8b..9b698503 100644 --- a/docs/governance_analysis_boundary.md +++ b/docs/governance_analysis_boundary.md @@ -2,24 +2,21 @@ SemaPact governance analysis is a pure, repeatable operation. It determines what a proposed contract change means; it does not perform the change or publish anything. -## Canonical Boundary +## Canonical boundary ```text -Analyze - base + candidate + ChangeContext +base + candidate + ChangeContext ↓ - GovernanceDecision - -Apply - explicit local/candidate mutation - -Publish / Deploy / CI - explicit protected side effects +GovernanceDecision + ↓ +application workflow + ↓ +explicit mutation / release / deployment boundary ``` -Governance analysis remains upstream of all mutation-capable workflows. Later phases consume its result rather than recomputing governance semantics. +Later phases consume the decision rather than recomputing governance semantics. -## Analysis Entry Points +## Analysis entry points The authoritative domain evaluator is: @@ -33,85 +30,63 @@ evaluate_governance_decision( ) ``` -Application clients should normally enter governance analysis through: +Application clients normally enter through: ```python GovernanceService.evaluate(...) ``` -`GovernanceService.merge_and_evaluate(...)` may construct an in-memory merged candidate before evaluation, but it must not persist that candidate or perform external publication side effects. +`GovernanceService.merge_and_evaluate(...)` may construct an in-memory merged candidate before evaluation, but it must not persist that candidate or perform publication/runtime side effects. -## Analysis Invariants +## Analysis invariants Governance analysis must not: -- mutate the base contract; -- mutate the candidate contract; -- write contract, manifest, or other files; -- create or modify Git branches, commits, or tags; +- mutate the base or candidate contract; +- write contract or artifact files; +- create Git branches, commits, or tags; - create pull requests; -- publish metadata or deployment artifacts; +- publish metadata; - apply a release version; -- invoke deployment or runtime mutation. +- invoke runtime mutation. -For the same normalized inputs and the same `ChangeContext`, repeated analysis must produce the same deterministic `GovernanceDecision`, including the same `decision_id`. +For identical normalized inputs and `ChangeContext`, repeated analysis must produce the same deterministic `GovernanceDecision`, including `decision_id`. -Analysis may compute evidence such as validation results, canonical `GovernanceChange` values, policy findings, breaking status, and the minimum required version bump. Computing a required version bump is analysis; applying a version is not. +Computing validation evidence, canonical `GovernanceChange` values, policy findings, breaking status, and minimum required version bump is analysis. Applying a version or changing runtime state is not. -## Policy Findings, Decisions, and Gate Results +## Policy findings, decisions, and execution authority -These three concepts have different meanings and must not be treated as interchangeable: +These are distinct: ```text policy.valid -= no policy-breaking findings were detected += whether policy-breaking findings were detected -decision +GovernanceDecision = authoritative governance disposition -gate result -= authoritative permission for a specific operation +execution authority += permission to cross a specific side-effect boundary ``` -A breaking change can therefore legitimately produce: +A breaking change may legitimately produce: ```text policy.valid = false decision = REVIEW ``` -This means the lifecycle policy found breaking evidence, not that every operation is prohibited. For example, analysis remains readable while a CI, publish, or deploy operation may require review before side effects are allowed. - -External consumers must use `GovernanceDecision` and the operation-specific governance gate for authorization. They must not use `policy.valid`, `breaking`, or individual reason codes as independent permission checks. - -## Side-Effecting Operations - -Mutation-capable workflows are downstream consumers of governance analysis. They must consume an already-produced `GovernanceDecision` (or an application result containing that decision) and enforce the appropriate `GovernanceOperation` gate before performing protected side effects. - -Conceptually: - -```text -GovernanceDecision - ↓ -GovernanceOperation gate - ↓ -explicit APPLY / PUBLISH / DEPLOY / CI boundary -``` +Consumers must use the authoritative `GovernanceDecision` and the workflow boundary that owns the side effect. They must not reinterpret `policy.valid`, `breaking`, or individual reason codes as independent permission checks. -A client or adapter must not independently reinterpret breaking changes, validation, lifecycle policy, or version requirements to bypass the authoritative decision. +Formal release REVIEW approval is scoped to the exact release artifact. Runtime deployment permission belongs to the protected CI/CD execution context; it is not inferred from `ContractRelease`. -## Regression Protection +## Regression protection -`tests/test_governance_analysis_boundary.py` protects this architecture contract by checking that: +`tests/test_governance_analysis_boundary.py` protects analysis purity and determinism. -- the authoritative evaluator does not import integration or mutation layers; -- decision evaluation performs no filesystem or process side effects; -- input contracts remain unchanged; -- repeated evaluation is deterministic; -- the `GovernanceService.evaluate(...)` application boundary preserves the same purity guarantees. +The two canonical GitHub Actions examples are architecture fitness tests for the public workflow boundary: -`tests/test_pipeline_manifest.py` additionally protects the CI boundary by checking that: +- `examples/github/data-product-ci-cd.yml` +- `examples/github/central-contract-repo-ci-cd.yml` -- legacy manifest breaking-change fields are projections of the authoritative decision rather than a parallel reason-code interpretation; -- reviewable non-breaking changes do not invent breaking evidence; -- the automation pipeline enforces the CI gate once at the artifact side-effect boundary. +They must use public CLI surfaces only. If they require internal artifact reconstruction, duplicate approval logic, unnecessary Git round-trips, or internal implementation IDs, that is treated as an application/CLI boundary defect. diff --git a/docs/lifecycle_semantics.md b/docs/lifecycle_semantics.md index 204a38c1..5748b9d1 100644 --- a/docs/lifecycle_semantics.md +++ b/docs/lifecycle_semantics.md @@ -1,156 +1,108 @@ # SemaPact Lifecycle Governance Semantics -This document describes the authoritative lifecycle model, resolution order, and governance scope rules for Open Data Contracts (ODCS) within SemaPact. +This document defines the canonical lifecycle model and governance scope rules for Open Data Contract Standard (ODCS) contracts. ---- +## Supported lifecycle states -## 1. Supported Lifecycle States +SemaPact recognizes exactly four lifecycle states: -SemaPact governance operates on four canonical lifecycle states: - -| State | Governance Meaning | Breaking Checks | Auto-Deprecation | Mutability | +| State | Governance meaning | Breaking checks | Auto-deprecation | Mutability | |---|---|---|---|---| -| `draft` | Development / non-production. | ❌ Skipped | ❌ Skipped | ✅ Free evolution | -| `active` | Production contract. Strict governance applies. | ✅ Enforced | ✅ Applied | ⚠ Governed | -| `deprecated` | Marked for decommissioning. | ❌ Skipped | ❌ Skipped | ⚠ Metadata only | -| `retired` | End of life / decommissioned. | ❌ Skipped | ❌ Skipped | ❌ Immutable (BLOCK) | - -### Read Normalization & Aliases - -Lifecycle status strings are normalized case-insensitively with leading/trailing whitespace removed: -- `"draft"` -> `LifecycleStatus.DRAFT` -- `"proposed"` -> `LifecycleStatus.DRAFT` - > [!NOTE] - > `"proposed"` is a read-only governance interpretation alias for ODCS compliance; SemaPact does not rewrite ODCS `status: proposed` to `status: draft` in the contract YAML merely during resolution. -- `"active"` -> `LifecycleStatus.ACTIVE` -- `"deprecated"` -> `LifecycleStatus.DEPRECATED` -- `"retired"` -> `LifecycleStatus.RETIRED` - -Any unknown or unsupported lifecycle status value is rejected by normalization and flagged by `ContractValidator` as a validation issue. +| `draft` | Development / non-production | skipped | skipped | free evolution | +| `active` | Production contract | enforced | applied | governed | +| `deprecated` | Marked for decommissioning | skipped | skipped | metadata only | +| `retired` | End of life | skipped | skipped | immutable | ---- +Lifecycle strings are normalized case-insensitively with surrounding whitespace removed. Any value other than `draft`, `active`, `deprecated`, or `retired` is invalid. -## 2. Canonical Authority & Fallback Order +## Canonical lifecycle authority -Authority is defined by entity level: +Lifecycle authority depends on entity level. -### Contract Root -1. Native `contract.status` (canonical ODCS root status field) -2. `contract.customProperties.lifecycleStatus` (legacy fallback) -3. Default: `LifecycleStatus.DRAFT` +For the contract root, only the native ODCS `contract.status` field is authoritative. When it is absent, the resolver returns `DRAFT`. An explicitly invalid value is reported by `ContractValidator` and governance fails closed. -### SchemaObject & SchemaProperty (including nested `properties[]` and `items`) -1. Entity's own `customProperties.lifecycleStatus` only (canonical ODCS extension point) -2. Note: Non-standard `schema.status` or `property.status` attributes are not governance authorities. +For `SchemaObject` and `SchemaProperty` entities, including nested properties/items, SemaPact uses the entity's `customProperties.lifecycleStatus` annotation. Non-standard `schema.status` or `property.status` attributes are not governance authorities. ---- +There is no second contract-root lifecycle representation. -## 3. Declared vs. Effective Lifecycle +## Declared and effective lifecycle -SemaPact strictly differentiates between **declared lifecycle** and **effective lifecycle**: +Declared lifecycle is the status explicitly attached to one schema/property entity through `customProperties.lifecycleStatus`. -### Declared Lifecycle -- The status explicitly annotated on an individual entity (`customProperties.lifecycleStatus`). -- Resolved via `resolve_declared_entity_lifecycle(entity)`. -- Used for release change classification to determine if an entity was newly marked deprecated. +Effective lifecycle includes parent governance scope: -### Effective Governance Lifecycle -- The status of an entity taking into account parent governance scope and hierarchy. -- Resolved via: - - `resolve_contract_lifecycle(contract)` - - `resolve_schema_lifecycle(schema_obj, contract=contract)` - - `resolve_property_lifecycle(prop, parent_lifecycle=...)` +```text +contract.status + ↓ +effective contract lifecycle + ↓ +schema declared lifecycle, when contract is ACTIVE + ↓ +property declared lifecycle, when parent is ACTIVE +``` -### Recursive Inheritance Invariant -> [!IMPORTANT] -> **Inactive Ancestor Invariant**: An inactive ancestor (`draft`, `deprecated`, `retired`) places its entire subtree outside active governance scope. A child entity cannot reactivate itself past an inactive parent. +The inactive-ancestor invariant is strict: a child cannot reactivate itself past a `draft`, `deprecated`, or `retired` ancestor. -#### Resolution Hierarchy: -1. **Contract**: `contract.status` -> `contract.customProperties.lifecycleStatus` -> `DRAFT`. -2. **Schema**: - - If `effective contract != ACTIVE` -> parent contract effective status wins. - - Else if schema has declared `lifecycleStatus` -> schema declared status. - - Else -> `ACTIVE`. -3. **Property (Top-level or Nested `properties[]` / `items`)**: - - If `parent_lifecycle != ACTIVE` -> `parent_lifecycle` wins. - - Else if property has declared `lifecycleStatus` -> property declared status. - - Else -> `ACTIVE`. +Resolution rules are therefore: ---- +1. Contract: native `contract.status`, otherwise `DRAFT`. +2. Schema: if the contract is not `ACTIVE`, inherit the contract lifecycle; otherwise use the schema declaration when present, else `ACTIVE`. +3. Property: if its parent is not `ACTIVE`, inherit the parent lifecycle; otherwise use the property declaration when present, else `ACTIVE`. -## 4. Governance Participation Matrix +## Governance participation matrix -| Contract Status | Schema Declared | Property Declared | Effective Schema | Effective Property | Breaking Checks Scope | +| Contract status | Schema declared | Property declared | Effective schema | Effective property | Breaking checks | |---|---|---|---|---|---| -| `active` | *(none)* | *(none)* | `active` | `active` | ✅ Included | -| `active` | `draft` | `active` | `draft` | `draft` | ❌ Excluded (parent inactive) | -| `active` | `deprecated` | `active` | `deprecated` | `deprecated` | ❌ Excluded (parent inactive) | -| `active` | `active` | `draft` | `active` | `draft` | ❌ Excluded (property draft) | -| `active` | `active` | `deprecated` | `active` | `deprecated` | ❌ Excluded (property deprecated) | -| `draft` | `active` | `active` | `draft` | `draft` | ❌ Excluded (contract draft) | -| `retired` | `active` | `active` | `retired` | `retired` | ❌ Excluded (contract retired) | - ---- +| `active` | none | none | `active` | `active` | included | +| `active` | `draft` | `active` | `draft` | `draft` | excluded | +| `active` | `deprecated` | `active` | `deprecated` | `deprecated` | excluded | +| `active` | `active` | `draft` | `active` | `draft` | excluded | +| `active` | `active` | `deprecated` | `active` | `deprecated` | excluded | +| `draft` | `active` | `active` | `draft` | `draft` | excluded | +| `deprecated` | `active` | `active` | `deprecated` | `deprecated` | excluded | +| `retired` | `active` | `active` | `retired` | `retired` | excluded | -## 5. Fail-Closed Validation Integration +## Fail-closed validation -> [!NOTE] -> **Total Resolvers vs. Validation Authority**: -> Lifecycle resolvers are intentionally total for deterministic analysis; lifecycle validity is enforced by `ContractValidator`. -> -> - `normalize_status()`: Strict parser (`ValueError` on unsupported values). -> - Resolvers (`resolve_contract_lifecycle`, `resolve_schema_lifecycle`, `resolve_property_lifecycle`): Total / tolerant functions with safe fallbacks (`DRAFT` / `None`), preventing crashes in downstream analysis pipelines. -> - `ContractValidator`: Authoritative validity check emitting `ValidationIssue(severity="error")` on invalid lifecycle values. -> - `evaluate_governance_decision()`: Blocks changes with `VALIDATION_FAILED` whenever `ContractValidator` reports issues. +Lifecycle resolvers are total so deterministic analysis can continue, while `ContractValidator` is the validity authority. -When an unknown or malformed lifecycle status is provided (e.g. `status: "unknown"`): -1. `ContractValidator` detects invalid status and records a `VALIDATION_FAILED` issue. -2. `evaluate_governance_decision()` receives `ValidationOutcome(valid=False)` and emits a deterministic `GovernanceDecision(decision=DecisionResult.BLOCK)`. -3. Evaluation and merge pipelines execute deterministically without leaking unhandled exceptions. +- `normalize_status()` raises on unsupported values. +- `resolve_contract_lifecycle()` returns `DRAFT` when root status is absent or invalid. +- schema/property declared-status resolution returns `None` when no valid declaration exists. +- `ContractValidator` reports malformed lifecycle values. +- `evaluate_governance_decision()` turns validation failure into `DecisionResult.BLOCK`. ---- +This separation prevents malformed input from crashing analysis without treating it as valid governed state. -## 6. Retired Contract Immutability +## Retired contract immutability -`RETIRED` represents a terminal, end-of-life governed state. A governed contract whose effective lifecycle is `retired` is permanently frozen and immutable. +`RETIRED` is terminal. -### Invariant: ```text -base effective lifecycle == RETIRED +base lifecycle == RETIRED AND base contract != candidate contract - ↓ + ↓ RETIRED_CONTRACT_MODIFIED - ↓ + ↓ DecisionResult.BLOCK ``` -Any semantic mutation against a retired base contract must be authoritatively blocked by `evaluate_governance_decision()` with `GovernanceReasonCode.RETIRED_CONTRACT_MODIFIED`. This applies to all differences, including: -- Technical schema changes (adding, removing, or modifying schemas/properties) -- Business metadata changes (descriptions, tags, business names) -- Schema and property lifecycle modifications -- Quality rules and relationship changes -- Root status changes (reactivation) -- Version changes -- Purely descriptive changes - -### Operation Gating Matrix: - -| Operation Category | Operation / Entrypoint | Gate Evaluated | Decision / Result on Retired Contract | -|---|---|---|---| -| Read-only / Inspect | `ContractLoader.load()` | *(none)* | ✅ Allowed (loading/inspecting preserved) | -| Analysis / Planning | `semapact plan`, `GovernanceService.evaluate()` | `ANALYZE` | ✅ Allowed (returns/displays `BLOCK` decision) | -| Release Classification | `semapact release classify` | `ANALYZE` | ✅ Allowed (returns classification with `BLOCK`) | -| Export Artifacts | `semapact export`, `export_ge` | *(none)* | ✅ Allowed (derives downstream views) | -| Technical Merge Output | `semapact merge` | `PROPOSE` | ❌ Blocked (`GovernanceBlockedError`) | -| Ingestion into Governed Contract | `semapact import --existing` | `PROPOSE` | ❌ Blocked (`GovernanceBlockedError`) | -| Semantic Lifecycle Edit | `semapact lifecycle promote / deprecate` | `APPLY` | ❌ Blocked (`GovernanceBlockedError`) | -| Release Preparation | `semapact release prepare` | `PROPOSE` | ❌ Blocked (`GovernanceBlockedError`) | -| Release PR Creation | `semapact release create-pr` | `PROPOSE` | ❌ Blocked (`GovernanceBlockedError`) | -| Batch Release Manifest | `semapact release build-manifest` | `PROPOSE` | ❌ Skipped from manifest tasks | -| Automation Pipeline | `ContractPipeline.run()` | `CI` | ❌ Blocked (`GovernanceBlockedError`) | - -### Lifecycle Transitions & Reactivations: -- **Transition into Retired**: `ACTIVE / DEPRECATED / DRAFT -> RETIRED` produces `DecisionResult.REVIEW` with `GovernanceReasonCode.CONTRACT_RETIRED_TRANSITION`. This is a valid lifecycle transition requiring review, not a mutation of an already-retired contract. -- **Reactivation**: `RETIRED -> ACTIVE / DRAFT / DEPRECATED` produces `DecisionResult.BLOCK` with `GovernanceReasonCode.RETIRED_CONTRACT_MODIFIED`. No unretire/reactivation semantics exist. -- **Unchanged Retired Contracts**: `retired base == retired candidate` produces `DecisionResult.ALLOW` (when no other violation exists), preserving history, export, and verification workflows. +This applies to technical schema, business metadata, lifecycle annotations, quality rules, relationships, root status, version, and descriptive changes. + +Transitioning an existing non-retired contract into `retired` is a reviewable transition. Reactivating a retired contract is blocked. An unchanged retired contract remains readable and analyzable. + +## Operation boundaries + +Read-only loading, analysis, classification, and export remain available when a decision is `BLOCK`; they report the decision rather than mutating state. + +Mutation-capable paths consume that decision at their canonical boundary: + +| Boundary | Canonical entry point | Authority | +|---|---|---| +| governed merge/import | merge/import application workflow | governance PROPOSE gate | +| formal release | `semapact release assess → approve (REVIEW only) → finalize` | PUBLISH approval + exact ReleaseBundle | +| candidate deployment | `semapact deployment assess → deploy` | governance provenance + protected runtime execution context | +| finalized release deployment | `ContractRelease → deployment assess → deploy` | exact release provenance + protected runtime execution context | + +A `ContractRelease` never grants runtime execution permission. diff --git a/examples/README.md b/examples/README.md index 37835c01..c8111f92 100644 --- a/examples/README.md +++ b/examples/README.md @@ -2,35 +2,47 @@ This folder contains reference assets for wiring SemaPact into CI/CD. -## Release Assets - -- `release/release-manifest.example.json` - - example per-contract batch release manifest - -## CI Shell Examples +## Pull-request validation - `ci/pr-check.example.sh` - - single-contract and multi-contract PR build examples -- `ci/release.example.sh` - - multi-contract release build example + - repository-level contract classification for PR validation +- `azure-devops/semapact-pr-validation.yml` + - the same classification pattern in Azure DevOps -## GitHub Actions Examples +These examples use `semapact release classify-repo` only to discover which governed +contracts changed. They do not create release branches, manifests, version bumps, or +pull requests. +## Canonical GitHub Actions CI/CD + +- `github/data-product-ci-cd.yml` + - end-to-end candidate CI + formal release + production deployment for a standard + data-product repo +- `github/central-contract-repo-ci-cd.yml` + - changed-contract discovery, matrix release finalization, immutable artifact handoff, + target fan-out, and a single governance-ledger write for a central contract repo - `github/semapact-enrich.yml` - on-demand LLM contract enrichment trigger -- `github/semapact-release.yml` - - release promotion trigger -## Azure DevOps Examples +The two bundle-driven CI/CD examples intentionally use only public SemaPact CLI surfaces +plus ordinary GitHub Actions/Git commands. They are architecture fitness tests: awkward +JSON extraction, internal IDs, Git round-trips, or duplicate approval plumbing indicate +a CLI/application-boundary problem. -- `azure-devops/semapact-pr-validation.yml` - - PR validation template -- `azure-devops/semapact-release.yml` - - release promotion template +Both CI/CD examples assume contracts define Databricks servers named `development` and +`production`; adapt those names to local contract conventions. + +Operational deployment history is config-first through `.semapact.yaml`. The examples +do not repeat `--operational-history` on every deployment. -Important rules reflected by these examples: +The examples use two distinct GitHub Environment boundaries. `contract-release` supplies +human PUBLISH approval for REVIEW releases. `semapact release approve --approval-out` +persists that fact to the governance ledger and emits the exact ApprovalRecord used by +`release finalize --approval`; Git is not used as the in-pipeline transport. +`production` protects whether the deployment job may run; SemaPact does not convert a +finalized ContractRelease into DEPLOY authorization. -- version governance is per contract, not per repo -- PR builds classify changes but do not bump contract versions -- release builds apply explicit release tags only for contracts that require a bump -- contracts with `required_bump = none` are skipped by default in batch release manifests +`release finalize --release-out` publishes the immutable ContractRelease as a pipeline +artifact. Deployment jobs consume that artifact directly with `deployment assess +--release`; Git history remains the governance ledger rather than the job-to-job +transport. diff --git a/examples/azure-devops/semapact-release.yml b/examples/azure-devops/semapact-release.yml deleted file mode 100644 index bb219fe8..00000000 --- a/examples/azure-devops/semapact-release.yml +++ /dev/null @@ -1,42 +0,0 @@ -trigger: none - -pool: - vmImage: ubuntu-latest - -variables: - UV_NO_PROGRESS: "1" - MANIFEST_PATH: "./artifacts/release_manifest.json" - -steps: - - checkout: self - - - task: UsePythonVersion@0 - inputs: - versionSpec: "3.11" - - - script: | - python -m pip install uv - uv sync --all-extras --group dev --frozen - displayName: Install dependencies - - - script: | - semapact release build-manifest \ - --base-root ./contracts-main \ - --candidate-root ./contracts-release \ - --output "$(MANIFEST_PATH)" - displayName: Build editable per-contract release manifest - - - script: | - cat "$(MANIFEST_PATH)" - displayName: Review manifest output - - - script: | - semapact release create-prs \ - --manifest "$(MANIFEST_PATH)" \ - --repo-path . \ - --organization "$(ADO_ORGANIZATION)" \ - --project "$(ADO_PROJECT)" \ - --repository-id "$(ADO_REPOSITORY_ID)" \ - --pat-token "$(ADO_PAT_TOKEN)" \ - --push - displayName: Create per-contract release PRs diff --git a/examples/ci/release.example.sh b/examples/ci/release.example.sh deleted file mode 100644 index 07214aaf..00000000 --- a/examples/ci/release.example.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Example release flow for a multi-contract repo. -# Review the generated manifest before creating PRs. - -BASE_ROOT="${BASE_ROOT:-./contracts-main}" -CANDIDATE_ROOT="${CANDIDATE_ROOT:-./contracts-release}" -MANIFEST_PATH="${MANIFEST_PATH:-./artifacts/release_manifest.json}" -REPO_PATH="${REPO_PATH:-.}" - -: "${ADO_ORGANIZATION:?Set ADO_ORGANIZATION}" -: "${ADO_PROJECT:?Set ADO_PROJECT}" -: "${ADO_REPOSITORY_ID:?Set ADO_REPOSITORY_ID}" -: "${ADO_PAT_TOKEN:?Set ADO_PAT_TOKEN}" - -semapact release build-manifest \ - --base-root "${BASE_ROOT}" \ - --candidate-root "${CANDIDATE_ROOT}" \ - --output "${MANIFEST_PATH}" - -echo "Review and edit ${MANIFEST_PATH} before continuing." - -semapact release create-prs \ - --manifest "${MANIFEST_PATH}" \ - --repo-path "${REPO_PATH}" \ - --organization "${ADO_ORGANIZATION}" \ - --project "${ADO_PROJECT}" \ - --repository-id "${ADO_REPOSITORY_ID}" \ - --pat-token "${ADO_PAT_TOKEN}" \ - --push diff --git a/examples/github/central-contract-repo-ci-cd.yml b/examples/github/central-contract-repo-ci-cd.yml new file mode 100644 index 00000000..2696b50d --- /dev/null +++ b/examples/github/central-contract-repo-ci-cd.yml @@ -0,0 +1,327 @@ +name: SemaPact - central contract repository CI/CD + +on: + pull_request: + paths: + - contracts/**/*.yaml + - contracts/**/*.yml + push: + branches: + - main + paths: + - contracts/**/*.yaml + - contracts/**/*.yml + +permissions: + contents: write + +concurrency: + group: semapact-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +env: + PYTHON_VERSION: "3.11" + CONTRACT_ROOT: contracts + DEV_SERVER: development + PROD_SERVER: production + +jobs: + detect-contracts: + if: > + github.event_name == 'pull_request' || + !contains(github.event.head_commit.message, '[semapact-release]') + runs-on: ubuntu-latest + outputs: + base_sha: ${{ steps.base.outputs.sha }} + matrix: ${{ steps.matrix.outputs.matrix }} + has_changes: ${{ steps.matrix.outputs.has_changes }} + steps: + - name: Checkout candidate repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install SemaPact + run: pip install semapact + + - name: Resolve comparison base + id: base + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + BASE_SHA="${{ github.event.pull_request.base.sha }}" + else + BASE_SHA="${{ github.event.before }}" + fi + echo "sha=${BASE_SHA}" >> "${GITHUB_OUTPUT}" + git worktree add --detach /tmp/semapact-base "${BASE_SHA}" + + - name: Classify repository contract changes + run: | + mkdir -p artifacts + semapact release classify-repo \ + --base-root "/tmp/semapact-base/${CONTRACT_ROOT}" \ + --candidate-root "${CONTRACT_ROOT}" \ + > artifacts/repository-changes.json + cat artifacts/repository-changes.json + + - name: Fail closed on unsupported repository changes + run: | + PROBLEMS=$(jq -c '[.contracts[] | select(.status == "added" or .status == "removed" or .status == "blocked")]' artifacts/repository-changes.json) + if [ "${PROBLEMS}" != "[]" ]; then + echo "Repository changes require explicit handling before deployment:" + echo "${PROBLEMS}" | jq . + exit 1 + fi + + - name: Build changed-contract matrix + id: matrix + run: | + MATRIX=$(jq -c '{include: [.contracts[] + | select(.status == "changed") + | {contract_path: .contractRepoPath, artifact: .artifactKey}]}' artifacts/repository-changes.json) + COUNT=$(echo "${MATRIX}" | jq '.include | length') + echo "matrix=${MATRIX}" >> "${GITHUB_OUTPUT}" + if [ "${COUNT}" -gt 0 ]; then + echo "has_changes=true" >> "${GITHUB_OUTPUT}" + else + echo "has_changes=false" >> "${GITHUB_OUTPUT}" + fi + + - name: Upload repository classification + uses: actions/upload-artifact@v4 + with: + name: semapact-repository-classification + path: artifacts/repository-changes.json + + candidate-assess: + if: github.event_name == 'pull_request' && needs.detect-contracts.outputs.has_changes == 'true' + needs: detect-contracts + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.detect-contracts.outputs.matrix) }} + env: + DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} + steps: + - name: Checkout candidate repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install SemaPact + run: pip install "semapact[databricks]" + + - name: Materialize base contract + env: + BASE_SHA: ${{ needs.detect-contracts.outputs.base_sha }} + CONTRACT_PATH: ${{ matrix.contract_path }} + run: | + git show "${BASE_SHA}:${CONTRACT_ROOT}/${CONTRACT_PATH}" > /tmp/base-contract.yaml + + - name: Assess candidate contract + env: + BASE_SHA: ${{ needs.detect-contracts.outputs.base_sha }} + CANDIDATE_SHA: ${{ github.event.pull_request.head.sha }} + CONTRACT_PATH: ${{ matrix.contract_path }} + ARTIFACT: ${{ matrix.artifact }} + run: | + mkdir -p artifacts + semapact deployment assess \ + --base /tmp/base-contract.yaml \ + --candidate "${CONTRACT_ROOT}/${CONTRACT_PATH}" \ + --base-revision-ref "git:${BASE_SHA}" \ + --candidate-revision-ref "git:${CANDIDATE_SHA}" \ + --server "${DEV_SERVER}" \ + --bundle-out "artifacts/${ARTIFACT}.candidate.deployment.bundle.json" + + - name: Upload candidate bundle + uses: actions/upload-artifact@v4 + with: + name: candidate-${{ matrix.artifact }} + path: artifacts/ + + release-assess: + if: github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.detect-contracts.outputs.has_changes == 'true' + needs: detect-contracts + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.detect-contracts.outputs.matrix) }} + steps: + - name: Checkout release candidate + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install SemaPact + run: pip install semapact + + - name: Materialize previous released contract + env: + BASE_SHA: ${{ needs.detect-contracts.outputs.base_sha }} + CONTRACT_PATH: ${{ matrix.contract_path }} + run: | + git show "${BASE_SHA}:${CONTRACT_ROOT}/${CONTRACT_PATH}" > /tmp/base-contract.yaml + + - name: Build target-neutral ReleaseBundle + env: + BASE_SHA: ${{ needs.detect-contracts.outputs.base_sha }} + CANDIDATE_SHA: ${{ github.sha }} + CONTRACT_PATH: ${{ matrix.contract_path }} + ARTIFACT: ${{ matrix.artifact }} + run: | + mkdir -p artifacts + semapact release assess \ + --base /tmp/base-contract.yaml \ + --candidate "${CONTRACT_ROOT}/${CONTRACT_PATH}" \ + --base-revision-ref "git:${BASE_SHA}" \ + --candidate-revision-ref "git:${CANDIDATE_SHA}" \ + --bundle-out "artifacts/${ARTIFACT}.release.bundle.json" + + - name: Upload exact release bundle + uses: actions/upload-artifact@v4 + with: + name: release-${{ matrix.artifact }} + path: artifacts/ + + finalize-releases: + if: github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.detect-contracts.outputs.has_changes == 'true' + needs: + - detect-contracts + - release-assess + runs-on: ubuntu-latest + environment: contract-release + steps: + - name: Checkout main + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install SemaPact + run: pip install semapact + + - name: Download all target-neutral release bundles + uses: actions/download-artifact@v4 + with: + pattern: release-* + path: artifacts/releases + merge-multiple: true + + - name: Approve REVIEW releases, finalize all releases, and materialize versions + env: + MATRIX: ${{ needs.detect-contracts.outputs.matrix }} + run: | + mkdir -p artifacts/finalized + echo "${MATRIX}" | jq -c '.include[]' | while read -r ITEM; do + CONTRACT_PATH=$(echo "${ITEM}" | jq -r '.contract_path') + ARTIFACT=$(echo "${ITEM}" | jq -r '.artifact') + BUNDLE="artifacts/releases/${ARTIFACT}.release.bundle.json" + + APPROVAL_ARGS=() + if [ "$(jq -r '.decision.decision' "${BUNDLE}")" = "REVIEW" ]; then + APPROVAL="artifacts/finalized/${ARTIFACT}.approval.json" + semapact release approve \ + --bundle "${BUNDLE}" \ + --actor-reference "github-environment:contract-release/run:${GITHUB_RUN_ID}" \ + --recorded-at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --approval-out "${APPROVAL}" + APPROVAL_ARGS=(--approval "${APPROVAL}") + fi + + semapact release finalize \ + --bundle "${BUNDLE}" \ + "${APPROVAL_ARGS[@]}" \ + --output-contract "${CONTRACT_ROOT}/${CONTRACT_PATH}" \ + --release-out "artifacts/finalized/${ARTIFACT}.contract-release.json" \ + > "/tmp/${ARTIFACT}.finalize.json" + done + + - name: Commit all released contracts and governance facts once + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add "${CONTRACT_ROOT}" .semapact/history + if ! git diff --cached --quiet; then + git commit -m "chore(semapact): finalize contract releases [semapact-release]" + git pull --rebase origin main + git push origin HEAD:main + fi + + - name: Upload finalized ContractRelease artifacts + uses: actions/upload-artifact@v4 + with: + name: semapact-finalized-releases + path: artifacts/finalized/ + + deploy-production: + needs: + - detect-contracts + - finalize-releases + runs-on: ubuntu-latest + environment: production + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.detect-contracts.outputs.matrix) }} + env: + DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} + DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} + steps: + - name: Checkout finalized governance ledger + uses: actions/checkout@v4 + with: + ref: main + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install SemaPact + run: pip install "semapact[databricks]" + + - name: Download finalized ContractRelease artifacts + uses: actions/download-artifact@v4 + with: + name: semapact-finalized-releases + path: artifacts/releases + + - name: Assess finalized release against production + env: + ARTIFACT: ${{ matrix.artifact }} + run: | + mkdir -p artifacts/deploy + semapact deployment assess \ + --release "artifacts/releases/${ARTIFACT}.contract-release.json" \ + --server "${PROD_SERVER}" \ + --bundle-out "artifacts/deploy/${ARTIFACT}.production.deployment.bundle.json" + + - name: Deploy exact target bundle + env: + ARTIFACT: ${{ matrix.artifact }} + run: | + semapact deployment deploy \ + --bundle "artifacts/deploy/${ARTIFACT}.production.deployment.bundle.json" \ + --warehouse-id "${DATABRICKS_WAREHOUSE_ID}" + + # Operational history is config-first via history.operational. diff --git a/examples/github/data-product-ci-cd.yml b/examples/github/data-product-ci-cd.yml new file mode 100644 index 00000000..4b53e963 --- /dev/null +++ b/examples/github/data-product-ci-cd.yml @@ -0,0 +1,235 @@ +name: SemaPact - data product CI/CD + +on: + pull_request: + paths: + - contract.yaml + push: + branches: + - main + paths: + - contract.yaml + +permissions: + contents: write + +concurrency: + group: semapact-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +env: + PYTHON_VERSION: "3.11" + CONTRACT_PATH: contract.yaml + DEV_SERVER: development + PROD_SERVER: production + +jobs: + candidate-assess: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + env: + DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} + steps: + - name: Checkout candidate + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install SemaPact + run: pip install "semapact[databricks]" + + - name: Materialize base contract + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + git show "${BASE_SHA}:${CONTRACT_PATH}" > /tmp/base-contract.yaml + + - name: Assess candidate against development + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + CANDIDATE_SHA: ${{ github.event.pull_request.head.sha }} + run: | + mkdir -p artifacts + semapact deployment assess \ + --base /tmp/base-contract.yaml \ + --candidate "${CONTRACT_PATH}" \ + --base-revision-ref "git:${BASE_SHA}" \ + --candidate-revision-ref "git:${CANDIDATE_SHA}" \ + --server "${DEV_SERVER}" \ + --bundle-out artifacts/candidate.deployment.bundle.json \ + --output json > artifacts/candidate.summary.json + + - name: Upload candidate assessment + uses: actions/upload-artifact@v4 + with: + name: semapact-candidate-assessment + path: artifacts/ + + release-assess: + if: > + github.event_name == 'push' && + github.ref == 'refs/heads/main' && + !contains(github.event.head_commit.message, '[semapact-release]') + runs-on: ubuntu-latest + steps: + - name: Checkout release candidate + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install SemaPact + run: pip install semapact + + - name: Materialize previous released contract + env: + BASE_SHA: ${{ github.event.before }} + run: | + git show "${BASE_SHA}:${CONTRACT_PATH}" > /tmp/base-contract.yaml + + - name: Build target-neutral formal release bundle + env: + BASE_SHA: ${{ github.event.before }} + CANDIDATE_SHA: ${{ github.sha }} + run: | + mkdir -p artifacts + semapact release assess \ + --base /tmp/base-contract.yaml \ + --candidate "${CONTRACT_PATH}" \ + --base-revision-ref "git:${BASE_SHA}" \ + --candidate-revision-ref "git:${CANDIDATE_SHA}" \ + --bundle-out artifacts/release.bundle.json \ + > artifacts/release.summary.json + + - name: Upload exact release bundle + uses: actions/upload-artifact@v4 + with: + name: semapact-release-bundle + path: artifacts/ + + finalize-release: + if: > + github.event_name == 'push' && + github.ref == 'refs/heads/main' && + !contains(github.event.head_commit.message, '[semapact-release]') + needs: release-assess + runs-on: ubuntu-latest + # Example policy: GitHub Environment protects formal release publication. + # SemaPact only persists an ApprovalRecord when its decision is REVIEW. + environment: contract-release + steps: + - name: Checkout main + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install SemaPact + run: pip install semapact + + - name: Download exact release bundle + uses: actions/download-artifact@v4 + with: + name: semapact-release-bundle + path: artifacts + + - name: Capture exact review approval when required + run: | + BUNDLE=artifacts/release.bundle.json + if [ "$(jq -r '.decision.decision' "${BUNDLE}")" = "REVIEW" ]; then + semapact release approve \ + --bundle "${BUNDLE}" \ + --actor-reference "github-environment:contract-release/run:${GITHUB_RUN_ID}" \ + --recorded-at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --approval-out artifacts/release.approval.json + fi + + - name: Finalize release and materialize released contract + id: finalize + run: | + APPROVAL_ARGS=() + if [ -f artifacts/release.approval.json ]; then + APPROVAL_ARGS=(--approval artifacts/release.approval.json) + fi + semapact release finalize \ + --bundle artifacts/release.bundle.json \ + "${APPROVAL_ARGS[@]}" \ + --output-contract "${CONTRACT_PATH}" \ + --release-out artifacts/contract-release.json \ + > artifacts/finalize.json + + - name: Commit released contract and governance ledger once + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add "${CONTRACT_PATH}" .semapact/history + if ! git diff --cached --quiet; then + git commit -m "chore(semapact): finalize contract release [semapact-release]" + git pull --rebase origin main + git push origin HEAD:main + fi + + - name: Upload release result + uses: actions/upload-artifact@v4 + with: + name: semapact-finalized-release + path: | + artifacts/finalize.json + artifacts/contract-release.json + + deploy-production: + needs: finalize-release + runs-on: ubuntu-latest + environment: production + env: + DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} + DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} + steps: + - name: Checkout finalized release + uses: actions/checkout@v4 + with: + ref: main + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install SemaPact + run: pip install "semapact[databricks]" + + - name: Download finalized release artifact + uses: actions/download-artifact@v4 + with: + name: semapact-finalized-release + path: artifacts + + - name: Assess finalized release against production + run: | + semapact deployment assess \ + --release artifacts/contract-release.json \ + --server "${PROD_SERVER}" \ + --bundle-out artifacts/production.deployment.bundle.json + + - name: Deploy exact target bundle + run: | + semapact deployment deploy \ + --bundle artifacts/production.deployment.bundle.json \ + --warehouse-id "${DATABRICKS_WAREHOUSE_ID}" + + # Operational deployment history is config-first. If history.operational is + # configured in .semapact.yaml, deploy writes there automatically. diff --git a/examples/github/semapact-release.yml b/examples/github/semapact-release.yml deleted file mode 100644 index 8c40cd1a..00000000 --- a/examples/github/semapact-release.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Release Contracts - -on: - workflow_dispatch: - inputs: - base_root: - description: 'Directory path for base contracts' - required: true - default: './contracts-main' - candidate_root: - description: 'Directory path for release candidates' - required: true - default: './contracts-release' - -jobs: - build-and-release: - runs-on: ubuntu-latest - env: - UV_NO_PROGRESS: "1" - SEMAPACT_RUNTIME_CONTEXT: "auto" - SEMAPACT_GIT_PROVIDER: "github" - SEMAPACT_GITHUB_OWNER: ${{ github.repository_owner }} - SEMAPACT_GITHUB_REPO: ${{ github.event.repository.name }} - SEMAPACT_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Set up uv - uses: astral-sh/setup-uv@v5 - - - name: Install dependencies - run: uv sync --all-extras --group dev --frozen - - - name: Build Release Manifest - run: | - mkdir -p artifacts - uv run semapact release build-manifest \ - --base-root ${{ github.event.inputs.base_root || './contracts-main' }} \ - --candidate-root ${{ github.event.inputs.candidate_root || './contracts-release' }} \ - --output ./artifacts/release_manifest.json - - - name: Upload Manifest Artifact - uses: actions/upload-artifact@v4 - with: - name: release_manifest - path: ./artifacts/release_manifest.json - - - name: Create Release PRs - run: | - uv run semapact release create-prs \ - --manifest ./artifacts/release_manifest.json \ - --repo-path . \ - --push diff --git a/examples/release/release-manifest.example.json b/examples/release/release-manifest.example.json deleted file mode 100644 index f624e56f..00000000 --- a/examples/release/release-manifest.example.json +++ /dev/null @@ -1,18 +0,0 @@ -[ - { - "base": "contracts-main/orders.yaml", - "candidate": "contracts-release/orders.yaml", - "contract_path": "contracts/orders.yaml", - "release_tag": "orders/v1.2.0", - "source_branch": "release/orders-v1.2.0", - "target_branch": "release" - }, - { - "base": "contracts-main/payments.yaml", - "candidate": "contracts-release/payments.yaml", - "contract_path": "contracts/payments.yaml", - "release_tag": "payments/v2.0.0", - "source_branch": "release/payments-v2.0.0", - "target_branch": "release" - } -] diff --git a/schemas/semapact-config.schema.json b/schemas/semapact-config.schema.json new file mode 100644 index 00000000..0a939a73 --- /dev/null +++ b/schemas/semapact-config.schema.json @@ -0,0 +1,101 @@ +{ + "$defs": { + "DeltaOperationalHistoryConfig": { + "additionalProperties": false, + "description": "Delta Lake operational history configuration.", + "properties": { + "backend": { + "const": "delta", + "title": "Backend", + "type": "string" + }, + "table_uri": { + "minLength": 1, + "title": "Table Uri", + "type": "string" + } + }, + "required": [ + "backend", + "table_uri" + ], + "title": "DeltaOperationalHistoryConfig", + "type": "object" + }, + "HistoryConfig": { + "additionalProperties": false, + "description": "Typed history configuration while unrelated configuration remains extensible.", + "properties": { + "operational": { + "anyOf": [ + { + "discriminator": { + "mapping": { + "delta": "#/$defs/DeltaOperationalHistoryConfig", + "sqlite": "#/$defs/SQLiteOperationalHistoryConfig" + }, + "propertyName": "backend" + }, + "oneOf": [ + { + "$ref": "#/$defs/SQLiteOperationalHistoryConfig" + }, + { + "$ref": "#/$defs/DeltaOperationalHistoryConfig" + } + ] + }, + { + "type": "null" + } + ], + "default": null, + "title": "Operational" + } + }, + "title": "HistoryConfig", + "type": "object" + }, + "SQLiteOperationalHistoryConfig": { + "additionalProperties": false, + "description": "Local SQLite operational history configuration.", + "properties": { + "backend": { + "const": "sqlite", + "title": "Backend", + "type": "string" + }, + "path": { + "minLength": 1, + "title": "Path", + "type": "string" + } + }, + "required": [ + "backend", + "path" + ], + "title": "SQLiteOperationalHistoryConfig", + "type": "object" + } + }, + "$id": "https://semapact.org/schemas/semapact-config.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": true, + "description": "Incremental root schema for project/global SemaPact configuration.", + "properties": { + "history": { + "anyOf": [ + { + "$ref": "#/$defs/HistoryConfig" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "title": "SemaPactConfigSchema", + "type": "object" +} diff --git a/semapact/__init__.py b/semapact/__init__.py index c6a5d3a8..31af619e 100644 --- a/semapact/__init__.py +++ b/semapact/__init__.py @@ -20,47 +20,6 @@ "ContractValidator": ("semapact.core.validator", "ContractValidator"), "AzureDevOpsConfig": ("semapact.devops.pr_creator", "AzureDevOpsConfig"), "PullRequestCreator": ("semapact.devops.pr_creator", "PullRequestCreator"), - "BatchReleaseManifestBuild": ( - "semapact.devops.release_workflow", - "BatchReleaseManifestBuild", - ), - "BatchReleaseTask": ("semapact.devops.release_workflow", "BatchReleaseTask"), - "ReleasePullRequestPlan": ( - "semapact.devops.release_workflow", - "ReleasePullRequestPlan", - ), - "RepositoryContractChange": ( - "semapact.devops.release_workflow", - "RepositoryContractChange", - ), - "batch_manifest_build_to_dict": ( - "semapact.devops.release_workflow", - "batch_manifest_build_to_dict", - ), - "build_batch_release_manifest": ( - "semapact.devops.release_workflow", - "build_batch_release_manifest", - ), - "build_release_pr_plan": ( - "semapact.devops.release_workflow", - "build_release_pr_plan", - ), - "create_release_pull_request": ( - "semapact.devops.release_workflow", - "create_release_pull_request", - ), - "create_release_pull_requests_from_manifest": ( - "semapact.devops.release_workflow", - "create_release_pull_requests_from_manifest", - ), - "load_batch_release_tasks": ( - "semapact.devops.release_workflow", - "load_batch_release_tasks", - ), - "repository_change_to_dict": ( - "semapact.devops.release_workflow", - "repository_change_to_dict", - ), "SparkSqlContractExporter": ( "semapact.exporters.sql_exporter", "SparkSqlContractExporter", diff --git a/semapact/application/models/__init__.py b/semapact/application/models/__init__.py index 34a27143..851e6335 100644 --- a/semapact/application/models/__init__.py +++ b/semapact/application/models/__init__.py @@ -1,12 +1,16 @@ """Typed application results composed from canonical domain artifacts.""" +from semapact.application.models.deployment_workflow import ( + DeploymentBundle, + DeploymentExecutionResult, + build_deployment_bundle, + compute_deployment_bundle_digest, +) from semapact.application.models.evolution import ( BrokenHistoryReference, ContractEvolution, - DeploymentEvolution, ProposalEvolution, ReleaseEvolution, - RuntimeEvolution, ) from semapact.application.models.governance import GovernanceAnalysis, GovernanceProposal from semapact.application.models.history_integrity import HistoryIntegrityReport @@ -16,13 +20,15 @@ __all__ = [ "BrokenHistoryReference", "ContractEvolution", - "DeploymentEvolution", + "DeploymentBundle", + "DeploymentExecutionResult", + "build_deployment_bundle", + "compute_deployment_bundle_digest", "GovernanceAnalysis", "GovernanceProposal", "HistoryIntegrityReport", "ProposalEvolution", "ReleaseEvolution", "ReleasePlanningResult", - "RuntimeEvolution", "RuntimeReconciliation", ] diff --git a/semapact/application/models/deployment_workflow.py b/semapact/application/models/deployment_workflow.py new file mode 100644 index 00000000..917a1acd --- /dev/null +++ b/semapact/application/models/deployment_workflow.py @@ -0,0 +1,199 @@ +"""Application artifacts for target-specific deployment workflows.""" + +from __future__ import annotations + +import hashlib +from typing import Literal + +from pydantic import BaseModel, ConfigDict, model_validator + +from semapact.contractops import ChangeSet, ContractRelease +from semapact.deployment import ( + DeploymentPlan, + DeploymentPreview, + DeploymentSourceSnapshot, +) +from semapact.governance import GovernanceDecision +from semapact.reconciliation import ReconciliationResult, RuntimeDriftStatus +from semapact.utils.deterministic import canonical_compact_json + + +class DeploymentExecutionResult(BaseModel): + """Application result for one bundle-driven CD execution.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + bundle_digest: str + deployment_plan_id: str + contract_release_id: str | None = None + fresh_preview: DeploymentPreview + reconciliation: ReconciliationResult + status: RuntimeDriftStatus + review_preview_changed: bool + + +class DeploymentBundle(BaseModel): + """Immutable target-specific CI-to-CD deployment package. + + Candidate versus formal-release mode is derived from deployment_source. + The bundle never stores a second mode flag. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + bundle_digest: str + deployment_source: DeploymentSourceSnapshot + deployment_plan: DeploymentPlan + review_preview: DeploymentPreview + decision: GovernanceDecision | None = None + change_set: ChangeSet | None = None + contract_release: ContractRelease | None = None + bundle_version: Literal["1"] = "1" + + @property + def is_release(self) -> bool: + return self.deployment_source.source_kind == "contract_release" + + @model_validator(mode="after") + def _validate_bundle_links(self) -> "DeploymentBundle": + source = self.deployment_source + plan = self.deployment_plan + + if self.is_release: + if self.contract_release is None: + raise ValueError( + "Release DeploymentBundle requires finalized ContractRelease" + ) + if self.decision is not None or self.change_set is not None: + raise ValueError( + "Release DeploymentBundle must not carry candidate governance artifacts" + ) + release = self.contract_release + if source.release_id != release.contract_release_id: + raise ValueError( + "DeploymentBundle source does not reference ContractRelease" + ) + if source.contract_id != release.contract_id: + raise ValueError( + "DeploymentBundle source/release contract mismatch" + ) + if source.contract_version != release.contract_version: + raise ValueError( + "DeploymentBundle source/release version mismatch" + ) + if source.revision_ref != release.source_revision_ref: + raise ValueError( + "DeploymentBundle source/release revision mismatch" + ) + else: + if source.source_kind != "candidate": + raise ValueError( + "Canonical DeploymentBundle supports candidate or finalized ContractRelease sources" + ) + if self.decision is None or self.change_set is None: + raise ValueError( + "Candidate DeploymentBundle requires decision and ChangeSet" + ) + if self.contract_release is not None: + raise ValueError( + "Candidate DeploymentBundle cannot contain ContractRelease" + ) + if self.change_set.contract_id != self.decision.contract_id: + raise ValueError( + "DeploymentBundle decision/change-set contract mismatch" + ) + if source.contract_id != self.decision.contract_id: + raise ValueError( + "DeploymentBundle source/decision contract mismatch" + ) + if self.change_set.candidate_revision_ref != source.revision_ref: + raise ValueError( + "DeploymentBundle candidate revision does not match ChangeSet" + ) + if self.change_set.changes != self.decision.changes: + raise ValueError( + "DeploymentBundle decision/change-set changes mismatch" + ) + + if plan.source_snapshot_id != source.source_snapshot_id: + raise ValueError( + "DeploymentBundle plan does not match deployment source" + ) + if plan.contract_id != source.contract_id: + raise ValueError("DeploymentBundle plan/source contract mismatch") + if plan.revision_ref != source.revision_ref: + raise ValueError("DeploymentBundle plan/source revision mismatch") + if plan.contract_version != source.contract_version: + raise ValueError("DeploymentBundle plan/source version mismatch") + if self.review_preview.deployment_plan_id != plan.deployment_plan_id: + raise ValueError("DeploymentBundle preview does not match DeploymentPlan") + + expected = compute_deployment_bundle_digest( + deployment_source=source, + deployment_plan=plan, + review_preview=self.review_preview, + decision=self.decision, + change_set=self.change_set, + contract_release=self.contract_release, + bundle_version=self.bundle_version, + ) + if self.bundle_digest != expected: + raise ValueError("DeploymentBundle digest does not match content") + return self + + +def build_deployment_bundle( + *, + deployment_source: DeploymentSourceSnapshot, + deployment_plan: DeploymentPlan, + review_preview: DeploymentPreview, + decision: GovernanceDecision | None = None, + change_set: ChangeSet | None = None, + contract_release: ContractRelease | None = None, +) -> DeploymentBundle: + """Package exact target-specific deployment material without release planning.""" + digest = compute_deployment_bundle_digest( + deployment_source=deployment_source, + deployment_plan=deployment_plan, + review_preview=review_preview, + decision=decision, + change_set=change_set, + contract_release=contract_release, + ) + return DeploymentBundle( + bundle_digest=digest, + deployment_source=deployment_source, + deployment_plan=deployment_plan, + review_preview=review_preview, + decision=decision, + change_set=change_set, + contract_release=contract_release, + ) + + +def compute_deployment_bundle_digest( + *, + deployment_source: DeploymentSourceSnapshot, + deployment_plan: DeploymentPlan, + review_preview: DeploymentPreview, + decision: GovernanceDecision | None = None, + change_set: ChangeSet | None = None, + contract_release: ContractRelease | None = None, + bundle_version: str = "1", +) -> str: + """Return the immutable digest for one target-specific deployment bundle.""" + payload = { + "bundle_version": bundle_version, + "deployment_source": deployment_source.model_dump(mode="json"), + "deployment_plan": deployment_plan.model_dump(mode="json"), + "review_preview": review_preview.model_dump(mode="json"), + "decision": decision.model_dump(mode="json") if decision is not None else None, + "change_set": change_set.model_dump(mode="json") if change_set is not None else None, + "contract_release": ( + contract_release.model_dump(mode="json") + if contract_release is not None + else None + ), + } + encoded = canonical_compact_json(payload).encode("utf-8") + return f"sha256:{hashlib.sha256(encoded).hexdigest()}" diff --git a/semapact/application/models/evolution.py b/semapact/application/models/evolution.py index 655c08a5..1d9021a0 100644 --- a/semapact/application/models/evolution.py +++ b/semapact/application/models/evolution.py @@ -4,20 +4,14 @@ from dataclasses import dataclass -from semapact.contractops import ChangeSet +from semapact.contractops import ChangeSet, ContractRelease from semapact.governance import GovernanceDecision -from semapact.history import ( - DeploymentRecord, - ReleaseRecord, - RuntimeObservationRecord, - RuntimeReconciliationRecord, -) from semapact.revision import ContractRevision @dataclass(frozen=True) class BrokenHistoryReference: - """One explicit history reference that could not be resolved consistently.""" + """One explicit governance-history reference that could not be resolved.""" source_kind: str source_id: str @@ -27,35 +21,16 @@ class BrokenHistoryReference: reason: str -@dataclass(frozen=True) -class RuntimeEvolution: - """One reconciliation occurrence with its exact observed-state evidence when present.""" - - reconciliation: RuntimeReconciliationRecord - observation: RuntimeObservationRecord | None - - -@dataclass(frozen=True) -class DeploymentEvolution: - """One deployment occurrence and runtime evidence explicitly linked to it.""" - - deployment: DeploymentRecord - runtime: tuple[RuntimeEvolution, ...] = () - - @dataclass(frozen=True) class ReleaseEvolution: - """One finalized release and its downstream execution/runtime history.""" + """One finalized formal ContractRelease linked to its governance path.""" - release: ReleaseRecord - released_revision: ContractRevision | None - deployments: tuple[DeploymentEvolution, ...] = () - runtime: tuple[RuntimeEvolution, ...] = () + release: ContractRelease @dataclass(frozen=True) class ProposalEvolution: - """One ChangeSet path through an optional recorded GovernanceDecision.""" + """One ChangeSet path through an optional GovernanceDecision and releases.""" change_set: ChangeSet base_revision: ContractRevision | None @@ -66,10 +41,9 @@ class ProposalEvolution: @dataclass(frozen=True) class ContractEvolution: - """Deterministic read-side reconstruction of one contract's persisted history.""" + """Deterministic read-side reconstruction of one contract governance history.""" contract_id: str proposals: tuple[ProposalEvolution, ...] = () unlinked_releases: tuple[ReleaseEvolution, ...] = () - unlinked_runtime: tuple[RuntimeEvolution, ...] = () broken_references: tuple[BrokenHistoryReference, ...] = () diff --git a/semapact/application/models/release.py b/semapact/application/models/release.py index fa4d4546..480cc12b 100644 --- a/semapact/application/models/release.py +++ b/semapact/application/models/release.py @@ -1,11 +1,23 @@ -"""Application result models for canonical release orchestration.""" +"""Application artifacts for target-neutral contract release workflows.""" from __future__ import annotations +import hashlib from dataclasses import dataclass +from typing import Literal -from semapact.contractops import ChangeSet, ReleasePlan, VersionResolution +from pydantic import BaseModel, ConfigDict, model_validator + +from semapact.contractops import ( + ChangeSet, + ContractRelease, + ReleasePlan, + ReleaseSnapshot, + VersionResolution, +) +from semapact.contractops.integrity import compute_contract_release_id from semapact.governance import GovernanceDecision +from semapact.utils.deterministic import canonical_compact_json @dataclass(frozen=True) @@ -16,3 +28,132 @@ class ReleasePlanningResult: decision: GovernanceDecision release_plan: ReleasePlan version_resolution: VersionResolution + + +class ReleaseBundle(BaseModel): + """Immutable target-neutral CI artifact for one formal contract release.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + bundle_digest: str + decision: GovernanceDecision + change_set: ChangeSet + release_plan: ReleasePlan + version_resolution: VersionResolution + release_snapshot: ReleaseSnapshot + bundle_version: Literal["1"] = "1" + + @model_validator(mode="after") + def _validate_links(self) -> "ReleaseBundle": + if self.change_set.contract_id != self.decision.contract_id: + raise ValueError("ReleaseBundle decision/change-set contract mismatch") + if self.release_plan.change_set_id != self.change_set.change_set_id: + raise ValueError("ReleaseBundle release plan does not match ChangeSet") + if self.release_plan.decision_id != self.decision.decision_id: + raise ValueError("ReleaseBundle release plan does not match decision") + if self.version_resolution.release_plan_id != self.release_plan.release_plan_id: + raise ValueError( + "ReleaseBundle version resolution does not match ReleasePlan" + ) + snapshot = self.release_snapshot + if snapshot.decision_id != self.decision.decision_id: + raise ValueError("ReleaseBundle snapshot does not match decision") + if snapshot.change_set_id != self.change_set.change_set_id: + raise ValueError("ReleaseBundle snapshot does not match ChangeSet") + if snapshot.release_plan_id != self.release_plan.release_plan_id: + raise ValueError("ReleaseBundle snapshot does not match ReleasePlan") + if ( + snapshot.version_resolution_id + != self.version_resolution.version_resolution_id + ): + raise ValueError( + "ReleaseBundle snapshot does not match VersionResolution" + ) + expected = compute_release_bundle_digest( + decision=self.decision, + change_set=self.change_set, + release_plan=self.release_plan, + version_resolution=self.version_resolution, + release_snapshot=self.release_snapshot, + bundle_version=self.bundle_version, + ) + if self.bundle_digest != expected: + raise ValueError("ReleaseBundle digest does not match content") + return self + + +def build_release_bundle( + *, + decision: GovernanceDecision, + change_set: ChangeSet, + release_plan: ReleasePlan, + version_resolution: VersionResolution, + release_snapshot: ReleaseSnapshot, +) -> ReleaseBundle: + """Build the exact target-neutral release artifact reviewed by CI/CD.""" + digest = compute_release_bundle_digest( + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + release_snapshot=release_snapshot, + ) + return ReleaseBundle( + bundle_digest=digest, + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + release_snapshot=release_snapshot, + ) + + +def compute_release_bundle_digest( + *, + decision: GovernanceDecision, + change_set: ChangeSet, + release_plan: ReleasePlan, + version_resolution: VersionResolution, + release_snapshot: ReleaseSnapshot, + bundle_version: str = "1", +) -> str: + """Return the immutable digest for one target-neutral release bundle.""" + payload = { + "bundle_version": bundle_version, + "decision": decision.model_dump(mode="json"), + "change_set": change_set.model_dump(mode="json"), + "release_plan": release_plan.model_dump(mode="json"), + "version_resolution": version_resolution.model_dump(mode="json"), + "release_snapshot": release_snapshot.model_dump(mode="json"), + } + encoded = canonical_compact_json(payload).encode("utf-8") + return f"sha256:{hashlib.sha256(encoded).hexdigest()}" + + +def build_contract_release(bundle: ReleaseBundle) -> ContractRelease: + """Materialize the canonical target-neutral release fact from one exact bundle.""" + snapshot = bundle.release_snapshot + resolution = bundle.version_resolution + release_id = compute_contract_release_id( + contract_id=snapshot.contract_id, + contract_version=snapshot.selected_version, + decision_id=bundle.decision.decision_id, + change_set_id=bundle.change_set.change_set_id, + release_plan_id=bundle.release_plan.release_plan_id, + version_resolution_id=resolution.version_resolution_id, + release_snapshot_id=snapshot.release_snapshot_id, + source_revision_ref=snapshot.release_revision_ref, + released_contract_json=snapshot.released_contract_json, + ) + return ContractRelease( + contract_release_id=release_id, + contract_id=snapshot.contract_id, + contract_version=snapshot.selected_version, + decision_id=bundle.decision.decision_id, + change_set_id=bundle.change_set.change_set_id, + release_plan_id=bundle.release_plan.release_plan_id, + version_resolution_id=resolution.version_resolution_id, + release_snapshot_id=snapshot.release_snapshot_id, + source_revision_ref=snapshot.release_revision_ref, + released_contract_json=snapshot.released_contract_json, + ) diff --git a/semapact/application/services/deployment.py b/semapact/application/services/deployment.py index fa5d62c9..5ca045f4 100644 --- a/semapact/application/services/deployment.py +++ b/semapact/application/services/deployment.py @@ -2,14 +2,13 @@ from __future__ import annotations -from semapact.contractops import AppliedContractRelease from semapact.deployment import ( DeploymentAdapter, - DeploymentAuthorization, DeploymentPlan, DeploymentPreview, + DeploymentSourceSnapshot, DeploymentTarget, - build_deployment_plan, + build_deployment_plan_from_source, ) from semapact.exceptions import ValidationError from semapact.reconciliation import ReconciliationResult @@ -20,10 +19,10 @@ class DeploymentService: def plan( self, - release: AppliedContractRelease, + source: DeploymentSourceSnapshot, target: DeploymentTarget, ) -> DeploymentPlan: - return build_deployment_plan(release, target) + return build_deployment_plan_from_source(source, target) def preview( self, @@ -34,16 +33,16 @@ def preview( _validate_component_key(adapter.key, plan.target.platform, "deployment adapter") return adapter.preview(plan) - def execute( + def apply( self, plan: DeploymentPlan, preview: DeploymentPreview, - authorization: DeploymentAuthorization, *, adapter: DeploymentAdapter, ) -> None: + """Apply one exact preview after the external execution boundary allows CD.""" _validate_component_key(adapter.key, plan.target.platform, "deployment adapter") - adapter.execute(plan, preview, authorization) + adapter.apply(plan, preview) def verify( self, diff --git a/semapact/application/services/deployment_history.py b/semapact/application/services/deployment_history.py deleted file mode 100644 index ef96a50d..00000000 --- a/semapact/application/services/deployment_history.py +++ /dev/null @@ -1,191 +0,0 @@ -"""Application orchestration for terminal deployment execution history.""" - -from __future__ import annotations - -from datetime import datetime, timezone - -from semapact.deployment import DeploymentAuthorization, DeploymentPlan, DeploymentPreview -from semapact.deployment.models import ( - validate_deployment_authorization_identity, - validate_deployment_plan_identity, - validate_deployment_preview_identity, -) -from semapact.history import ( - DeploymentAuthorizationHistoryRepository, - DeploymentPlanHistoryRepository, - DeploymentPreviewHistoryRepository, - DeploymentRecord, - DeploymentRecordHistoryRepository, - DeploymentStatus, - ReleaseRecord, - ReleaseRecordHistoryRepository, -) -from semapact.history.integrity import compute_deployment_record_id - - -class DeploymentHistoryService: - """Record already-observed provider execution outcomes without executing them.""" - - def __init__( - self, - *, - releases: ReleaseRecordHistoryRepository, - deployment_plans: DeploymentPlanHistoryRepository, - deployment_previews: DeploymentPreviewHistoryRepository, - deployment_authorizations: DeploymentAuthorizationHistoryRepository, - deployment_records: DeploymentRecordHistoryRepository, - ) -> None: - self._releases = releases - self._deployment_plans = deployment_plans - self._deployment_previews = deployment_previews - self._deployment_authorizations = deployment_authorizations - self._deployment_records = deployment_records - - def record_execution( - self, - *, - plan: DeploymentPlan, - preview: DeploymentPreview, - authorization: DeploymentAuthorization, - status: DeploymentStatus, - started_at: datetime, - completed_at: datetime, - actor_reference: str | None = None, - external_reference: str | None = None, - ) -> DeploymentRecord: - """Persist one terminal execution occurrence against exact deployment artifacts. - - The caller supplies the outcome after the provider execution boundary has - completed. This method never invokes a DeploymentAdapter and never infers - runtime convergence from execution success. - """ - _validate_types(plan, preview, authorization, status) - validate_deployment_plan_identity(plan) - validate_deployment_preview_identity(preview) - validate_deployment_authorization_identity(authorization) - - release = self._releases.get_release_record_by_version( - plan.contract_id, - plan.selected_version, - ) - _validate_release_link(release, plan) - _validate_preview_link(preview, plan) - _validate_authorization_link(authorization, plan) - - started = _utc_timestamp(started_at, "started_at") - completed = _utc_timestamp(completed_at, "completed_at") - if completed < started: - raise ValueError("completed_at must not be earlier than started_at") - - actor = _optional_text(actor_reference) - external = _optional_text(external_reference) - deployment_record_id = compute_deployment_record_id( - release_record_id=release.release_record_id, - deployment_plan_id=plan.deployment_plan_id, - deployment_preview_id=preview.deployment_preview_id, - deployment_authorization_id=authorization.deployment_authorization_id, - platform=plan.target.platform, - runtime_target=plan.target.runtime_target, - source_reference=plan.target.source_reference, - status=status.value, - started_at=started.isoformat(), - completed_at=completed.isoformat(), - actor_reference=actor, - external_reference=external, - ) - record = DeploymentRecord( - deployment_record_id=deployment_record_id, - release_record_id=release.release_record_id, - deployment_plan_id=plan.deployment_plan_id, - deployment_preview_id=preview.deployment_preview_id, - deployment_authorization_id=authorization.deployment_authorization_id, - platform=plan.target.platform, - runtime_target=plan.target.runtime_target, - source_reference=plan.target.source_reference, - status=status, - started_at=started, - completed_at=completed, - actor_reference=actor, - external_reference=external, - ) - - # Persist exact referenced artifacts before the occurrence record so a - # partial failure cannot leave history pointing at absent dependencies. - self._deployment_plans.put_deployment_plan(plan) - self._deployment_previews.put_deployment_preview(preview) - self._deployment_authorizations.put_deployment_authorization(authorization) - self._deployment_records.put_deployment_record(record) - return record - - -def _validate_types( - plan: DeploymentPlan, - preview: DeploymentPreview, - authorization: DeploymentAuthorization, - status: DeploymentStatus, -) -> None: - expected = ( - (plan, DeploymentPlan, "plan"), - (preview, DeploymentPreview, "preview"), - (authorization, DeploymentAuthorization, "authorization"), - (status, DeploymentStatus, "status"), - ) - for value, expected_type, name in expected: - if not isinstance(value, expected_type): - raise TypeError( - f"{name} must be {expected_type.__name__}, got {type(value).__name__}" - ) - - -def _validate_release_link(release: ReleaseRecord, plan: DeploymentPlan) -> None: - if ( - release.contract_id != plan.contract_id - or release.contract_version != plan.selected_version - or release.release_plan_id != plan.release_plan_id - or release.applied_release_id != plan.applied_release_id - ): - raise ValueError("DeploymentPlan does not match the finalized ReleaseRecord") - - -def _validate_preview_link(preview: DeploymentPreview, plan: DeploymentPlan) -> None: - if preview.deployment_plan_id != plan.deployment_plan_id: - raise ValueError("DeploymentPreview does not reference the supplied DeploymentPlan") - if preview.platform.strip().casefold() != plan.target.platform: - raise ValueError("DeploymentPreview platform does not match DeploymentPlan target") - if preview.runtime_target != plan.target.runtime_target: - raise ValueError("DeploymentPreview runtime target does not match DeploymentPlan") - if preview.source_identifier != plan.target.source_reference: - raise ValueError("DeploymentPreview source does not match DeploymentPlan source reference") - - -def _validate_authorization_link( - authorization: DeploymentAuthorization, - plan: DeploymentPlan, -) -> None: - if authorization.deployment_plan_id != plan.deployment_plan_id: - raise ValueError( - "DeploymentAuthorization does not reference the supplied DeploymentPlan" - ) - if authorization.applied_release_id != plan.applied_release_id: - raise ValueError( - "DeploymentAuthorization does not reference the plan's applied release" - ) - if not authorization.allowed: - raise ValueError("deployment history requires an allowed DeploymentAuthorization") - - -def _utc_timestamp(value: datetime, field_name: str) -> datetime: - if not isinstance(value, datetime): - raise TypeError(f"{field_name} must be datetime") - if value.tzinfo is None or value.utcoffset() is None: - raise ValueError(f"{field_name} must be timezone-aware") - return value.astimezone(timezone.utc) - - -def _optional_text(value: str | None) -> str | None: - if value is None: - return None - if not isinstance(value, str): - raise TypeError("optional deployment references must be strings") - cleaned = value.strip() - return cleaned or None diff --git a/semapact/application/services/deployment_workflow.py b/semapact/application/services/deployment_workflow.py new file mode 100644 index 00000000..5abe20d8 --- /dev/null +++ b/semapact/application/services/deployment_workflow.py @@ -0,0 +1,249 @@ +"""Canonical target-specific deployment workflow orchestration.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Literal + +from open_data_contract_standard.model import OpenDataContractStandard + +from semapact.application.models.deployment_workflow import ( + DeploymentBundle, + DeploymentExecutionResult, + build_deployment_bundle, +) +from semapact.application.services.deployment import DeploymentService +from semapact.application.services.governance import GovernanceService +from semapact.contractops import ContractRelease +from semapact.deployment import ( + DeploymentAdapter, + DeploymentTarget, + RuntimeReleaseMetadata, + RuntimeReleaseMetadataProjector, + build_candidate_deployment_source, + build_contract_release_deployment_source, + validate_candidate_deployment_context, + validate_contract_release_deployment_context, +) +from semapact.history import ( + OperationalHistorySink, + build_operational_deployment_event, +) +from semapact.reconciliation import RuntimeDriftStatus, classify_reconciliation_status + + +class DeploymentWorkflowService: + """Assess desired state and converge one target after the external CD gate runs.""" + + def __init__( + self, + *, + governance_service: GovernanceService | None = None, + deployment_service: DeploymentService | None = None, + ) -> None: + self._governance = governance_service or GovernanceService() + self._deployment = deployment_service or DeploymentService() + + def assess( + self, + base_contract: OpenDataContractStandard, + candidate_contract: OpenDataContractStandard, + *, + base_revision_ref: str, + candidate_revision_ref: str, + target: DeploymentTarget, + adapter: DeploymentAdapter, + ) -> DeploymentBundle: + """Build an immutable candidate deployment bundle.""" + proposal = self._governance.evaluate_proposal( + base_contract, + candidate_contract, + base_revision_ref=base_revision_ref, + candidate_revision_ref=candidate_revision_ref, + ) + source = build_candidate_deployment_source( + candidate_contract, + revision_ref=candidate_revision_ref, + ) + plan = self._deployment.plan(source, target) + preview = self._deployment.preview(plan, adapter=adapter) + return build_deployment_bundle( + decision=proposal.decision, + change_set=proposal.change_set, + deployment_source=source, + deployment_plan=plan, + review_preview=preview, + ) + + def assess_release( + self, + release: ContractRelease, + *, + target: DeploymentTarget, + adapter: DeploymentAdapter, + ) -> DeploymentBundle: + """Build one target-specific bundle from an already-finalized release.""" + source = build_contract_release_deployment_source(release) + plan = self._deployment.plan(source, target) + preview = self._deployment.preview(plan, adapter=adapter) + return build_deployment_bundle( + contract_release=release, + deployment_source=source, + deployment_plan=plan, + review_preview=preview, + ) + + def deploy( + self, + bundle: DeploymentBundle, + *, + adapter: DeploymentAdapter, + operational_history: OperationalHistorySink | None = None, + ) -> DeploymentExecutionResult: + """Apply one exact bundle after the caller's execution boundary allows CD. + + SemaPact validates governance/release provenance, runtime freshness and + deterministic operations. It does not turn ContractRelease into DEPLOY + authorization; protected CI/CD environments control whether this operation + may be invoked. + """ + if not isinstance(bundle, DeploymentBundle): + raise TypeError( + f"bundle must be DeploymentBundle, got {type(bundle).__name__}" + ) + + release_record = None + if bundle.is_release: + assert bundle.contract_release is not None + validate_contract_release_deployment_context( + bundle.deployment_plan, + bundle.deployment_source, + bundle.contract_release, + ) + release_record = bundle.contract_release + else: + assert bundle.decision is not None + validate_candidate_deployment_context( + bundle.deployment_plan, + bundle.deployment_source, + bundle.decision, + ) + + started_at = datetime.now(timezone.utc) + fresh_preview = None + reconciliation = None + status = None + try: + fresh_preview = self._deployment.preview( + bundle.deployment_plan, + adapter=adapter, + ) + self._deployment.apply( + bundle.deployment_plan, + fresh_preview, + adapter=adapter, + ) + reconciliation = self._deployment.verify( + bundle.deployment_plan, + adapter=adapter, + ) + status = classify_reconciliation_status(reconciliation) + + if ( + release_record is not None + and status is RuntimeDriftStatus.IN_SYNC + and isinstance(adapter, RuntimeReleaseMetadataProjector) + ): + adapter.project_release_metadata( + bundle.deployment_plan, + RuntimeReleaseMetadata( + contract_id=release_record.contract_id, + contract_version=release_record.contract_version, + contract_release_id=release_record.contract_release_id, + source_revision_ref=release_record.source_revision_ref, + ), + ) + except Exception as exc: + _record_operational_event( + operational_history, + bundle=bundle, + contract_release=release_record, + deployment_preview_id=( + fresh_preview.deployment_preview_id + if fresh_preview is not None + else None + ), + status="FAILED", + reconciliation_status=status, + started_at=started_at, + error_message=str(exc) or type(exc).__name__, + ) + raise + + assert fresh_preview is not None + assert reconciliation is not None + assert status is not None + result = DeploymentExecutionResult( + bundle_digest=bundle.bundle_digest, + deployment_plan_id=bundle.deployment_plan.deployment_plan_id, + contract_release_id=( + release_record.contract_release_id + if release_record is not None + else None + ), + fresh_preview=fresh_preview, + reconciliation=reconciliation, + status=status, + review_preview_changed=fresh_preview != bundle.review_preview, + ) + _record_operational_event( + operational_history, + bundle=bundle, + contract_release=release_record, + deployment_preview_id=fresh_preview.deployment_preview_id, + status="SUCCEEDED", + reconciliation_status=status, + started_at=started_at, + error_message=None, + ) + return result + + +def _record_operational_event( + sink: OperationalHistorySink | None, + *, + bundle: DeploymentBundle, + contract_release: ContractRelease | None, + deployment_preview_id: str | None, + status: Literal["SUCCEEDED", "FAILED"], + reconciliation_status: RuntimeDriftStatus | None, + started_at: datetime, + error_message: str | None, +) -> None: + """Persist optional deployment telemetry without influencing execution semantics.""" + if sink is None: + return + plan = bundle.deployment_plan + sink.record_deployment( + build_operational_deployment_event( + bundle_digest=bundle.bundle_digest, + contract_release_id=( + contract_release.contract_release_id + if contract_release is not None + else None + ), + contract_id=plan.contract_id, + contract_version=plan.contract_version, + revision_ref=plan.revision_ref, + deployment_plan_id=plan.deployment_plan_id, + deployment_preview_id=deployment_preview_id, + platform=plan.target.platform, + runtime_target=plan.target.runtime_target, + source_reference=plan.target.source_reference, + status=status, + reconciliation_status=reconciliation_status, + started_at=started_at, + completed_at=datetime.now(timezone.utc), + error_message=error_message, + ) + ) diff --git a/semapact/application/services/evolution.py b/semapact/application/services/evolution.py index 2daede25..92460824 100644 --- a/semapact/application/services/evolution.py +++ b/semapact/application/services/evolution.py @@ -1,35 +1,28 @@ -"""Read-only reconstruction of persisted governance evolution history.""" +"""Read-only reconstruction of canonical governance history.""" from __future__ import annotations from semapact.application.models.evolution import ( BrokenHistoryReference, ContractEvolution, - DeploymentEvolution, ProposalEvolution, ReleaseEvolution, - RuntimeEvolution, ) -from semapact.contractops import ChangeSet +from semapact.contractops import ChangeSet, ContractRelease from semapact.governance import GovernanceDecision from semapact.history import ( ChangeSetDecisionLinkHistoryRepository, ChangeSetHistoryRepository, + ContractReleaseHistoryRepository, ContractRevisionHistoryRepository, DecisionHistoryRepository, - DeploymentRecordHistoryRepository, HistoryNotFoundError, - ReleaseRecord, - ReleaseRecordHistoryRepository, - RuntimeObservationHistoryRepository, - RuntimeReconciliationHistoryRepository, - RuntimeReconciliationRecord, ) from semapact.revision import ContractRevision class EvolutionChainService: - """Reconstruct one contract's history from existing stable artifact references.""" + """Reconstruct proposal → decision → ContractRelease governance history.""" def __init__( self, @@ -38,22 +31,15 @@ def __init__( change_sets: ChangeSetHistoryRepository, decisions: DecisionHistoryRepository, decision_links: ChangeSetDecisionLinkHistoryRepository, - releases: ReleaseRecordHistoryRepository, - deployments: DeploymentRecordHistoryRepository, - observations: RuntimeObservationHistoryRepository, - runtime_reconciliations: RuntimeReconciliationHistoryRepository, + releases: ContractReleaseHistoryRepository, ) -> None: self._revisions = revisions self._change_sets = change_sets self._decisions = decisions self._decision_links = decision_links self._releases = releases - self._deployments = deployments - self._observations = observations - self._runtime_reconciliations = runtime_reconciliations def reconstruct(self, contract_id: str) -> ContractEvolution: - """Return a deterministic read model without persisting a second history graph.""" contract_id = _required_text(contract_id, "contract_id") issues: list[BrokenHistoryReference] = [] @@ -63,46 +49,20 @@ def reconstruct(self, contract_id: str) -> ContractEvolution: key=lambda item: item.change_set_id, ) ) - release_records = tuple( + releases = tuple( sorted( - self._releases.list_release_records(contract_id), - key=lambda item: item.release_record_id, + self._releases.list_contract_releases(contract_id), + key=lambda item: item.contract_release_id, ) ) - runtime_records = tuple( - sorted( - self._runtime_reconciliations.list_runtime_reconciliation_records( - contract_id - ), - key=lambda item: item.runtime_reconciliation_record_id, - ) - ) - - runtime_by_id = { - record.runtime_reconciliation_record_id: self._runtime_evolution( - record, - issues, - ) - for record in runtime_records - } - runtime_by_release: dict[str, list[RuntimeReconciliationRecord]] = {} - runtime_by_deployment: dict[str, list[RuntimeReconciliationRecord]] = {} - for record in runtime_records: - if record.deployment_record_id is not None: - runtime_by_deployment.setdefault(record.deployment_record_id, []).append(record) - elif record.release_record_id is not None: - runtime_by_release.setdefault(record.release_record_id, []).append(record) - - releases_by_path: dict[tuple[str, str], list[ReleaseRecord]] = {} - for release in release_records: + releases_by_path: dict[tuple[str, str], list[ContractRelease]] = {} + for release in releases: releases_by_path.setdefault( (release.change_set_id, release.decision_id), [], ).append(release) consumed_release_ids: set[str] = set() - consumed_runtime_ids: set[str] = set() - known_deployment_ids: set[str] = set() linked_decision_pairs: set[tuple[str, str]] = set() proposals: list[ProposalEvolution] = [] @@ -149,24 +109,15 @@ def reconstruct(self, contract_id: str) -> ContractEvolution: contract_id=contract_id, issues=issues, ) - releases = tuple( - self._build_release( - release, - contract_id=contract_id, - runtime_by_id=runtime_by_id, - runtime_by_release=runtime_by_release, - runtime_by_deployment=runtime_by_deployment, - consumed_runtime_ids=consumed_runtime_ids, - known_deployment_ids=known_deployment_ids, - issues=issues, - ) + linked_releases = tuple( + ReleaseEvolution(release=release) for release in sorted( releases_by_path.get(pair, ()), - key=lambda item: item.release_record_id, + key=lambda item: item.contract_release_id, ) ) consumed_release_ids.update( - item.release.release_record_id for item in releases + item.release.contract_release_id for item in linked_releases ) proposals.append( ProposalEvolution( @@ -174,21 +125,21 @@ def reconstruct(self, contract_id: str) -> ContractEvolution: base_revision=base_revision, candidate_revision=candidate_revision, decision=decision, - releases=releases, + releases=linked_releases, ) ) change_set_ids = {item.change_set_id for item in change_sets} unlinked_releases: list[ReleaseEvolution] = [] - for release in release_records: - if release.release_record_id in consumed_release_ids: + for release in releases: + if release.contract_release_id in consumed_release_ids: continue pair = (release.change_set_id, release.decision_id) if release.change_set_id not in change_set_ids: _add_issue( issues, - source_kind="ReleaseRecord", - source_id=release.release_record_id, + source_kind="ContractRelease", + source_id=release.contract_release_id, reference_field="change_set_id", target_kind="ChangeSet", target_id=release.change_set_id, @@ -197,59 +148,14 @@ def reconstruct(self, contract_id: str) -> ContractEvolution: elif pair not in linked_decision_pairs: _add_issue( issues, - source_kind="ReleaseRecord", - source_id=release.release_record_id, + source_kind="ContractRelease", + source_id=release.contract_release_id, reference_field="decision_id", target_kind="ChangeSetDecisionLink", target_id=f"{release.change_set_id}:{release.decision_id}", reason="RELATION_MISSING", ) - unlinked_releases.append( - self._build_release( - release, - contract_id=contract_id, - runtime_by_id=runtime_by_id, - runtime_by_release=runtime_by_release, - runtime_by_deployment=runtime_by_deployment, - consumed_runtime_ids=consumed_runtime_ids, - known_deployment_ids=known_deployment_ids, - issues=issues, - ) - ) - - release_ids = {item.release_record_id for item in release_records} - unlinked_runtime: list[RuntimeEvolution] = [] - for record in runtime_records: - record_id = record.runtime_reconciliation_record_id - if record_id in consumed_runtime_ids: - continue - if ( - record.release_record_id is not None - and record.release_record_id not in release_ids - ): - _add_issue( - issues, - source_kind="RuntimeReconciliationRecord", - source_id=record_id, - reference_field="release_record_id", - target_kind="ReleaseRecord", - target_id=record.release_record_id, - reason="NOT_FOUND", - ) - if ( - record.deployment_record_id is not None - and record.deployment_record_id not in known_deployment_ids - ): - _add_issue( - issues, - source_kind="RuntimeReconciliationRecord", - source_id=record_id, - reference_field="deployment_record_id", - target_kind="DeploymentRecord", - target_id=record.deployment_record_id, - reason="NOT_FOUND", - ) - unlinked_runtime.append(runtime_by_id[record_id]) + unlinked_releases.append(ReleaseEvolution(release=release)) proposals.sort( key=lambda item: ( @@ -257,10 +163,7 @@ def reconstruct(self, contract_id: str) -> ContractEvolution: item.decision.decision_id if item.decision is not None else "", ) ) - unlinked_releases.sort(key=lambda item: item.release.release_record_id) - unlinked_runtime.sort( - key=lambda item: item.reconciliation.runtime_reconciliation_record_id - ) + unlinked_releases.sort(key=lambda item: item.release.contract_release_id) issues.sort( key=lambda item: ( item.source_kind, @@ -275,7 +178,6 @@ def reconstruct(self, contract_id: str) -> ContractEvolution: contract_id=contract_id, proposals=tuple(proposals), unlinked_releases=tuple(unlinked_releases), - unlinked_runtime=tuple(unlinked_runtime), broken_references=tuple(issues), ) @@ -348,138 +250,6 @@ def _resolve_decision( return None return decision - def _build_release( - self, - release: ReleaseRecord, - *, - contract_id: str, - runtime_by_id: dict[str, RuntimeEvolution], - runtime_by_release: dict[str, list[RuntimeReconciliationRecord]], - runtime_by_deployment: dict[str, list[RuntimeReconciliationRecord]], - consumed_runtime_ids: set[str], - known_deployment_ids: set[str], - issues: list[BrokenHistoryReference], - ) -> ReleaseEvolution: - released_revision: ContractRevision | None - try: - released_revision = self._revisions.get_revision(release.released_revision_id) - except HistoryNotFoundError: - released_revision = None - _add_issue( - issues, - source_kind="ReleaseRecord", - source_id=release.release_record_id, - reference_field="released_revision_id", - target_kind="ContractRevision", - target_id=release.released_revision_id, - reason="NOT_FOUND", - ) - else: - if ( - str(released_revision.contract.id or "") != contract_id - or str(released_revision.contract.version or "") != release.contract_version - ): - _add_issue( - issues, - source_kind="ReleaseRecord", - source_id=release.release_record_id, - reference_field="released_revision_id", - target_kind="ContractRevision", - target_id=release.released_revision_id, - reason="RELEASE_MISMATCH", - ) - released_revision = None - - deployments = tuple( - sorted( - self._deployments.list_deployment_records_for_release( - release.release_record_id - ), - key=lambda item: item.deployment_record_id, - ) - ) - deployment_paths: list[DeploymentEvolution] = [] - for deployment in deployments: - known_deployment_ids.add(deployment.deployment_record_id) - linked_runtime: list[RuntimeEvolution] = [] - for record in sorted( - runtime_by_deployment.get(deployment.deployment_record_id, ()), - key=lambda item: item.runtime_reconciliation_record_id, - ): - if record.release_record_id != release.release_record_id: - _add_issue( - issues, - source_kind="RuntimeReconciliationRecord", - source_id=record.runtime_reconciliation_record_id, - reference_field="release_record_id", - target_kind="ReleaseRecord", - target_id=str(record.release_record_id or ""), - reason="DEPLOYMENT_RELEASE_MISMATCH", - ) - continue - linked_runtime.append(runtime_by_id[record.runtime_reconciliation_record_id]) - consumed_runtime_ids.add(record.runtime_reconciliation_record_id) - deployment_paths.append( - DeploymentEvolution( - deployment=deployment, - runtime=tuple(linked_runtime), - ) - ) - - release_runtime = tuple( - runtime_by_id[record.runtime_reconciliation_record_id] - for record in sorted( - runtime_by_release.get(release.release_record_id, ()), - key=lambda item: item.runtime_reconciliation_record_id, - ) - ) - consumed_runtime_ids.update( - item.reconciliation.runtime_reconciliation_record_id for item in release_runtime - ) - return ReleaseEvolution( - release=release, - released_revision=released_revision, - deployments=tuple(deployment_paths), - runtime=release_runtime, - ) - - def _runtime_evolution( - self, - record: RuntimeReconciliationRecord, - issues: list[BrokenHistoryReference], - ) -> RuntimeEvolution: - try: - observation = self._observations.get_runtime_observation_record( - record.observation_record_id - ) - except HistoryNotFoundError: - _add_issue( - issues, - source_kind="RuntimeReconciliationRecord", - source_id=record.runtime_reconciliation_record_id, - reference_field="observation_record_id", - target_kind="RuntimeObservationRecord", - target_id=record.observation_record_id, - reason="NOT_FOUND", - ) - return RuntimeEvolution(reconciliation=record, observation=None) - - observed = observation.observation - if ( - observed.source_identifier != record.result.observation_source_identifier - or observed.fingerprint != record.result.observation_fingerprint - ): - _add_issue( - issues, - source_kind="RuntimeReconciliationRecord", - source_id=record.runtime_reconciliation_record_id, - reference_field="observation_record_id", - target_kind="RuntimeObservationRecord", - target_id=record.observation_record_id, - reason="EVIDENCE_MISMATCH", - ) - return RuntimeEvolution(reconciliation=record, observation=observation) - def _add_issue( issues: list[BrokenHistoryReference], diff --git a/semapact/application/services/governance.py b/semapact/application/services/governance.py index 12ba4cf2..b33aaad4 100644 --- a/semapact/application/services/governance.py +++ b/semapact/application/services/governance.py @@ -41,15 +41,12 @@ def evaluate( base_contract: OpenDataContractStandard, candidate_contract: OpenDataContractStandard, *, - effective_date: date | str, merge_conflicts: Sequence[MergeConflict] = (), ) -> GovernanceDecision: - """Evaluate one contract change from an application-level effective date.""" - context = self.create_context(effective_date) + """Evaluate one contract change without execution-time inputs.""" return evaluate_governance_decision( base_contract, candidate_contract, - context=context, merge_conflicts=merge_conflicts, ) @@ -58,7 +55,6 @@ def evaluate_proposal( base_contract: OpenDataContractStandard, candidate_contract: OpenDataContractStandard, *, - effective_date: date | str, base_revision_ref: str, candidate_revision_ref: str, merge_conflicts: Sequence[MergeConflict] = (), @@ -66,11 +62,9 @@ def evaluate_proposal( actor_reference: str | None = None, ) -> GovernanceProposal: """Evaluate once and project the authoritative decision into a ChangeSet.""" - context = self.create_context(effective_date) decision = evaluate_governance_decision( base_contract, candidate_contract, - context=context, merge_conflicts=merge_conflicts, ) change_set = build_change_set_from_decision( @@ -101,7 +95,6 @@ def merge_and_evaluate( decision = evaluate_governance_decision( business_contract, merge_result.contract, - context=context, merge_conflicts=merge_result.conflicts, ) return GovernanceAnalysis( diff --git a/semapact/application/services/history.py b/semapact/application/services/history.py index b1ab1d0d..4d48a1c7 100644 --- a/semapact/application/services/history.py +++ b/semapact/application/services/history.py @@ -96,7 +96,5 @@ def _validate_proposal_links( "ChangeSet candidate_revision_ref must equal candidate ContractRevision ID" ) - if change_set.context != decision.context: - raise ValueError("ChangeSet context does not match GovernanceDecision context") if change_set.changes != decision.changes: raise ValueError("ChangeSet changes do not match GovernanceDecision changes") diff --git a/semapact/application/services/release_approval.py b/semapact/application/services/release_approval.py new file mode 100644 index 00000000..a72ff07e --- /dev/null +++ b/semapact/application/services/release_approval.py @@ -0,0 +1,51 @@ +"""Resolve exact formal-release approval from immutable approval history.""" + +from __future__ import annotations + +from semapact.approval import ApprovalRecord +from semapact.application.models.release import ReleaseBundle +from semapact.contractops import ReviewEvidenceAction +from semapact.governance import GovernanceOperation +from semapact.history import ApprovalHistoryRepository + + +class ReleaseApprovalResolver: + """Resolve exact PUBLISH approval for one immutable ReleaseBundle.""" + + def __init__(self, approvals: ApprovalHistoryRepository) -> None: + self._approvals = approvals + + def resolve(self, bundle: ReleaseBundle) -> ApprovalRecord | None: + scoped = self._scoped_records(bundle) + if not scoped or _contains_conflict(scoped): + return None + return min(scoped, key=lambda record: record.approval_id) + + def has_conflict(self, bundle: ReleaseBundle) -> bool: + """Return whether exact persisted review evidence contains a non-approval.""" + return _contains_conflict(self._scoped_records(bundle)) + + def _scoped_records( + self, + bundle: ReleaseBundle, + ) -> tuple[ApprovalRecord, ...]: + records = self._approvals.list_approval_records_for_context( + decision_id=bundle.decision.decision_id, + change_set_id=bundle.change_set.change_set_id, + release_plan_id=bundle.release_plan.release_plan_id, + version_resolution_id=bundle.version_resolution.version_resolution_id, + operation=GovernanceOperation.PUBLISH, + ) + return tuple( + record + for record in records + if record.scope_reference == bundle.release_snapshot.release_snapshot_id + and bundle.bundle_digest in record.evidence_references + ) + + +def _contains_conflict(records: tuple[ApprovalRecord, ...]) -> bool: + return any( + record.action is not ReviewEvidenceAction.APPROVE + for record in records + ) diff --git a/semapact/application/services/release_history.py b/semapact/application/services/release_history.py deleted file mode 100644 index bd8305b7..00000000 --- a/semapact/application/services/release_history.py +++ /dev/null @@ -1,214 +0,0 @@ -"""Application orchestration for finalized contract release history.""" - -from __future__ import annotations - -from semapact.contractops.context import validate_release_context -from semapact.contractops.execution_models import AppliedContractRelease -from semapact.contractops.integrity import ( - validate_applied_release_identity, - validate_contractops_authorization_identity, -) -from semapact.contractops.models import ( - ContractOpsAuthorization, - ReleasePlan, - VersionResolution, -) -from semapact.governance.gate import GovernanceOperation -from semapact.history import ( - ChangeSetDecisionLinkHistoryRepository, - ChangeSetHistoryRepository, - ContractRevisionHistoryRepository, - DecisionHistoryRepository, - ReleasePlanHistoryRepository, - ReleaseRecord, - ReleaseRecordHistoryRepository, -) -from semapact.history.integrity import compute_release_record_id -from semapact.revision import build_contract_revision -from semapact.revision.integrity import validate_contract_revision_identity - - -class ReleaseHistoryService: - """Record one already-applied release without recomputing release semantics.""" - - def __init__( - self, - *, - revisions: ContractRevisionHistoryRepository, - change_sets: ChangeSetHistoryRepository, - decisions: DecisionHistoryRepository, - decision_links: ChangeSetDecisionLinkHistoryRepository, - release_plans: ReleasePlanHistoryRepository, - release_records: ReleaseRecordHistoryRepository, - ) -> None: - self._revisions = revisions - self._change_sets = change_sets - self._decisions = decisions - self._decision_links = decision_links - self._release_plans = release_plans - self._release_records = release_records - - def record_release( - self, - *, - release_plan: ReleasePlan, - version_resolution: VersionResolution, - authorization: ContractOpsAuthorization, - applied_release: AppliedContractRelease, - ) -> ReleaseRecord: - """Persist the exact proposal → plan → released-revision audit chain. - - All supplied ContractOps artifacts must already exist as valid outputs from - their owning M2 stages. This use case validates linkage only; it never reruns - governance, version authority, authorization, or APPLY. - """ - decision = self._decisions.get_decision(release_plan.decision_id) - change_set = self._change_sets.get_change_set(release_plan.change_set_id) - candidate_revision = self._revisions.get_revision(release_plan.release_revision_ref) - - _require_decision_link( - self._decision_links, - change_set_id=change_set.change_set_id, - decision_id=decision.decision_id, - ) - validate_contract_revision_identity(candidate_revision) - validate_release_context( - decision, - change_set, - release_plan, - version_resolution, - ) - _validate_candidate_revision(candidate_revision, release_plan, version_resolution) - _validate_apply_authorization( - authorization, - release_plan=release_plan, - version_resolution=version_resolution, - ) - _validate_applied_release( - applied_release, - release_plan=release_plan, - version_resolution=version_resolution, - authorization=authorization, - ) - - released_revision = build_contract_revision(applied_release.to_contract()) - if released_revision.revision_id == candidate_revision.revision_id: - raise ValueError( - "released revision must differ from candidate revision after APPLY versioning" - ) - - record_id = compute_release_record_id( - contract_id=release_plan.contract_id, - contract_version=version_resolution.selected_version, - decision_id=release_plan.decision_id, - change_set_id=release_plan.change_set_id, - release_plan_id=release_plan.release_plan_id, - version_resolution_id=version_resolution.version_resolution_id, - authorization_id=authorization.authorization_id, - applied_release_id=applied_release.applied_release_id, - released_revision_id=released_revision.revision_id, - required_version_bump=version_resolution.required_version_bump, - actual_version_bump=version_resolution.actual_bump, - version_authority=version_resolution.authority.value, - authority_reference=version_resolution.authority_reference, - review_evidence_reference=authorization.evidence_reference, - review_evidence_action=( - authorization.evidence_action.value - if authorization.evidence_action is not None - else None - ), - ) - record = ReleaseRecord( - release_record_id=record_id, - contract_id=release_plan.contract_id, - contract_version=version_resolution.selected_version, - decision_id=release_plan.decision_id, - change_set_id=release_plan.change_set_id, - release_plan_id=release_plan.release_plan_id, - version_resolution_id=version_resolution.version_resolution_id, - authorization_id=authorization.authorization_id, - applied_release_id=applied_release.applied_release_id, - released_revision_id=released_revision.revision_id, - required_version_bump=version_resolution.required_version_bump, - actual_version_bump=version_resolution.actual_bump, - version_authority=version_resolution.authority, - authority_reference=version_resolution.authority_reference, - review_evidence_reference=authorization.evidence_reference, - review_evidence_action=authorization.evidence_action, - ) - - # Persist referenced artifacts before the final record so a partial failure - # cannot leave a ReleaseRecord pointing at absent release/revision history. - self._release_plans.put_release_plan(release_plan) - self._revisions.put_revision(released_revision) - self._release_records.put_release_record(record) - return record - - -def _require_decision_link( - repository: ChangeSetDecisionLinkHistoryRepository, - *, - change_set_id: str, - decision_id: str, -) -> None: - links = repository.list_change_set_decision_links(change_set_id) - if not any(link.decision_id == decision_id for link in links): - raise ValueError( - "release history requires the ChangeSet-to-GovernanceDecision history link" - ) - - -def _validate_candidate_revision( - candidate_revision, - release_plan: ReleasePlan, - version_resolution: VersionResolution, -) -> None: - contract_id = str(candidate_revision.contract.id or "").strip() - contract_version = str(candidate_revision.contract.version or "").strip() - if contract_id != release_plan.contract_id: - raise ValueError("candidate ContractRevision does not match ReleasePlan contract") - if contract_version != version_resolution.current_version: - raise ValueError( - "candidate ContractRevision version does not match VersionResolution current_version" - ) - - -def _validate_apply_authorization( - authorization: ContractOpsAuthorization, - *, - release_plan: ReleasePlan, - version_resolution: VersionResolution, -) -> None: - validate_contractops_authorization_identity(authorization) - if authorization.operation is not GovernanceOperation.APPLY: - raise ValueError("release history requires APPLY-scoped authorization") - if not authorization.allowed: - raise ValueError("release history requires an allowed APPLY authorization") - if ( - authorization.decision_id != release_plan.decision_id - or authorization.change_set_id != release_plan.change_set_id - or authorization.release_plan_id != release_plan.release_plan_id - or authorization.version_resolution_id != version_resolution.version_resolution_id - ): - raise ValueError("APPLY authorization does not match the exact release context") - - -def _validate_applied_release( - applied_release: AppliedContractRelease, - *, - release_plan: ReleasePlan, - version_resolution: VersionResolution, - authorization: ContractOpsAuthorization, -) -> None: - validate_applied_release_identity(applied_release) - if ( - applied_release.contract_id != release_plan.contract_id - or applied_release.decision_id != release_plan.decision_id - or applied_release.change_set_id != release_plan.change_set_id - or applied_release.release_plan_id != release_plan.release_plan_id - or applied_release.version_resolution_id != version_resolution.version_resolution_id - or applied_release.release_revision_ref != release_plan.release_revision_ref - or applied_release.selected_version != version_resolution.selected_version - or applied_release.authorization_id != authorization.authorization_id - ): - raise ValueError("AppliedContractRelease does not match the exact release context") diff --git a/semapact/application/services/release_planning.py b/semapact/application/services/release_planning.py index 6fd33f58..5b1a5e5d 100644 --- a/semapact/application/services/release_planning.py +++ b/semapact/application/services/release_planning.py @@ -2,8 +2,6 @@ from __future__ import annotations -from datetime import date - from open_data_contract_standard.model import OpenDataContractStandard from semapact.application.models.release import ReleasePlanningResult @@ -29,7 +27,6 @@ def plan( base_contract: OpenDataContractStandard, candidate_contract: OpenDataContractStandard, *, - effective_date: date | str, base_revision_ref: str, candidate_revision_ref: str, authority_reference: str | None = None, @@ -38,7 +35,6 @@ def plan( proposal = self._governance.evaluate_proposal( base_contract, candidate_contract, - effective_date=effective_date, base_revision_ref=base_revision_ref, candidate_revision_ref=candidate_revision_ref, ) diff --git a/semapact/application/services/release_workflow.py b/semapact/application/services/release_workflow.py new file mode 100644 index 00000000..ea7d61e9 --- /dev/null +++ b/semapact/application/services/release_workflow.py @@ -0,0 +1,139 @@ +"""Application workflow for target-neutral formal contract releases.""" + +from __future__ import annotations + +from datetime import datetime + +from open_data_contract_standard.model import OpenDataContractStandard + +from semapact.approval import ApprovalRecord, build_approval_record +from semapact.approval.evidence import project_review_authorization_evidence +from semapact.application.models.release import ( + ReleaseBundle, + build_contract_release, + build_release_bundle, +) +from semapact.application.services.release_planning import ReleasePlanningService +from semapact.contractops import ( + ContractRelease, + ReviewEvidenceAction, + authorize_contract_operation, + build_release_snapshot, +) +from semapact.exceptions import ContractOpsAuthorizationError, ValidationError +from semapact.governance import DecisionResult, GovernanceOperation + + +class ReleaseWorkflowService: + """Assess and approve target-neutral formal contract releases.""" + + def __init__( + self, + *, + planning_service: ReleasePlanningService | None = None, + ) -> None: + self._planning = planning_service or ReleasePlanningService() + + def assess( + self, + base_contract: OpenDataContractStandard, + candidate_contract: OpenDataContractStandard, + *, + base_revision_ref: str, + candidate_revision_ref: str, + authority_reference: str | None = None, + ) -> ReleaseBundle: + """Build one immutable target-neutral formal release bundle.""" + planning = self._planning.plan( + base_contract, + candidate_contract, + base_revision_ref=base_revision_ref, + candidate_revision_ref=candidate_revision_ref, + authority_reference=authority_reference, + ) + snapshot = build_release_snapshot( + candidate_contract, + candidate_revision_ref=candidate_revision_ref, + decision=planning.decision, + change_set=planning.change_set, + release_plan=planning.release_plan, + version_resolution=planning.version_resolution, + ) + return build_release_bundle( + decision=planning.decision, + change_set=planning.change_set, + release_plan=planning.release_plan, + version_resolution=planning.version_resolution, + release_snapshot=snapshot, + ) + + def approve( + self, + bundle: ReleaseBundle, + *, + actor_reference: str, + recorded_at: datetime, + comment: str | None = None, + ) -> ApprovalRecord: + """Create explicit approval evidence for custom/manual REVIEW workflows.""" + if bundle.decision.decision is not DecisionResult.REVIEW: + raise ValidationError( + "Explicit release approval is only required for REVIEW releases" + ) + return build_approval_record( + decision_id=bundle.decision.decision_id, + change_set_id=bundle.change_set.change_set_id, + release_plan_id=bundle.release_plan.release_plan_id, + version_resolution_id=bundle.version_resolution.version_resolution_id, + operation=GovernanceOperation.PUBLISH, + action=ReviewEvidenceAction.APPROVE, + actor_reference=actor_reference, + recorded_at=recorded_at, + scope_reference=bundle.release_snapshot.release_snapshot_id, + comment=comment, + evidence_references=(bundle.bundle_digest,), + ) + + +class ReleaseFinalizer: + """Authorize one exact ReleaseBundle and construct its ContractRelease fact.""" + + def finalize( + self, + bundle: ReleaseBundle, + *, + approval: ApprovalRecord | None = None, + ) -> ContractRelease: + """Finalize domain state without persisting or materializing projections.""" + evidence = None + if bundle.decision.decision is DecisionResult.REVIEW: + if approval is None: + raise ContractOpsAuthorizationError( + "Formal release requires approval for REVIEW" + ) + if approval.scope_reference != bundle.release_snapshot.release_snapshot_id: + raise ValidationError( + "ApprovalRecord is not scoped to the exact ReleaseSnapshot" + ) + if bundle.bundle_digest not in approval.evidence_references: + raise ValidationError( + "ApprovalRecord does not reference the exact ReleaseBundle digest" + ) + evidence = project_review_authorization_evidence(approval) + elif approval is not None: + raise ValidationError("ALLOW release does not consume an ApprovalRecord") + + authorization = authorize_contract_operation( + bundle.decision, + bundle.change_set, + bundle.release_plan, + bundle.version_resolution, + GovernanceOperation.PUBLISH, + evidence=evidence, + ) + if not authorization.allowed: + raise ContractOpsAuthorizationError( + "Formal release is not authorized: " + f"{authorization.reason.value}" + ) + return build_contract_release(bundle) diff --git a/semapact/application/services/repository_classification.py b/semapact/application/services/repository_classification.py new file mode 100644 index 00000000..65a80586 --- /dev/null +++ b/semapact/application/services/repository_classification.py @@ -0,0 +1,147 @@ +"""Repository-level contract change classification for central contract repositories.""" + +from __future__ import annotations + +from dataclasses import dataclass +import hashlib +from pathlib import Path +from typing import Any + +from semapact.governance import DecisionResult, GovernanceDecision, evaluate_governance_decision +from semapact.utils.schema_utils import contract_to_model +from semapact.utils.yaml_utils import list_yaml_documents, load_yaml +from semapact.versioning import RequiredBump, suggest_release_version + + +@dataclass(slots=True) +class RepositoryContractChange: + """Per-contract change status within a repository comparison.""" + + contract_repo_path: str + status: str + contract_id: str | None = None + current_version: str | None = None + candidate_version: str | None = None + required_bump: RequiredBump | None = "none" + suggested_release_version: str | None = None + reasons: list[str] | None = None + governance_decision: GovernanceDecision | None = None + + +def classify_contracts_in_repo( + *, + base_root: str | Path, + candidate_root: str | Path, +) -> list[RepositoryContractChange]: + """Compare two contract roots using the canonical governance evaluator.""" + base_root_path = Path(base_root).expanduser().resolve() + candidate_root_path = Path(candidate_root).expanduser().resolve() + + base_index = _relative_contract_index(base_root_path) + candidate_index = _relative_contract_index(candidate_root_path) + + results: list[RepositoryContractChange] = [] + for relative_path in sorted(set(base_index) | set(candidate_index)): + base_path = base_index.get(relative_path) + candidate_path = candidate_index.get(relative_path) + + if base_path is None: + assert candidate_path is not None + candidate = contract_to_model(load_yaml(candidate_path)) + results.append( + RepositoryContractChange( + contract_repo_path=relative_path, + status="added", + contract_id=str(candidate.id or ""), + candidate_version=str(candidate.version or ""), + required_bump=None, + reasons=["New governed contract"], + ) + ) + continue + + if candidate_path is None: + base = contract_to_model(load_yaml(base_path)) + results.append( + RepositoryContractChange( + contract_repo_path=relative_path, + status="removed", + contract_id=str(base.id or ""), + current_version=str(base.version or ""), + required_bump=None, + reasons=["Governed contract missing from candidate root"], + ) + ) + continue + + base = contract_to_model(load_yaml(base_path)) + candidate = contract_to_model(load_yaml(candidate_path)) + decision = evaluate_governance_decision(base, candidate) + status = ( + "blocked" + if decision.decision is DecisionResult.BLOCK + else ("changed" if decision.evidence.has_changes else "unchanged") + ) + required_bump = decision.required_version_bump + results.append( + RepositoryContractChange( + contract_repo_path=relative_path, + status=status, + contract_id=str(base.id or ""), + current_version=str(base.version or ""), + candidate_version=str(candidate.version or ""), + required_bump=required_bump, + suggested_release_version=( + suggest_release_version(str(base.version or ""), required_bump) + if required_bump != "none" + else None + ), + reasons=[reason.message for reason in decision.reasons] + or ["No contract changes detected"], + governance_decision=decision, + ) + ) + return results + + +def repository_change_to_dict(change: RepositoryContractChange) -> dict[str, Any]: + """Serialize repository classification for CLI and CI matrix consumers.""" + decision = ( + change.governance_decision.model_dump(mode="json") + if change.governance_decision is not None + else None + ) + return { + "contract_repo_path": change.contract_repo_path, + "contractRepoPath": change.contract_repo_path, + "artifact_key": _artifact_key(change.contract_repo_path), + "artifactKey": _artifact_key(change.contract_repo_path), + "status": change.status, + "contract_id": change.contract_id, + "contractId": change.contract_id, + "current_version": change.current_version, + "currentVersion": change.current_version, + "candidate_version": change.candidate_version, + "candidateVersion": change.candidate_version, + "required_bump": change.required_bump, + "requiredBump": change.required_bump, + "suggested_release_version": change.suggested_release_version, + "suggestedReleaseVersion": change.suggested_release_version, + "reasons": change.reasons, + "governance_decision": decision, + "governanceDecision": decision, + } + + +def _relative_contract_index(root: Path) -> dict[str, Path]: + if not root.exists(): + return {} + return { + str(path.relative_to(root)): path + for path in (Path(item) for item in list_yaml_documents(root)) + } + + +def _artifact_key(contract_repo_path: str) -> str: + """Return a collision-resistant filesystem/artifact key for one repo-relative path.""" + return hashlib.sha256(contract_repo_path.encode("utf-8")).hexdigest() diff --git a/semapact/application/services/runtime_history.py b/semapact/application/services/runtime_history.py deleted file mode 100644 index ce059a1d..00000000 --- a/semapact/application/services/runtime_history.py +++ /dev/null @@ -1,207 +0,0 @@ -"""Application orchestration for point-in-time runtime evidence history.""" - -from __future__ import annotations - -from semapact.history import ( - DeploymentRecord, - DeploymentRecordHistoryRepository, - ReleaseRecord, - ReleaseRecordHistoryRepository, - RuntimeObservationHistoryRepository, - RuntimeObservationRecord, - RuntimeReconciliationHistoryRepository, - RuntimeReconciliationRecord, -) -from semapact.history.integrity import ( - compute_runtime_observation_record_id, - compute_runtime_reconciliation_record_id, -) -from semapact.observation import ( - ObservedPlatformState, - fingerprint_observed_state, - with_observed_state_fingerprint, -) -from semapact.reconciliation import ( - ReconciliationResult, - classify_reconciliation_status, -) - - -class RuntimeHistoryService: - """Persist canonical M1 runtime evidence with optional exact lifecycle links.""" - - def __init__( - self, - *, - observations: RuntimeObservationHistoryRepository, - reconciliations: RuntimeReconciliationHistoryRepository, - releases: ReleaseRecordHistoryRepository, - deployments: DeploymentRecordHistoryRepository, - ) -> None: - self._observations = observations - self._reconciliations = reconciliations - self._releases = releases - self._deployments = deployments - - def record_reconciliation( - self, - observation: ObservedPlatformState, - result: ReconciliationResult, - *, - release_record_id: str | None = None, - deployment_record_id: str | None = None, - ) -> RuntimeReconciliationRecord: - """Record one canonical observation/result pair without recomputing M1 semantics.""" - if not isinstance(observation, ObservedPlatformState): - raise TypeError( - "observation must be ObservedPlatformState, " - f"got {type(observation).__name__}" - ) - if not isinstance(result, ReconciliationResult): - raise TypeError( - f"result must be ReconciliationResult, got {type(result).__name__}" - ) - - canonical_observation = _canonical_observation(observation) - _validate_observation_result_link(canonical_observation, result) - release = _load_optional_release(self._releases, release_record_id) - deployment = _load_optional_deployment( - self._deployments, - deployment_record_id, - ) - _validate_optional_history_links( - canonical_observation, - result, - release=release, - deployment=deployment, - ) - - observation_record_id = compute_runtime_observation_record_id(canonical_observation) - observation_record = RuntimeObservationRecord( - observation_record_id=observation_record_id, - observation=canonical_observation, - ) - - status = classify_reconciliation_status(result) - runtime_reconciliation_record_id = compute_runtime_reconciliation_record_id( - observation_record_id=observation_record_id, - result=result, - status=status.value, - release_record_id=( - release.release_record_id if release is not None else None - ), - deployment_record_id=( - deployment.deployment_record_id if deployment is not None else None - ), - ) - record = RuntimeReconciliationRecord( - runtime_reconciliation_record_id=runtime_reconciliation_record_id, - observation_record_id=observation_record_id, - result=result, - status=status, - release_record_id=( - release.release_record_id if release is not None else None - ), - deployment_record_id=( - deployment.deployment_record_id if deployment is not None else None - ), - ) - - # Persist the canonical observation first so the final linkage record never - # points at absent runtime evidence after a partial storage failure. - self._observations.put_runtime_observation_record(observation_record) - self._reconciliations.put_runtime_reconciliation_record(record) - return record - - -def _canonical_observation(observation: ObservedPlatformState) -> ObservedPlatformState: - semantic_fingerprint = fingerprint_observed_state(observation) - if ( - observation.fingerprint is not None - and observation.fingerprint != semantic_fingerprint - ): - raise ValueError( - "ObservedPlatformState fingerprint does not match canonical semantic content" - ) - if observation.fingerprint is not None: - return observation - return with_observed_state_fingerprint(observation) - - -def _validate_observation_result_link( - observation: ObservedPlatformState, - result: ReconciliationResult, -) -> None: - if observation.source_identifier != result.observation_source_identifier: - raise ValueError( - "ReconciliationResult source does not match ObservedPlatformState" - ) - if observation.fingerprint != result.observation_fingerprint: - raise ValueError( - "ReconciliationResult fingerprint does not match ObservedPlatformState" - ) - - -def _load_optional_release( - repository: ReleaseRecordHistoryRepository, - release_record_id: str | None, -) -> ReleaseRecord | None: - if release_record_id is None: - return None - return repository.get_release_record(_required_text(release_record_id, "release_record_id")) - - -def _load_optional_deployment( - repository: DeploymentRecordHistoryRepository, - deployment_record_id: str | None, -) -> DeploymentRecord | None: - if deployment_record_id is None: - return None - return repository.get_deployment_record( - _required_text(deployment_record_id, "deployment_record_id") - ) - - -def _validate_optional_history_links( - observation: ObservedPlatformState, - result: ReconciliationResult, - *, - release: ReleaseRecord | None, - deployment: DeploymentRecord | None, -) -> None: - if release is not None: - if ( - release.contract_id != result.contract_id - or release.contract_version != result.contract_version - ): - raise ValueError( - "Runtime reconciliation does not match the linked ReleaseRecord" - ) - - if deployment is None: - return - if release is None: - raise ValueError( - "deployment-linked runtime history requires an explicit ReleaseRecord link" - ) - if deployment.release_record_id != release.release_record_id: - raise ValueError( - "DeploymentRecord does not belong to the linked ReleaseRecord" - ) - if deployment.platform.strip().casefold() != observation.platform.strip().casefold(): - raise ValueError( - "DeploymentRecord platform does not match ObservedPlatformState" - ) - if deployment.source_reference != observation.source_identifier: - raise ValueError( - "DeploymentRecord source does not match ObservedPlatformState" - ) - - -def _required_text(value: str, field_name: str) -> str: - if not isinstance(value, str): - raise TypeError(f"{field_name} must be str") - cleaned = value.strip() - if not cleaned: - raise ValueError(f"{field_name} must not be empty") - return cleaned diff --git a/semapact/change_context.py b/semapact/change_context.py index 3bf95336..6f72d568 100644 --- a/semapact/change_context.py +++ b/semapact/change_context.py @@ -1,4 +1,4 @@ -"""Explicit contextual inputs for deterministic contract change analysis.""" +"""Explicit business-effective context for lifecycle mutations.""" from __future__ import annotations @@ -8,12 +8,12 @@ class ChangeContext(BaseModel): - """Governance-relevant context supplied explicitly by the caller. + """Business-effective inputs used when a mutation materializes dated state. - ``effective_date`` is required and intentionally has no wall-clock default. - Callers must create the context at an upstream workflow boundary and pass the - same instance through merge and governance evaluation. Lower layers must not - regenerate or overwrite this date. + The context is deliberately explicit and has no wall-clock default. It belongs at + lifecycle/merge mutation boundaries where SemaPact may write facts such as + `deprecationDate`. Pure governance, release planning, and deployment assessment + must not depend on it. """ model_config = ConfigDict(frozen=True, extra="forbid") diff --git a/semapact/contractops/__init__.py b/semapact/contractops/__init__.py index b95e5f8f..8e04024a 100644 --- a/semapact/contractops/__init__.py +++ b/semapact/contractops/__init__.py @@ -5,20 +5,16 @@ build_change_set, build_change_set_from_decision, ) -from semapact.contractops.execution import ( - ContractReleasePublisher, - apply_contract_release, - publish_contract_release, -) +from semapact.contractops.execution import build_release_snapshot from semapact.contractops.execution_models import ( - AppliedContractRelease, - PublicationResult, + ContractRelease, + ReleaseSnapshot, ) from semapact.contractops.integrity import ( - validate_applied_release_identity, + validate_contract_release_identity, + validate_release_snapshot_identity, validate_change_set_identity, validate_contractops_authorization_identity, - validate_publication_result_identity, validate_release_plan_identity, validate_version_resolution_identity, ) @@ -41,12 +37,11 @@ ) __all__ = [ - "AppliedContractRelease", + "ContractRelease", + "ReleaseSnapshot", "AuthorizationReason", "ChangeSet", "ContractOpsAuthorization", - "ContractReleasePublisher", - "PublicationResult", "ReleasePlan", "ReleasePrecondition", "ReviewAuthorizationEvidence", @@ -54,18 +49,17 @@ "VersionAuthority", "VersionAuthorityConfig", "VersionResolution", - "apply_contract_release", + "build_release_snapshot", "authorize_contract_operation", "build_change_set", "build_change_set_from_decision", "build_release_plan", "extract_version_from_release_reference", - "publish_contract_release", "resolve_release_version", - "validate_applied_release_identity", + "validate_contract_release_identity", + "validate_release_snapshot_identity", "validate_change_set_identity", "validate_contractops_authorization_identity", - "validate_publication_result_identity", "validate_release_plan_identity", "validate_version_resolution_identity", ] diff --git a/semapact/contractops/changeset.py b/semapact/contractops/changeset.py index 2faa7fc7..afc8a91b 100644 --- a/semapact/contractops/changeset.py +++ b/semapact/contractops/changeset.py @@ -4,7 +4,6 @@ from collections.abc import Sequence -from semapact.change_context import ChangeContext from semapact.contractops.integrity import ( SEMAPACT_CHANGESET_NAMESPACE, compute_change_set_id, @@ -20,7 +19,6 @@ def build_change_set( base_revision_ref: str, candidate_revision_ref: str, changes: Sequence[GovernanceChange], - context: ChangeContext, source: str | None = None, actor_reference: str | None = None, ) -> ChangeSet: @@ -49,7 +47,6 @@ def build_change_set( base_revision_ref=cleaned_base_ref, candidate_revision_ref=cleaned_candidate_ref, changes=canonical_changes, - context=context, source=cleaned_source, actor_reference=cleaned_actor_reference, ) @@ -60,7 +57,6 @@ def build_change_set( base_revision_ref=cleaned_base_ref, candidate_revision_ref=cleaned_candidate_ref, changes=canonical_changes, - context=context, source=cleaned_source, actor_reference=cleaned_actor_reference, ) @@ -80,7 +76,6 @@ def build_change_set_from_decision( base_revision_ref=base_revision_ref, candidate_revision_ref=candidate_revision_ref, changes=decision.changes, - context=decision.context, source=source, actor_reference=actor_reference, ) diff --git a/semapact/contractops/context.py b/semapact/contractops/context.py index 37df8ec2..1a3f90ee 100644 --- a/semapact/contractops/context.py +++ b/semapact/contractops/context.py @@ -29,10 +29,6 @@ def validate_proposal_context( raise ReleaseValidationError( "ChangeSet and GovernanceDecision contract IDs do not match" ) - if change_set.context != decision.context: - raise ReleaseValidationError( - "ChangeSet and GovernanceDecision governance contexts do not match" - ) if change_set.changes != decision.changes: raise ReleaseValidationError( "ChangeSet changes do not match authoritative GovernanceDecision changes" diff --git a/semapact/contractops/execution.py b/semapact/contractops/execution.py index 93b3ec4b..c30443b9 100644 --- a/semapact/contractops/execution.py +++ b/semapact/contractops/execution.py @@ -1,43 +1,24 @@ -"""Explicit ContractOps APPLY and PUBLISH execution boundaries.""" +"""Pure ContractOps release snapshot construction.""" from __future__ import annotations -from typing import Protocol - from open_data_contract_standard.model import OpenDataContractStandard from semapact.contractops.context import validate_release_context -from semapact.contractops.execution_models import AppliedContractRelease, PublicationResult -from semapact.contractops.integrity import ( - SEMAPACT_APPLIED_RELEASE_NAMESPACE, - SEMAPACT_PUBLICATION_NAMESPACE, - compute_applied_release_id, - compute_publication_id, - validate_applied_release_identity, - validate_contractops_authorization_identity, -) +from semapact.contractops.execution_models import ReleaseSnapshot +from semapact.contractops.integrity import compute_release_snapshot_id from semapact.contractops.models import ( ChangeSet, - ContractOpsAuthorization, ReleasePlan, VersionResolution, ) -from semapact.exceptions import ContractOpsAuthorizationError, ReleaseValidationError -from semapact.governance.gate import GovernanceOperation +from semapact.exceptions import ReleaseValidationError from semapact.governance.models import GovernanceDecision from semapact.odcs.serialization import canonical_contract_json from semapact.versioning import normalize_semver -class ContractReleasePublisher(Protocol): - """Narrow external publication port for one applied contract release.""" - - def publish(self, release: AppliedContractRelease) -> str: - """Publish the exact applied release and return an opaque reference.""" - ... - - -def apply_contract_release( +def build_release_snapshot( candidate_contract: OpenDataContractStandard, *, candidate_revision_ref: str, @@ -45,13 +26,8 @@ def apply_contract_release( change_set: ChangeSet, release_plan: ReleasePlan, version_resolution: VersionResolution, - authorization: ContractOpsAuthorization, -) -> AppliedContractRelease: - """Materialize the exact released ODCS state after explicit APPLY authorization. - - This is the canonical M2 apply path. It never re-runs governance, change - classification, or version authority. The input candidate is not mutated. - """ +) -> ReleaseSnapshot: + """Freeze one exact governed release without requiring side-effect authorization.""" if not isinstance(candidate_contract, OpenDataContractStandard): raise TypeError( "candidate_contract must be OpenDataContractStandard, " @@ -61,14 +37,6 @@ def apply_contract_release( raise TypeError("candidate_revision_ref must be str") validate_release_context(decision, change_set, release_plan, version_resolution) - _validate_authorization( - authorization, - decision=decision, - change_set=change_set, - release_plan=release_plan, - version_resolution=version_resolution, - operation=GovernanceOperation.APPLY, - ) supplied_revision_ref = candidate_revision_ref.strip() if not supplied_revision_ref: @@ -104,7 +72,7 @@ def apply_contract_release( released_contract.version = selected_version released_contract_json = canonical_contract_json(released_contract) - applied_release_id = compute_applied_release_id( + release_snapshot_id = compute_release_snapshot_id( contract_id=release_plan.contract_id, decision_id=decision.decision_id, change_set_id=change_set.change_set_id, @@ -112,12 +80,10 @@ def apply_contract_release( version_resolution_id=version_resolution.version_resolution_id, release_revision_ref=release_plan.release_revision_ref, selected_version=selected_version, - authorization_id=authorization.authorization_id, released_contract_json=released_contract_json, ) - - return AppliedContractRelease( - applied_release_id=applied_release_id, + return ReleaseSnapshot( + release_snapshot_id=release_snapshot_id, contract_id=release_plan.contract_id, decision_id=decision.decision_id, change_set_id=change_set.change_set_id, @@ -125,122 +91,17 @@ def apply_contract_release( version_resolution_id=version_resolution.version_resolution_id, release_revision_ref=release_plan.release_revision_ref, selected_version=selected_version, - authorization_id=authorization.authorization_id, released_contract_json=released_contract_json, ) -def publish_contract_release( - release: AppliedContractRelease, - *, - authorization: ContractOpsAuthorization, - publisher: ContractReleasePublisher, -) -> PublicationResult: - """Invoke one external publisher only after exact PUBLISH authorization.""" - if not isinstance(release, AppliedContractRelease): - raise TypeError( - f"release must be AppliedContractRelease, got {type(release).__name__}" - ) - if not isinstance(authorization, ContractOpsAuthorization): - raise TypeError( - "authorization must be ContractOpsAuthorization, " - f"got {type(authorization).__name__}" - ) - if not hasattr(publisher, "publish") or not callable(publisher.publish): - raise TypeError("publisher must provide a callable publish(release) method") - - validate_applied_release_identity(release) - _validate_publication_authorization(release, authorization) - - publication_reference = publisher.publish(release) - if not isinstance(publication_reference, str): - raise TypeError("publisher.publish() must return str") - publication_reference = publication_reference.strip() - if not publication_reference: - raise ReleaseValidationError( - "publisher.publish() returned an empty publication reference" - ) - - publication_id = compute_publication_id( - applied_release_id=release.applied_release_id, - authorization_id=authorization.authorization_id, - publication_reference=publication_reference, - ) - return PublicationResult( - publication_id=publication_id, - applied_release_id=release.applied_release_id, - authorization_id=authorization.authorization_id, - publication_reference=publication_reference, - ) - - -def _validate_authorization( - authorization: ContractOpsAuthorization, - *, - decision: GovernanceDecision, - change_set: ChangeSet, - release_plan: ReleasePlan, - version_resolution: VersionResolution, - operation: GovernanceOperation, -) -> None: - if not isinstance(authorization, ContractOpsAuthorization): - raise TypeError( - "authorization must be ContractOpsAuthorization, " - f"got {type(authorization).__name__}" - ) - validate_contractops_authorization_identity(authorization) - if authorization.operation is not operation: - raise ReleaseValidationError( - f"Authorization operation must be {operation.value}, " - f"got {authorization.operation.value}" - ) - if ( - authorization.decision_id != decision.decision_id - or authorization.change_set_id != change_set.change_set_id - or authorization.release_plan_id != release_plan.release_plan_id - or authorization.version_resolution_id - != version_resolution.version_resolution_id - ): - raise ReleaseValidationError( - "Authorization does not match the exact version-resolved release context" - ) - if not authorization.allowed: - raise ContractOpsAuthorizationError( - f"ContractOps {operation.value} is not authorized: " - f"{authorization.reason.value}" - ) - - -def _validate_publication_authorization( - release: AppliedContractRelease, - authorization: ContractOpsAuthorization, -) -> None: - validate_contractops_authorization_identity(authorization) - if authorization.operation is not GovernanceOperation.PUBLISH: - raise ReleaseValidationError( - "Publication requires operation-scoped PUBLISH authorization" - ) - if ( - authorization.decision_id != release.decision_id - or authorization.change_set_id != release.change_set_id - or authorization.release_plan_id != release.release_plan_id - or authorization.version_resolution_id != release.version_resolution_id - ): - raise ReleaseValidationError( - "PUBLISH authorization does not match the applied release context" - ) - if not authorization.allowed: - raise ContractOpsAuthorizationError( - "ContractOps PUBLISH is not authorized: " - f"{authorization.reason.value}" - ) - - def _canonical_version(version: str, *, field_name: str) -> str: try: canonical = normalize_semver(version) except ValueError as exc: - raise ReleaseValidationError(f"{field_name} is not valid semantic version") from exc + raise ReleaseValidationError( + f"{field_name} is not valid semantic version" + ) from exc if canonical != str(version).strip(): raise ReleaseValidationError( f"{field_name} must use canonical major.minor.patch form" diff --git a/semapact/contractops/execution_models.py b/semapact/contractops/execution_models.py index 0604dfb1..68b2c3f6 100644 --- a/semapact/contractops/execution_models.py +++ b/semapact/contractops/execution_models.py @@ -1,4 +1,4 @@ -"""Immutable artifacts for explicit ContractOps APPLY and PUBLISH phases.""" +"""Immutable artifacts for canonical ContractOps release flow.""" from __future__ import annotations @@ -8,10 +8,10 @@ from semapact.contractops.models import ContractOpsModel -class AppliedContractRelease(ContractOpsModel): - """Exact released ODCS state produced by an authorized APPLY operation.""" +class ReleaseSnapshot(ContractOpsModel): + """Exact immutable released ODCS state before any external side effect.""" - applied_release_id: str + release_snapshot_id: str contract_id: str decision_id: str change_set_id: str @@ -19,11 +19,10 @@ class AppliedContractRelease(ContractOpsModel): version_resolution_id: str release_revision_ref: str selected_version: str - authorization_id: str released_contract_json: str @field_validator( - "applied_release_id", + "release_snapshot_id", "contract_id", "decision_id", "change_set_id", @@ -31,21 +30,20 @@ class AppliedContractRelease(ContractOpsModel): "version_resolution_id", "release_revision_ref", "selected_version", - "authorization_id", "released_contract_json", ) @classmethod - def _require_non_empty_text(cls, value: str) -> str: + def _require_snapshot_text(cls, value: str) -> str: cleaned = value.strip() if not cleaned: raise ValueError("value must not be empty") return cleaned @model_validator(mode="after") - def _validate_released_contract_snapshot(self) -> AppliedContractRelease: + def _validate_released_contract_snapshot(self) -> "ReleaseSnapshot": contract = OpenDataContractStandard.model_validate_json(self.released_contract_json) if str(contract.id or "") != self.contract_id: - raise ValueError("released contract ID does not match AppliedContractRelease") + raise ValueError("released contract ID does not match ReleaseSnapshot") if str(contract.version or "") != self.selected_version: raise ValueError("released contract version does not match selected_version") return self @@ -55,23 +53,54 @@ def to_contract(self) -> OpenDataContractStandard: return OpenDataContractStandard.model_validate_json(self.released_contract_json) -class PublicationResult(ContractOpsModel): - """Successful result of one explicitly authorized external publication.""" +class ContractRelease(ContractOpsModel): + """Finalized target-neutral formal contract release.""" - publication_id: str - applied_release_id: str - authorization_id: str - publication_reference: str + contract_release_id: str + contract_id: str + contract_version: str + decision_id: str + change_set_id: str + release_plan_id: str + version_resolution_id: str + release_snapshot_id: str + source_revision_ref: str + released_contract_json: str @field_validator( - "publication_id", - "applied_release_id", - "authorization_id", - "publication_reference", + "contract_release_id", + "contract_id", + "contract_version", + "decision_id", + "change_set_id", + "release_plan_id", + "version_resolution_id", + "release_snapshot_id", + "source_revision_ref", + "released_contract_json", ) @classmethod - def _require_publication_text(cls, value: str) -> str: + def _require_contract_release_text(cls, value: str) -> str: cleaned = value.strip() if not cleaned: raise ValueError("value must not be empty") return cleaned + + @model_validator(mode="after") + def _validate_contract_snapshot(self) -> "ContractRelease": + contract = OpenDataContractStandard.model_validate_json( + self.released_contract_json + ) + if str(contract.id or "").strip() != self.contract_id: + raise ValueError("released contract ID does not match ContractRelease") + if str(contract.version or "").strip() != self.contract_version: + raise ValueError( + "released contract version does not match ContractRelease" + ) + return self + + def to_contract(self) -> OpenDataContractStandard: + """Materialize a fresh ODCS model from the finalized release.""" + return OpenDataContractStandard.model_validate_json( + self.released_contract_json + ) diff --git a/semapact/contractops/integrity.py b/semapact/contractops/integrity.py index 1b606459..794f9981 100644 --- a/semapact/contractops/integrity.py +++ b/semapact/contractops/integrity.py @@ -12,8 +12,10 @@ import uuid from collections.abc import Sequence -from semapact.change_context import ChangeContext -from semapact.contractops.execution_models import AppliedContractRelease, PublicationResult +from semapact.contractops.execution_models import ( + ContractRelease, + ReleaseSnapshot, +) from semapact.contractops.models import ( ChangeSet, ContractOpsAuthorization, @@ -37,11 +39,11 @@ SEMAPACT_CONTRACTOPS_AUTHORIZATION_NAMESPACE = uuid.UUID( "b6218d0c-3f9d-44a2-8d68-e3b0ee170948" ) -SEMAPACT_APPLIED_RELEASE_NAMESPACE = uuid.UUID( - "a5de2e65-aee7-48ac-9cb6-6a785f4cdf33" +SEMAPACT_RELEASE_SNAPSHOT_NAMESPACE = uuid.UUID( + "d4248396-5ec7-4e6a-a238-b2db1abf76e8" ) -SEMAPACT_PUBLICATION_NAMESPACE = uuid.UUID( - "f776fc77-b37f-43ef-bf6d-d8dcf38d264f" +SEMAPACT_CONTRACT_RELEASE_NAMESPACE = uuid.UUID( + "0f95c8c5-4957-43f7-a38c-2756605c2df6" ) @@ -60,10 +62,10 @@ def validate_contractops_artifact_identity(artifact: object) -> None: validate_version_resolution_identity(artifact) elif isinstance(artifact, ContractOpsAuthorization): validate_contractops_authorization_identity(artifact) - elif isinstance(artifact, AppliedContractRelease): - validate_applied_release_identity(artifact) - elif isinstance(artifact, PublicationResult): - validate_publication_result_identity(artifact) + elif isinstance(artifact, ReleaseSnapshot): + validate_release_snapshot_identity(artifact) + elif isinstance(artifact, ContractRelease): + validate_contract_release_identity(artifact) def compute_change_set_id( @@ -72,7 +74,6 @@ def compute_change_set_id( base_revision_ref: str, candidate_revision_ref: str, changes: Sequence[GovernanceChange], - context: ChangeContext, source: str | None, actor_reference: str | None, ) -> str: @@ -81,7 +82,6 @@ def compute_change_set_id( "contract_id": contract_id, "base_revision_ref": base_revision_ref, "candidate_revision_ref": candidate_revision_ref, - "context": context.model_dump(mode="json"), "changes": [change.model_dump(mode="json") for change in canonical_changes], "source": source, "actor_reference": actor_reference, @@ -98,7 +98,6 @@ def validate_change_set_identity(change_set: ChangeSet) -> None: base_revision_ref=change_set.base_revision_ref, candidate_revision_ref=change_set.candidate_revision_ref, changes=change_set.changes, - context=change_set.context, source=change_set.source, actor_reference=change_set.actor_reference, ) @@ -202,10 +201,7 @@ def compute_contractops_authorization_id( "evidence_reference": evidence_reference, "evidence_action": evidence_action, } - # Compatibility invariant: scope_reference did not participate in pre-#122 IDs - # when it was absent. Keep that byte-for-byte behavior. - if scope_reference is not None: - payload["scope_reference"] = scope_reference + payload["scope_reference"] = scope_reference return deterministic_uuid5(SEMAPACT_CONTRACTOPS_AUTHORIZATION_NAMESPACE, payload) @@ -235,7 +231,7 @@ def validate_contractops_authorization_identity( ) -def compute_applied_release_id( +def compute_release_snapshot_id( *, contract_id: str, decision_id: str, @@ -244,7 +240,6 @@ def compute_applied_release_id( version_resolution_id: str, release_revision_ref: str, selected_version: str, - authorization_id: str, released_contract_json: str, ) -> str: payload = { @@ -255,51 +250,73 @@ def compute_applied_release_id( "version_resolution_id": version_resolution_id, "release_revision_ref": release_revision_ref, "selected_version": selected_version, - "authorization_id": authorization_id, "released_contract_json": released_contract_json, } - return deterministic_uuid5(SEMAPACT_APPLIED_RELEASE_NAMESPACE, payload) - - -def validate_applied_release_identity(release: AppliedContractRelease) -> None: - _require_canonical_json(release.released_contract_json, "AppliedContractRelease snapshot") - expected = compute_applied_release_id( - contract_id=release.contract_id, - decision_id=release.decision_id, - change_set_id=release.change_set_id, - release_plan_id=release.release_plan_id, - version_resolution_id=release.version_resolution_id, - release_revision_ref=release.release_revision_ref, - selected_version=release.selected_version, - authorization_id=release.authorization_id, - released_contract_json=release.released_contract_json, + return deterministic_uuid5(SEMAPACT_RELEASE_SNAPSHOT_NAMESPACE, payload) + + +def validate_release_snapshot_identity(snapshot: ReleaseSnapshot) -> None: + _require_canonical_json(snapshot.released_contract_json, "ReleaseSnapshot snapshot") + expected = compute_release_snapshot_id( + contract_id=snapshot.contract_id, + decision_id=snapshot.decision_id, + change_set_id=snapshot.change_set_id, + release_plan_id=snapshot.release_plan_id, + version_resolution_id=snapshot.version_resolution_id, + release_revision_ref=snapshot.release_revision_ref, + selected_version=snapshot.selected_version, + released_contract_json=snapshot.released_contract_json, ) - _require_identity(release.applied_release_id, expected, "AppliedContractRelease") + _require_identity(snapshot.release_snapshot_id, expected, "ReleaseSnapshot") -def compute_publication_id( +def compute_contract_release_id( *, - applied_release_id: str, - authorization_id: str, - publication_reference: str, + contract_id: str, + contract_version: str, + decision_id: str, + change_set_id: str, + release_plan_id: str, + version_resolution_id: str, + release_snapshot_id: str, + source_revision_ref: str, + released_contract_json: str, ) -> str: + """Derive deterministic identity for one finalized formal contract release.""" return deterministic_uuid5( - SEMAPACT_PUBLICATION_NAMESPACE, + SEMAPACT_CONTRACT_RELEASE_NAMESPACE, { - "applied_release_id": applied_release_id, - "authorization_id": authorization_id, - "publication_reference": publication_reference, + "contract_id": contract_id, + "contract_version": contract_version, + "decision_id": decision_id, + "change_set_id": change_set_id, + "release_plan_id": release_plan_id, + "version_resolution_id": version_resolution_id, + "release_snapshot_id": release_snapshot_id, + "source_revision_ref": source_revision_ref, + "released_contract_json": released_contract_json, }, ) -def validate_publication_result_identity(result: PublicationResult) -> None: - expected = compute_publication_id( - applied_release_id=result.applied_release_id, - authorization_id=result.authorization_id, - publication_reference=result.publication_reference, +def validate_contract_release_identity(release: ContractRelease) -> None: + """Fail closed when finalized release identity/content diverge.""" + _require_canonical_json( + release.released_contract_json, + "ContractRelease snapshot", + ) + expected = compute_contract_release_id( + contract_id=release.contract_id, + contract_version=release.contract_version, + decision_id=release.decision_id, + change_set_id=release.change_set_id, + release_plan_id=release.release_plan_id, + version_resolution_id=release.version_resolution_id, + release_snapshot_id=release.release_snapshot_id, + source_revision_ref=release.source_revision_ref, + released_contract_json=release.released_contract_json, ) - _require_identity(result.publication_id, expected, "PublicationResult") + _require_identity(release.contract_release_id, expected, "ContractRelease") def _require_identity(actual: str, expected: str, artifact: str) -> None: diff --git a/semapact/contractops/models.py b/semapact/contractops/models.py index 1135b1d0..7e164529 100644 --- a/semapact/contractops/models.py +++ b/semapact/contractops/models.py @@ -7,7 +7,6 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator -from semapact.change_context import ChangeContext from semapact.governance.gate import GovernanceOperation from semapact.lifecycle.changes import GovernanceChange from semapact.versioning import ActualVersionBump, RequiredBump @@ -39,7 +38,6 @@ class ChangeSet(ContractOpsModel): base_revision_ref: str candidate_revision_ref: str changes: tuple[GovernanceChange, ...] - context: ChangeContext source: str | None = None actor_reference: str | None = None @@ -190,8 +188,8 @@ class ReviewAuthorizationEvidence(ContractOpsModel): """Opaque review evidence projected onto one exact version-resolved action. ``scope_reference`` is optional downstream scope provenance. ContractOps preserves - but does not interpret it. For example, deployment review can bind the evidence - to an exact ``DeploymentPlan`` without making ContractOps depend on deployment. + but does not interpret it. The formal release workflow uses it to bind PUBLISH + approval to the exact ``ReleaseSnapshot``. """ evidence_reference: str diff --git a/semapact/core/__init__.py b/semapact/core/__init__.py index a92a6658..08ab9f06 100644 --- a/semapact/core/__init__.py +++ b/semapact/core/__init__.py @@ -6,16 +6,6 @@ set_contract_tags_list, ) from semapact.core.loader import ContractLoader, load_contract -from semapact.core.release import ( - ContractChangeAssessment, - PromotionResult, - apply_release_candidate, - classify_contract_change, - classify_version_bump, - parse_release_tag_version, - prepare_release_candidate, - suggest_release_version, -) from semapact.core.validator import ( ContractValidator, ValidationIssue, @@ -23,22 +13,14 @@ ) __all__ = [ - "ContractChangeAssessment", "ContractLoader", "ContractValidator", - "PromotionResult", - "apply_release_candidate", - "classify_contract_change", - "classify_version_bump", "ValidationIssue", "ValidationReport", "load_contract", "normalize_draft_contract", "normalize_tags", - "parse_release_tag_version", - "prepare_release_candidate", "schema_items", "set_contract_description_part", "set_contract_tags_list", - "suggest_release_version", ] diff --git a/semapact/core/config_schema.py b/semapact/core/config_schema.py new file mode 100644 index 00000000..d0252535 --- /dev/null +++ b/semapact/core/config_schema.py @@ -0,0 +1,101 @@ +"""Typed configuration schema for optional operational deployment history.""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, field_validator + + +class _OperationalHistoryConfig(BaseModel): + """Shared immutable configuration contract for operational history backends.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + backend: str + + +class SQLiteOperationalHistoryConfig(_OperationalHistoryConfig): + """Local SQLite operational history configuration.""" + + backend: Literal["sqlite"] + path: str = Field(min_length=1) + + @field_validator("path") + @classmethod + def _require_path(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("SQLite operational history path must not be empty") + return cleaned + + def as_uri(self) -> str: + return f"sqlite:///{self.path}" + + +class DeltaOperationalHistoryConfig(_OperationalHistoryConfig): + """Delta Lake operational history configuration.""" + + backend: Literal["delta"] + table_uri: str = Field(min_length=1) + + @field_validator("table_uri") + @classmethod + def _require_table_uri(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("Delta operational history table_uri must not be empty") + return cleaned + + def as_uri(self) -> str: + return f"delta:///{self.table_uri}" + + +OperationalHistoryConfig = Annotated[ + SQLiteOperationalHistoryConfig | DeltaOperationalHistoryConfig, + Field(discriminator="backend"), +] + + +class HistoryConfig(BaseModel): + """Typed history configuration while unrelated configuration remains extensible.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + operational: OperationalHistoryConfig | None = None + + +class SemaPactConfigSchema(BaseModel): + """Incremental root schema for project/global SemaPact configuration.""" + + model_config = ConfigDict( + extra="allow", + json_schema_extra={ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://semapact.org/schemas/semapact-config.schema.json", + }, + ) + + history: HistoryConfig | None = None + +_OPERATIONAL_HISTORY_ADAPTER = TypeAdapter(OperationalHistoryConfig) + + +def parse_operational_history_config( + value: object, +) -> OperationalHistoryConfig | None: + """Validate the history.operational config subsection fail closed.""" + if value is None: + return None + return _OPERATIONAL_HISTORY_ADAPTER.validate_python(value) + + +def operational_history_uri_from_config(value: object) -> str | None: + """Return the normalized sink URI represented by typed configuration.""" + config = parse_operational_history_config(value) + return config.as_uri() if config is not None else None + + +def published_config_json_schema() -> dict[str, object]: + """Return the canonical machine-readable SemaPact configuration schema.""" + return SemaPactConfigSchema.model_json_schema() diff --git a/semapact/core/editor_contract.py b/semapact/core/editor_contract.py deleted file mode 100644 index 0827a634..00000000 --- a/semapact/core/editor_contract.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Compatibility facade for editor helpers. - -New code should import from: -- `semapact.core.editor_semantics` -- `semapact.core.editor_rows` - -This module remains as a stable compatibility layer while the editor codebase -is being split into ODCS-aware semantics and dict-based UI row adapters. -""" - -from __future__ import annotations - -from semapact.core.editor_rows import ( - add_field, - add_quality_rule, - apply_field_detail, - apply_quality_rows, - apply_quick_field_rows, - field_by_name, - is_blank_quality_row, - is_blank_quick_field_row, - optional_int, - quality_rows, - rule_condition, - selected_schema_field_names, - tags_to_text, - text_to_tags, -) -from semapact.core.editor_semantics import ( - contract_description_part, - contract_tags, - description_mapping, - field_declared_lifecycle_status, - field_examples_text, - field_lifecycle_status, - normalize_tags, - schema_items, - set_contract_description_part, - set_contract_tags_list, - set_field_examples, - set_field_lifecycle_status, - set_mapping_text, -) - -__all__ = [ - "add_field", - "add_quality_rule", - "apply_field_detail", - "apply_quality_rows", - "apply_quick_field_rows", - "contract_description_part", - "contract_tags", - "description_mapping", - "field_by_name", - "field_declared_lifecycle_status", - "field_examples_text", - "field_lifecycle_status", - - "is_blank_quality_row", - "is_blank_quick_field_row", - "normalize_tags", - "optional_int", - "quality_rows", - "rule_condition", - "schema_items", - "selected_schema_field_names", - "set_contract_description_part", - "set_contract_tags_list", - "set_field_examples", - "set_field_lifecycle_status", - "set_mapping_text", - "tags_to_text", - "text_to_tags", -] diff --git a/semapact/core/editor_semantics.py b/semapact/core/editor_semantics.py index 4ea5ac20..ec62d2f6 100644 --- a/semapact/core/editor_semantics.py +++ b/semapact/core/editor_semantics.py @@ -168,10 +168,6 @@ def field_declared_lifecycle_status(field_obj: PropertyInput) -> str: return "" -# Compatibility alias -field_lifecycle_status = field_declared_lifecycle_status - - def field_option_label(field_obj: PropertyInput, index: int) -> str: """Format a field label for display.""" field_name = str(_property_model(field_obj).name or "").strip() diff --git a/semapact/core/lifecycle_cli.py b/semapact/core/lifecycle_cli.py index d126c3b4..8031d5b0 100644 --- a/semapact/core/lifecycle_cli.py +++ b/semapact/core/lifecycle_cli.py @@ -7,8 +7,8 @@ ) from semapact.core.loader import ContractLoader +from semapact.change_context import ChangeContext from semapact.governance import ( - ChangeContext, GovernanceOperation, enforce_governance_gate, evaluate_governance_decision, @@ -56,7 +56,6 @@ def apply_lifecycle( decision = evaluate_governance_decision( base_contract, candidate_contract, - context=context, ) enforce_governance_gate(decision, GovernanceOperation.APPLY) diff --git a/semapact/core/release.py b/semapact/core/release.py deleted file mode 100644 index 17212933..00000000 --- a/semapact/core/release.py +++ /dev/null @@ -1,166 +0,0 @@ -from __future__ import annotations - -import logging -import re -from dataclasses import dataclass, field -from typing import Any - -from open_data_contract_standard.model import OpenDataContractStandard - -from semapact.lifecycle.change_classification import ( - ContractChangeAssessment, - classify_contract_change, -) -from semapact.lifecycle.policy import BreakingChange -from semapact.utils.schema_utils import contract_to_model -from semapact.versioning import ( - ActualVersionBump, - RequiredBump, - classify_version_bump, - increment_version, - normalize_semver, - suggest_release_version, - version_bump_satisfies, -) - - -LOGGER = logging.getLogger(__name__) - -SEMVER_TAG_RE = re.compile(r"(?:^|[/-])v?(?P\d+\.\d+\.\d+)$") - - -@dataclass(slots=True) -class PromotionResult: - """Prepared release candidate for a single governed contract.""" - - contract: OpenDataContractStandard - required_bump: RequiredBump - current_version: str - target_version: str - actual_bump: ActualVersionBump - release_tag: str - reasons: list[str] = field(default_factory=list) - breaking_changes: list[BreakingChange] = field(default_factory=list) - - -def apply_release_candidate( - base_contract: OpenDataContractStandard, - candidate_contract: OpenDataContractStandard, - release_tag: str, - *, - required_bump: RequiredBump, -) -> PromotionResult: - """Apply a legacy release candidate using a pre-calculated required bump. - - This compatibility workflow preserves its historical semantics. Canonical M2 - ContractOps release execution lives outside this module. - """ - if not isinstance(base_contract, OpenDataContractStandard): - raise TypeError( - f"base_contract must be OpenDataContractStandard, got {type(base_contract).__name__}" - ) - if not isinstance(candidate_contract, OpenDataContractStandard): - raise TypeError( - "candidate_contract must be OpenDataContractStandard, " - f"got {type(candidate_contract).__name__}" - ) - - base_model = base_contract - candidate_model = candidate_contract.model_copy(deep=True) - - candidate_model.id = base_model.id - candidate_model.version = base_model.version - - LOGGER.info( - "Applying release candidate for contract %s with tag %s (required bump: %s)", - base_model.id, - release_tag, - required_bump, - ) - - if required_bump == "none": - from semapact.exceptions import ReleaseValidationError - - raise ReleaseValidationError("Contract changes do not require a release version bump") - - target_version = parse_release_tag_version(release_tag) - actual_bump = classify_version_bump(str(base_model.version or ""), target_version) - if not version_bump_satisfies(actual_bump, required_bump): - from semapact.exceptions import ReleaseValidationError - - raise ReleaseValidationError( - f"Release tag '{release_tag}' applies a {actual_bump} bump, but contract requires at least a " - f"{required_bump} bump" - ) - - promoted = candidate_model.model_copy(deep=True) - promoted.version = target_version - return PromotionResult( - contract=promoted, - required_bump=required_bump, - current_version=str(base_model.version or ""), - target_version=target_version, - actual_bump=actual_bump, - release_tag=release_tag, - reasons=[], - breaking_changes=[], - ) - - -def prepare_release_candidate( - base_contract: OpenDataContractStandard | dict[str, Any], - candidate_contract: OpenDataContractStandard | dict[str, Any], - release_tag: str, -) -> PromotionResult: - """Prepare a promoted contract candidate through the legacy compatibility path.""" - base_model = contract_to_model(base_contract) - candidate_model = contract_to_model(candidate_contract) - - assessment = classify_contract_change(base_model, candidate_model) - if not assessment.has_changes: - LOGGER.error("Preparation failed: contract %s has no changes", base_model.id) - raise ValueError("Cannot promote a contract with no changes") - - result = apply_release_candidate( - base_model, - candidate_model, - release_tag, - required_bump=assessment.required_bump, - ) - return PromotionResult( - contract=result.contract, - required_bump=result.required_bump, - current_version=result.current_version, - target_version=result.target_version, - actual_bump=result.actual_bump, - release_tag=result.release_tag, - reasons=assessment.reasons, - breaking_changes=assessment.breaking_changes, - ) - - -def parse_release_tag_version(release_tag: str) -> str: - """Extract semantic version from an explicit legacy release tag.""" - text = str(release_tag or "").strip() - match = SEMVER_TAG_RE.search(text) - if not match: - raise ValueError( - f"Release tag '{release_tag}' must end with a semantic version like v1.2.3" - ) - return match.group("version") - - -__all__ = [ - "ActualVersionBump", - "ContractChangeAssessment", - "PromotionResult", - "RequiredBump", - "apply_release_candidate", - "classify_contract_change", - "classify_version_bump", - "increment_version", - "normalize_semver", - "parse_release_tag_version", - "prepare_release_candidate", - "suggest_release_version", -] diff --git a/semapact/deployment/__init__.py b/semapact/deployment/__init__.py index 8a3a7c45..e80bc177 100644 --- a/semapact/deployment/__init__.py +++ b/semapact/deployment/__init__.py @@ -1,12 +1,18 @@ -"""Provider-neutral deployment planning, orchestration and authorization boundary.""" +"""Provider-neutral deployment planning and orchestration boundary.""" -from semapact.deployment.adapters import DeploymentAdapter -from semapact.deployment.authorization import authorize_deployment +from semapact.deployment.adapters import ( + DeploymentAdapter, + RuntimeReleaseMetadata, + RuntimeReleaseMetadataProjector, +) +from semapact.deployment.provenance import ( + validate_candidate_deployment_context, + validate_contract_release_deployment_context, +) from semapact.deployment.compilers import TransitionCompiler from semapact.deployment.models import ( DeploymentAction, DeploymentActionKind, - DeploymentAuthorization, DeploymentPlan, DeploymentPreview, DeploymentTarget, @@ -14,7 +20,15 @@ NativeOperationKind, ) from semapact.deployment.orchestrator import DeploymentOrchestrator -from semapact.deployment.planner import build_deployment_plan +from semapact.deployment.planner import ( + build_deployment_actions, + build_deployment_plan_from_source, +) +from semapact.deployment.source import ( + DeploymentSourceSnapshot, + build_candidate_deployment_source, + build_contract_release_deployment_source, +) from semapact.deployment.providers import ( DeploymentExecutionConfig, NativeOperationExecutor, @@ -29,7 +43,8 @@ "DeploymentAction", "DeploymentActionKind", "DeploymentAdapter", - "DeploymentAuthorization", + "RuntimeReleaseMetadata", + "RuntimeReleaseMetadataProjector", "DeploymentExecutionConfig", "DeploymentOrchestrator", "DeploymentPlan", @@ -41,7 +56,12 @@ "NativeOperationKind", "TransitionCompiler", "SchemaTransitionPlanner", - "authorize_deployment", - "build_deployment_plan", + "validate_candidate_deployment_context", + "validate_contract_release_deployment_context", + "build_deployment_actions", + "build_deployment_plan_from_source", + "build_candidate_deployment_source", + "build_contract_release_deployment_source", + "DeploymentSourceSnapshot", "verify_deployment_convergence", ] diff --git a/semapact/deployment/adapters.py b/semapact/deployment/adapters.py index 5bfbe689..6c50dc8d 100644 --- a/semapact/deployment/adapters.py +++ b/semapact/deployment/adapters.py @@ -3,15 +3,39 @@ from __future__ import annotations from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Protocol, runtime_checkable from semapact.deployment.models import ( - DeploymentAuthorization, DeploymentPlan, DeploymentPreview, ) from semapact.reconciliation import ReconciliationResult + + + +@dataclass(frozen=True) +class RuntimeReleaseMetadata: + """Release provenance projected into a runtime provider after convergence.""" + + contract_id: str + contract_version: str + contract_release_id: str + source_revision_ref: str + + +@runtime_checkable +class RuntimeReleaseMetadataProjector(Protocol): + """Optional provider capability for projecting formal release provenance.""" + + def project_release_metadata( + self, + plan: DeploymentPlan, + metadata: RuntimeReleaseMetadata, + ) -> None: ... + class DeploymentAdapter(ABC): """Own the complete provider-neutral deployment lifecycle entrypoints.""" @@ -33,11 +57,10 @@ def verify(self, plan: DeploymentPlan) -> ReconciliationResult: raise NotImplementedError @abstractmethod - def execute( + def apply( self, plan: DeploymentPlan, preview: DeploymentPreview, - authorization: DeploymentAuthorization, ) -> None: - """Execute only the exact authorized preview.""" + """Apply one exact preview after the application boundary authorizes execution.""" raise NotImplementedError diff --git a/semapact/deployment/authorization.py b/semapact/deployment/authorization.py deleted file mode 100644 index 70f822e0..00000000 --- a/semapact/deployment/authorization.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Bind release-level DEPLOY authorization to one exact DeploymentPlan.""" - -from __future__ import annotations - -from semapact.contractops.execution_models import AppliedContractRelease -from semapact.contractops.integrity import ( - validate_applied_release_identity, - validate_contractops_authorization_identity, -) -from semapact.contractops.models import AuthorizationReason, ContractOpsAuthorization -from semapact.deployment.models import ( - DeploymentAuthorization, - DeploymentPlan, - compute_deployment_authorization_id, - validate_deployment_plan_identity, -) -from semapact.exceptions import ReleaseValidationError -from semapact.governance.gate import GovernanceOperation - - -def authorize_deployment( - plan: DeploymentPlan, - release: AppliedContractRelease, - authorization: ContractOpsAuthorization, -) -> DeploymentAuthorization: - """Bind one DEPLOY authorization to the exact deployment target/plan.""" - if not isinstance(plan, DeploymentPlan): - raise TypeError(f"plan must be DeploymentPlan, got {type(plan).__name__}") - if not isinstance(release, AppliedContractRelease): - raise TypeError( - f"release must be AppliedContractRelease, got {type(release).__name__}" - ) - if not isinstance(authorization, ContractOpsAuthorization): - raise TypeError( - "authorization must be ContractOpsAuthorization, " - f"got {type(authorization).__name__}" - ) - - validate_deployment_plan_identity(plan) - validate_applied_release_identity(release) - validate_contractops_authorization_identity(authorization) - - if authorization.operation is not GovernanceOperation.DEPLOY: - raise ReleaseValidationError( - "Deployment requires operation-scoped DEPLOY authorization" - ) - - _validate_plan_release_context(plan, release) - _validate_authorization_release_context(authorization, release) - - if ( - authorization.allowed - and authorization.reason is AuthorizationReason.ALLOWED_BY_REVIEW - and authorization.scope_reference != plan.deployment_plan_id - ): - raise ReleaseValidationError( - "Review-based DEPLOY authorization is not scoped to this DeploymentPlan" - ) - - deployment_authorization_id = compute_deployment_authorization_id( - contract_ops_authorization_id=authorization.authorization_id, - deployment_plan_id=plan.deployment_plan_id, - applied_release_id=release.applied_release_id, - allowed=authorization.allowed, - ) - return DeploymentAuthorization( - deployment_authorization_id=deployment_authorization_id, - contract_ops_authorization_id=authorization.authorization_id, - deployment_plan_id=plan.deployment_plan_id, - applied_release_id=release.applied_release_id, - allowed=authorization.allowed, - ) - - -def _validate_plan_release_context( - plan: DeploymentPlan, - release: AppliedContractRelease, -) -> None: - if plan.applied_release_id != release.applied_release_id: - raise ReleaseValidationError( - "DeploymentPlan does not reference the supplied AppliedContractRelease" - ) - if plan.contract_id != release.contract_id: - raise ReleaseValidationError( - "DeploymentPlan and AppliedContractRelease contract IDs do not match" - ) - if plan.release_plan_id != release.release_plan_id: - raise ReleaseValidationError( - "DeploymentPlan and AppliedContractRelease release plan IDs do not match" - ) - if plan.released_revision_ref != release.release_revision_ref: - raise ReleaseValidationError( - "DeploymentPlan released revision does not match AppliedContractRelease" - ) - if plan.selected_version != release.selected_version: - raise ReleaseValidationError( - "DeploymentPlan selected version does not match AppliedContractRelease" - ) - - -def _validate_authorization_release_context( - authorization: ContractOpsAuthorization, - release: AppliedContractRelease, -) -> None: - if authorization.decision_id != release.decision_id: - raise ReleaseValidationError( - "DEPLOY authorization decision does not match AppliedContractRelease" - ) - if authorization.change_set_id != release.change_set_id: - raise ReleaseValidationError( - "DEPLOY authorization ChangeSet does not match AppliedContractRelease" - ) - if authorization.release_plan_id != release.release_plan_id: - raise ReleaseValidationError( - "DEPLOY authorization ReleasePlan does not match AppliedContractRelease" - ) - if authorization.version_resolution_id != release.version_resolution_id: - raise ReleaseValidationError( - "DEPLOY authorization version resolution does not match AppliedContractRelease" - ) diff --git a/semapact/deployment/models.py b/semapact/deployment/models.py index a32cefe8..2ffb8d13 100644 --- a/semapact/deployment/models.py +++ b/semapact/deployment/models.py @@ -1,4 +1,4 @@ -"""Provider-neutral immutable deployment planning, preview, and authorization artifacts.""" +"""Provider-neutral immutable deployment planning and preview artifacts.""" from __future__ import annotations @@ -7,7 +7,13 @@ from typing import Literal, Sequence from open_data_contract_standard.model import SchemaObject -from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + field_validator, + model_validator, +) from semapact.lifecycle.identity import normalize_identity_name from semapact.utils.deterministic import deterministic_uuid5 @@ -16,9 +22,6 @@ SEMAPACT_DEPLOYMENT_PLAN_NAMESPACE = uuid.UUID( "d59eaa31-997a-478d-9978-4659beee673d" ) -SEMAPACT_DEPLOYMENT_AUTHORIZATION_NAMESPACE = uuid.UUID( - "c1dd6b40-cb67-44c1-b0f2-5f133ba6a3f5" -) SEMAPACT_DEPLOYMENT_PREVIEW_NAMESPACE = uuid.UUID( "0ee613b9-a9ef-4a95-ac87-25d6c706b16c" ) @@ -119,25 +122,23 @@ def _validate_desired_state_identity(self) -> DeploymentAction: class DeploymentPlan(DeploymentModel): - """Pure deterministic runtime convergence plan for one applied release.""" + """Canonical deterministic runtime convergence plan for one exact source snapshot.""" deployment_plan_id: str - applied_release_id: str + source_snapshot_id: str contract_id: str - release_plan_id: str - released_revision_ref: str - selected_version: str + revision_ref: str + contract_version: str target: DeploymentTarget actions: tuple[DeploymentAction, ...] - plan_version: Literal["2"] = "2" + plan_version: Literal["1"] = "1" @field_validator( "deployment_plan_id", - "applied_release_id", + "source_snapshot_id", "contract_id", - "release_plan_id", - "released_revision_ref", - "selected_version", + "revision_ref", + "contract_version", ) @classmethod def _require_plan_text(cls, value: str) -> str: @@ -147,12 +148,13 @@ def _require_plan_text(cls, value: str) -> str: return cleaned @model_validator(mode="after") - def _validate_action_order_and_identity(self) -> DeploymentPlan: + def _validate_action_order_and_identity(self) -> "DeploymentPlan": governed_assets = [action.governed_asset for action in self.actions] if governed_assets != sorted(governed_assets): raise ValueError("DeploymentPlan actions must be ordered by governed_asset") if len(governed_assets) != len(set(governed_assets)): raise ValueError("DeploymentPlan cannot contain duplicate governed assets") + validate_deployment_plan_identity(self) return self @@ -216,53 +218,25 @@ def _validate_identity(self) -> DeploymentPreview: return self -class DeploymentAuthorization(DeploymentModel): - """Authorization bound to one exact DeploymentPlan and applied release.""" - - deployment_authorization_id: str - contract_ops_authorization_id: str - deployment_plan_id: str - applied_release_id: str - allowed: bool = Field(strict=True) - - @field_validator( - "deployment_authorization_id", - "contract_ops_authorization_id", - "deployment_plan_id", - "applied_release_id", - ) - @classmethod - def _require_authorization_text(cls, value: str) -> str: - cleaned = value.strip() - if not cleaned: - raise ValueError("value must not be empty") - return cleaned - - @model_validator(mode="after") - def _validate_identity(self) -> DeploymentAuthorization: - validate_deployment_authorization_identity(self) - return self - - def compute_deployment_plan_id( *, - applied_release_id: str, + source_snapshot_id: str, contract_id: str, - release_plan_id: str, - released_revision_ref: str, - selected_version: str, + revision_ref: str, + contract_version: str, target: DeploymentTarget, actions: Sequence[DeploymentAction], - plan_version: str = "2", + plan_version: str = "1", ) -> str: + if plan_version != "1": + raise ValueError(f"Unsupported canonical DeploymentPlan version: {plan_version}") return deterministic_uuid5( SEMAPACT_DEPLOYMENT_PLAN_NAMESPACE, { - "applied_release_id": applied_release_id, + "source_snapshot_id": source_snapshot_id, "contract_id": contract_id, - "release_plan_id": release_plan_id, - "released_revision_ref": released_revision_ref, - "selected_version": selected_version, + "revision_ref": revision_ref, + "contract_version": contract_version, "target": target.model_dump(mode="json"), "actions": [action.model_dump(mode="json") for action in actions], "plan_version": plan_version, @@ -270,24 +244,6 @@ def compute_deployment_plan_id( ) -def compute_deployment_authorization_id( - *, - contract_ops_authorization_id: str, - deployment_plan_id: str, - applied_release_id: str, - allowed: bool, -) -> str: - return deterministic_uuid5( - SEMAPACT_DEPLOYMENT_AUTHORIZATION_NAMESPACE, - { - "contract_ops_authorization_id": contract_ops_authorization_id, - "deployment_plan_id": deployment_plan_id, - "applied_release_id": applied_release_id, - "allowed": allowed, - }, - ) - - def compute_deployment_preview_id( *, deployment_plan_id: str, @@ -314,11 +270,10 @@ def compute_deployment_preview_id( def validate_deployment_plan_identity(plan: DeploymentPlan) -> None: expected = compute_deployment_plan_id( - applied_release_id=plan.applied_release_id, + source_snapshot_id=plan.source_snapshot_id, contract_id=plan.contract_id, - release_plan_id=plan.release_plan_id, - released_revision_ref=plan.released_revision_ref, - selected_version=plan.selected_version, + revision_ref=plan.revision_ref, + contract_version=plan.contract_version, target=plan.target, actions=plan.actions, plan_version=plan.plan_version, @@ -327,21 +282,6 @@ def validate_deployment_plan_identity(plan: DeploymentPlan) -> None: raise ValueError("DeploymentPlan deterministic identity does not match content") -def validate_deployment_authorization_identity( - authorization: DeploymentAuthorization, -) -> None: - expected = compute_deployment_authorization_id( - contract_ops_authorization_id=authorization.contract_ops_authorization_id, - deployment_plan_id=authorization.deployment_plan_id, - applied_release_id=authorization.applied_release_id, - allowed=authorization.allowed, - ) - if expected != authorization.deployment_authorization_id: - raise ValueError( - "DeploymentAuthorization deterministic identity does not match content" - ) - - def validate_deployment_preview_identity(preview: DeploymentPreview) -> None: expected = compute_deployment_preview_id( deployment_plan_id=preview.deployment_plan_id, diff --git a/semapact/deployment/orchestrator.py b/semapact/deployment/orchestrator.py index 6e95b994..78a3d060 100644 --- a/semapact/deployment/orchestrator.py +++ b/semapact/deployment/orchestrator.py @@ -7,21 +7,21 @@ from semapact.deployment.adapters import DeploymentAdapter from semapact.deployment.compilers import TransitionCompiler from semapact.deployment.models import ( + DeploymentAction, DeploymentActionKind, - DeploymentAuthorization, DeploymentPlan, DeploymentPreview, + DeploymentTarget, NativeOperation, NativeOperationKind, compute_deployment_preview_id, - validate_deployment_authorization_identity, validate_deployment_plan_identity, validate_deployment_preview_identity, ) from semapact.deployment.providers import NativeOperationExecutor from semapact.deployment.schema_transitions import SchemaTransitionPlanner from semapact.deployment.verification import verify_deployment_convergence -from semapact.exceptions import ContractOpsAuthorizationError, ValidationError +from semapact.exceptions import ValidationError from semapact.observation.fingerprint import fingerprint_observed_state from semapact.observation.models import ObservedAssetIdentity, ObservedPlatformState from semapact.observation.providers import RuntimeAssetBinding, RuntimeProvider @@ -92,29 +92,20 @@ def verify(self, plan: DeploymentPlan) -> ReconciliationResult: schema_mapper=self._schema_mapper, ) - def execute( + def apply( self, plan: DeploymentPlan, preview: DeploymentPreview, - authorization: DeploymentAuthorization, ) -> None: - """Execute only the exact authorized preview against unchanged runtime state.""" + """Apply the exact preview against unchanged runtime state. + + The application/CI boundary decides whether execution may run. This adapter + only enforces plan/preview/runtime integrity and never interprets release + facts as deployment authorization. + """ validate_deployment_preview_identity(preview) - validate_deployment_authorization_identity(authorization) desired_by_action = self._validate_and_map_plan(plan) - if not authorization.allowed: - raise ContractOpsAuthorizationError( - "DeploymentAuthorization is not allowed" - ) - if authorization.deployment_plan_id != plan.deployment_plan_id: - raise ContractOpsAuthorizationError( - "DeploymentAuthorization is not bound to this DeploymentPlan" - ) - if authorization.applied_release_id != plan.applied_release_id: - raise ContractOpsAuthorizationError( - "DeploymentAuthorization release does not match DeploymentPlan" - ) if preview.deployment_plan_id != plan.deployment_plan_id: raise ValidationError( "DeploymentPreview is not bound to this DeploymentPlan" @@ -152,7 +143,7 @@ def execute( if expected != preview: raise ValidationError( "DeploymentPreview no longer equals the deterministic preview for " - "the authorized plan and runtime evidence" + "the plan and runtime evidence" ) for operation in preview.operations: @@ -170,15 +161,54 @@ def _preview_from_observation( ) -> DeploymentPreview: if desired_by_action is None: desired_by_action = self._validate_and_map_plan(plan) - self._validate_observation(plan, observed_state, bindings=bindings) + operations = self._operations_for_scope( + plan.target, + plan.actions, + observed_state, + bindings=bindings, + desired_by_action=desired_by_action, + ) + if observed_state.fingerprint is None: + raise ValidationError("Runtime observation fingerprint is required") + return DeploymentPreview( + deployment_preview_id=compute_deployment_preview_id( + deployment_plan_id=plan.deployment_plan_id, + platform=self.key, + runtime_target=plan.target.runtime_target, + source_identifier=observed_state.source_identifier, + observation_fingerprint=observed_state.fingerprint, + operations=operations, + ), + deployment_plan_id=plan.deployment_plan_id, + platform=self.key, + runtime_target=plan.target.runtime_target, + source_identifier=observed_state.source_identifier, + observation_fingerprint=observed_state.fingerprint, + operations=operations, + ) + + def _operations_for_scope( + self, + target: DeploymentTarget, + actions: tuple[DeploymentAction, ...], + observed_state: ObservedPlatformState, + *, + bindings: tuple[RuntimeAssetBinding, ...], + desired_by_action: dict[str, SchemaAssetState], + ) -> tuple[NativeOperation, ...]: + self._validate_observation_for_scope( + target, + observed_state, + bindings=bindings, + ) observed_by_asset = { asset.identity.asset.casefold(): asset for asset in observed_state.assets } operations: list[NativeOperation] = [] - for action in plan.actions: + for action in actions: desired_asset = desired_by_action[action.governed_asset] observed = observed_by_asset.get(action.physical_name.casefold()) observed_assets = ( @@ -191,7 +221,6 @@ def _preview_from_observation( ), ) ) - comparison = compare_schema_snapshots( SchemaSnapshot(assets=(desired_asset,)), SchemaSnapshot(assets=observed_assets), @@ -205,46 +234,32 @@ def _preview_from_observation( ) operations.append( self._transition_compiler.compile( - runtime_target=plan.target.runtime_target, + runtime_target=target.runtime_target, transition=transition, ) ) - - if observed_state.fingerprint is None: - raise ValidationError("Runtime observation fingerprint is required") - ordered = tuple(operations) - return DeploymentPreview( - deployment_preview_id=compute_deployment_preview_id( - deployment_plan_id=plan.deployment_plan_id, - platform=self.key, - runtime_target=plan.target.runtime_target, - source_identifier=observed_state.source_identifier, - observation_fingerprint=observed_state.fingerprint, - operations=ordered, - ), - deployment_plan_id=plan.deployment_plan_id, - platform=self.key, - runtime_target=plan.target.runtime_target, - source_identifier=observed_state.source_identifier, - observation_fingerprint=observed_state.fingerprint, - operations=ordered, - ) + return tuple(operations) def _validate_and_map_plan( self, plan: DeploymentPlan, ) -> dict[str, SchemaAssetState]: validate_deployment_plan_identity(plan) - if plan.target.platform.casefold() != self.key.casefold(): + return self._validate_and_map_actions(plan.target, plan.actions) + + def _validate_and_map_actions( + self, + target: DeploymentTarget, + actions: tuple[DeploymentAction, ...], + ) -> dict[str, SchemaAssetState]: + if target.platform.casefold() != self.key.casefold(): raise ValidationError( - f"Runtime provider '{self.key}' cannot deploy " - f"'{plan.target.platform}'" + f"Runtime provider '{self.key}' cannot deploy '{target.platform}'" ) physical_assets: set[str] = set() desired_by_action: dict[str, SchemaAssetState] = {} - - for action in plan.actions: + for action in actions: if action.kind is not DeploymentActionKind.ENSURE_ASSET_STATE: raise ValidationError( f"Unsupported deployment action kind: {action.kind.value}" @@ -269,30 +284,35 @@ def _validate_and_map_plan( ) desired_by_action[action.governed_asset] = mapped - # Binding resolution is pure provider-local identity resolution. Calling it - # here validates runtime-target syntax and exact plan bindings without I/O. - self._resolve_plan_bindings(plan) + self._resolve_bindings(target, actions) return desired_by_action def _resolve_plan_bindings( self, plan: DeploymentPlan, + ) -> tuple[RuntimeAssetBinding, ...]: + return self._resolve_bindings(plan.target, plan.actions) + + def _resolve_bindings( + self, + target: DeploymentTarget, + actions: tuple[DeploymentAction, ...], ) -> tuple[RuntimeAssetBinding, ...]: assets = tuple( RuntimeAssetSpec( governed_asset=action.governed_asset, physical_name=action.physical_name, ) - for action in plan.actions + for action in actions ) bindings = self._runtime_provider.resolve_bindings( - runtime_target=plan.target.runtime_target, + runtime_target=target.runtime_target, assets=assets, ) expected = { action.governed_asset: action.physical_name.casefold() - for action in plan.actions + for action in actions } if len(bindings) != len(expected): raise ValidationError( @@ -337,6 +357,19 @@ def _validate_observation( observed_state: ObservedPlatformState, *, bindings: tuple[RuntimeAssetBinding, ...], + ) -> None: + self._validate_observation_for_scope( + plan.target, + observed_state, + bindings=bindings, + ) + + def _validate_observation_for_scope( + self, + target: DeploymentTarget, + observed_state: ObservedPlatformState, + *, + bindings: tuple[RuntimeAssetBinding, ...], ) -> None: if observed_state.platform.casefold() != self.key.casefold(): raise ValidationError( @@ -346,9 +379,9 @@ def _validate_observation( raise ValidationError( "Runtime observation source_identifier is required" ) - if observed_state.source_identifier != plan.target.source_reference: + if observed_state.source_identifier != target.source_reference: raise ValidationError( - "Runtime observation source does not match DeploymentPlan " + "Runtime observation source does not match deployment target " "source reference" ) if observed_state.fingerprint is None: @@ -379,7 +412,14 @@ def _observe_plan_scope( self, plan: DeploymentPlan, ) -> tuple[ObservedPlatformState, tuple[RuntimeAssetBinding, ...]]: - bindings = self._resolve_plan_bindings(plan) + return self._observe_scope(plan.target, plan.actions) + + def _observe_scope( + self, + target: DeploymentTarget, + actions: tuple[DeploymentAction, ...], + ) -> tuple[ObservedPlatformState, tuple[RuntimeAssetBinding, ...]]: + bindings = self._resolve_bindings(target, actions) return ( self._runtime_provider.observe(bindings=bindings), bindings, diff --git a/semapact/deployment/planner.py b/semapact/deployment/planner.py index 10894c1f..4e4b7367 100644 --- a/semapact/deployment/planner.py +++ b/semapact/deployment/planner.py @@ -1,11 +1,9 @@ -"""Pure compilation of applied contract releases into deployment convergence plans.""" +"""Pure compilation of canonical deployment sources into convergence plans.""" from __future__ import annotations -from open_data_contract_standard.model import SchemaObject +from open_data_contract_standard.model import OpenDataContractStandard, SchemaObject -from semapact.contractops.execution_models import AppliedContractRelease -from semapact.contractops.integrity import validate_applied_release_identity from semapact.deployment.models import ( DeploymentAction, DeploymentActionKind, @@ -13,31 +11,63 @@ DeploymentTarget, compute_deployment_plan_id, ) +from semapact.deployment.source import DeploymentSourceSnapshot from semapact.lifecycle.identity import normalize_identity_name from semapact.runtime import runtime_asset_specs_from_contract from semapact.utils.deterministic import canonical_compact_json -def build_deployment_plan( - release: AppliedContractRelease, +def build_deployment_plan_from_source( + source: DeploymentSourceSnapshot, target: DeploymentTarget, ) -> DeploymentPlan: - """Build one deterministic provider-neutral convergence plan. - - Planning consumes exact released desired state only. It does not observe runtime, - choose CREATE/ALTER/DROP operations, contact a provider, or recompute governance. - """ - if not isinstance(release, AppliedContractRelease): + """Build the canonical plan from one exact deployment source snapshot.""" + if not isinstance(source, DeploymentSourceSnapshot): raise TypeError( - f"release must be AppliedContractRelease, got {type(release).__name__}" + "source must be DeploymentSourceSnapshot, " + f"got {type(source).__name__}" ) if not isinstance(target, DeploymentTarget): raise TypeError( f"target must be DeploymentTarget, got {type(target).__name__}" ) - validate_applied_release_identity(release) - contract = release.to_contract() + contract = source.to_contract() + ordered_actions = build_deployment_actions(contract) + deployment_plan_id = compute_deployment_plan_id( + source_snapshot_id=source.source_snapshot_id, + contract_id=source.contract_id, + revision_ref=source.revision_ref, + contract_version=source.contract_version, + target=target, + actions=ordered_actions, + plan_version="1", + ) + return DeploymentPlan( + deployment_plan_id=deployment_plan_id, + source_snapshot_id=source.source_snapshot_id, + contract_id=source.contract_id, + revision_ref=source.revision_ref, + contract_version=source.contract_version, + target=target, + actions=ordered_actions, + plan_version="1", + ) + +def build_deployment_actions( + contract: OpenDataContractStandard, +) -> tuple[DeploymentAction, ...]: + """Project candidate/released ODCS state into provider-neutral desired actions. + + Actions are desired-state facts only. They are not an execution permission; + they become target-specific planning input only through an exact DeploymentPlan. + """ + if not isinstance(contract, OpenDataContractStandard): + raise TypeError( + "contract must be OpenDataContractStandard, " + f"got {type(contract).__name__}" + ) + asset_specs = runtime_asset_specs_from_contract(contract) specs_by_asset = {spec.governed_asset: spec for spec in asset_specs} @@ -57,27 +87,7 @@ def build_deployment_plan( ) ) - ordered_actions = tuple(sorted(actions, key=lambda action: action.governed_asset)) - deployment_plan_id = compute_deployment_plan_id( - applied_release_id=release.applied_release_id, - contract_id=release.contract_id, - release_plan_id=release.release_plan_id, - released_revision_ref=release.release_revision_ref, - selected_version=release.selected_version, - target=target, - actions=ordered_actions, - ) - - return DeploymentPlan( - deployment_plan_id=deployment_plan_id, - applied_release_id=release.applied_release_id, - contract_id=release.contract_id, - release_plan_id=release.release_plan_id, - released_revision_ref=release.release_revision_ref, - selected_version=release.selected_version, - target=target, - actions=ordered_actions, - ) + return tuple(sorted(actions, key=lambda action: action.governed_asset)) def _canonical_schema_json(schema: SchemaObject) -> str: diff --git a/semapact/deployment/provenance.py b/semapact/deployment/provenance.py new file mode 100644 index 00000000..e425b913 --- /dev/null +++ b/semapact/deployment/provenance.py @@ -0,0 +1,75 @@ +"""Canonical deployment provenance validation.""" + +from __future__ import annotations + +from semapact.deployment.models import ( + DeploymentPlan, + validate_deployment_plan_identity, +) +from semapact.deployment.source import DeploymentSourceSnapshot +from semapact.contractops import ContractRelease +from semapact.governance import DecisionResult, GovernanceDecision +from semapact.exceptions import ReleaseValidationError + + +def validate_candidate_deployment_context( + plan: DeploymentPlan, + source: DeploymentSourceSnapshot, + decision: GovernanceDecision, +) -> None: + """Validate candidate deployment provenance and fail closed on BLOCK.""" + if source.source_kind != "candidate": + raise ReleaseValidationError( + "Candidate deployment requires candidate source without release provenance" + ) + validate_deployment_plan_identity(plan) + if plan.source_snapshot_id != source.source_snapshot_id: + raise ReleaseValidationError( + "DeploymentPlan does not reference the supplied candidate source" + ) + if plan.contract_id != source.contract_id: + raise ReleaseValidationError( + "DeploymentPlan and candidate source contract IDs do not match" + ) + if decision.contract_id != source.contract_id: + raise ReleaseValidationError( + "GovernanceDecision does not match candidate deployment contract" + ) + if decision.decision is DecisionResult.BLOCK: + raise ReleaseValidationError("Candidate deployment is blocked by governance") + + +def validate_contract_release_deployment_context( + plan: DeploymentPlan, + source: DeploymentSourceSnapshot, + release: ContractRelease, +) -> None: + """Validate that one plan is derived from the exact finalized release.""" + if source.source_kind != "contract_release": + raise ReleaseValidationError( + "Finalized release deployment requires ContractRelease source provenance" + ) + validate_deployment_plan_identity(plan) + if plan.source_snapshot_id != source.source_snapshot_id: + raise ReleaseValidationError( + "DeploymentPlan does not reference the supplied release source" + ) + if source.release_id != release.contract_release_id: + raise ReleaseValidationError( + "Deployment source does not reference the finalized ContractRelease" + ) + if source.contract_id != release.contract_id: + raise ReleaseValidationError( + "Deployment source and ContractRelease contract IDs do not match" + ) + if source.contract_version != release.contract_version: + raise ReleaseValidationError( + "Deployment source and ContractRelease versions do not match" + ) + if source.revision_ref != release.source_revision_ref: + raise ReleaseValidationError( + "Deployment source and ContractRelease revisions do not match" + ) + + + diff --git a/semapact/deployment/source.py b/semapact/deployment/source.py new file mode 100644 index 00000000..3c60ba17 --- /dev/null +++ b/semapact/deployment/source.py @@ -0,0 +1,181 @@ +"""Exact desired-state snapshots consumed by deployment planning.""" + +from __future__ import annotations + +import uuid +from typing import Literal + +from open_data_contract_standard.model import OpenDataContractStandard +from pydantic import BaseModel, ConfigDict, field_validator, model_validator + +from semapact.contractops import ContractRelease +from semapact.utils.deterministic import canonical_compact_json, deterministic_uuid5 + + +SEMAPACT_DEPLOYMENT_SOURCE_NAMESPACE = uuid.UUID( + "598e0c13-d9af-4cf9-a8a7-9fc7628e17c1" +) + +DeploymentSourceKind = Literal["candidate", "contract_release"] + + +class DeploymentSourceSnapshot(BaseModel): + """Immutable desired state plus the provenance that produced it. + + source_kind is the single canonical discriminator for candidate versus + released desired state. Downstream plans/bundles derive release mode from this + source instead of persisting duplicate booleans. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + source_snapshot_id: str + source_kind: DeploymentSourceKind + contract_id: str + revision_ref: str + contract_version: str + contract_json: str + release_id: str | None = None + + @field_validator( + "source_snapshot_id", + "contract_id", + "revision_ref", + "contract_version", + "contract_json", + ) + @classmethod + def _require_text(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("value must not be empty") + return cleaned + + @model_validator(mode="after") + def _validate_source(self) -> "DeploymentSourceSnapshot": + contract = self.to_contract() + if str(contract.id or "").strip() != self.contract_id: + raise ValueError("DeploymentSourceSnapshot contract ID mismatch") + if str(contract.version or "").strip() != self.contract_version: + raise ValueError("DeploymentSourceSnapshot contract version mismatch") + + if self.source_kind == "candidate": + if self.release_id is not None: + raise ValueError( + "Candidate deployment source cannot contain release provenance" + ) + elif self.release_id is None: + raise ValueError( + "ContractRelease source requires contract_release_id provenance" + ) + + expected = compute_deployment_source_id( + source_kind=self.source_kind, + contract_id=self.contract_id, + revision_ref=self.revision_ref, + contract_version=self.contract_version, + contract_json=self.contract_json, + release_id=self.release_id, + ) + if self.source_snapshot_id != expected: + raise ValueError( + "DeploymentSourceSnapshot deterministic identity does not match content" + ) + return self + + @property + def is_release(self) -> bool: + return self.source_kind != "candidate" + + def to_contract(self) -> OpenDataContractStandard: + return OpenDataContractStandard.model_validate_json(self.contract_json) + + +def build_candidate_deployment_source( + contract: OpenDataContractStandard, + *, + revision_ref: str, +) -> DeploymentSourceSnapshot: + """Freeze candidate desired state without calculating or changing its version.""" + if not isinstance(contract, OpenDataContractStandard): + raise TypeError( + "contract must be OpenDataContractStandard, " + f"got {type(contract).__name__}" + ) + contract_id = str(contract.id or "").strip() + contract_version = str(contract.version or "").strip() + revision = revision_ref.strip() + if not contract_id or not contract_version or not revision: + raise ValueError( + "Candidate deployment source requires contract ID, version, and revision" + ) + contract_json = canonical_compact_json( + contract.model_dump(mode="json", by_alias=True, exclude_none=True) + ) + source_id = compute_deployment_source_id( + source_kind="candidate", + contract_id=contract_id, + revision_ref=revision, + contract_version=contract_version, + contract_json=contract_json, + release_id=None, + ) + return DeploymentSourceSnapshot( + source_snapshot_id=source_id, + source_kind="candidate", + contract_id=contract_id, + revision_ref=revision, + contract_version=contract_version, + contract_json=contract_json, + ) + + +def build_contract_release_deployment_source( + release: ContractRelease, +) -> DeploymentSourceSnapshot: + """Project one finalized ContractRelease into deployment desired state.""" + if not isinstance(release, ContractRelease): + raise TypeError( + "release must be ContractRelease, " + f"got {type(release).__name__}" + ) + contract_json = release.released_contract_json + source_id = compute_deployment_source_id( + source_kind="contract_release", + contract_id=release.contract_id, + revision_ref=release.source_revision_ref, + contract_version=release.contract_version, + contract_json=contract_json, + release_id=release.contract_release_id, + ) + return DeploymentSourceSnapshot( + source_snapshot_id=source_id, + source_kind="contract_release", + contract_id=release.contract_id, + revision_ref=release.source_revision_ref, + contract_version=release.contract_version, + contract_json=contract_json, + release_id=release.contract_release_id, + ) + + +def compute_deployment_source_id( + *, + source_kind: DeploymentSourceKind, + contract_id: str, + revision_ref: str, + contract_version: str, + contract_json: str, + release_id: str | None, +) -> str: + return deterministic_uuid5( + SEMAPACT_DEPLOYMENT_SOURCE_NAMESPACE, + { + "source_kind": source_kind, + "contract_id": contract_id, + "revision_ref": revision_ref, + "contract_version": contract_version, + "contract_json": contract_json, + "release_id": release_id, + }, + ) diff --git a/semapact/deployment/verification.py b/semapact/deployment/verification.py index 2d1ae047..07829e39 100644 --- a/semapact/deployment/verification.py +++ b/semapact/deployment/verification.py @@ -79,6 +79,6 @@ def _contract_projection_from_plan( # deterministic plan identity was revalidated above. return OpenDataContractStandard.model_construct( id=plan.contract_id, - version=plan.selected_version, + version=plan.contract_version, schema_=schemas, ) diff --git a/semapact/devops/__init__.py b/semapact/devops/__init__.py index 5a3fd63d..1a7dac6d 100644 --- a/semapact/devops/__init__.py +++ b/semapact/devops/__init__.py @@ -1,19 +1,6 @@ from semapact.devops.audit import AuditMetadata, build_audit_metadata from semapact.devops.ci_cd import CIDecision, evaluate_ci_gate, write_ci_summary from semapact.devops.pr_creator import AzureDevOpsConfig, PullRequestCreator -from semapact.devops.release_workflow import ( - BatchReleaseManifestBuild, - BatchReleaseTask, - ReleasePullRequestPlan, - RepositoryContractChange, - batch_manifest_build_to_dict, - build_batch_release_manifest, - build_release_pr_plan, - create_release_pull_request, - create_release_pull_requests_from_manifest, - load_batch_release_tasks, - repository_change_to_dict, -) __all__ = [ "AuditMetadata", @@ -23,15 +10,4 @@ "write_ci_summary", "AzureDevOpsConfig", "PullRequestCreator", - "BatchReleaseManifestBuild", - "BatchReleaseTask", - "ReleasePullRequestPlan", - "RepositoryContractChange", - "batch_manifest_build_to_dict", - "build_batch_release_manifest", - "build_release_pr_plan", - "create_release_pull_request", - "create_release_pull_requests_from_manifest", - "load_batch_release_tasks", - "repository_change_to_dict", ] diff --git a/semapact/devops/release_workflow.py b/semapact/devops/release_workflow.py deleted file mode 100644 index 8c98de1d..00000000 --- a/semapact/devops/release_workflow.py +++ /dev/null @@ -1,465 +0,0 @@ -from __future__ import annotations - -import json -from dataclasses import asdict, dataclass -from pathlib import Path -from typing import Any - -from semapact.core.release import ( - PromotionResult, - RequiredBump, - apply_release_candidate, - suggest_release_version, -) -from semapact.devops.pr_creator import ( - GitProviderConfig, - PullRequestCreator, -) -from semapact.exceptions import GovernanceBlockedError -from semapact.governance import ( - ChangeContext, - DecisionResult, - GovernanceDecision, - GovernanceOperation, - enforce_governance_gate, - evaluate_governance_decision, -) -from open_data_contract_standard.model import OpenDataContractStandard -from semapact.utils.schema_utils import contract_to_model -from semapact.utils.yaml_utils import dump_yaml, list_yaml_documents, load_yaml - - -@dataclass(slots=True) -class ReleasePullRequestPlan: - """Prepared per-contract release PR payload.""" - - contract_id: str - current_version: str - target_version: str - required_bump: str - actual_bump: str - release_tag: str - contract_repo_path: str - source_branch: str - target_branch: str - commit_message: str - title: str - description: str - - -@dataclass(slots=True) -class RepositoryContractChange: - """Per-contract change status within a multi-contract repo comparison.""" - - contract_repo_path: str - status: str - contract_id: str | None = None - current_version: str | None = None - candidate_version: str | None = None - required_bump: RequiredBump | None = "none" - suggested_release_version: str | None = None - reasons: list[str] | None = None - governance_decision: GovernanceDecision | None = None - - -@dataclass(slots=True) -class BatchReleaseTask: - """Explicit per-contract release task for batch orchestration.""" - - base: str - candidate: str - contract_path: str - release_tag: str - source_branch: str - target_branch: str - effective_date: str - title: str | None = None - description: str | None = None - commit_message: str | None = None - - -@dataclass(slots=True) -class BatchReleaseManifestBuild: - """Generated batch manifest plus skipped contract summary.""" - - tasks: list[BatchReleaseTask] - skipped: list[RepositoryContractChange] - - -def build_release_pr_plan( - *, - promotion: PromotionResult, - contract_repo_path: str, - source_branch: str, - target_branch: str, - title: str | None = None, - description: str | None = None, - commit_message: str | None = None, -) -> ReleasePullRequestPlan: - """Build a per-contract release PR plan from a prepared promotion result.""" - contract_id = str(promotion.contract.id or "") - target_version = promotion.target_version - default_title = f"Release {contract_id} {target_version}" - default_commit = f"release({contract_id}): prepare {target_version}" - default_description = ( - f"Prepare release for contract `{contract_id}`.\n\n" - f"- current version: `{promotion.current_version}`\n" - f"- target version: `{promotion.target_version}`\n" - f"- required bump: `{promotion.required_bump}`\n" - f"- actual bump: `{promotion.actual_bump}`\n" - f"- release tag: `{promotion.release_tag}`\n" - ) - return ReleasePullRequestPlan( - contract_id=contract_id, - current_version=promotion.current_version, - target_version=target_version, - required_bump=promotion.required_bump, - actual_bump=promotion.actual_bump, - release_tag=promotion.release_tag, - contract_repo_path=contract_repo_path, - source_branch=source_branch, - target_branch=target_branch, - commit_message=commit_message or default_commit, - title=title or default_title, - description=description or default_description, - ) - - -def create_release_pull_request( - *, - config: GitProviderConfig, - repo_path: str, - contract_repo_path: str, - base_contract: OpenDataContractStandard, - candidate_contract: OpenDataContractStandard, - release_tag: str, - source_branch: str, - target_branch: str, - context: ChangeContext, - title: str | None = None, - description: str | None = None, - commit_message: str | None = None, - push: bool = False, -) -> dict[str, Any]: - """Prepare one promoted contract and open a release PR for it.""" - # 1. Authoritative decision evaluation and PROPOSE gate enforcement before file write or Git/PR actions - decision = evaluate_governance_decision( - base_contract, - candidate_contract, - context=context, - ) - enforce_governance_gate(decision, GovernanceOperation.PROPOSE) - - promotion = apply_release_candidate( - base_contract, - candidate_contract, - release_tag, - required_bump=decision.required_version_bump, - ) - repo_root = Path(repo_path).expanduser().resolve() - contract_path = repo_root / contract_repo_path - dump_yaml(promotion.contract, contract_path) - - plan = build_release_pr_plan( - promotion=promotion, - contract_repo_path=contract_repo_path, - source_branch=source_branch, - target_branch=target_branch, - title=title, - description=description, - commit_message=commit_message, - ) - - creator = PullRequestCreator(config=config) - pr_payload = creator.create_update_pr( - repo_path=str(repo_root), - source_branch=source_branch, - target_branch=target_branch, - commit_message=plan.commit_message, - title=plan.title, - description=plan.description, - paths=[contract_repo_path], - push=push, - ) - return { - "promotion": { - "contractId": plan.contract_id, - "currentVersion": plan.current_version, - "targetVersion": plan.target_version, - "requiredBump": plan.required_bump, - "actualBump": plan.actual_bump, - "releaseTag": plan.release_tag, - "contractPath": plan.contract_repo_path, - "sourceBranch": plan.source_branch, - "targetBranch": plan.target_branch, - }, - "pullRequest": pr_payload, - "governanceDecision": decision.model_dump(mode="json"), - } - - -def release_plan_to_dict(plan: ReleasePullRequestPlan) -> dict[str, Any]: - """Serialize a release PR plan for CLI/JSON output.""" - return asdict(plan) - - -def classify_contracts_in_repo( - *, - base_root: str | Path, - candidate_root: str | Path, - context: ChangeContext, -) -> list[RepositoryContractChange]: - """Compare two contract roots and classify changes per contract file based on GovernanceDecision.""" - base_root_path = Path(base_root).expanduser().resolve() - candidate_root_path = Path(candidate_root).expanduser().resolve() - - base_index = _relative_contract_index(base_root_path) - candidate_index = _relative_contract_index(candidate_root_path) - - results: list[RepositoryContractChange] = [] - for relative_path in sorted(set(base_index) | set(candidate_index)): - base_path = base_index.get(relative_path) - candidate_path = candidate_index.get(relative_path) - - if base_path is None: - assert candidate_path is not None - candidate_model = contract_to_model(load_yaml(candidate_path)) - results.append( - RepositoryContractChange( - contract_repo_path=relative_path, - status="added", - contract_id=str(candidate_model.id or ""), - current_version=None, - candidate_version=str(candidate_model.version or ""), - required_bump=None, - suggested_release_version=None, - reasons=[ - "New governed contract; initial release handled separately" - ], - governance_decision=None, - ) - ) - continue - - if candidate_path is None: - base_model = contract_to_model(load_yaml(base_path)) - results.append( - RepositoryContractChange( - contract_repo_path=relative_path, - status="removed", - contract_id=str(base_model.id or ""), - current_version=str(base_model.version or ""), - candidate_version=None, - required_bump=None, - suggested_release_version=None, - reasons=[ - "Governed contract missing from candidate root; manual review required" - ], - governance_decision=None, - ) - ) - continue - - base_model = contract_to_model(load_yaml(base_path)) - candidate_model = contract_to_model(load_yaml(candidate_path)) - - decision = evaluate_governance_decision( - base_model, - candidate_model, - context=context, - ) - reasons_list = [r.message for r in decision.reasons] or ["No contract changes detected"] - - status = ( - "blocked" - if decision.decision == DecisionResult.BLOCK - else ("changed" if decision.evidence.has_changes else "unchanged") - ) - - results.append( - RepositoryContractChange( - contract_repo_path=relative_path, - status=status, - contract_id=str(base_model.id or ""), - current_version=str(base_model.version or ""), - candidate_version=str(candidate_model.version or ""), - required_bump=decision.required_version_bump, - suggested_release_version=( - suggest_release_version( - str(base_model.version or ""), - decision.required_version_bump, - ) - if decision.required_version_bump != "none" - else None - ), - reasons=reasons_list, - governance_decision=decision, - ) - ) - - return results - - -def create_release_pull_requests_from_manifest( - *, - config: GitProviderConfig, - repo_path: str, - tasks: list[BatchReleaseTask], - push: bool = False, -) -> list[dict[str, Any]]: - """Run explicit per-contract release PR automation from a batch manifest.""" - results: list[dict[str, Any]] = [] - for task in tasks: - base_contract = contract_to_model(load_yaml(task.base)) - candidate_contract = contract_to_model(load_yaml(task.candidate)) - task_context = ChangeContext(effective_date=task.effective_date) - - results.append( - create_release_pull_request( - config=config, - repo_path=repo_path, - contract_repo_path=task.contract_path, - base_contract=base_contract, - candidate_contract=candidate_contract, - release_tag=task.release_tag, - source_branch=task.source_branch, - target_branch=task.target_branch, - context=task_context, - title=task.title, - description=task.description, - commit_message=task.commit_message, - push=push, - ) - ) - return results - - -def build_batch_release_manifest( - *, - base_root: str | Path, - candidate_root: str | Path, - context: ChangeContext, - target_branch: str = "release", - source_branch_prefix: str = "release/", -) -> BatchReleaseManifestBuild: - """Build an editable batch manifest from repo-level contract changes.""" - changes = classify_contracts_in_repo( - base_root=base_root, - candidate_root=candidate_root, - context=context, - ) - base_root_path = Path(base_root).expanduser().resolve() - candidate_root_path = Path(candidate_root).expanduser().resolve() - - tasks: list[BatchReleaseTask] = [] - skipped: list[RepositoryContractChange] = [] - for change in changes: - if change.status != "changed" or change.required_bump == "none": - skipped.append(change) - continue - - # Enforce PROPOSE gate using GovernanceBlockedError exception catching to handle blocked changes - if change.governance_decision is not None: - try: - enforce_governance_gate( - change.governance_decision, - GovernanceOperation.PROPOSE, - ) - except GovernanceBlockedError: - skipped.append(change) - continue - - contract_key = _contract_release_key(change) - next_version = change.suggested_release_version or suggest_release_version( - str(change.current_version or "0.0.0"), - change.required_bump or "none", - ) - release_tag = f"{contract_key}/v{next_version}" - source_branch = ( - f"{source_branch_prefix}{_branch_safe_name(contract_key)}-v{next_version}" - ) - - tasks.append( - BatchReleaseTask( - base=str(base_root_path / change.contract_repo_path), - candidate=str(candidate_root_path / change.contract_repo_path), - contract_path=change.contract_repo_path, - release_tag=release_tag, - source_branch=source_branch, - target_branch=target_branch, - effective_date=context.effective_date.isoformat(), - ) - ) - - return BatchReleaseManifestBuild(tasks=tasks, skipped=skipped) - - -def load_batch_release_tasks(path: str | Path) -> list[BatchReleaseTask]: - """Load a JSON batch manifest for per-contract release orchestration.""" - manifest_path = Path(path).expanduser().resolve() - payload = json.loads(manifest_path.read_text(encoding="utf-8")) - if not isinstance(payload, list): - raise ValueError("Batch release manifest must be a JSON array") - return [BatchReleaseTask(**item) for item in payload] - - -def repository_change_to_dict(change: RepositoryContractChange) -> dict[str, Any]: - """Serialize repo-level change result for CLI/JSON output.""" - gov_dec_dict = ( - change.governance_decision.model_dump(mode="json") - if change.governance_decision is not None - else None - ) - - return { - "contract_repo_path": change.contract_repo_path, - "contractRepoPath": change.contract_repo_path, - "status": change.status, - "contract_id": change.contract_id, - "contractId": change.contract_id, - "current_version": change.current_version, - "currentVersion": change.current_version, - "candidate_version": change.candidate_version, - "candidateVersion": change.candidate_version, - "required_bump": change.required_bump, - "requiredBump": change.required_bump, - "suggested_release_version": change.suggested_release_version, - "suggestedReleaseVersion": change.suggested_release_version, - "reasons": change.reasons, - "governance_decision": gov_dec_dict, - "governanceDecision": gov_dec_dict, - } - - -def batch_task_to_dict(task: BatchReleaseTask) -> dict[str, Any]: - """Serialize a batch release task for debugging/output.""" - return asdict(task) - - -def batch_manifest_build_to_dict(build: BatchReleaseManifestBuild) -> dict[str, Any]: - """Serialize manifest build result for CLI/JSON output.""" - return { - "tasks": [batch_task_to_dict(task) for task in build.tasks], - "skipped": [repository_change_to_dict(change) for change in build.skipped], - } - - -def _relative_contract_index(root: Path) -> dict[str, Path]: - if not root.exists(): - return {} - documents = [Path(path) for path in list_yaml_documents(root)] - return {str(path.relative_to(root)): path for path in documents} - - -def _contract_release_key(change: RepositoryContractChange) -> str: - contract_id = str(change.contract_id or "").strip() - if contract_id: - return contract_id - return Path(change.contract_repo_path).stem - - -def _branch_safe_name(value: str) -> str: - cleaned = "".join( - char if char.isalnum() or char in {"-", "_"} else "-" for char in value.strip() - ) - return cleaned.strip("-") or "contract" diff --git a/semapact/governance/__init__.py b/semapact/governance/__init__.py index bd9be777..a6499272 100644 --- a/semapact/governance/__init__.py +++ b/semapact/governance/__init__.py @@ -1,6 +1,5 @@ """SemaPact Governance Kernel package.""" -from semapact.change_context import ChangeContext from semapact.governance_codes import ( GOVERNANCE_REASON_REGISTRY, GovernanceReasonCode, @@ -24,7 +23,6 @@ evaluate_governance_gate, ) from semapact.governance.public import ( - PublicChangeContextV1, PublicChangeDomain, PublicChangeEvidenceV1, PublicChangeType, @@ -45,7 +43,6 @@ ) __all__ = [ - "ChangeContext", "DecisionResult", "GovernanceReasonCode", "GovernanceSeverity", @@ -70,7 +67,6 @@ "PublicEntityType", "PublicChangeDomain", "PublicEvidenceSource", - "PublicChangeContextV1", "PublicGovernanceReasonV1", "PublicValidationOutcomeV1", "PublicPolicyOutcomeV1", diff --git a/semapact/governance/evaluator.py b/semapact/governance/evaluator.py index a4263af1..9356aec4 100644 --- a/semapact/governance/evaluator.py +++ b/semapact/governance/evaluator.py @@ -9,7 +9,6 @@ from open_data_contract_standard.model import OpenDataContractStandard -from semapact.change_context import ChangeContext from semapact.core.validator import ContractValidator from semapact.exceptions import ValidationError from semapact.governance.change_classification import ( @@ -45,7 +44,6 @@ def evaluate_governance_decision( base_contract: OpenDataContractStandard, candidate_contract: OpenDataContractStandard, *, - context: ChangeContext, merge_conflicts: Sequence[MergeConflict] = (), ) -> GovernanceDecision: """Evaluate an authoritative, deterministic governance decision for a contract change.""" @@ -92,7 +90,6 @@ def evaluate_governance_decision( decision_id = _generate_decision_id( base_contract=base_contract, candidate_contract=candidate_contract, - context=context, merge_conflicts=merge_conflicts, reasons=reasons, ) @@ -107,7 +104,6 @@ def evaluate_governance_decision( decision_id=decision_id, decision=decision_result, contract_id=contract_id, - context=context, breaking=bool(change_assessment.breaking_changes), required_version_bump=change_assessment.required_bump, reasons=reasons, @@ -386,7 +382,6 @@ def _determine_decision( def _generate_decision_id( base_contract: OpenDataContractStandard, candidate_contract: OpenDataContractStandard, - context: ChangeContext, merge_conflicts: Sequence[MergeConflict], reasons: tuple[GovernanceReason, ...], ) -> str: @@ -402,7 +397,6 @@ def _generate_decision_id( "alg": "v1", "base": base_fp, "candidate": candidate_fp, - "effective_date": context.effective_date.isoformat(), "conflict_count": len(merge_conflicts), "conflict_paths": conflict_paths, "reason_codes": reason_codes, diff --git a/semapact/governance/models.py b/semapact/governance/models.py index 00fad906..b6333156 100644 --- a/semapact/governance/models.py +++ b/semapact/governance/models.py @@ -7,7 +7,6 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator -from semapact.change_context import ChangeContext from semapact.governance_codes import GovernanceReasonCode, GovernanceSeverity from semapact.lifecycle.changes import GovernanceChange from semapact.lifecycle.policy import BreakingChange @@ -77,7 +76,6 @@ class GovernanceDecision(GovernanceModel): decision_id: str decision: DecisionResult contract_id: str - context: ChangeContext breaking: bool = Field(strict=True) required_version_bump: RequiredBump validation: ValidationOutcome diff --git a/semapact/governance/public.py b/semapact/governance/public.py index ca9205a2..c84c3f8a 100644 --- a/semapact/governance/public.py +++ b/semapact/governance/public.py @@ -6,7 +6,7 @@ from typing import Any, Literal from pydantic import AliasChoices, BaseModel, ConfigDict, Field, JsonValue -from semapact.core.release import RequiredBump +from semapact.versioning import RequiredBump from semapact.governance.models import ( DecisionResult, GovernanceDecision, @@ -169,15 +169,6 @@ class PublicGovernanceModel(BaseModel): ) -class PublicChangeContextV1(PublicGovernanceModel): - """Public representation of contextual evaluation parameters.""" - - effective_date: str = Field( - validation_alias=AliasChoices("effective_date", "effectiveDate"), - serialization_alias="effectiveDate", - ) - - class PublicGovernanceReasonV1(PublicGovernanceModel): """Public structured reason for a governance outcome or violation.""" @@ -282,7 +273,6 @@ class PublicGovernanceDecisionV1(PublicGovernanceModel): validation_alias=AliasChoices("contract_id", "contractId"), serialization_alias="contractId", ) - context: PublicChangeContextV1 breaking: bool required_version_bump: PublicRequiredVersionBump = Field( validation_alias=AliasChoices("required_version_bump", "requiredVersionBump"), @@ -379,22 +369,17 @@ def to_public_governance_decision(decision: GovernanceDecision) -> PublicGoverna f"to_public_governance_decision requires GovernanceDecision, got {type(decision).__name__}" ) - # 1. Project context - context = PublicChangeContextV1( - effective_date=decision.context.effective_date.isoformat() - ) - - # 2. Project reasons + # 1. Project reasons projected_reasons = tuple(_project_reason(r) for r in decision.reasons) - # 3. Project validation outcome + # 2. Project validation outcome validation_issues = tuple(_project_reason(r) for r in decision.validation.issues) validation = PublicValidationOutcomeV1( valid=decision.validation.valid, issues=validation_issues, ) - # 4. Project policy outcome (omits internal BreakingChange structs) + # 3. Project policy outcome (omits internal BreakingChange structs) policy_violations = tuple(_project_reason(r) for r in decision.policy.violations) policy = PublicPolicyOutcomeV1( valid=decision.policy.valid, @@ -404,16 +389,16 @@ def to_public_governance_decision(decision: GovernanceDecision) -> PublicGoverna violations=policy_violations, ) - # 5. Project evidence + # 4. Project evidence evidence = PublicChangeEvidenceV1( has_changes=decision.evidence.has_changes, merge_conflicts_count=decision.evidence.merge_conflicts_count, ) - # 6. Project canonical changes + # 5. Project canonical changes projected_changes = tuple(_project_change(c) for c in decision.changes) - # 7. Aggregate stable unique reason codes in deterministic alphabetical order + # 6. Aggregate stable unique reason codes in deterministic alphabetical order all_reason_codes: set[str] = set() for r in projected_reasons: all_reason_codes.add(r.code) @@ -428,7 +413,6 @@ def to_public_governance_decision(decision: GovernanceDecision) -> PublicGoverna decision_id=decision.decision_id, decision=decision_val, contract_id=decision.contract_id, - context=context, breaking=decision.breaking, required_version_bump=bump_val, reason_codes=tuple(sorted(all_reason_codes)), diff --git a/semapact/history/__init__.py b/semapact/history/__init__.py index 6ed31053..35b8e114 100644 --- a/semapact/history/__init__.py +++ b/semapact/history/__init__.py @@ -1,39 +1,24 @@ -"""Durable governance-history persistence boundary. - -History stores canonical domain artifacts without becoming a second source of -revision, governance, ContractOps, deployment, or reconciliation semantics. -""" +"""Durable governance-history persistence boundary.""" from semapact.history.models import ( ChangeSetDecisionLink, - DeploymentRecord, - DeploymentStatus, HistoryIntegrityIssueCode, HistoryStorageIntegrityIssue, - ReleaseRecord, - RuntimeObservationRecord, - RuntimeReconciliationRecord, ) from semapact.history.repository import ( ApprovalHistoryRepository, ChangeSetDecisionLinkHistoryRepository, ChangeSetHistoryRepository, + ContractReleaseHistoryRepository, ContractRevisionHistoryRepository, ContractRevisionSourceHistoryRepository, DecisionHistoryRepository, - DeploymentAuthorizationHistoryRepository, - DeploymentPlanHistoryRepository, - DeploymentPreviewHistoryRepository, - DeploymentRecordHistoryRepository, HistoryConflictError, HistoryCorruptionError, HistoryIntegrityRepository, HistoryNotFoundError, HistoryRepositoryError, ReleasePlanHistoryRepository, - ReleaseRecordHistoryRepository, - RuntimeObservationHistoryRepository, - RuntimeReconciliationHistoryRepository, ) __all__ = [ @@ -41,15 +26,10 @@ "ChangeSetDecisionLink", "ChangeSetDecisionLinkHistoryRepository", "ChangeSetHistoryRepository", + "ContractReleaseHistoryRepository", "ContractRevisionHistoryRepository", "ContractRevisionSourceHistoryRepository", "DecisionHistoryRepository", - "DeploymentAuthorizationHistoryRepository", - "DeploymentPlanHistoryRepository", - "DeploymentPreviewHistoryRepository", - "DeploymentRecord", - "DeploymentRecordHistoryRepository", - "DeploymentStatus", "HistoryConflictError", "HistoryCorruptionError", "HistoryIntegrityIssueCode", @@ -58,10 +38,21 @@ "HistoryRepositoryError", "HistoryStorageIntegrityIssue", "ReleasePlanHistoryRepository", - "ReleaseRecord", - "ReleaseRecordHistoryRepository", - "RuntimeObservationHistoryRepository", - "RuntimeObservationRecord", - "RuntimeReconciliationHistoryRepository", - "RuntimeReconciliationRecord", ] + + +from semapact.history.operational import ( + OperationalDeploymentEvent, + OperationalHistorySink, + build_operational_deployment_event, +) +from semapact.history.operational_registry import create_operational_history_sink + +__all__.extend( + [ + "OperationalDeploymentEvent", + "OperationalHistorySink", + "build_operational_deployment_event", + "create_operational_history_sink", + ] +) diff --git a/semapact/history/integrity.py b/semapact/history/integrity.py deleted file mode 100644 index bb8e82e5..00000000 --- a/semapact/history/integrity.py +++ /dev/null @@ -1,222 +0,0 @@ -"""Deterministic identity for history-owned audit records.""" - -from __future__ import annotations - -import uuid - -from semapact.history.models import ( - DeploymentRecord, - ReleaseRecord, - RuntimeObservationRecord, - RuntimeReconciliationRecord, -) -from semapact.observation import ObservedPlatformState -from semapact.reconciliation import ReconciliationResult -from semapact.utils.deterministic import deterministic_uuid5 - - -SEMAPACT_RELEASE_RECORD_NAMESPACE = uuid.UUID( - "9f750d1c-8f0a-491a-b861-8e349fc351cb" -) -SEMAPACT_DEPLOYMENT_RECORD_NAMESPACE = uuid.UUID( - "93db6770-90f0-4ac5-a185-2f21e818d18e" -) -SEMAPACT_RUNTIME_OBSERVATION_RECORD_NAMESPACE = uuid.UUID( - "8a3c8c10-e7d0-4df6-8b1a-3f087ef7e8e2" -) -SEMAPACT_RUNTIME_RECONCILIATION_RECORD_NAMESPACE = uuid.UUID( - "11969598-1f11-4c03-8ab0-f99f012a5041" -) - - -def compute_release_record_id( - *, - contract_id: str, - contract_version: str, - decision_id: str, - change_set_id: str, - release_plan_id: str, - version_resolution_id: str, - authorization_id: str, - applied_release_id: str, - released_revision_id: str, - required_version_bump: str, - actual_version_bump: str, - version_authority: str, - authority_reference: str | None, - review_evidence_reference: str | None, - review_evidence_action: str | None, -) -> str: - """Derive one stable identity from the complete immutable release audit record.""" - return deterministic_uuid5( - SEMAPACT_RELEASE_RECORD_NAMESPACE, - { - "contract_id": contract_id, - "contract_version": contract_version, - "decision_id": decision_id, - "change_set_id": change_set_id, - "release_plan_id": release_plan_id, - "version_resolution_id": version_resolution_id, - "authorization_id": authorization_id, - "applied_release_id": applied_release_id, - "released_revision_id": released_revision_id, - "required_version_bump": required_version_bump, - "actual_version_bump": actual_version_bump, - "version_authority": version_authority, - "authority_reference": authority_reference, - "review_evidence_reference": review_evidence_reference, - "review_evidence_action": review_evidence_action, - }, - ) - - -def validate_release_record_identity(record: ReleaseRecord) -> None: - """Fail closed when a persisted release record ID does not match its content.""" - expected = compute_release_record_id( - contract_id=record.contract_id, - contract_version=record.contract_version, - decision_id=record.decision_id, - change_set_id=record.change_set_id, - release_plan_id=record.release_plan_id, - version_resolution_id=record.version_resolution_id, - authorization_id=record.authorization_id, - applied_release_id=record.applied_release_id, - released_revision_id=record.released_revision_id, - required_version_bump=record.required_version_bump, - actual_version_bump=record.actual_version_bump, - version_authority=record.version_authority.value, - authority_reference=record.authority_reference, - review_evidence_reference=record.review_evidence_reference, - review_evidence_action=( - record.review_evidence_action.value - if record.review_evidence_action is not None - else None - ), - ) - if record.release_record_id != expected: - raise ValueError("ReleaseRecord deterministic identity does not match its content") - - -def compute_deployment_record_id( - *, - release_record_id: str, - deployment_plan_id: str, - deployment_preview_id: str, - deployment_authorization_id: str, - platform: str, - runtime_target: str, - source_reference: str, - status: str, - started_at: str, - completed_at: str, - actor_reference: str | None, - external_reference: str | None, -) -> str: - """Derive one stable identity for a concrete deployment execution occurrence.""" - return deterministic_uuid5( - SEMAPACT_DEPLOYMENT_RECORD_NAMESPACE, - { - "release_record_id": release_record_id, - "deployment_plan_id": deployment_plan_id, - "deployment_preview_id": deployment_preview_id, - "deployment_authorization_id": deployment_authorization_id, - "platform": platform, - "runtime_target": runtime_target, - "source_reference": source_reference, - "status": status, - "started_at": started_at, - "completed_at": completed_at, - "actor_reference": actor_reference, - "external_reference": external_reference, - }, - ) - - -def validate_deployment_record_identity(record: DeploymentRecord) -> None: - """Fail closed when a deployment occurrence ID does not match its content.""" - expected = compute_deployment_record_id( - release_record_id=record.release_record_id, - deployment_plan_id=record.deployment_plan_id, - deployment_preview_id=record.deployment_preview_id, - deployment_authorization_id=record.deployment_authorization_id, - platform=record.platform, - runtime_target=record.runtime_target, - source_reference=record.source_reference, - status=record.status.value, - started_at=record.started_at.isoformat(), - completed_at=record.completed_at.isoformat(), - actor_reference=record.actor_reference, - external_reference=record.external_reference, - ) - if record.deployment_record_id != expected: - raise ValueError( - "DeploymentRecord deterministic identity does not match its content" - ) - - -def compute_runtime_observation_record_id( - observation: ObservedPlatformState, -) -> str: - """Derive stable history identity from the exact canonical M1 observation.""" - if not isinstance(observation, ObservedPlatformState): - raise TypeError( - "observation must be ObservedPlatformState, " - f"got {type(observation).__name__}" - ) - return deterministic_uuid5( - SEMAPACT_RUNTIME_OBSERVATION_RECORD_NAMESPACE, - {"observation": observation.model_dump(mode="json")}, - ) - - -def validate_runtime_observation_record_identity( - record: RuntimeObservationRecord, -) -> None: - """Fail closed when observation history identity does not match exact evidence.""" - expected = compute_runtime_observation_record_id(record.observation) - if record.observation_record_id != expected: - raise ValueError( - "RuntimeObservationRecord deterministic identity does not match content" - ) - - -def compute_runtime_reconciliation_record_id( - *, - observation_record_id: str, - result: ReconciliationResult, - status: str, - release_record_id: str | None, - deployment_record_id: str | None, -) -> str: - """Derive stable identity for one persisted runtime reconciliation conclusion.""" - if not isinstance(result, ReconciliationResult): - raise TypeError( - f"result must be ReconciliationResult, got {type(result).__name__}" - ) - return deterministic_uuid5( - SEMAPACT_RUNTIME_RECONCILIATION_RECORD_NAMESPACE, - { - "observation_record_id": observation_record_id, - "result": result.model_dump(mode="json"), - "status": status, - "release_record_id": release_record_id, - "deployment_record_id": deployment_record_id, - }, - ) - - -def validate_runtime_reconciliation_record_identity( - record: RuntimeReconciliationRecord, -) -> None: - """Fail closed when runtime reconciliation history identity is stale/tampered.""" - expected = compute_runtime_reconciliation_record_id( - observation_record_id=record.observation_record_id, - result=record.result, - status=record.status.value, - release_record_id=record.release_record_id, - deployment_record_id=record.deployment_record_id, - ) - if record.runtime_reconciliation_record_id != expected: - raise ValueError( - "RuntimeReconciliationRecord deterministic identity does not match content" - ) diff --git a/semapact/history/models.py b/semapact/history/models.py index 2fbad67a..edc21cf5 100644 --- a/semapact/history/models.py +++ b/semapact/history/models.py @@ -6,19 +6,10 @@ from __future__ import annotations -from datetime import datetime, timezone from enum import Enum from pydantic import BaseModel, ConfigDict, field_validator, model_validator -from semapact.contractops.models import ReviewEvidenceAction, VersionAuthority -from semapact.observation import ObservedPlatformState -from semapact.reconciliation import ( - ReconciliationResult, - RuntimeDriftStatus, - classify_reconciliation_status, -) -from semapact.versioning import ActualVersionBump, RequiredBump class HistoryModel(BaseModel): @@ -72,192 +63,6 @@ def _require_non_empty_text(cls, value: str) -> str: return _required_text(value) -class ReleaseRecord(HistoryModel): - """Durable audit projection for one finalized governed contract release. - - Canonical ContractOps artifacts remain authoritative for planning, versioning, - authorization, and APPLY semantics. This record freezes the stable links and - final facts required to reconstruct one released semantic version. - """ - - release_record_id: str - contract_id: str - contract_version: str - decision_id: str - change_set_id: str - release_plan_id: str - version_resolution_id: str - authorization_id: str - applied_release_id: str - released_revision_id: str - required_version_bump: RequiredBump - actual_version_bump: ActualVersionBump - version_authority: VersionAuthority - authority_reference: str | None = None - review_evidence_reference: str | None = None - review_evidence_action: ReviewEvidenceAction | None = None - - @field_validator( - "release_record_id", - "contract_id", - "contract_version", - "decision_id", - "change_set_id", - "release_plan_id", - "version_resolution_id", - "authorization_id", - "applied_release_id", - "released_revision_id", - ) - @classmethod - def _require_release_text(cls, value: str) -> str: - return _required_text(value) - - @field_validator( - "authority_reference", - "review_evidence_reference", - ) - @classmethod - def _normalize_optional_text(cls, value: str | None) -> str | None: - return _optional_text(value) - - @model_validator(mode="after") - def _validate_provenance_pairs(self) -> ReleaseRecord: - if self.version_authority is VersionAuthority.GIT: - if self.authority_reference is None: - raise ValueError("git release history requires authority_reference") - elif self.authority_reference is not None: - raise ValueError( - "SemaPact release history must not contain authority_reference" - ) - - has_reference = self.review_evidence_reference is not None - has_action = self.review_evidence_action is not None - if has_reference != has_action: - raise ValueError( - "review evidence reference and action must either both be present or both be absent" - ) - return self - - -class DeploymentStatus(str, Enum): - """Terminal provider-execution outcome for one deployment occurrence.""" - - SUCCEEDED = "SUCCEEDED" - FAILED = "FAILED" - - -class DeploymentRecord(HistoryModel): - """Immutable audit record for one completed deployment execution occurrence. - - This records provider execution only. ``SUCCEEDED`` never implies runtime - convergence; reconciliation remains a separate observation/history concern. - """ - - deployment_record_id: str - release_record_id: str - deployment_plan_id: str - deployment_preview_id: str - deployment_authorization_id: str - platform: str - runtime_target: str - source_reference: str - status: DeploymentStatus - started_at: datetime - completed_at: datetime - actor_reference: str | None = None - external_reference: str | None = None - - @field_validator( - "deployment_record_id", - "release_record_id", - "deployment_plan_id", - "deployment_preview_id", - "deployment_authorization_id", - "runtime_target", - "source_reference", - ) - @classmethod - def _require_deployment_text(cls, value: str) -> str: - return _required_text(value) - - @field_validator("platform") - @classmethod - def _normalize_platform(cls, value: str) -> str: - return _required_text(value).casefold() - - @field_validator("actor_reference", "external_reference") - @classmethod - def _normalize_optional_deployment_text(cls, value: str | None) -> str | None: - return _optional_text(value) - - @field_validator("started_at", "completed_at") - @classmethod - def _normalize_timestamp(cls, value: datetime) -> datetime: - if value.tzinfo is None or value.utcoffset() is None: - raise ValueError("deployment timestamps must be timezone-aware") - return value.astimezone(timezone.utc) - - @model_validator(mode="after") - def _validate_timestamp_order(self) -> DeploymentRecord: - if self.completed_at < self.started_at: - raise ValueError("completed_at must not be earlier than started_at") - return self - - -class RuntimeObservationRecord(HistoryModel): - """Content-addressed history envelope around canonical M1 runtime evidence.""" - - observation_record_id: str - observation: ObservedPlatformState - - @field_validator("observation_record_id") - @classmethod - def _require_observation_id(cls, value: str) -> str: - return _required_text(value) - - -class RuntimeReconciliationRecord(HistoryModel): - """Immutable linkage from canonical runtime evidence to optional lifecycle context. - - ``result`` remains the canonical M1 ReconciliationResult. This record adds only - durable identity, point-in-time classification, and optional exact history links. - """ - - runtime_reconciliation_record_id: str - observation_record_id: str - result: ReconciliationResult - status: RuntimeDriftStatus - release_record_id: str | None = None - deployment_record_id: str | None = None - - @field_validator( - "runtime_reconciliation_record_id", - "observation_record_id", - ) - @classmethod - def _require_runtime_history_text(cls, value: str) -> str: - return _required_text(value) - - @field_validator("release_record_id", "deployment_record_id") - @classmethod - def _normalize_runtime_links(cls, value: str | None) -> str | None: - return _optional_text(value) - - @model_validator(mode="after") - def _validate_runtime_history_semantics(self) -> RuntimeReconciliationRecord: - expected_status = classify_reconciliation_status(self.result) - if self.status is not expected_status: - raise ValueError( - "runtime reconciliation status does not match canonical classification" - ) - if self.deployment_record_id is not None and self.release_record_id is None: - raise ValueError( - "deployment-linked runtime history requires release_record_id" - ) - return self - - def _required_text(value: str) -> str: if not isinstance(value, str): raise TypeError("history identifiers must be strings") diff --git a/semapact/history/operational.py b/semapact/history/operational.py new file mode 100644 index 00000000..16082648 --- /dev/null +++ b/semapact/history/operational.py @@ -0,0 +1,194 @@ +"""High-frequency operational deployment history boundary. + +Operational history is optional telemetry. It is deliberately separate from the +Git-friendly governance ledger because deployment/reconciliation events can be +high-volume in CI/CD environments. +""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from typing import Literal, Protocol + +from pydantic import BaseModel, ConfigDict, field_validator, model_validator + +from semapact.reconciliation import RuntimeDriftStatus +from semapact.utils.deterministic import deterministic_uuid5 + + +SEMAPACT_OPERATIONAL_DEPLOYMENT_NAMESPACE = uuid.UUID( + "9c0c69eb-6294-48d3-93b7-0959a122168e" +) + + +class OperationalDeploymentEvent(BaseModel): + """One immutable deployment occurrence for optional operational persistence.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + event_id: str + event_version: Literal["1"] = "1" + bundle_digest: str + contract_release_id: str | None = None + contract_id: str + contract_version: str + revision_ref: str + deployment_plan_id: str + deployment_preview_id: str | None = None + platform: str + runtime_target: str + source_reference: str + status: Literal["SUCCEEDED", "FAILED"] + reconciliation_status: RuntimeDriftStatus | None = None + started_at: datetime + completed_at: datetime + error_message: str | None = None + + @field_validator( + "event_id", + "bundle_digest", + "contract_id", + "contract_version", + "revision_ref", + "deployment_plan_id", + "platform", + "runtime_target", + "source_reference", + ) + @classmethod + def _require_text(cls, value: str) -> str: + cleaned = value.strip() + if not cleaned: + raise ValueError("value must not be empty") + return cleaned + + @field_validator( + "contract_release_id", + "deployment_preview_id", + "error_message", + ) + @classmethod + def _normalize_optional_text(cls, value: str | None) -> str | None: + if value is None: + return None + cleaned = value.strip() + return cleaned or None + + @field_validator("platform") + @classmethod + def _normalize_platform(cls, value: str) -> str: + return value.strip().casefold() + + @field_validator("started_at", "completed_at") + @classmethod + def _normalize_time(cls, value: datetime) -> datetime: + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError("operational history timestamps must be timezone-aware") + return value.astimezone(timezone.utc) + + @model_validator(mode="after") + def _validate_event(self) -> "OperationalDeploymentEvent": + if self.completed_at < self.started_at: + raise ValueError("completed_at must not be earlier than started_at") + if self.status == "SUCCEEDED" and self.error_message is not None: + raise ValueError("Successful deployment event must not contain error_message") + if self.status == "FAILED" and self.error_message is None: + raise ValueError("Failed deployment event requires error_message") + expected = compute_operational_deployment_event_id( + event_version=self.event_version, + bundle_digest=self.bundle_digest, + contract_release_id=self.contract_release_id, + contract_id=self.contract_id, + contract_version=self.contract_version, + revision_ref=self.revision_ref, + deployment_plan_id=self.deployment_plan_id, + deployment_preview_id=self.deployment_preview_id, + platform=self.platform, + runtime_target=self.runtime_target, + source_reference=self.source_reference, + status=self.status, + reconciliation_status=( + self.reconciliation_status.value + if self.reconciliation_status is not None + else None + ), + started_at=self.started_at.isoformat(), + completed_at=self.completed_at.isoformat(), + error_message=self.error_message, + ) + if self.event_id != expected: + raise ValueError( + "OperationalDeploymentEvent deterministic identity does not match content" + ) + return self + + +class OperationalHistorySink(Protocol): + """Storage-neutral sink for optional high-frequency deployment telemetry.""" + + def record_deployment(self, event: OperationalDeploymentEvent) -> None: ... + + +def build_operational_deployment_event( + *, + bundle_digest: str, + contract_release_id: str | None, + contract_id: str, + contract_version: str, + revision_ref: str, + deployment_plan_id: str, + deployment_preview_id: str | None, + platform: str, + runtime_target: str, + source_reference: str, + status: Literal["SUCCEEDED", "FAILED"], + reconciliation_status: RuntimeDriftStatus | None, + started_at: datetime, + completed_at: datetime, + error_message: str | None, +) -> OperationalDeploymentEvent: + normalized_platform = platform.strip().casefold() + event_id = compute_operational_deployment_event_id( + event_version="1", + bundle_digest=bundle_digest, + contract_release_id=contract_release_id, + contract_id=contract_id, + contract_version=contract_version, + revision_ref=revision_ref, + deployment_plan_id=deployment_plan_id, + deployment_preview_id=deployment_preview_id, + platform=normalized_platform, + runtime_target=runtime_target, + source_reference=source_reference, + status=status, + reconciliation_status=( + reconciliation_status.value if reconciliation_status is not None else None + ), + started_at=started_at.astimezone(timezone.utc).isoformat(), + completed_at=completed_at.astimezone(timezone.utc).isoformat(), + error_message=error_message, + ) + return OperationalDeploymentEvent( + event_id=event_id, + event_version="1", + bundle_digest=bundle_digest, + contract_release_id=contract_release_id, + contract_id=contract_id, + contract_version=contract_version, + revision_ref=revision_ref, + deployment_plan_id=deployment_plan_id, + deployment_preview_id=deployment_preview_id, + platform=normalized_platform, + runtime_target=runtime_target, + source_reference=source_reference, + status=status, + reconciliation_status=reconciliation_status, + started_at=started_at, + completed_at=completed_at, + error_message=error_message, + ) + + +def compute_operational_deployment_event_id(**payload: object) -> str: + return deterministic_uuid5(SEMAPACT_OPERATIONAL_DEPLOYMENT_NAMESPACE, payload) diff --git a/semapact/history/operational_registry.py b/semapact/history/operational_registry.py new file mode 100644 index 00000000..00c7a807 --- /dev/null +++ b/semapact/history/operational_registry.py @@ -0,0 +1,34 @@ +"""Composition helper for optional operational history backends.""" + +from __future__ import annotations + +from semapact.history.operational import OperationalHistorySink + + +def create_operational_history_sink( + uri: str | None, +) -> OperationalHistorySink | None: + """Create a configured operational sink, or disable persistence when omitted.""" + if uri is None or not uri.strip(): + return None + cleaned = uri.strip() + + if cleaned.startswith("sqlite:///"): + from semapact.platforms.sqlite import SQLiteOperationalHistorySink + + path = cleaned.removeprefix("sqlite:///") + if not path: + raise ValueError("SQLite operational history URI requires a path") + return SQLiteOperationalHistorySink(path) + + if cleaned.startswith("delta:///"): + from semapact.platforms.delta import DeltaOperationalHistorySink + + table_uri = cleaned.removeprefix("delta:///") + if not table_uri: + raise ValueError("Delta operational history URI requires a table path") + return DeltaOperationalHistorySink(table_uri) + + raise ValueError( + "Unsupported operational history URI. Use sqlite:///... or delta:///..." + ) diff --git a/semapact/history/repository.py b/semapact/history/repository.py index a17f2408..cab60e0b 100644 --- a/semapact/history/repository.py +++ b/semapact/history/repository.py @@ -5,17 +5,12 @@ from typing import Protocol from semapact.approval.models import ApprovalRecord -from semapact.contractops import ChangeSet, ReleasePlan -from semapact.deployment import DeploymentAuthorization, DeploymentPlan, DeploymentPreview +from semapact.contractops import ChangeSet, ContractRelease, ReleasePlan from semapact.governance import GovernanceDecision from semapact.governance.gate import GovernanceOperation from semapact.history.models import ( ChangeSetDecisionLink, - DeploymentRecord, HistoryStorageIntegrityIssue, - ReleaseRecord, - RuntimeObservationRecord, - RuntimeReconciliationRecord, ) from semapact.revision.models import ContractRevision, ContractRevisionSource @@ -113,99 +108,17 @@ def put_release_plan(self, release_plan: ReleasePlan) -> None: ... def get_release_plan(self, release_plan_id: str) -> ReleasePlan: ... -class ReleaseRecordHistoryRepository(Protocol): - """Persistence capability for finalized release audit records only.""" +class ContractReleaseHistoryRepository(Protocol): + """Persistence capability for finalized formal contract release facts.""" - def put_release_record(self, record: ReleaseRecord) -> None: ... - def get_release_record(self, release_record_id: str) -> ReleaseRecord: ... - def list_release_records(self, contract_id: str) -> tuple[ReleaseRecord, ...]: ... - - def get_release_record_by_version( + def put_contract_release(self, record: ContractRelease) -> None: ... + def get_contract_release(self, contract_release_id: str) -> ContractRelease: ... + def list_contract_releases( self, contract_id: str, - contract_version: str, - ) -> ReleaseRecord: ... - - -class DeploymentPlanHistoryRepository(Protocol): - """Persistence capability for canonical DeploymentPlan history only.""" - - def put_deployment_plan(self, plan: DeploymentPlan) -> None: ... - def get_deployment_plan(self, deployment_plan_id: str) -> DeploymentPlan: ... - - -class DeploymentPreviewHistoryRepository(Protocol): - """Persistence capability for canonical DeploymentPreview history only.""" - - def put_deployment_preview(self, preview: DeploymentPreview) -> None: ... - def get_deployment_preview(self, deployment_preview_id: str) -> DeploymentPreview: ... - - -class DeploymentAuthorizationHistoryRepository(Protocol): - """Persistence capability for canonical DeploymentAuthorization history only.""" - - def put_deployment_authorization( - self, - authorization: DeploymentAuthorization, - ) -> None: ... - - def get_deployment_authorization( - self, - deployment_authorization_id: str, - ) -> DeploymentAuthorization: ... - - -class DeploymentRecordHistoryRepository(Protocol): - """Persistence capability for terminal deployment execution occurrences only.""" - - def put_deployment_record(self, record: DeploymentRecord) -> None: ... - def get_deployment_record(self, deployment_record_id: str) -> DeploymentRecord: ... - - def list_deployment_records_for_release( - self, - release_record_id: str, - ) -> tuple[DeploymentRecord, ...]: ... - - def list_deployment_records_for_plan( - self, - deployment_plan_id: str, - ) -> tuple[DeploymentRecord, ...]: ... - - -class RuntimeObservationHistoryRepository(Protocol): - """Persistence capability for canonical M1 observation evidence envelopes.""" - - def put_runtime_observation_record(self, record: RuntimeObservationRecord) -> None: ... - def get_runtime_observation_record( - self, - observation_record_id: str, - ) -> RuntimeObservationRecord: ... - - def list_runtime_observation_records( - self, - source_identifier: str, - ) -> tuple[RuntimeObservationRecord, ...]: ... - - -class RuntimeReconciliationHistoryRepository(Protocol): - """Persistence capability for point-in-time runtime reconciliation history.""" - - def put_runtime_reconciliation_record( - self, - record: RuntimeReconciliationRecord, - ) -> None: ... - - def get_runtime_reconciliation_record( - self, - runtime_reconciliation_record_id: str, - ) -> RuntimeReconciliationRecord: ... - - def list_runtime_reconciliation_records( + ) -> tuple[ContractRelease, ...]: ... + def get_contract_release_by_version( self, contract_id: str, - ) -> tuple[RuntimeReconciliationRecord, ...]: ... - - def list_runtime_reconciliation_records_for_source( - self, - source_identifier: str, - ) -> tuple[RuntimeReconciliationRecord, ...]: ... + contract_version: str, + ) -> ContractRelease: ... diff --git a/semapact/interfaces/cli.py b/semapact/interfaces/cli.py index abc20f60..522ced02 100644 --- a/semapact/interfaces/cli.py +++ b/semapact/interfaces/cli.py @@ -275,6 +275,73 @@ def _build_parser() -> argparse.ArgumentParser: dest="release_command", required=True ) + release_assess_parser = release_subparsers.add_parser( + "assess", + help="Build one immutable target-neutral formal ReleaseBundle", + ) + release_assess_parser.add_argument("--base", required=True) + release_assess_parser.add_argument("--candidate", required=True) + release_assess_parser.add_argument("--base-revision-ref", required=True) + release_assess_parser.add_argument("--candidate-revision-ref", required=True) + release_assess_parser.add_argument( + "--authority-reference", + help="Explicit Git release reference when release.versionAuthority=git", + ) + release_assess_parser.add_argument("--runtime-context", default="auto") + release_assess_parser.add_argument( + "--bundle-out", + help="Write the immutable ReleaseBundle JSON artifact to this path", + ) + + release_approve_parser = release_subparsers.add_parser( + "approve", + help="Record explicit REVIEW approval for one exact ReleaseBundle", + ) + release_approve_parser.add_argument("--bundle", required=True) + release_approve_parser.add_argument("--actor-reference", required=True) + release_approve_parser.add_argument( + "--recorded-at", + required=True, + help="Approval timestamp as timezone-aware ISO-8601", + ) + release_approve_parser.add_argument("--comment") + release_approve_parser.add_argument( + "--repository-root", + default=".", + help="Repository root containing the Git governance ledger", + ) + release_approve_parser.add_argument( + "--approval-out", + help="Optional standalone ApprovalRecord JSON output", + ) + + release_finalize_parser = release_subparsers.add_parser( + "finalize", + help="Finalize one ReleaseBundle, record it, and materialize the versioned contract", + ) + release_finalize_parser.add_argument("--bundle", required=True) + release_finalize_parser.add_argument( + "--approval", + help=( + "Exact ApprovalRecord JSON artifact for a REVIEW release. " + "When omitted, SemaPact may resolve approval from the Git governance ledger." + ), + ) + release_finalize_parser.add_argument( + "--output-contract", + required=True, + help="Write the exact released/versioned ODCS contract to this path", + ) + release_finalize_parser.add_argument( + "--release-out", + help="Write the finalized immutable ContractRelease JSON artifact to this path", + ) + release_finalize_parser.add_argument( + "--repository-root", + default=".", + help="Repository root containing the Git governance ledger", + ) + release_classify_parser = release_subparsers.add_parser( "classify", help="Analyze the required version bump without creating release artifacts", @@ -282,7 +349,6 @@ def _build_parser() -> argparse.ArgumentParser: release_classify_parser.add_argument("--base", required=True) release_classify_parser.add_argument("--candidate", required=True) release_classify_parser.add_argument("--runtime-context", default="auto") - _add_effective_date_argument(release_classify_parser) release_plan_parser = release_subparsers.add_parser( "plan", @@ -297,7 +363,6 @@ def _build_parser() -> argparse.ArgumentParser: help="Explicit Git release reference when release.versionAuthority=git", ) release_plan_parser.add_argument("--runtime-context", default="auto") - _add_effective_date_argument(release_plan_parser) release_classify_repo_parser = release_subparsers.add_parser( "classify-repo", @@ -305,79 +370,6 @@ def _build_parser() -> argparse.ArgumentParser: ) release_classify_repo_parser.add_argument("--base-root", required=True) release_classify_repo_parser.add_argument("--candidate-root", required=True) - _add_effective_date_argument(release_classify_repo_parser) - - release_build_manifest_parser = release_subparsers.add_parser( - "build-manifest", - help="Build an editable per-contract release manifest from two contract roots", - ) - release_build_manifest_parser.add_argument("--base-root", required=True) - release_build_manifest_parser.add_argument("--candidate-root", required=True) - release_build_manifest_parser.add_argument("--output", required=True) - release_build_manifest_parser.add_argument("--target-branch", default="release") - release_build_manifest_parser.add_argument( - "--source-branch-prefix", default="release/" - ) - _add_effective_date_argument(release_build_manifest_parser) - - release_prepare_parser = release_subparsers.add_parser( - "prepare", - help="Compatibility helper: prepare a candidate using an explicit release tag", - ) - release_prepare_parser.add_argument("--base", required=True) - release_prepare_parser.add_argument("--candidate", required=True) - release_prepare_parser.add_argument("--release-tag", required=True) - release_prepare_parser.add_argument("--output", required=True) - release_prepare_parser.add_argument("--runtime-context", default="auto") - _add_effective_date_argument(release_prepare_parser) - - release_pr_parser = release_subparsers.add_parser( - "create-pr", - help="Compatibility Git workflow: prepare a candidate and open a release PR", - ) - release_pr_parser.add_argument("--base", required=True) - release_pr_parser.add_argument("--candidate", required=True) - release_pr_parser.add_argument("--release-tag", required=True) - release_pr_parser.add_argument("--repo-path", help="Local repository path") - release_pr_parser.add_argument("--contract-path", required=True) - release_pr_parser.add_argument("--source-branch", required=True) - release_pr_parser.add_argument("--target-branch", required=True) - release_pr_parser.add_argument( - "--git-provider", - choices=["azure", "github"], - ) - release_pr_parser.add_argument("--organization") - release_pr_parser.add_argument("--github-owner") - release_pr_parser.add_argument("--github-repo") - release_pr_parser.add_argument("--github-token") - release_pr_parser.add_argument("--project") - release_pr_parser.add_argument("--repository-id") - release_pr_parser.add_argument("--pat-token") - release_pr_parser.add_argument("--title") - release_pr_parser.add_argument("--description") - release_pr_parser.add_argument("--commit-message") - release_pr_parser.add_argument("--push", action="store_true") - release_pr_parser.add_argument("--runtime-context", default="auto") - _add_effective_date_argument(release_pr_parser) - - release_prs_parser = release_subparsers.add_parser( - "create-prs", - help="Run explicit per-contract release PR automation from a batch manifest", - ) - release_prs_parser.add_argument("--manifest", required=True) - release_prs_parser.add_argument("--repo-path", help="Local repository path") - release_prs_parser.add_argument( - "--git-provider", - choices=["azure", "github"], - ) - release_prs_parser.add_argument("--organization") - release_prs_parser.add_argument("--github-owner") - release_prs_parser.add_argument("--github-repo") - release_prs_parser.add_argument("--github-token") - release_prs_parser.add_argument("--project") - release_prs_parser.add_argument("--repository-id") - release_prs_parser.add_argument("--pat-token") - release_prs_parser.add_argument("--push", action="store_true") doctor_parser = subparsers.add_parser( @@ -443,52 +435,93 @@ def _build_parser() -> argparse.ArgumentParser: deployment_parser = subparsers.add_parser( "deployment", - help="Plan, preview, execute, and verify governed runtime deployment", + help="Assess and deploy immutable target-specific runtime bundles", ) deployment_subparsers = deployment_parser.add_subparsers( dest="deployment_command", required=True ) - deployment_plan_parser = deployment_subparsers.add_parser( - "plan", help="Build a DeploymentPlan from an exact AppliedContractRelease" + deployment_assess_parser = deployment_subparsers.add_parser( + "assess", + help=( + "Assess either a base/candidate change or an already-finalized release " + "against one fresh runtime target" + ), ) - deployment_plan_parser.add_argument("--release", required=True) - deployment_plan_parser.add_argument("--platform", required=True) - deployment_plan_parser.add_argument("--runtime", required=True) - deployment_plan_parser.add_argument( - "--source-reference", - required=True, - help="Stable runtime source identity (for Databricks, the workspace host; never credentials)", + deployment_assess_parser.add_argument("--base") + deployment_assess_parser.add_argument("--candidate") + deployment_assess_parser.add_argument("--base-revision-ref") + deployment_assess_parser.add_argument("--candidate-revision-ref") + release_source_group = deployment_assess_parser.add_mutually_exclusive_group() + release_source_group.add_argument( + "--release", + help="Finalized ContractRelease JSON artifact to deploy instead of a candidate", ) - deployment_plan_parser.add_argument("--server") - - deployment_preview_parser = deployment_subparsers.add_parser( - "preview", help="Observe runtime and derive provider-native deployment operations" + release_source_group.add_argument( + "--release-id", + help="Convenience fallback: resolve a finalized ContractRelease ID from Git history", ) - deployment_preview_parser.add_argument("--plan", required=True) - - deployment_execute_parser = deployment_subparsers.add_parser( - "execute", help="Execute an exact authorized DeploymentPreview" + deployment_assess_parser.add_argument( + "--repository-root", + default=".", + help="Repository root containing finalized release history", ) - deployment_execute_parser.add_argument("--plan", required=True) - deployment_execute_parser.add_argument("--preview", required=True) - deployment_execute_parser.add_argument("--authorization", required=True) - deployment_execute_parser.add_argument( - "--warehouse-id", - help="Databricks SQL warehouse required only when preview contains mutation operations", + deployment_assess_parser.add_argument( + "--server", + help="Contract server identifier when the deployment source defines multiple servers", + ) + deployment_assess_parser.add_argument( + "--platform", + help="Fallback runtime provider when the deployment source defines no servers", + ) + deployment_assess_parser.add_argument( + "--runtime", + help="Fallback provider-local runtime target when the deployment source defines no servers", + ) + deployment_assess_parser.add_argument( + "--source-reference", + help="Fallback stable runtime source identity when the source defines no server host", + ) + deployment_assess_parser.add_argument("--runtime-context", default="auto") + deployment_assess_parser.add_argument( + "--bundle-out", + help="Write the immutable DeploymentBundle JSON artifact to this path", + ) + deployment_assess_parser.add_argument( + "--output", + choices=["text", "json"], + default="text", + help="Output format (default: text)", ) - deployment_verify_parser = deployment_subparsers.add_parser( - "verify", help="Verify DeploymentPlan convergence through M1 reconciliation" + deployment_deploy_parser = deployment_subparsers.add_parser( + "deploy", + help="Consume an immutable DeploymentBundle, execute against fresh runtime, and verify", ) - deployment_verify_parser.add_argument("--plan", required=True) - deployment_verify_parser.add_argument( + deployment_deploy_parser.add_argument("--bundle", required=True) + deployment_deploy_parser.add_argument( + "--warehouse-id", + help=( + "Databricks SQL warehouse required for runtime mutation and formal-release " + "Unity Catalog provenance tag projection" + ), + ) + deployment_deploy_parser.add_argument( + "--operational-history", + help=( + "Optional operational-history URI override. When omitted, SemaPact reads " + "typed history.operational configuration; if neither is configured, " + "deployment telemetry persistence is disabled." + ), + ) + deployment_deploy_parser.add_argument( "--output", choices=["text", "json"], default="text", help="Output format (default: text)", ) + return parser @@ -607,21 +640,15 @@ def main() -> int: if args.command == "deployment": from semapact.interfaces.commands.deployment_cmd import ( - run_deployment_execute, - run_deployment_plan, - run_deployment_preview, - run_deployment_verify, + run_deployment_assess, + run_deployment_deploy, ) from semapact.interfaces.outcomes import exit_code_from_outcome - if args.deployment_command == "plan": - result = run_deployment_plan(args) - elif args.deployment_command == "preview": - result = run_deployment_preview(args) - elif args.deployment_command == "execute": - result = run_deployment_execute(args) - elif args.deployment_command == "verify": - result = run_deployment_verify(args) + if args.deployment_command == "assess": + result = run_deployment_assess(args) + elif args.deployment_command == "deploy": + result = run_deployment_deploy(args) else: parser.error(f"Unknown deployment command: {args.deployment_command}") print(result.output) @@ -629,40 +656,35 @@ def main() -> int: if args.command == "release": from semapact.interfaces.commands.release_cmd import ( - run_release_build_manifest, + run_release_approve, + run_release_assess, run_release_classify, run_release_classify_repo, - run_release_create_pr, - run_release_create_prs, + run_release_finalize, run_release_plan, - run_release_prepare, ) - if args.release_command == "classify": - payload = run_release_classify(args) - print(json.dumps(payload, indent=2, sort_keys=True)) - return 0 - if args.release_command == "plan": - payload = run_release_plan(args) + if args.release_command == "assess": + payload = run_release_assess(args) print(json.dumps(payload, indent=2, sort_keys=True)) return 0 - if args.release_command == "classify-repo": - payload = run_release_classify_repo(args) + if args.release_command == "approve": + payload = run_release_approve(args) print(json.dumps(payload, indent=2, sort_keys=True)) return 0 - if args.release_command == "build-manifest": - payload = run_release_build_manifest(args) + if args.release_command == "finalize": + payload = run_release_finalize(args) print(json.dumps(payload, indent=2, sort_keys=True)) return 0 - if args.release_command == "prepare": - payload = run_release_prepare(args) + if args.release_command == "classify": + payload = run_release_classify(args) print(json.dumps(payload, indent=2, sort_keys=True)) return 0 - if args.release_command == "create-pr": - payload = run_release_create_pr(args) + if args.release_command == "plan": + payload = run_release_plan(args) print(json.dumps(payload, indent=2, sort_keys=True)) return 0 - if args.release_command == "create-prs": - payload = run_release_create_prs(args) + if args.release_command == "classify-repo": + payload = run_release_classify_repo(args) print(json.dumps(payload, indent=2, sort_keys=True)) return 0 diff --git a/semapact/interfaces/commands/approval_cmd.py b/semapact/interfaces/commands/approval_cmd.py index 87b1534c..026f28a6 100644 --- a/semapact/interfaces/commands/approval_cmd.py +++ b/semapact/interfaces/commands/approval_cmd.py @@ -3,17 +3,16 @@ from __future__ import annotations from argparse import Namespace -from datetime import datetime - from semapact.application.services.approval_record import ApprovalRecordService from semapact.contractops import ReviewEvidenceAction from semapact.governance import GovernanceOperation +from semapact.interfaces.parsing import parse_iso_timestamp from semapact.platforms.git import GitWorkingTreeHistoryRepository def run_approval_record(args: Namespace) -> dict[str, object]: """Record one explicit review event through the application service boundary.""" - recorded_at = _parse_timestamp(args.recorded_at) + recorded_at = parse_iso_timestamp(args.recorded_at) service = ApprovalRecordService( GitWorkingTreeHistoryRepository(args.repository_root), ) @@ -34,13 +33,3 @@ def run_approval_record(args: Namespace) -> dict[str, object]: return record.model_dump(mode="json", by_alias=True) -def _parse_timestamp(value: str) -> datetime: - if not isinstance(value, str): - raise TypeError("recorded_at must be an ISO-8601 string") - try: - parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) - except ValueError as exc: - raise ValueError("recorded_at must be an ISO-8601 timestamp") from exc - if parsed.tzinfo is None or parsed.utcoffset() is None: - raise ValueError("recorded_at must include an explicit timezone offset") - return parsed diff --git a/semapact/interfaces/commands/deployment_cmd.py b/semapact/interfaces/commands/deployment_cmd.py index 26f14e6d..3f456475 100644 --- a/semapact/interfaces/commands/deployment_cmd.py +++ b/semapact/interfaces/commands/deployment_cmd.py @@ -1,4 +1,4 @@ -"""CLI adapter for canonical deployment planning, execution, and verification.""" +"""CLI adapter for target-specific deployment workflows.""" from __future__ import annotations @@ -12,20 +12,14 @@ from pydantic import BaseModel from pydantic import ValidationError as PydanticValidationError -from semapact.application.services.deployment import DeploymentService -from semapact.contractops import AppliedContractRelease -from semapact.deployment import ( - DeploymentAuthorization, - DeploymentPlan, - DeploymentPreview, - DeploymentTarget, -) +from semapact.application.models.deployment_workflow import DeploymentBundle +from semapact.deployment import DeploymentTarget from semapact.exceptions import ValidationError from semapact.interfaces.outcomes import ( ProcessOutcome, + outcome_from_gate_result, outcome_from_reconciliation_status, ) -from semapact.reconciliation import classify_reconciliation_status _ModelT = TypeVar("_ModelT", bound=BaseModel) @@ -33,54 +27,124 @@ @dataclass(frozen=True) class DeploymentCommandResult: - """Rendered CLI output plus its existing semantic process outcome.""" + """Rendered CLI output plus its semantic process outcome.""" output: str outcome: ProcessOutcome -def run_deployment_plan(args: argparse.Namespace) -> DeploymentCommandResult: - """Build one provider-neutral DeploymentPlan from an exact applied release.""" - release = _load_model(args.release, AppliedContractRelease) - target = DeploymentTarget( - platform=args.platform, - runtime_target=args.runtime, - source_reference=args.source_reference, - server_name=args.server, +def run_deployment_assess(args: argparse.Namespace) -> DeploymentCommandResult: + """Build one target-specific candidate or finalized-release deployment bundle.""" + from open_data_contract_standard.model import OpenDataContractStandard + + from semapact.application.services.deployment_workflow import ( + DeploymentWorkflowService, ) - plan = DeploymentService().plan(release, target) - return DeploymentCommandResult( - output=_model_json(plan), - outcome=ProcessOutcome.SUCCESS, + from semapact.core.loader import ContractLoader + from semapact.contractops import ContractRelease + from semapact.governance import GovernanceOperation, evaluate_governance_gate + from semapact.platforms.git import GitWorkingTreeHistoryRepository + from semapact.platforms.runtime_registry import ( + create_deployment_adapter, + resolve_runtime_location, ) + service = DeploymentWorkflowService() + loader = ContractLoader(runtime_context=args.runtime_context) + + if args.release or args.release_id: + _reject_candidate_args_for_release(args) + release = ( + _load_model(args.release, ContractRelease) + if args.release + else GitWorkingTreeHistoryRepository( + args.repository_root + ).get_contract_release(args.release_id) + ) + source_contract = OpenDataContractStandard.model_validate_json( + release.released_contract_json + ) + location = resolve_runtime_location( + source_contract, + server_name=args.server, + fallback_platform=args.platform, + fallback_runtime_target=args.runtime, + ) + target = _deployment_target( + location, + source_reference=_assessment_source_reference( + location.contract_server, + args.source_reference, + ), + ) + adapter = create_deployment_adapter( + location.platform, + contract_server=location.contract_server, + ) + bundle = service.assess_release( + release, + target=target, + adapter=adapter, + ) + outcome = ProcessOutcome.SUCCESS + else: + _require_candidate_args(args) + base_contract = loader.load(args.base) + candidate_contract = loader.load(args.candidate) + location = resolve_runtime_location( + candidate_contract, + server_name=args.server, + fallback_platform=args.platform, + fallback_runtime_target=args.runtime, + ) + target = _deployment_target( + location, + source_reference=_assessment_source_reference( + location.contract_server, + args.source_reference, + ), + ) + adapter = create_deployment_adapter( + location.platform, + contract_server=location.contract_server, + ) + bundle = service.assess( + base_contract, + candidate_contract, + base_revision_ref=args.base_revision_ref, + candidate_revision_ref=args.candidate_revision_ref, + target=target, + adapter=adapter, + ) + assert bundle.decision is not None + gate = evaluate_governance_gate( + bundle.decision, + GovernanceOperation.PROPOSE, + ) + outcome = outcome_from_gate_result(gate) -def run_deployment_preview(args: argparse.Namespace) -> DeploymentCommandResult: - """Observe exact runtime scope and render the adapter's canonical preview.""" - plan = _load_model(args.plan, DeploymentPlan) - from semapact.platforms.runtime_registry import create_deployment_adapter + if args.bundle_out: + _write_model_artifact(args.bundle_out, bundle) - adapter = create_deployment_adapter(plan.target.platform) - preview = DeploymentService().preview( - plan, - adapter=adapter, - ) - return DeploymentCommandResult( - output=_model_json(preview), - outcome=ProcessOutcome.SUCCESS, + rendered = ( + _model_json(bundle) + if args.output == "json" + else _bundle_text(bundle, artifact_path=args.bundle_out) ) + return DeploymentCommandResult(output=rendered, outcome=outcome) -def run_deployment_execute(args: argparse.Namespace) -> DeploymentCommandResult: - """Execute only an exact plan/preview/authorization tuple.""" - plan = _load_model(args.plan, DeploymentPlan) - preview = _load_model(args.preview, DeploymentPreview) - authorization = _load_model(args.authorization, DeploymentAuthorization) - +def run_deployment_deploy(args: argparse.Namespace) -> DeploymentCommandResult: + """Consume one exact DeploymentBundle and run the canonical CD workflow.""" + from semapact.application.services.deployment_workflow import ( + DeploymentWorkflowService, + ) from semapact.platforms.runtime_registry import create_deployment_adapter + bundle = _load_model(args.bundle, DeploymentBundle) + execution_config = None - if plan.target.platform == "databricks": + if bundle.deployment_plan.target.platform == "databricks": from semapact.platforms.databricks.deployment import ( DatabricksDeploymentExecutionConfig, ) @@ -94,52 +158,91 @@ def run_deployment_execute(args: argparse.Namespace) -> DeploymentCommandResult: ) adapter = create_deployment_adapter( - plan.target.platform, + bundle.deployment_plan.target.platform, execution_config=execution_config, ) - DeploymentService().execute( - plan, - preview, - authorization, - adapter=adapter, - ) - return DeploymentCommandResult( - output=json.dumps( - { - "convergenceVerified": False, - "deploymentPlanId": plan.deployment_plan_id, - "deploymentPreviewId": preview.deployment_preview_id, - "providerExecution": "SUCCEEDED", - }, - indent=2, - sort_keys=True, - ), - outcome=ProcessOutcome.SUCCESS, - ) + from semapact.history import create_operational_history_sink -def run_deployment_verify(args: argparse.Namespace) -> DeploymentCommandResult: - """Verify exact DeploymentPlan convergence through the unified deployment adapter.""" - plan = _load_model(args.plan, DeploymentPlan) - from semapact.platforms.runtime_registry import create_deployment_adapter - - adapter = create_deployment_adapter(plan.target.platform) - result = DeploymentService().verify( - plan, + operational_history = create_operational_history_sink( + _resolve_operational_history_uri(args.operational_history) + ) + result = DeploymentWorkflowService().deploy( + bundle, adapter=adapter, + operational_history=operational_history, ) - status = classify_reconciliation_status(result) rendered = ( - _verification_json(status.value, result) + _model_json(result) if args.output == "json" - else _verification_text(status.value, result) + else _deployment_result_text(result) ) return DeploymentCommandResult( output=rendered, - outcome=outcome_from_reconciliation_status(status), + outcome=outcome_from_reconciliation_status(result.status), ) +def _require_candidate_args(args: argparse.Namespace) -> None: + required = { + "--base": args.base, + "--candidate": args.candidate, + "--base-revision-ref": args.base_revision_ref, + "--candidate-revision-ref": args.candidate_revision_ref, + } + missing = [name for name, value in required.items() if not value] + if missing: + raise ValidationError( + "Candidate deployment assessment requires " + ", ".join(missing) + ) + + +def _reject_candidate_args_for_release(args: argparse.Namespace) -> None: + supplied = [ + name + for name, value in ( + ("--base", args.base), + ("--candidate", args.candidate), + ("--base-revision-ref", args.base_revision_ref), + ("--candidate-revision-ref", args.candidate_revision_ref), + ) + if value + ] + if supplied: + raise ValidationError( + "Release deployment cannot be combined with candidate assessment arguments: " + + ", ".join(supplied) + ) + + +def _deployment_target(location, *, source_reference: str) -> DeploymentTarget: + return DeploymentTarget( + platform=location.platform, + runtime_target=location.runtime_target, + source_reference=source_reference, + server_name=location.server_name, + ) + + +def _resolve_operational_history_uri(cli_override: str | None) -> str | None: + """Resolve CLI override first, then typed project/global configuration.""" + if cli_override is not None and cli_override.strip(): + return cli_override.strip() + + from pydantic import ValidationError as PydanticConfigValidationError + + from semapact.core.config import config_manager + from semapact.core.config_schema import operational_history_uri_from_config + + raw_config = config_manager.get("history.operational", default=None) + try: + return operational_history_uri_from_config(raw_config) + except PydanticConfigValidationError as exc: + raise ValidationError( + f"Invalid history.operational configuration: {exc}" + ) from exc + + def _load_model(path: str, model_type: type[_ModelT]) -> _ModelT: try: raw = ( @@ -154,6 +257,12 @@ def _load_model(path: str, model_type: type[_ModelT]) -> _ModelT: ) from exc +def _write_model_artifact(path: str, model: BaseModel) -> None: + artifact_path = Path(path) + artifact_path.parent.mkdir(parents=True, exist_ok=True) + artifact_path.write_text(_model_json(model) + "\n", encoding="utf-8") + + def _model_json(model: BaseModel) -> str: return json.dumps( model.model_dump(mode="json"), @@ -163,46 +272,93 @@ def _model_json(model: BaseModel) -> str: ) -def _verification_json(status: str, result: BaseModel) -> str: - return json.dumps( - { - "reconciliation": result.model_dump(mode="json"), - "status": status, - }, - indent=2, - ensure_ascii=False, - sort_keys=True, +def _assessment_source_reference(contract_server, cli_source_reference: str | None) -> str: + cli_value = (cli_source_reference or "").strip() + if contract_server is None: + if not cli_value: + raise ValidationError( + "Contract defines no server host; provide --source-reference" + ) + return cli_value + + host = str(getattr(contract_server, "host", "") or "").strip() + if not host: + raise ValidationError( + "Selected contract server must define host for deployment assessment" + ) + if cli_value and cli_value != host: + raise ValidationError( + "--source-reference cannot override the selected contract server host" + ) + return host + + +def _bundle_text( + bundle: DeploymentBundle, + *, + artifact_path: str | None, +) -> str: + plan = bundle.deployment_plan + preview = bundle.review_preview + lines = [ + f"Contract: {bundle.deployment_source.contract_id}@" + f"{bundle.deployment_source.contract_version}", + f"Mode: {'release' if bundle.is_release else 'candidate'}", + f"Target: {plan.target.platform}/{plan.target.runtime_target}", + f"Deployment plan: {plan.deployment_plan_id}", + f"Bundle digest: {bundle.bundle_digest}", + "Execution authority: none (CI/read-only bundle)", + ] + if bundle.is_release: + assert bundle.contract_release is not None + lines.append( + f"Contract release: {bundle.contract_release.contract_release_id}" + ) + else: + assert bundle.decision is not None + lines.append(f"Governance: {bundle.decision.decision.value}") + lines.append("Review operations:") + for operation in preview.operations: + detail = f" - {operation.kind.value} {operation.governed_asset}" + if operation.statement is not None: + detail += f": {operation.statement}" + lines.append(detail) + if not preview.operations: + lines.append(" - none") + lines.extend( + [ + f"Observation source: {preview.source_identifier}", + f"Observation fingerprint: {preview.observation_fingerprint}", + ] ) + if artifact_path: + lines.append(f"Bundle artifact: {artifact_path}") + return "\n".join(lines) -def _verification_text(status: str, result: BaseModel) -> str: - differences = getattr(result, "differences") - unverified_paths = getattr(result, "unverified_paths") +def _deployment_result_text(result) -> str: lines = [ - f"Status: {status}", + f"Bundle digest: {result.bundle_digest}", + f"Deployment plan: {result.deployment_plan_id}", + "Execution authority: surrounding protected deployment context", + *( + [f"Contract release: {result.contract_release_id}"] + if result.contract_release_id is not None + else [] + ), + "Provider execution: SUCCEEDED", + f"Verification: {result.status.value}", ( - f"Contract: {getattr(result, 'contract_id')}@" - f"{getattr(result, 'contract_version')}" + "CI review preview changed: " + f"{'yes' if result.review_preview_changed else 'no'}" ), - f"Observation source: {getattr(result, 'observation_source_identifier')}", - f"Observation fingerprint: {getattr(result, 'observation_fingerprint')}", + "Fresh operations:", ] - if differences: - lines.append("Differences:") - for difference in differences: - detail = f" - {difference.reason_code.value} {difference.path}" - if difference.expected is not None or difference.observed is not None: - detail += ( - f" expected={difference.expected!r}" - f" observed={difference.observed!r}" - ) - lines.append(detail) - else: - lines.append("Differences: none") - - if unverified_paths: - lines.append("Unverified paths:") - lines.extend(f" - {path}" for path in unverified_paths) - else: - lines.append("Unverified paths: none") + for operation in result.fresh_preview.operations: + detail = f" - {operation.kind.value} {operation.governed_asset}" + if operation.statement is not None: + detail += f": {operation.statement}" + lines.append(detail) + if not result.fresh_preview.operations: + lines.append(" - none") return "\n".join(lines) diff --git a/semapact/interfaces/commands/import_cmd.py b/semapact/interfaces/commands/import_cmd.py index c00cfacf..23eb0fa6 100644 --- a/semapact/interfaces/commands/import_cmd.py +++ b/semapact/interfaces/commands/import_cmd.py @@ -10,7 +10,7 @@ _resolve_adls_oauth_token_from_config, _split_discovered_delta_tables, ) -from semapact.services import GovernanceService +from semapact.application.services.governance import GovernanceService def run_import(args: argparse.Namespace) -> Path: from semapact.core.loader import ContractLoader diff --git a/semapact/interfaces/commands/lifecycle_cmd.py b/semapact/interfaces/commands/lifecycle_cmd.py index 4f498876..88b8ab16 100644 --- a/semapact/interfaces/commands/lifecycle_cmd.py +++ b/semapact/interfaces/commands/lifecycle_cmd.py @@ -1,7 +1,7 @@ import argparse from typing import Any -from semapact.services import GovernanceService +from semapact.application.services.governance import GovernanceService def run_lifecycle_promote(args: argparse.Namespace) -> dict[str, Any]: diff --git a/semapact/interfaces/commands/plan_cmd.py b/semapact/interfaces/commands/plan_cmd.py index 3e3972a6..b2f8a4d6 100644 --- a/semapact/interfaces/commands/plan_cmd.py +++ b/semapact/interfaces/commands/plan_cmd.py @@ -83,7 +83,6 @@ def run_plan(args: argparse.Namespace) -> None: decision = evaluate_governance_decision( base_contract, merged, - context=change_context, merge_conflicts=merge_result.conflicts, ) gate_res = evaluate_governance_gate(decision, GovernanceOperation.ANALYZE) diff --git a/semapact/interfaces/commands/release_cmd.py b/semapact/interfaces/commands/release_cmd.py index dd70b21e..5a93c0c1 100644 --- a/semapact/interfaces/commands/release_cmd.py +++ b/semapact/interfaces/commands/release_cmd.py @@ -11,6 +11,121 @@ ) + +def run_release_assess(args: argparse.Namespace) -> dict[str, Any]: + """Build one immutable target-neutral ReleaseBundle.""" + from semapact.application.services.release_workflow import ReleaseWorkflowService + from semapact.core.loader import ContractLoader + + loader = ContractLoader(runtime_context=args.runtime_context) + base_contract = loader.load(args.base) + candidate_contract = loader.load(args.candidate) + bundle = ReleaseWorkflowService().assess( + base_contract, + candidate_contract, + base_revision_ref=args.base_revision_ref, + candidate_revision_ref=args.candidate_revision_ref, + authority_reference=args.authority_reference, + ) + if args.bundle_out: + _write_model_artifact(args.bundle_out, bundle) + return bundle.model_dump(mode="json") + + +def run_release_approve(args: argparse.Namespace) -> dict[str, Any]: + """Record explicit REVIEW approval for one exact ReleaseBundle.""" + from semapact.application.models.release import ReleaseBundle + from semapact.application.services.release_workflow import ReleaseWorkflowService + from semapact.interfaces.parsing import parse_iso_timestamp + from semapact.platforms.git import GitWorkingTreeHistoryRepository + + bundle = _load_model(args.bundle, ReleaseBundle) + approval = ReleaseWorkflowService().approve( + bundle, + actor_reference=args.actor_reference, + recorded_at=parse_iso_timestamp(args.recorded_at), + comment=args.comment, + ) + GitWorkingTreeHistoryRepository(args.repository_root).put_approval_record(approval) + if args.approval_out: + _write_model_artifact(args.approval_out, approval) + return approval.model_dump(mode="json") + + +def run_release_finalize(args: argparse.Namespace) -> dict[str, Any]: + """Finalize one exact release, persist ledger fact, and materialize versioned ODCS.""" + from semapact.application.models.release import ReleaseBundle + from semapact.application.services.release_approval import ReleaseApprovalResolver + from semapact.application.services.release_workflow import ReleaseFinalizer + from semapact.governance import DecisionResult + from semapact.platforms.git import GitWorkingTreeHistoryRepository + from semapact.utils.yaml_utils import dump_yaml + + from semapact.approval import ApprovalRecord + + bundle = _load_model(args.bundle, ReleaseBundle) + repository = GitWorkingTreeHistoryRepository(args.repository_root) + approval = None + resolver = ReleaseApprovalResolver(repository) + approval_path = getattr(args, "approval", None) + if approval_path: + approval = _load_model(approval_path, ApprovalRecord) + if resolver.has_conflict(bundle): + from semapact.exceptions import ValidationError + + raise ValidationError( + "Persisted release approval history contains conflicting exact review evidence" + ) + elif bundle.decision.decision is DecisionResult.REVIEW: + approval = resolver.resolve(bundle) + + record = ReleaseFinalizer().finalize( + bundle, + approval=approval, + ) + repository.put_contract_release(record) + output_path = dump_yaml(bundle.release_snapshot.to_contract(), args.output_contract) + release_out = getattr(args, "release_out", None) + if release_out: + _write_model_artifact(release_out, record) + return { + "contractReleaseId": record.contract_release_id, + "contractId": record.contract_id, + "contractVersion": record.contract_version, + "sourceRevisionRef": record.source_revision_ref, + "releaseBundleDigest": bundle.bundle_digest, + "outputContract": str(output_path), + "releaseArtifact": release_out, + } + + +def _load_model(path: str, model_type): + from pydantic import ValidationError as PydanticValidationError + + from semapact.exceptions import ValidationError + + try: + return model_type.model_validate_json(Path(path).read_text(encoding="utf-8")) + except (OSError, PydanticValidationError) as exc: + raise ValidationError( + f"Invalid {model_type.__name__} artifact '{path}': {exc}" + ) from exc + + +def _write_model_artifact(path: str, model) -> None: + artifact_path = Path(path) + artifact_path.parent.mkdir(parents=True, exist_ok=True) + artifact_path.write_text( + json.dumps( + model.model_dump(mode="json"), + indent=2, + ensure_ascii=False, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + def run_release_classify(args: argparse.Namespace) -> dict[str, Any]: """Analyze one change without creating canonical release artifacts.""" from dataclasses import asdict @@ -26,7 +141,6 @@ def run_release_classify(args: argparse.Namespace) -> dict[str, Any]: decision = GovernanceService().evaluate( base_contract, candidate_contract, - effective_date=args.effective_date, ) evaluate_governance_gate(decision, GovernanceOperation.ANALYZE) @@ -64,7 +178,6 @@ def run_release_plan(args: argparse.Namespace) -> dict[str, Any]: result = ReleasePlanningService().plan( base_contract, candidate_contract, - effective_date=args.effective_date, base_revision_ref=args.base_revision_ref, candidate_revision_ref=args.candidate_revision_ref, authority_reference=args.authority_reference, @@ -77,136 +190,16 @@ def run_release_plan(args: argparse.Namespace) -> dict[str, Any]: } -def run_release_prepare(args: argparse.Namespace) -> dict[str, Any]: - """Compatibility release-tag helper retained for existing Git workflows.""" - from dataclasses import asdict - - from semapact.core.loader import ContractLoader - from semapact.core.release import apply_release_candidate - from semapact.governance import GovernanceOperation, enforce_governance_gate - from semapact.utils.schema_utils import contract_to_dict - from semapact.utils.yaml_utils import dump_yaml - - loader = ContractLoader(runtime_context=args.runtime_context) - base_contract = loader.load(args.base) - candidate_contract = loader.load(args.candidate) - - decision = GovernanceService().evaluate( - base_contract, - candidate_contract, - effective_date=args.effective_date, - ) - enforce_governance_gate(decision, GovernanceOperation.PROPOSE) - - result = apply_release_candidate( - base_contract, - candidate_contract, - args.release_tag, - required_bump=decision.required_version_bump, - ) - output_path = dump_yaml(contract_to_dict(result.contract), args.output) - return { - "contractId": str(result.contract.id or ""), - "currentVersion": result.current_version, - "targetVersion": result.target_version, - "requiredBump": result.required_bump, - "actualBump": result.actual_bump, - "releaseTag": result.release_tag, - "reasons": [r.message for r in decision.reasons], - "breakingChanges": [asdict(change) for change in decision.policy.breaking_changes], - "output": str(output_path), - "governanceDecision": decision.model_dump(mode="json"), - } - def run_release_classify_repo(args: argparse.Namespace) -> dict[str, Any]: - from semapact.devops.release_workflow import ( + from semapact.application.services.repository_classification import ( classify_contracts_in_repo, repository_change_to_dict, ) - change_context = GovernanceService.create_context(args.effective_date) results = classify_contracts_in_repo( base_root=args.base_root, candidate_root=args.candidate_root, - context=change_context, ) return {"contracts": [repository_change_to_dict(item) for item in results]} - -def run_release_build_manifest(args: argparse.Namespace) -> dict[str, Any]: - from semapact.devops.release_workflow import ( - batch_manifest_build_to_dict, - batch_task_to_dict, - build_batch_release_manifest, - ) - - change_context = GovernanceService.create_context(args.effective_date) - build = build_batch_release_manifest( - base_root=args.base_root, - candidate_root=args.candidate_root, - context=change_context, - target_branch=args.target_branch, - source_branch_prefix=args.source_branch_prefix, - ) - output_path = Path(args.output).expanduser().resolve() - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text( - json.dumps( - [batch_task_to_dict(item) for item in build.tasks], indent=2, sort_keys=True - ), - encoding="utf-8", - ) - payload = batch_manifest_build_to_dict(build) - payload["output"] = str(output_path) - return payload - - -def run_release_create_pr(args: argparse.Namespace) -> dict[str, Any]: - """Compatibility Git publication workflow retained until a publisher is selected.""" - from semapact.core.loader import ContractLoader - from semapact.devops.release_workflow import create_release_pull_request - - loader = ContractLoader(runtime_context=args.runtime_context) - base_contract = loader.load(args.base) - candidate_contract = loader.load(args.candidate) - change_context = GovernanceService.create_context(args.effective_date) - - config = _build_git_config(args) - payload = create_release_pull_request( - config=config, - repo_path=_get_repo_path(args), - contract_repo_path=args.contract_path, - base_contract=base_contract, - candidate_contract=candidate_contract, - release_tag=args.release_tag, - source_branch=args.source_branch, - target_branch=args.target_branch, - context=change_context, - title=args.title, - description=args.description, - commit_message=args.commit_message, - push=args.push, - ) - return payload - - -def run_release_create_prs(args: argparse.Namespace) -> dict[str, Any]: - from semapact.devops.release_workflow import ( - batch_task_to_dict, - create_release_pull_requests_from_manifest, - load_batch_release_tasks, - ) - - config = _build_git_config(args) - tasks = load_batch_release_tasks(args.manifest) - payload = create_release_pull_requests_from_manifest( - config=config, - repo_path=_get_repo_path(args), - tasks=tasks, - push=args.push, - ) - return { - "tasks": [batch_task_to_dict(item) for item in tasks], - "results": payload, - } diff --git a/semapact/interfaces/parsing.py b/semapact/interfaces/parsing.py new file mode 100644 index 00000000..be4e36e7 --- /dev/null +++ b/semapact/interfaces/parsing.py @@ -0,0 +1,18 @@ +"""Shared interface parsing helpers with no domain ownership.""" + +from __future__ import annotations + +from datetime import datetime + + +def parse_iso_timestamp(value: str) -> datetime: + """Parse one timezone-aware ISO-8601 timestamp.""" + if not isinstance(value, str): + raise TypeError("timestamp must be an ISO-8601 string") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise ValueError("timestamp must be an ISO-8601 timestamp") from exc + if parsed.tzinfo is None or parsed.utcoffset() is None: + raise ValueError("timestamp must include an explicit timezone offset") + return parsed diff --git a/semapact/lifecycle/__init__.py b/semapact/lifecycle/__init__.py index 7045adb5..b75523ab 100644 --- a/semapact/lifecycle/__init__.py +++ b/semapact/lifecycle/__init__.py @@ -1,7 +1,4 @@ -from semapact.lifecycle.helpers import ( - allows_breaking_changes, - schema_items, -) +from semapact.lifecycle.helpers import schema_items from semapact.lifecycle.status import ( LIFECYCLE_STATUS_PROPERTY, LifecycleStatus, @@ -61,7 +58,6 @@ "is_retired_contract", "participates_in_breaking_checks", "is_explicitly_deprecated", - "allows_breaking_changes", "schema_items", "SchemaIdentity", diff --git a/semapact/lifecycle/helpers.py b/semapact/lifecycle/helpers.py index 73b03bcc..00e47b10 100644 --- a/semapact/lifecycle/helpers.py +++ b/semapact/lifecycle/helpers.py @@ -23,7 +23,6 @@ __all__ = [ "LIFECYCLE_STATUS_PROPERTY", "LifecycleStatus", - "allows_breaking_changes", "decimal_precision_reduction", "decimal_scale_reduction", "is_active_contract", @@ -45,34 +44,6 @@ def schema_items(contract: OpenDataContractStandard) -> list[Any]: return list(contract.schema_ or []) -def allows_breaking_changes(entity: Any) -> bool: - """Deprecated compatibility wrapper: returns True if entity participates in breaking checks.""" - import warnings - - warnings.warn( - "allows_breaking_changes is deprecated; use participates_in_breaking_checks(resolve_*_lifecycle(...)) instead.", - DeprecationWarning, - stacklevel=2, - ) - if isinstance(entity, OpenDataContractStandard): - return participates_in_breaking_checks(resolve_contract_lifecycle(entity)) - declared = resolve_declared_entity_lifecycle(entity) - if declared is not None: - return participates_in_breaking_checks(declared) - declared_val = getattr(entity, "lifecycleStatus", None) - if declared_val is not None: - try: - return participates_in_breaking_checks(normalize_status(declared_val)) - except ValueError: - return False - return True - - - -# Keep private alias for backwards compatibility -_lifecycle_from_custom_properties = lifecycle_from_custom_properties - - # --------------------------------------------------------------------------- diff --git a/semapact/lifecycle/policy.py b/semapact/lifecycle/policy.py index bde5dcf0..16c15b17 100644 --- a/semapact/lifecycle/policy.py +++ b/semapact/lifecycle/policy.py @@ -102,7 +102,7 @@ def evaluate_merge_policy( brk = BreakingChange( code=GovernanceReasonCode.CONTRACT_VERSION_MANUALLY_CHANGED, path="version", - message="Contract version mismatch. Contract versions are release-managed and cannot be manually updated during normal import/merge. Please revert the version change and use 'semapact release prepare'.", + message="Contract version mismatch. Contract versions are release-managed and cannot be manually updated during normal import/merge. Please revert the version change; SemaPact sets the release version during 'semapact release finalize'.", ) change_breaks.append(brk) change_reasons.append(brk.code) diff --git a/semapact/lifecycle/status.py b/semapact/lifecycle/status.py index 8d92f4dc..cfc9a0e1 100644 --- a/semapact/lifecycle/status.py +++ b/semapact/lifecycle/status.py @@ -37,10 +37,8 @@ class LifecycleStatus(StrEnum): def normalize_status(value: Any) -> LifecycleStatus: """Normalize status string to canonical LifecycleStatus enum. - Supported values & aliases: - - 'draft', 'proposed' -> LifecycleStatus.DRAFT - (Note: 'proposed' is a read-only governance interpretation alias; - SemaPact does not rewrite ODCS YAML status during resolution) + Supported values: + - 'draft' -> LifecycleStatus.DRAFT - 'active' -> LifecycleStatus.ACTIVE - 'deprecated' -> LifecycleStatus.DEPRECATED - 'retired' -> LifecycleStatus.RETIRED @@ -58,7 +56,7 @@ def normalize_status(value: Any) -> LifecycleStatus: if not text: raise ValueError("Lifecycle status value cannot be empty") - if text in ("draft", "proposed"): + if text == "draft": return LifecycleStatus.DRAFT if text == "active": return LifecycleStatus.ACTIVE @@ -109,10 +107,9 @@ def resolve_contract_lifecycle( ) -> LifecycleStatus: """Resolve authoritative contract root lifecycle status. - Fallback order: + Resolution: 1. contract.status (ODCS native canonical root status) - 2. contract.customProperties.lifecycleStatus (legacy fallback) - 3. LifecycleStatus.DRAFT (canonical default) + 2. LifecycleStatus.DRAFT when root status is absent or invalid Note: Lifecycle resolvers are intentionally total for deterministic analysis; invalid statuses fall back to DRAFT while lifecycle validity is enforced by ContractValidator. @@ -121,7 +118,7 @@ def resolve_contract_lifecycle( return LifecycleStatus.DRAFT - # 1. Native root status + # Native root status raw_status = contract.status if raw_status is not None and str(raw_status).strip() != "": try: @@ -129,12 +126,6 @@ def resolve_contract_lifecycle( except ValueError: return LifecycleStatus.DRAFT - # 2. Legacy customProperties fallback - declared = lifecycle_from_custom_properties(contract.customProperties) - if declared is not None: - return declared - - # 3. Canonical default return LifecycleStatus.DRAFT diff --git a/semapact/orchestrator/pipeline.py b/semapact/orchestrator/pipeline.py index 8e6b1866..aa97f80e 100644 --- a/semapact/orchestrator/pipeline.py +++ b/semapact/orchestrator/pipeline.py @@ -25,8 +25,8 @@ from datacontract.data_contract import DataContract +from semapact.change_context import ChangeContext from semapact.governance import ( - ChangeContext, GovernanceDecision, GovernanceOperation, enforce_governance_gate, @@ -293,12 +293,10 @@ def run( if merged_contract is None: raise ValueError("Merge did not produce a contract") - # Single-pass governance decision evaluation using the same explicit - # context as merge. + # Governance evaluation is independent of the mutation effective date. decision = evaluate_governance_decision( business_contract, merged_contract, - context=change_context, merge_conflicts=conflicts, ) diff --git a/semapact/platforms/databricks/deployment.py b/semapact/platforms/databricks/deployment.py index 10da032d..fd013dbb 100644 --- a/semapact/platforms/databricks/deployment.py +++ b/semapact/platforms/databricks/deployment.py @@ -7,7 +7,12 @@ from pydantic import field_validator -from semapact.deployment.models import NativeOperation +from semapact.deployment import RuntimeReleaseMetadata +from semapact.deployment.models import ( + DeploymentPlan, + NativeOperation, + NativeOperationKind, +) from semapact.deployment.orchestrator import DeploymentOrchestrator from semapact.deployment.providers import ( DeploymentExecutionConfig, @@ -18,10 +23,11 @@ from semapact.platforms.databricks.transition_compiler import ( DatabricksTransitionCompiler, ) +from semapact.platforms.databricks.target import parse_databricks_runtime_target from semapact.platforms.databricks.transition_planner import ( DatabricksSchemaTransitionPlanner, ) -from semapact.schema import SqlSchemaMapper +from semapact.schema import SqlSchemaMapper, validate_simple_sql_identifier _TERMINAL_STATES = {"SUCCEEDED", "FAILED", "CANCELED", "CLOSED"} @@ -146,6 +152,54 @@ def __init__( ) + def project_release_metadata( + self, + plan: DeploymentPlan, + metadata: RuntimeReleaseMetadata, + ) -> None: + """Project SemaPact release provenance into Unity Catalog table tags.""" + if metadata.contract_id != plan.contract_id: + raise ValidationError( + "Release metadata contract does not match DeploymentPlan" + ) + if metadata.contract_version != plan.contract_version: + raise ValidationError( + "Release metadata version does not match DeploymentPlan" + ) + + catalog, schema_name = parse_databricks_runtime_target( + plan.target.runtime_target + ) + validate_simple_sql_identifier(catalog, "catalog") + validate_simple_sql_identifier(schema_name, "schema") + + tags = { + "semapact_contract_id": metadata.contract_id, + "semapact_contract_version": metadata.contract_version, + "semapact_release_id": metadata.contract_release_id, + "semapact_source_revision": metadata.source_revision_ref, + } + rendered_tags = ", ".join( + f"{_sql_string(key)} = {_sql_string(value)}" + for key, value in sorted(tags.items()) + ) + for action in plan.actions: + validate_simple_sql_identifier(action.physical_name, "asset") + qualified = ".".join( + f"`{part}`" + for part in (catalog, schema_name, action.physical_name) + ) + self._executor.execute( + NativeOperation( + kind=NativeOperationKind.ALTER, + governed_asset=action.governed_asset, + statement=( + f"ALTER TABLE {qualified} SET TAGS ({rendered_tags})" + ), + ) + ) + + def _statement_state(response: Any) -> str: status = getattr(response, "status", None) state = getattr(status, "state", None) @@ -155,3 +209,7 @@ def _statement_state(response: Any) -> str: ) value = getattr(state, "value", state) return str(value).upper() + + +def _sql_string(value: str) -> str: + return "'" + value.replace("'", "''") + "'" diff --git a/semapact/platforms/delta/__init__.py b/semapact/platforms/delta/__init__.py new file mode 100644 index 00000000..359e2761 --- /dev/null +++ b/semapact/platforms/delta/__init__.py @@ -0,0 +1,5 @@ +"""Delta operational history adapter.""" + +from semapact.platforms.delta.operational_history import DeltaOperationalHistorySink + +__all__ = ["DeltaOperationalHistorySink"] diff --git a/semapact/platforms/delta/operational_history.py b/semapact/platforms/delta/operational_history.py new file mode 100644 index 00000000..b8c5c2e4 --- /dev/null +++ b/semapact/platforms/delta/operational_history.py @@ -0,0 +1,48 @@ +"""Delta Lake persistence for optional operational deployment history.""" + +from __future__ import annotations + +from semapact.history.operational import OperationalDeploymentEvent +from semapact.utils.deterministic import canonical_compact_json + + +class DeltaOperationalHistorySink: + """Append deployment telemetry to a Delta table. + + Consumers should use event_id as the idempotency/deduplication key when pipeline + retry semantics can append the same immutable event more than once. + """ + + def __init__(self, table_uri: str) -> None: + cleaned = table_uri.strip() + if not cleaned: + raise ValueError("Delta operational history table URI is required") + self._table_uri = cleaned + + def record_deployment(self, event: OperationalDeploymentEvent) -> None: + try: + import pyarrow as pa + from deltalake import write_deltalake + except ImportError as exc: + raise RuntimeError( + 'Delta operational history requires: pip install "semapact[delta]"' + ) from exc + + payload = canonical_compact_json(event.model_dump(mode="json")) + table = pa.table( + { + "event_id": [event.event_id], + "completed_at": [event.completed_at.isoformat()], + "contract_id": [event.contract_id], + "deployment_plan_id": [event.deployment_plan_id], + "platform": [event.platform], + "runtime_target": [event.runtime_target], + "status": [event.status], + "payload_json": [payload], + } + ) + write_deltalake( + self._table_uri, + table, + mode="append", + ) diff --git a/semapact/platforms/git/history_repository.py b/semapact/platforms/git/history_repository.py index dfcdaf40..b7558758 100644 --- a/semapact/platforms/git/history_repository.py +++ b/semapact/platforms/git/history_repository.py @@ -7,6 +7,7 @@ from __future__ import annotations import hashlib +import json import os import re import secrets @@ -19,36 +20,21 @@ from semapact.approval.integrity import validate_approval_record_identity from semapact.approval.models import ApprovalRecord -from semapact.contractops import ChangeSet, ReleasePlan +from semapact.contractops import ChangeSet, ContractRelease, ReleasePlan from semapact.contractops.integrity import ( validate_change_set_identity, + validate_contract_release_identity, validate_release_plan_identity, ) -from semapact.deployment import DeploymentAuthorization, DeploymentPlan, DeploymentPreview -from semapact.deployment.models import ( - validate_deployment_authorization_identity, - validate_deployment_plan_identity, - validate_deployment_preview_identity, -) from semapact.governance import GovernanceDecision from semapact.governance.gate import GovernanceOperation from semapact.history import ( ChangeSetDecisionLink, - DeploymentRecord, HistoryConflictError, HistoryCorruptionError, HistoryIntegrityIssueCode, HistoryNotFoundError, HistoryStorageIntegrityIssue, - ReleaseRecord, - RuntimeObservationRecord, - RuntimeReconciliationRecord, -) -from semapact.history.integrity import ( - validate_deployment_record_identity, - validate_release_record_identity, - validate_runtime_observation_record_identity, - validate_runtime_reconciliation_record_identity, ) from semapact.revision.integrity import ( validate_contract_revision_identity, @@ -113,47 +99,11 @@ class _HistoryKindSpec(Generic[T]): "release_plan_id", validate_release_plan_identity, ) -_RELEASE_RECORDS = _HistoryKindSpec( - "release_records", - ReleaseRecord, - "release_record_id", - validate_release_record_identity, -) -_DEPLOYMENT_PLANS = _HistoryKindSpec( - "deployment_plans", - DeploymentPlan, - "deployment_plan_id", - validate_deployment_plan_identity, -) -_DEPLOYMENT_PREVIEWS = _HistoryKindSpec( - "deployment_previews", - DeploymentPreview, - "deployment_preview_id", - validate_deployment_preview_identity, -) -_DEPLOYMENT_AUTHORIZATIONS = _HistoryKindSpec( - "deployment_authorizations", - DeploymentAuthorization, - "deployment_authorization_id", - validate_deployment_authorization_identity, -) -_DEPLOYMENT_RECORDS = _HistoryKindSpec( - "deployment_records", - DeploymentRecord, - "deployment_record_id", - validate_deployment_record_identity, -) -_RUNTIME_OBSERVATIONS = _HistoryKindSpec( - "runtime_observations", - RuntimeObservationRecord, - "observation_record_id", - validate_runtime_observation_record_identity, -) -_RUNTIME_RECONCILIATIONS = _HistoryKindSpec( - "runtime_reconciliations", - RuntimeReconciliationRecord, - "runtime_reconciliation_record_id", - validate_runtime_reconciliation_record_identity, +_CONTRACT_RELEASES = _HistoryKindSpec( + "contract_releases", + ContractRelease, + "contract_release_id", + validate_contract_release_identity, ) _HISTORY_KIND_SPECS: dict[str, _HistoryKindSpec[BaseModel]] = { @@ -166,13 +116,7 @@ class _HistoryKindSpec(Generic[T]): _CONTRACT_REVISIONS, _CONTRACT_REVISION_SOURCES, _RELEASE_PLANS, - _RELEASE_RECORDS, - _DEPLOYMENT_PLANS, - _DEPLOYMENT_PREVIEWS, - _DEPLOYMENT_AUTHORIZATIONS, - _DEPLOYMENT_RECORDS, - _RUNTIME_OBSERVATIONS, - _RUNTIME_RECONCILIATIONS, + _CONTRACT_RELEASES, ) } @@ -330,173 +274,65 @@ def put_release_plan(self, release_plan: ReleasePlan) -> None: def get_release_plan(self, release_plan_id: str) -> ReleasePlan: return self._get(_RELEASE_PLANS, release_plan_id) - def put_release_record(self, record: ReleaseRecord) -> None: - if not isinstance(record, ReleaseRecord): + def put_contract_release(self, record: ContractRelease) -> None: + if not isinstance(record, ContractRelease): raise TypeError( - f"record must be ReleaseRecord, got {type(record).__name__}" + "record must be ContractRelease, " + f"got {type(record).__name__}" ) - validate_release_record_identity(record) - for existing in self.list_release_records(record.contract_id): + validate_contract_release_identity(record) + for existing in self.list_contract_releases(record.contract_id): if ( existing.contract_version == record.contract_version - and existing.release_record_id != record.release_record_id + and existing.contract_release_id != record.contract_release_id ): raise HistoryConflictError( - "A different ReleaseRecord already exists for " + "A different ContractRelease already exists for " f"{record.contract_id!r} version {record.contract_version!r}" ) - self._put(_RELEASE_RECORDS, record.release_record_id, record) + self._put(_CONTRACT_RELEASES, record.contract_release_id, record) - def get_release_record(self, release_record_id: str) -> ReleaseRecord: - return self._get(_RELEASE_RECORDS, release_record_id) + def get_contract_release( + self, + contract_release_id: str, + ) -> ContractRelease: + return self._get(_CONTRACT_RELEASES, contract_release_id) - def list_release_records(self, contract_id: str) -> tuple[ReleaseRecord, ...]: + def list_contract_releases( + self, + contract_id: str, + ) -> tuple[ContractRelease, ...]: contract_id = _required_text(contract_id, "contract_id") return tuple( record - for record in self._list(_RELEASE_RECORDS) + for record in self._list(_CONTRACT_RELEASES) if record.contract_id == contract_id ) - def get_release_record_by_version( + def get_contract_release_by_version( self, contract_id: str, contract_version: str, - ) -> ReleaseRecord: + ) -> ContractRelease: contract_id = _required_text(contract_id, "contract_id") contract_version = _required_text(contract_version, "contract_version") matches = tuple( record - for record in self.list_release_records(contract_id) + for record in self.list_contract_releases(contract_id) if record.contract_version == contract_version ) if not matches: raise HistoryNotFoundError( - f"ReleaseRecord for {contract_id!r} version {contract_version!r} was not found" + f"ContractRelease for {contract_id!r} " + f"version {contract_version!r} was not found" ) if len(matches) != 1: raise HistoryCorruptionError( - f"Multiple ReleaseRecords exist for {contract_id!r} version {contract_version!r}" + f"Multiple ContractReleases exist for {contract_id!r} " + f"version {contract_version!r}" ) return matches[0] - def put_deployment_plan(self, plan: DeploymentPlan) -> None: - self._put(_DEPLOYMENT_PLANS, plan.deployment_plan_id, plan) - - def get_deployment_plan(self, deployment_plan_id: str) -> DeploymentPlan: - return self._get(_DEPLOYMENT_PLANS, deployment_plan_id) - - def put_deployment_preview(self, preview: DeploymentPreview) -> None: - self._put(_DEPLOYMENT_PREVIEWS, preview.deployment_preview_id, preview) - - def get_deployment_preview(self, deployment_preview_id: str) -> DeploymentPreview: - return self._get(_DEPLOYMENT_PREVIEWS, deployment_preview_id) - - def put_deployment_authorization( - self, - authorization: DeploymentAuthorization, - ) -> None: - self._put( - _DEPLOYMENT_AUTHORIZATIONS, - authorization.deployment_authorization_id, - authorization, - ) - - def get_deployment_authorization( - self, - deployment_authorization_id: str, - ) -> DeploymentAuthorization: - return self._get(_DEPLOYMENT_AUTHORIZATIONS, deployment_authorization_id) - - def put_deployment_record(self, record: DeploymentRecord) -> None: - self._put(_DEPLOYMENT_RECORDS, record.deployment_record_id, record) - - def get_deployment_record(self, deployment_record_id: str) -> DeploymentRecord: - return self._get(_DEPLOYMENT_RECORDS, deployment_record_id) - - def list_deployment_records_for_release( - self, - release_record_id: str, - ) -> tuple[DeploymentRecord, ...]: - release_record_id = _required_text(release_record_id, "release_record_id") - return tuple( - record - for record in self._list(_DEPLOYMENT_RECORDS) - if record.release_record_id == release_record_id - ) - - def list_deployment_records_for_plan( - self, - deployment_plan_id: str, - ) -> tuple[DeploymentRecord, ...]: - deployment_plan_id = _required_text(deployment_plan_id, "deployment_plan_id") - return tuple( - record - for record in self._list(_DEPLOYMENT_RECORDS) - if record.deployment_plan_id == deployment_plan_id - ) - - def put_runtime_observation_record(self, record: RuntimeObservationRecord) -> None: - self._put(_RUNTIME_OBSERVATIONS, record.observation_record_id, record) - - def get_runtime_observation_record( - self, - observation_record_id: str, - ) -> RuntimeObservationRecord: - return self._get(_RUNTIME_OBSERVATIONS, observation_record_id) - - def list_runtime_observation_records( - self, - source_identifier: str, - ) -> tuple[RuntimeObservationRecord, ...]: - source_identifier = _required_text(source_identifier, "source_identifier") - return tuple( - record - for record in self._list(_RUNTIME_OBSERVATIONS) - if record.observation.source_identifier == source_identifier - ) - - def put_runtime_reconciliation_record( - self, - record: RuntimeReconciliationRecord, - ) -> None: - self._put( - _RUNTIME_RECONCILIATIONS, - record.runtime_reconciliation_record_id, - record, - ) - - def get_runtime_reconciliation_record( - self, - runtime_reconciliation_record_id: str, - ) -> RuntimeReconciliationRecord: - return self._get( - _RUNTIME_RECONCILIATIONS, - runtime_reconciliation_record_id, - ) - - def list_runtime_reconciliation_records( - self, - contract_id: str, - ) -> tuple[RuntimeReconciliationRecord, ...]: - contract_id = _required_text(contract_id, "contract_id") - return tuple( - record - for record in self._list(_RUNTIME_RECONCILIATIONS) - if record.result.contract_id == contract_id - ) - - def list_runtime_reconciliation_records_for_source( - self, - source_identifier: str, - ) -> tuple[RuntimeReconciliationRecord, ...]: - source_identifier = _required_text(source_identifier, "source_identifier") - return tuple( - record - for record in self._list(_RUNTIME_RECONCILIATIONS) - if record.result.observation_source_identifier == source_identifier - ) - def inspect_history_integrity(self) -> tuple[HistoryStorageIntegrityIssue, ...]: """Inspect physical history artifacts and checksum evidence without mutation.""" root_issue = self._inspect_history_root() @@ -576,7 +412,7 @@ def _inspect_history_artifact( ) try: - artifact = spec.model_type.model_validate_json(raw) + artifact = _parse_artifact_json(spec, raw) if spec.integrity_validator is not None: spec.integrity_validator(artifact) except (PydanticValidationError, ValueError, TypeError) as exc: @@ -762,7 +598,7 @@ def _require_idempotent_existing( expected_id: str, ) -> None: existing = self._read_validated(spec, path, expected_id=expected_id) - existing_canonical = _canonical_model_json(existing) + existing_canonical = _canonical_artifact_json(spec, existing) if existing_canonical != canonical: raise HistoryConflictError( f"{spec.model_type.__name__} {expected_id!r} already exists with different content" @@ -779,7 +615,7 @@ def _read_validated( try: raw = path.read_text(encoding="utf-8") self._verify_checksum_if_present(path, raw) - artifact = spec.model_type.model_validate_json(raw) + artifact = _parse_artifact_json(spec, raw) if spec.integrity_validator is not None: spec.integrity_validator(artifact) except HistoryCorruptionError: @@ -808,8 +644,8 @@ def _validated_canonical_json( f"artifact must be {spec.model_type.__name__}, got {type(artifact).__name__}" ) try: - canonical = _canonical_model_json(artifact) - validated = spec.model_type.model_validate_json(canonical) + canonical = _canonical_artifact_json(spec, artifact) + validated = _parse_artifact_json(spec, canonical) if spec.integrity_validator is not None: spec.integrity_validator(validated) except (PydanticValidationError, ValueError, TypeError) as exc: @@ -1056,6 +892,23 @@ def _path_name(path: Path) -> str: return path.name +def _canonical_artifact_json( + spec: _HistoryKindSpec[T], + artifact: T, +) -> str: + """Serialize one canonical persisted history artifact.""" + del spec + return _canonical_model_json(artifact) + + +def _parse_artifact_json( + spec: _HistoryKindSpec[T], + raw: str, +) -> T: + """Rehydrate one canonical persisted history artifact.""" + return spec.model_type.model_validate_json(raw) + + def _canonical_model_json(artifact: BaseModel) -> str: """Serialize persisted models with aliases so nested ODCS models round-trip.""" return canonical_compact_json(artifact.model_dump(mode="json", by_alias=True)) diff --git a/semapact/platforms/sqlite/__init__.py b/semapact/platforms/sqlite/__init__.py new file mode 100644 index 00000000..3bd40c81 --- /dev/null +++ b/semapact/platforms/sqlite/__init__.py @@ -0,0 +1,5 @@ +"""SQLite operational history adapter.""" + +from semapact.platforms.sqlite.operational_history import SQLiteOperationalHistorySink + +__all__ = ["SQLiteOperationalHistorySink"] diff --git a/semapact/platforms/sqlite/operational_history.py b/semapact/platforms/sqlite/operational_history.py new file mode 100644 index 00000000..30b16328 --- /dev/null +++ b/semapact/platforms/sqlite/operational_history.py @@ -0,0 +1,70 @@ +"""SQLite persistence for optional operational deployment history.""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +from semapact.history.operational import OperationalDeploymentEvent +from semapact.utils.deterministic import canonical_compact_json + + +class SQLiteOperationalHistorySink: + """Persist high-frequency deployment events without touching Git history.""" + + def __init__(self, path: str | Path) -> None: + self._path = Path(path).expanduser().resolve(strict=False) + + def record_deployment(self, event: OperationalDeploymentEvent) -> None: + self._path.parent.mkdir(parents=True, exist_ok=True) + payload = canonical_compact_json(event.model_dump(mode="json")) + with sqlite3.connect(self._path) as connection: + connection.execute( + """ + CREATE TABLE IF NOT EXISTS deployment_events ( + event_id TEXT PRIMARY KEY, + completed_at TEXT NOT NULL, + contract_id TEXT NOT NULL, + deployment_plan_id TEXT NOT NULL, + platform TEXT NOT NULL, + runtime_target TEXT NOT NULL, + status TEXT NOT NULL, + payload_json TEXT NOT NULL + ) + """ + ) + existing = connection.execute( + "SELECT payload_json FROM deployment_events WHERE event_id = ?", + (event.event_id,), + ).fetchone() + if existing is not None: + if existing[0] != payload: + raise RuntimeError( + "Operational deployment event identity already exists " + "with different content" + ) + return + connection.execute( + """ + INSERT INTO deployment_events ( + event_id, + completed_at, + contract_id, + deployment_plan_id, + platform, + runtime_target, + status, + payload_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + event.event_id, + event.completed_at.isoformat(), + event.contract_id, + event.deployment_plan_id, + event.platform, + event.runtime_target, + event.status, + payload, + ), + ) diff --git a/semapact/services/__init__.py b/semapact/services/__init__.py deleted file mode 100644 index a9db5846..00000000 --- a/semapact/services/__init__.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Backward-compatible imports for the pre-application package layout. - -New code must import application services and application result models from -``semapact.application``. This package owns no business logic or data models. -""" - -from semapact.application.models import ( - GovernanceAnalysis, - GovernanceProposal, - ReleasePlanningResult, - RuntimeReconciliation, -) -from semapact.application.services.deployment import DeploymentService -from semapact.application.services.governance import GovernanceService -from semapact.application.services.reconciliation import ReconciliationService -from semapact.application.services.release_planning import ReleasePlanningService -from semapact.application.services.version_authority import VersionAuthorityService - -__all__ = [ - "DeploymentService", - "GovernanceAnalysis", - "GovernanceProposal", - "GovernanceService", - "ReconciliationService", - "ReleasePlanningResult", - "ReleasePlanningService", - "RuntimeReconciliation", - "VersionAuthorityService", -] diff --git a/semapact/services/deployment_service.py b/semapact/services/deployment_service.py deleted file mode 100644 index 64806ad9..00000000 --- a/semapact/services/deployment_service.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Compatibility import for the former deployment service module.""" - -from semapact.application.services.deployment import DeploymentService - -__all__ = ["DeploymentService"] diff --git a/semapact/services/governance_service.py b/semapact/services/governance_service.py deleted file mode 100644 index 3b1ada83..00000000 --- a/semapact/services/governance_service.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Compatibility imports for the former governance service module.""" - -from semapact.application.models.governance import GovernanceAnalysis, GovernanceProposal -from semapact.application.services.governance import GovernanceService - -__all__ = ["GovernanceAnalysis", "GovernanceProposal", "GovernanceService"] diff --git a/semapact/services/reconciliation_service.py b/semapact/services/reconciliation_service.py deleted file mode 100644 index 1f52bc0a..00000000 --- a/semapact/services/reconciliation_service.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Compatibility imports for the former reconciliation service module.""" - -from semapact.application.models.reconciliation import RuntimeReconciliation -from semapact.application.services.reconciliation import ReconciliationService - -__all__ = ["ReconciliationService", "RuntimeReconciliation"] diff --git a/semapact/services/release_models.py b/semapact/services/release_models.py deleted file mode 100644 index 39b1a9f0..00000000 --- a/semapact/services/release_models.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Compatibility import for the former release application-model module.""" - -from semapact.application.models.release import ReleasePlanningResult - -__all__ = ["ReleasePlanningResult"] diff --git a/semapact/services/release_planning_service.py b/semapact/services/release_planning_service.py deleted file mode 100644 index e78cc0c7..00000000 --- a/semapact/services/release_planning_service.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Compatibility import for the former release-planning service module.""" - -from semapact.application.services.release_planning import ReleasePlanningService - -__all__ = ["ReleasePlanningService"] diff --git a/semapact/services/version_authority_service.py b/semapact/services/version_authority_service.py deleted file mode 100644 index 825a2671..00000000 --- a/semapact/services/version_authority_service.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Compatibility import for the former version-authority service module.""" - -from semapact.application.services.version_authority import VersionAuthorityService - -__all__ = ["VersionAuthorityService"] diff --git a/tests/fixtures/governance_decisions/allow_clean_decision.json b/tests/fixtures/governance_decisions/allow_clean_decision.json deleted file mode 100644 index 9145b0b6..00000000 --- a/tests/fixtures/governance_decisions/allow_clean_decision.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "breaking": false, - "changes": [], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "my-data-contract", - "decision": "ALLOW", - "decisionId": "0480ed51-8b7f-57aa-93f2-b09c135004c1", - "evidence": { - "hasChanges": false, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": false, - "retiredViolation": false, - "valid": true, - "versionViolation": false, - "violations": [] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "No contract changes detected", - "path": null, - "severity": "INFO" - } - ], - "requiredVersionBump": "none", - "schemaVersion": "1", - "validation": { - "issues": [], - "valid": true - } -} diff --git a/tests/fixtures/governance_decisions/block_retired_decision.json b/tests/fixtures/governance_decisions/block_retired_decision.json deleted file mode 100644 index 4f2b4cb6..00000000 --- a/tests/fixtures/governance_decisions/block_retired_decision.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "breaking": false, - "changes": [ - { - "after": "Retired update", - "before": null, - "breaking": false, - "changeType": "MODIFY", - "domain": "METADATA", - "entityType": "PROPERTY", - "evidence": [], - "field": "description", - "identity": [ - "orders", - "amount" - ], - "path": "schema[orders].properties[amount].description", - "reasonCodes": [] - } - ], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "my-data-contract", - "decision": "BLOCK", - "decisionId": "e8296105-5b21-5c7b-b5f1-99004a632224", - "evidence": { - "hasChanges": true, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": false, - "retiredViolation": true, - "valid": false, - "versionViolation": false, - "violations": [ - { - "code": "RETIRED_CONTRACT_MODIFIED", - "details": {}, - "message": "Cannot modify a retired contract.", - "path": "status", - "severity": "ERROR" - } - ] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT", - "RETIRED_CONTRACT_MODIFIED" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "Only descriptive metadata changed; no required version bump", - "path": null, - "severity": "INFO" - }, - { - "code": "RETIRED_CONTRACT_MODIFIED", - "details": {}, - "message": "Cannot modify a retired contract.", - "path": "status", - "severity": "ERROR" - } - ], - "requiredVersionBump": "none", - "schemaVersion": "1", - "validation": { - "issues": [], - "valid": true - } -} diff --git a/tests/fixtures/governance_decisions/block_validation_decision.json b/tests/fixtures/governance_decisions/block_validation_decision.json deleted file mode 100644 index 41f06a3a..00000000 --- a/tests/fixtures/governance_decisions/block_validation_decision.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "breaking": false, - "changes": [], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "my-data-contract", - "decision": "BLOCK", - "decisionId": "3de328dd-f6f1-5bff-b805-c3f1ff2325c4", - "evidence": { - "hasChanges": true, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": false, - "retiredViolation": false, - "valid": false, - "versionViolation": false, - "violations": [] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT", - "VALIDATION_FAILED" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "No contract changes detected", - "path": null, - "severity": "INFO" - }, - { - "code": "VALIDATION_FAILED", - "details": {}, - "message": "Property name cannot be empty or whitespace-only", - "path": "schema", - "severity": "ERROR" - } - ], - "requiredVersionBump": "none", - "schemaVersion": "1", - "validation": { - "issues": [ - { - "code": "VALIDATION_FAILED", - "details": {}, - "message": "Property name cannot be empty or whitespace-only", - "path": "schema", - "severity": "ERROR" - } - ], - "valid": false - } -} diff --git a/tests/fixtures/governance_decisions/breaking_review_decision.json b/tests/fixtures/governance_decisions/breaking_review_decision.json deleted file mode 100644 index bda1b129..00000000 --- a/tests/fixtures/governance_decisions/breaking_review_decision.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "breaking": true, - "changes": [ - { - "after": "decimal(8,2)", - "before": "decimal(10,2)", - "breaking": true, - "changeType": "MODIFY", - "domain": "STRUCTURE", - "entityType": "PROPERTY", - "evidence": [], - "field": "physicalType", - "identity": [ - "orders", - "amount" - ], - "path": "schema[orders].properties[amount].physicalType", - "reasonCodes": [ - "DECIMAL_PRECISION_REDUCED" - ] - } - ], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "my-data-contract", - "decision": "REVIEW", - "decisionId": "dc341a61-b19f-5c01-a250-0fda5ea7d061", - "evidence": { - "hasChanges": true, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": false, - "retiredViolation": false, - "valid": false, - "versionViolation": false, - "violations": [ - { - "code": "DECIMAL_PRECISION_REDUCED", - "details": {}, - "message": "Decimal precision reduced from 'decimal(10,2)' to 'decimal(8,2)'", - "path": "schema[orders].properties[amount].physicalType", - "severity": "WARNING" - } - ] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT", - "DECIMAL_PRECISION_REDUCED" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "Breaking lifecycle changes require a major version bump", - "path": null, - "severity": "INFO" - }, - { - "code": "DECIMAL_PRECISION_REDUCED", - "details": {}, - "message": "Decimal precision reduced from 'decimal(10,2)' to 'decimal(8,2)'", - "path": "schema[orders].properties[amount].physicalType", - "severity": "WARNING" - } - ], - "requiredVersionBump": "major", - "schemaVersion": "1", - "validation": { - "issues": [], - "valid": true - } -} diff --git a/tests/fixtures/governance_decisions/review_deprecate_decision.json b/tests/fixtures/governance_decisions/review_deprecate_decision.json deleted file mode 100644 index 6d040922..00000000 --- a/tests/fixtures/governance_decisions/review_deprecate_decision.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "breaking": false, - "changes": [ - { - "after": "deprecated", - "before": null, - "breaking": false, - "changeType": "DEPRECATE", - "domain": "LIFECYCLE", - "entityType": "PROPERTY", - "evidence": [], - "field": "lifecycleStatus", - "identity": [ - "orders", - "amount" - ], - "path": "schema[orders].properties[amount]", - "reasonCodes": [] - } - ], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "my-data-contract", - "decision": "REVIEW", - "decisionId": "43daa6d0-6925-5b00-ad33-18e871dadfa9", - "evidence": { - "hasChanges": true, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": false, - "retiredViolation": false, - "valid": true, - "versionViolation": false, - "violations": [] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "New schema/property deprecations require a minor version bump", - "path": null, - "severity": "INFO" - } - ], - "requiredVersionBump": "minor", - "schemaVersion": "1", - "validation": { - "issues": [], - "valid": true - } -} diff --git a/tests/fixtures/governance_scenarios/active_to_retired_transition/expected.json b/tests/fixtures/governance_scenarios/active_to_retired_transition/expected.json deleted file mode 100644 index d800c96c..00000000 --- a/tests/fixtures/governance_scenarios/active_to_retired_transition/expected.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "breaking": false, - "changes": [ - { - "after": "retired", - "before": "active", - "breaking": false, - "changeType": "DEPRECATE", - "domain": "LIFECYCLE", - "entityType": "CONTRACT", - "evidence": [], - "field": "status", - "identity": [ - "orders-contract" - ], - "path": "status", - "reasonCodes": [] - } - ], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "orders-contract", - "decision": "REVIEW", - "decisionId": "b18506a1-f428-510f-9c5b-848054f2b03f", - "evidence": { - "hasChanges": true, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": false, - "retiredViolation": false, - "valid": true, - "versionViolation": false, - "violations": [ - { - "code": "CONTRACT_RETIRED_TRANSITION", - "details": {}, - "message": "Contract transition to retired status requires governance review.", - "path": "status", - "severity": "WARNING" - } - ] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT", - "CONTRACT_RETIRED_TRANSITION" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "Only descriptive metadata changed; no required version bump", - "path": null, - "severity": "INFO" - }, - { - "code": "CONTRACT_RETIRED_TRANSITION", - "details": {}, - "message": "Contract transition to retired status requires governance review.", - "path": "status", - "severity": "WARNING" - } - ], - "requiredVersionBump": "none", - "schemaVersion": "1", - "validation": { - "issues": [], - "valid": true - } -} diff --git a/tests/fixtures/governance_scenarios/contract_id_change/expected.json b/tests/fixtures/governance_scenarios/contract_id_change/expected.json deleted file mode 100644 index 60ce02bc..00000000 --- a/tests/fixtures/governance_scenarios/contract_id_change/expected.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "breaking": true, - "changes": [ - { - "after": "renamed-orders-contract", - "before": "orders-contract", - "breaking": true, - "changeType": "MODIFY", - "domain": "IDENTITY", - "entityType": "CONTRACT", - "evidence": [], - "field": "id", - "identity": [ - "orders-contract" - ], - "path": "id", - "reasonCodes": [ - "CONTRACT_ID_CHANGED" - ] - } - ], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "orders-contract", - "decision": "BLOCK", - "decisionId": "750bd41f-1b63-5de6-a47c-47d9515c4d10", - "evidence": { - "hasChanges": true, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": true, - "retiredViolation": false, - "valid": false, - "versionViolation": false, - "violations": [ - { - "code": "CONTRACT_ID_CHANGED", - "details": {}, - "message": "Contract ID mismatch. You changed the root ID of the contract, which is immutable. If you want to create a new contract, use 'semapact import --new' or change the ID back.", - "path": "id", - "severity": "ERROR" - } - ] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT", - "CONTRACT_ID_CHANGED" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "Breaking lifecycle changes require a major version bump", - "path": null, - "severity": "INFO" - }, - { - "code": "CONTRACT_ID_CHANGED", - "details": {}, - "message": "Contract ID mismatch. You changed the root ID of the contract, which is immutable. If you want to create a new contract, use 'semapact import --new' or change the ID back.", - "path": "id", - "severity": "ERROR" - } - ], - "requiredVersionBump": "major", - "schemaVersion": "1", - "validation": { - "issues": [], - "valid": true - } -} diff --git a/tests/fixtures/governance_scenarios/decimal_precision_reduction/expected.json b/tests/fixtures/governance_scenarios/decimal_precision_reduction/expected.json deleted file mode 100644 index ab4d9b7b..00000000 --- a/tests/fixtures/governance_scenarios/decimal_precision_reduction/expected.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "breaking": true, - "changes": [ - { - "after": "decimal(8,2)", - "before": "decimal(10,2)", - "breaking": true, - "changeType": "MODIFY", - "domain": "STRUCTURE", - "entityType": "PROPERTY", - "evidence": [], - "field": "physicalType", - "identity": [ - "orders", - "amount" - ], - "path": "schema[orders].properties[amount].physicalType", - "reasonCodes": [ - "DECIMAL_PRECISION_REDUCED" - ] - } - ], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "orders-contract", - "decision": "REVIEW", - "decisionId": "2300dfb4-71cc-5601-8b7c-eeb1cdad2306", - "evidence": { - "hasChanges": true, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": false, - "retiredViolation": false, - "valid": false, - "versionViolation": false, - "violations": [ - { - "code": "DECIMAL_PRECISION_REDUCED", - "details": {}, - "message": "Decimal precision reduced from 'decimal(10,2)' to 'decimal(8,2)'", - "path": "schema[orders].properties[amount].physicalType", - "severity": "WARNING" - } - ] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT", - "DECIMAL_PRECISION_REDUCED" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "Breaking lifecycle changes require a major version bump", - "path": null, - "severity": "INFO" - }, - { - "code": "DECIMAL_PRECISION_REDUCED", - "details": {}, - "message": "Decimal precision reduced from 'decimal(10,2)' to 'decimal(8,2)'", - "path": "schema[orders].properties[amount].physicalType", - "severity": "WARNING" - } - ], - "requiredVersionBump": "major", - "schemaVersion": "1", - "validation": { - "issues": [], - "valid": true - } -} diff --git a/tests/fixtures/governance_scenarios/decimal_scale_reduction/expected.json b/tests/fixtures/governance_scenarios/decimal_scale_reduction/expected.json deleted file mode 100644 index 9e26be98..00000000 --- a/tests/fixtures/governance_scenarios/decimal_scale_reduction/expected.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "breaking": true, - "changes": [ - { - "after": "decimal(10,2)", - "before": "decimal(10,4)", - "breaking": true, - "changeType": "MODIFY", - "domain": "STRUCTURE", - "entityType": "PROPERTY", - "evidence": [], - "field": "physicalType", - "identity": [ - "orders", - "amount" - ], - "path": "schema[orders].properties[amount].physicalType", - "reasonCodes": [ - "DECIMAL_SCALE_REDUCED" - ] - } - ], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "orders-contract", - "decision": "REVIEW", - "decisionId": "d0c6689c-2f79-5a9f-999d-6b1a2e75fefc", - "evidence": { - "hasChanges": true, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": false, - "retiredViolation": false, - "valid": false, - "versionViolation": false, - "violations": [ - { - "code": "DECIMAL_SCALE_REDUCED", - "details": {}, - "message": "Decimal scale reduced from 'decimal(10,4)' to 'decimal(10,2)'", - "path": "schema[orders].properties[amount].physicalType", - "severity": "WARNING" - } - ] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT", - "DECIMAL_SCALE_REDUCED" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "Breaking lifecycle changes require a major version bump", - "path": null, - "severity": "INFO" - }, - { - "code": "DECIMAL_SCALE_REDUCED", - "details": {}, - "message": "Decimal scale reduced from 'decimal(10,4)' to 'decimal(10,2)'", - "path": "schema[orders].properties[amount].physicalType", - "severity": "WARNING" - } - ], - "requiredVersionBump": "major", - "schemaVersion": "1", - "validation": { - "issues": [], - "valid": true - } -} diff --git a/tests/fixtures/governance_scenarios/decimal_widening/expected.json b/tests/fixtures/governance_scenarios/decimal_widening/expected.json deleted file mode 100644 index 92a01c8d..00000000 --- a/tests/fixtures/governance_scenarios/decimal_widening/expected.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "breaking": false, - "changes": [ - { - "after": "decimal(14,4)", - "before": "decimal(10,2)", - "breaking": false, - "changeType": "MODIFY", - "domain": "STRUCTURE", - "entityType": "PROPERTY", - "evidence": [], - "field": "physicalType", - "identity": [ - "orders", - "amount" - ], - "path": "schema[orders].properties[amount].physicalType", - "reasonCodes": [] - } - ], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "orders-contract", - "decision": "REVIEW", - "decisionId": "04ce4d23-79dd-54cd-ae92-c7e65744412c", - "evidence": { - "hasChanges": true, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": false, - "retiredViolation": false, - "valid": true, - "versionViolation": false, - "violations": [] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "Non-breaking structural or quality changes require a minor version bump", - "path": null, - "severity": "INFO" - } - ], - "requiredVersionBump": "minor", - "schemaVersion": "1", - "validation": { - "issues": [], - "valid": true - } -} diff --git a/tests/fixtures/governance_scenarios/deprecated_entity_change/expected.json b/tests/fixtures/governance_scenarios/deprecated_entity_change/expected.json deleted file mode 100644 index 0bdb8660..00000000 --- a/tests/fixtures/governance_scenarios/deprecated_entity_change/expected.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "breaking": false, - "changes": [ - { - "after": "integer", - "before": "string", - "breaking": false, - "changeType": "MODIFY", - "domain": "STRUCTURE", - "entityType": "PROPERTY", - "evidence": [], - "field": "logicalType", - "identity": [ - "orders", - "legacy_code" - ], - "path": "schema[orders].properties[legacy_code].logicalType", - "reasonCodes": [] - }, - { - "after": "bigint", - "before": "varchar(255)", - "breaking": false, - "changeType": "MODIFY", - "domain": "STRUCTURE", - "entityType": "PROPERTY", - "evidence": [], - "field": "physicalType", - "identity": [ - "orders", - "legacy_code" - ], - "path": "schema[orders].properties[legacy_code].physicalType", - "reasonCodes": [] - }, - { - "after": true, - "before": false, - "breaking": false, - "changeType": "MODIFY", - "domain": "STRUCTURE", - "entityType": "PROPERTY", - "evidence": [], - "field": "required", - "identity": [ - "orders", - "legacy_code" - ], - "path": "schema[orders].properties[legacy_code].required", - "reasonCodes": [] - } - ], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "orders-contract", - "decision": "REVIEW", - "decisionId": "17fec941-afae-54ce-9764-4d2efab07418", - "evidence": { - "hasChanges": true, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": false, - "retiredViolation": false, - "valid": true, - "versionViolation": false, - "violations": [] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "Non-breaking structural or quality changes require a minor version bump", - "path": null, - "severity": "INFO" - } - ], - "requiredVersionBump": "minor", - "schemaVersion": "1", - "validation": { - "issues": [], - "valid": true - } -} diff --git a/tests/fixtures/governance_scenarios/descriptive_metadata_only/expected.json b/tests/fixtures/governance_scenarios/descriptive_metadata_only/expected.json deleted file mode 100644 index bc8e2c3f..00000000 --- a/tests/fixtures/governance_scenarios/descriptive_metadata_only/expected.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "breaking": false, - "changes": [ - { - "after": "Orders table description update", - "before": "Orders table description", - "breaking": false, - "changeType": "MODIFY", - "domain": "METADATA", - "entityType": "SCHEMA", - "evidence": [], - "field": "description", - "identity": [ - "orders" - ], - "path": "schema[orders].description", - "reasonCodes": [] - }, - { - "after": "Order total amount in USD", - "before": "Order total amount", - "breaking": false, - "changeType": "MODIFY", - "domain": "METADATA", - "entityType": "PROPERTY", - "evidence": [], - "field": "description", - "identity": [ - "orders", - "amount" - ], - "path": "schema[orders].properties[amount].description", - "reasonCodes": [] - }, - { - "after": "Primary identifier updated notes", - "before": "Primary identifier", - "breaking": false, - "changeType": "MODIFY", - "domain": "METADATA", - "entityType": "PROPERTY", - "evidence": [], - "field": "description", - "identity": [ - "orders", - "id" - ], - "path": "schema[orders].properties[id].description", - "reasonCodes": [] - }, - { - "after": { - "purpose": "Updated descriptive documentation for orders contract" - }, - "before": { - "purpose": "Authoritative orders contract" - }, - "breaking": false, - "changeType": "MODIFY", - "domain": "METADATA", - "entityType": "CONTRACT", - "evidence": [], - "field": "description", - "identity": [ - "orders-contract" - ], - "path": "description", - "reasonCodes": [] - } - ], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "orders-contract", - "decision": "ALLOW", - "decisionId": "0a749876-f87f-5348-8b7b-4694ffd3aba9", - "evidence": { - "hasChanges": true, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": false, - "retiredViolation": false, - "valid": true, - "versionViolation": false, - "violations": [] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "Only descriptive metadata changed; no required version bump", - "path": null, - "severity": "INFO" - } - ], - "requiredVersionBump": "none", - "schemaVersion": "1", - "validation": { - "issues": [], - "valid": true - } -} diff --git a/tests/fixtures/governance_scenarios/draft_entity_change/expected.json b/tests/fixtures/governance_scenarios/draft_entity_change/expected.json deleted file mode 100644 index 1a3dbf87..00000000 --- a/tests/fixtures/governance_scenarios/draft_entity_change/expected.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "breaking": false, - "changes": [ - { - "after": "integer", - "before": "string", - "breaking": false, - "changeType": "MODIFY", - "domain": "STRUCTURE", - "entityType": "PROPERTY", - "evidence": [], - "field": "logicalType", - "identity": [ - "orders", - "experimental_feature" - ], - "path": "schema[orders].properties[experimental_feature].logicalType", - "reasonCodes": [] - }, - { - "after": "bigint", - "before": "varchar(255)", - "breaking": false, - "changeType": "MODIFY", - "domain": "STRUCTURE", - "entityType": "PROPERTY", - "evidence": [], - "field": "physicalType", - "identity": [ - "orders", - "experimental_feature" - ], - "path": "schema[orders].properties[experimental_feature].physicalType", - "reasonCodes": [] - }, - { - "after": true, - "before": false, - "breaking": false, - "changeType": "MODIFY", - "domain": "STRUCTURE", - "entityType": "PROPERTY", - "evidence": [], - "field": "required", - "identity": [ - "orders", - "experimental_feature" - ], - "path": "schema[orders].properties[experimental_feature].required", - "reasonCodes": [] - } - ], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "orders-contract", - "decision": "REVIEW", - "decisionId": "87f536c6-c55c-5111-b573-366b5a5fe71a", - "evidence": { - "hasChanges": true, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": false, - "retiredViolation": false, - "valid": true, - "versionViolation": false, - "violations": [] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "Non-breaking structural or quality changes require a minor version bump", - "path": null, - "severity": "INFO" - } - ], - "requiredVersionBump": "minor", - "schemaVersion": "1", - "validation": { - "issues": [], - "valid": true - } -} diff --git a/tests/fixtures/governance_scenarios/enum_reduction/expected.json b/tests/fixtures/governance_scenarios/enum_reduction/expected.json deleted file mode 100644 index 1b17e053..00000000 --- a/tests/fixtures/governance_scenarios/enum_reduction/expected.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "breaking": false, - "changes": [], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "orders-contract", - "decision": "ALLOW", - "decisionId": "59d98075-e496-52c7-9a72-6045ccb21b0d", - "evidence": { - "hasChanges": false, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": false, - "retiredViolation": false, - "valid": true, - "versionViolation": false, - "violations": [] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "No contract changes detected", - "path": null, - "severity": "INFO" - } - ], - "requiredVersionBump": "none", - "schemaVersion": "1", - "validation": { - "issues": [], - "valid": true - } -} diff --git a/tests/fixtures/governance_scenarios/logical_type_change/expected.json b/tests/fixtures/governance_scenarios/logical_type_change/expected.json deleted file mode 100644 index d967e1b8..00000000 --- a/tests/fixtures/governance_scenarios/logical_type_change/expected.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "breaking": true, - "changes": [ - { - "after": "string", - "before": "number", - "breaking": true, - "changeType": "MODIFY", - "domain": "STRUCTURE", - "entityType": "PROPERTY", - "evidence": [], - "field": "logicalType", - "identity": [ - "orders", - "amount" - ], - "path": "schema[orders].properties[amount].logicalType", - "reasonCodes": [ - "LOGICAL_TYPE_CHANGED" - ] - }, - { - "after": "varchar(64)", - "before": "decimal(10,2)", - "breaking": false, - "changeType": "MODIFY", - "domain": "STRUCTURE", - "entityType": "PROPERTY", - "evidence": [], - "field": "physicalType", - "identity": [ - "orders", - "amount" - ], - "path": "schema[orders].properties[amount].physicalType", - "reasonCodes": [] - } - ], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "orders-contract", - "decision": "REVIEW", - "decisionId": "106e6a6f-aee7-500e-bc67-1da9acba847b", - "evidence": { - "hasChanges": true, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": false, - "retiredViolation": false, - "valid": false, - "versionViolation": false, - "violations": [ - { - "code": "LOGICAL_TYPE_CHANGED", - "details": {}, - "message": "Logical type changed from 'number' to 'string'", - "path": "schema[orders].properties[amount].logicalType", - "severity": "WARNING" - } - ] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT", - "LOGICAL_TYPE_CHANGED" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "Breaking lifecycle changes require a major version bump", - "path": null, - "severity": "INFO" - }, - { - "code": "LOGICAL_TYPE_CHANGED", - "details": {}, - "message": "Logical type changed from 'number' to 'string'", - "path": "schema[orders].properties[amount].logicalType", - "severity": "WARNING" - } - ], - "requiredVersionBump": "major", - "schemaVersion": "1", - "validation": { - "issues": [], - "valid": true - } -} diff --git a/tests/fixtures/governance_scenarios/manual_version_change/expected.json b/tests/fixtures/governance_scenarios/manual_version_change/expected.json deleted file mode 100644 index cf3cc1c7..00000000 --- a/tests/fixtures/governance_scenarios/manual_version_change/expected.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "breaking": true, - "changes": [ - { - "after": "2.0.0", - "before": "1.0.0", - "breaking": true, - "changeType": "MODIFY", - "domain": "VERSION", - "entityType": "CONTRACT", - "evidence": [], - "field": "version", - "identity": [ - "orders-contract" - ], - "path": "version", - "reasonCodes": [ - "CONTRACT_VERSION_MANUALLY_CHANGED" - ] - } - ], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "orders-contract", - "decision": "BLOCK", - "decisionId": "642819bc-96eb-551c-abd0-ed2ba0187d27", - "evidence": { - "hasChanges": true, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": false, - "retiredViolation": false, - "valid": false, - "versionViolation": true, - "violations": [ - { - "code": "CONTRACT_VERSION_MANUALLY_CHANGED", - "details": {}, - "message": "Contract version mismatch. Contract versions are release-managed and cannot be manually updated during normal import/merge. Please revert the version change and use 'semapact release prepare'.", - "path": "version", - "severity": "ERROR" - } - ] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT", - "CONTRACT_VERSION_MANUALLY_CHANGED" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "Breaking lifecycle changes require a major version bump", - "path": null, - "severity": "INFO" - }, - { - "code": "CONTRACT_VERSION_MANUALLY_CHANGED", - "details": {}, - "message": "Contract version mismatch. Contract versions are release-managed and cannot be manually updated during normal import/merge. Please revert the version change and use 'semapact release prepare'.", - "path": "version", - "severity": "ERROR" - } - ], - "requiredVersionBump": "major", - "schemaVersion": "1", - "validation": { - "issues": [], - "valid": true - } -} diff --git a/tests/fixtures/governance_scenarios/merge_conflict/expected.json b/tests/fixtures/governance_scenarios/merge_conflict/expected.json deleted file mode 100644 index c9a2591d..00000000 --- a/tests/fixtures/governance_scenarios/merge_conflict/expected.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "breaking": false, - "changes": [], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "orders-contract", - "decision": "REVIEW", - "decisionId": "1678a050-8f17-5739-bfb2-83ab7aea05ad", - "evidence": { - "hasChanges": false, - "mergeConflictsCount": 1 - }, - "policy": { - "idViolation": false, - "retiredViolation": false, - "valid": true, - "versionViolation": false, - "violations": [] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT", - "MERGE_CONFLICT" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "No contract changes detected", - "path": null, - "severity": "INFO" - }, - { - "code": "MERGE_CONFLICT", - "details": { - "property_name": "amount", - "rule": "description_conflict", - "schema_id": "orders" - }, - "message": "Conflicting description updates from upstream branches", - "path": "schema[orders].properties[amount]", - "severity": "WARNING" - } - ], - "requiredVersionBump": "none", - "schemaVersion": "1", - "validation": { - "issues": [], - "valid": true - } -} diff --git a/tests/fixtures/governance_scenarios/no_change/expected.json b/tests/fixtures/governance_scenarios/no_change/expected.json deleted file mode 100644 index 3c5908c0..00000000 --- a/tests/fixtures/governance_scenarios/no_change/expected.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "breaking": false, - "changes": [], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "orders-contract", - "decision": "ALLOW", - "decisionId": "e496a148-d48f-528f-b007-3a9dd1d7f060", - "evidence": { - "hasChanges": false, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": false, - "retiredViolation": false, - "valid": true, - "versionViolation": false, - "violations": [] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "No contract changes detected", - "path": null, - "severity": "INFO" - } - ], - "requiredVersionBump": "none", - "schemaVersion": "1", - "validation": { - "issues": [], - "valid": true - } -} diff --git a/tests/fixtures/governance_scenarios/physical_name_identity_stability/expected.json b/tests/fixtures/governance_scenarios/physical_name_identity_stability/expected.json deleted file mode 100644 index d6ebcbbe..00000000 --- a/tests/fixtures/governance_scenarios/physical_name_identity_stability/expected.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "breaking": false, - "changes": [ - { - "after": "tbl_orders_renamed_in_db", - "before": "tbl_orders", - "breaking": false, - "changeType": "MODIFY", - "domain": "STRUCTURE", - "entityType": "SCHEMA", - "evidence": [], - "field": "physicalName", - "identity": [ - "orders" - ], - "path": "schema[orders].physicalName", - "reasonCodes": [] - }, - { - "after": "col_id_renamed_in_db", - "before": "col_id", - "breaking": false, - "changeType": "MODIFY", - "domain": "STRUCTURE", - "entityType": "PROPERTY", - "evidence": [], - "field": "physicalName", - "identity": [ - "orders", - "id" - ], - "path": "schema[orders].properties[id].physicalName", - "reasonCodes": [] - } - ], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "orders-contract", - "decision": "REVIEW", - "decisionId": "a9b4d271-2c80-56db-a119-c87110dc40f4", - "evidence": { - "hasChanges": true, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": false, - "retiredViolation": false, - "valid": true, - "versionViolation": false, - "violations": [] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "Non-breaking structural or quality changes require a minor version bump", - "path": null, - "severity": "INFO" - } - ], - "requiredVersionBump": "minor", - "schemaVersion": "1", - "validation": { - "issues": [], - "valid": true - } -} diff --git a/tests/fixtures/governance_scenarios/physical_type_narrowing/expected.json b/tests/fixtures/governance_scenarios/physical_type_narrowing/expected.json deleted file mode 100644 index 735f108a..00000000 --- a/tests/fixtures/governance_scenarios/physical_type_narrowing/expected.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "breaking": true, - "changes": [ - { - "after": "varchar(32)", - "before": "varchar(64)", - "breaking": true, - "changeType": "MODIFY", - "domain": "STRUCTURE", - "entityType": "PROPERTY", - "evidence": [], - "field": "physicalType", - "identity": [ - "orders", - "id" - ], - "path": "schema[orders].properties[id].physicalType", - "reasonCodes": [ - "PHYSICAL_TYPE_NARROWED" - ] - } - ], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "orders-contract", - "decision": "REVIEW", - "decisionId": "4f94025b-d059-5dd6-a97f-80a2f124761b", - "evidence": { - "hasChanges": true, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": false, - "retiredViolation": false, - "valid": false, - "versionViolation": false, - "violations": [ - { - "code": "PHYSICAL_TYPE_NARROWED", - "details": {}, - "message": "Physical type narrowed from 'varchar(64)' to 'varchar(32)'", - "path": "schema[orders].properties[id].physicalType", - "severity": "WARNING" - } - ] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT", - "PHYSICAL_TYPE_NARROWED" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "Breaking lifecycle changes require a major version bump", - "path": null, - "severity": "INFO" - }, - { - "code": "PHYSICAL_TYPE_NARROWED", - "details": {}, - "message": "Physical type narrowed from 'varchar(64)' to 'varchar(32)'", - "path": "schema[orders].properties[id].physicalType", - "severity": "WARNING" - } - ], - "requiredVersionBump": "major", - "schemaVersion": "1", - "validation": { - "issues": [], - "valid": true - } -} diff --git a/tests/fixtures/governance_scenarios/property_addition/expected.json b/tests/fixtures/governance_scenarios/property_addition/expected.json deleted file mode 100644 index 1cfada20..00000000 --- a/tests/fixtures/governance_scenarios/property_addition/expected.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "breaking": false, - "changes": [ - { - "after": { - "description": "Foreign reference to customer", - "id": "customer_id", - "logicalType": "string", - "name": "customer_id", - "physicalName": "col_customer_id", - "physicalType": "varchar(64)", - "required": false - }, - "before": null, - "breaking": false, - "changeType": "ADD", - "domain": "STRUCTURE", - "entityType": "PROPERTY", - "evidence": [], - "field": null, - "identity": [ - "orders", - "customer_id" - ], - "path": "schema[orders].properties[customer_id]", - "reasonCodes": [] - } - ], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "orders-contract", - "decision": "REVIEW", - "decisionId": "6aca9196-551a-52f4-a814-e1a86f6ba3f3", - "evidence": { - "hasChanges": true, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": false, - "retiredViolation": false, - "valid": true, - "versionViolation": false, - "violations": [] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "Non-breaking structural or quality changes require a minor version bump", - "path": null, - "severity": "INFO" - }, - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "Schema or property additions require a minor version bump", - "path": null, - "severity": "INFO" - } - ], - "requiredVersionBump": "minor", - "schemaVersion": "1", - "validation": { - "issues": [], - "valid": true - } -} diff --git a/tests/fixtures/governance_scenarios/property_removal/expected.json b/tests/fixtures/governance_scenarios/property_removal/expected.json deleted file mode 100644 index b12a3400..00000000 --- a/tests/fixtures/governance_scenarios/property_removal/expected.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "breaking": true, - "changes": [ - { - "after": null, - "before": { - "description": "Order total amount", - "id": "amount", - "logicalType": "number", - "name": "amount", - "physicalName": "col_amount", - "physicalType": "decimal(10,2)", - "required": false - }, - "breaking": true, - "changeType": "REMOVE", - "domain": "STRUCTURE", - "entityType": "PROPERTY", - "evidence": [], - "field": null, - "identity": [ - "orders", - "amount" - ], - "path": "schema[orders].properties[amount]", - "reasonCodes": [ - "PROPERTY_REMOVED" - ] - } - ], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "orders-contract", - "decision": "REVIEW", - "decisionId": "2ac271d9-2c86-578a-aafb-394ce63f3a4b", - "evidence": { - "hasChanges": true, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": false, - "retiredViolation": false, - "valid": false, - "versionViolation": false, - "violations": [ - { - "code": "PROPERTY_REMOVED", - "details": {}, - "message": "Property removed from active lifecycle scope", - "path": "schema[orders].properties[amount]", - "severity": "WARNING" - } - ] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT", - "PROPERTY_REMOVED" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "Breaking lifecycle changes require a major version bump", - "path": null, - "severity": "INFO" - }, - { - "code": "PROPERTY_REMOVED", - "details": {}, - "message": "Property removed from active lifecycle scope", - "path": "schema[orders].properties[amount]", - "severity": "WARNING" - } - ], - "requiredVersionBump": "major", - "schemaVersion": "1", - "validation": { - "issues": [], - "valid": true - } -} diff --git a/tests/fixtures/governance_scenarios/relationship_removal/expected.json b/tests/fixtures/governance_scenarios/relationship_removal/expected.json deleted file mode 100644 index 453a76df..00000000 --- a/tests/fixtures/governance_scenarios/relationship_removal/expected.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "breaking": true, - "changes": [ - { - "after": null, - "before": { - "to": "customers.id", - "type": "foreignKey" - }, - "breaking": true, - "changeType": "REMOVE", - "domain": "RELATIONSHIP", - "entityType": "RELATIONSHIP", - "evidence": [], - "field": null, - "identity": [ - "orders", - "foreignKey:->customers.id" - ], - "path": "schema[orders].relationships", - "reasonCodes": [ - "RELATIONSHIP_REMOVED" - ] - } - ], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "orders-contract", - "decision": "REVIEW", - "decisionId": "1c53c03d-c05c-5416-9406-63703c8d5594", - "evidence": { - "hasChanges": true, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": false, - "retiredViolation": false, - "valid": false, - "versionViolation": false, - "violations": [ - { - "code": "RELATIONSHIP_REMOVED", - "details": {}, - "message": "Relationship 'foreignKey:->customers.id' removed from active lifecycle scope. Downstream joins may fail.", - "path": "schema[orders].relationships", - "severity": "WARNING" - } - ] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT", - "RELATIONSHIP_REMOVED" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "Breaking lifecycle changes require a major version bump", - "path": null, - "severity": "INFO" - }, - { - "code": "RELATIONSHIP_REMOVED", - "details": {}, - "message": "Relationship 'foreignKey:->customers.id' removed from active lifecycle scope. Downstream joins may fail.", - "path": "schema[orders].relationships", - "severity": "WARNING" - } - ], - "requiredVersionBump": "major", - "schemaVersion": "1", - "validation": { - "issues": [], - "valid": true - } -} diff --git a/tests/fixtures/governance_scenarios/required_tightening/expected.json b/tests/fixtures/governance_scenarios/required_tightening/expected.json deleted file mode 100644 index 8fe443c4..00000000 --- a/tests/fixtures/governance_scenarios/required_tightening/expected.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "breaking": true, - "changes": [ - { - "after": true, - "before": false, - "breaking": true, - "changeType": "MODIFY", - "domain": "STRUCTURE", - "entityType": "PROPERTY", - "evidence": [], - "field": "required", - "identity": [ - "orders", - "amount" - ], - "path": "schema[orders].properties[amount].required", - "reasonCodes": [ - "REQUIRED_TIGHTENED" - ] - } - ], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "orders-contract", - "decision": "REVIEW", - "decisionId": "4ea6efa1-1081-56c1-8b31-a2645a4feabc", - "evidence": { - "hasChanges": true, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": false, - "retiredViolation": false, - "valid": false, - "versionViolation": false, - "violations": [ - { - "code": "REQUIRED_TIGHTENED", - "details": {}, - "message": "Required flag tightened from False to True", - "path": "schema[orders].properties[amount].required", - "severity": "WARNING" - } - ] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT", - "REQUIRED_TIGHTENED" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "Breaking lifecycle changes require a major version bump", - "path": null, - "severity": "INFO" - }, - { - "code": "REQUIRED_TIGHTENED", - "details": {}, - "message": "Required flag tightened from False to True", - "path": "schema[orders].properties[amount].required", - "severity": "WARNING" - } - ], - "requiredVersionBump": "major", - "schemaVersion": "1", - "validation": { - "issues": [], - "valid": true - } -} diff --git a/tests/fixtures/governance_scenarios/retired_contract_mutation/expected.json b/tests/fixtures/governance_scenarios/retired_contract_mutation/expected.json deleted file mode 100644 index 990a5539..00000000 --- a/tests/fixtures/governance_scenarios/retired_contract_mutation/expected.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "breaking": false, - "changes": [ - { - "after": { - "purpose": "Retired orders contract - forbidden modification" - }, - "before": { - "purpose": "Retired orders contract" - }, - "breaking": false, - "changeType": "MODIFY", - "domain": "METADATA", - "entityType": "CONTRACT", - "evidence": [], - "field": "description", - "identity": [ - "orders-contract" - ], - "path": "description", - "reasonCodes": [] - } - ], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "orders-contract", - "decision": "BLOCK", - "decisionId": "cbc08d31-312b-548a-9619-51dddd337642", - "evidence": { - "hasChanges": true, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": false, - "retiredViolation": true, - "valid": false, - "versionViolation": false, - "violations": [ - { - "code": "RETIRED_CONTRACT_MODIFIED", - "details": {}, - "message": "Cannot modify a retired contract.", - "path": "status", - "severity": "ERROR" - } - ] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT", - "RETIRED_CONTRACT_MODIFIED" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "Only descriptive metadata changed; no required version bump", - "path": null, - "severity": "INFO" - }, - { - "code": "RETIRED_CONTRACT_MODIFIED", - "details": {}, - "message": "Cannot modify a retired contract.", - "path": "status", - "severity": "ERROR" - } - ], - "requiredVersionBump": "none", - "schemaVersion": "1", - "validation": { - "issues": [], - "valid": true - } -} diff --git a/tests/fixtures/governance_scenarios/schema_addition/expected.json b/tests/fixtures/governance_scenarios/schema_addition/expected.json deleted file mode 100644 index 7f1827b1..00000000 --- a/tests/fixtures/governance_scenarios/schema_addition/expected.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "breaking": false, - "changes": [ - { - "after": { - "description": "Line items table", - "id": "line_items", - "name": "line_items", - "physicalName": "tbl_line_items", - "properties": [ - { - "description": "Line item identifier", - "id": "item_id", - "logicalType": "string", - "name": "item_id", - "physicalName": "col_item_id", - "physicalType": "varchar(64)", - "required": true - } - ] - }, - "before": null, - "breaking": false, - "changeType": "ADD", - "domain": "STRUCTURE", - "entityType": "SCHEMA", - "evidence": [], - "field": null, - "identity": [ - "line_items" - ], - "path": "schema[line_items]", - "reasonCodes": [] - } - ], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "orders-contract", - "decision": "REVIEW", - "decisionId": "54299462-9f11-563c-a152-f70551d6c5ea", - "evidence": { - "hasChanges": true, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": false, - "retiredViolation": false, - "valid": true, - "versionViolation": false, - "violations": [] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "Non-breaking structural or quality changes require a minor version bump", - "path": null, - "severity": "INFO" - }, - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "Schema or property additions require a minor version bump", - "path": null, - "severity": "INFO" - } - ], - "requiredVersionBump": "minor", - "schemaVersion": "1", - "validation": { - "issues": [], - "valid": true - } -} diff --git a/tests/fixtures/governance_scenarios/schema_removal/expected.json b/tests/fixtures/governance_scenarios/schema_removal/expected.json deleted file mode 100644 index d3151205..00000000 --- a/tests/fixtures/governance_scenarios/schema_removal/expected.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "breaking": true, - "changes": [ - { - "after": null, - "before": { - "id": "audit_log", - "name": "audit_log", - "physicalName": "tbl_audit_log", - "properties": [ - { - "id": "log_id", - "logicalType": "string", - "name": "log_id", - "physicalName": "col_log_id", - "physicalType": "varchar(64)", - "required": true - } - ] - }, - "breaking": true, - "changeType": "REMOVE", - "domain": "STRUCTURE", - "entityType": "SCHEMA", - "evidence": [], - "field": null, - "identity": [ - "audit_log" - ], - "path": "schema[audit_log]", - "reasonCodes": [ - "SCHEMA_REMOVED" - ] - } - ], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "orders-contract", - "decision": "REVIEW", - "decisionId": "e3aeba92-dbbf-5ea6-8f3f-5cbbf0bf2449", - "evidence": { - "hasChanges": true, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": false, - "retiredViolation": false, - "valid": false, - "versionViolation": false, - "violations": [ - { - "code": "SCHEMA_REMOVED", - "details": {}, - "message": "Schema removed from active contract", - "path": "schema[audit_log]", - "severity": "WARNING" - } - ] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT", - "SCHEMA_REMOVED" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "Breaking lifecycle changes require a major version bump", - "path": null, - "severity": "INFO" - }, - { - "code": "SCHEMA_REMOVED", - "details": {}, - "message": "Schema removed from active contract", - "path": "schema[audit_log]", - "severity": "WARNING" - } - ], - "requiredVersionBump": "major", - "schemaVersion": "1", - "validation": { - "issues": [], - "valid": true - } -} diff --git a/tests/fixtures/governance_scenarios/validation_failure/expected.json b/tests/fixtures/governance_scenarios/validation_failure/expected.json deleted file mode 100644 index 01eb8f5a..00000000 --- a/tests/fixtures/governance_scenarios/validation_failure/expected.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "breaking": false, - "changes": [], - "context": { - "effectiveDate": "2026-01-01" - }, - "contractId": "orders-contract", - "decision": "BLOCK", - "decisionId": "d887c18a-5d1f-57de-b374-7bb1fe5a4ae8", - "evidence": { - "hasChanges": true, - "mergeConflictsCount": 0 - }, - "policy": { - "idViolation": false, - "retiredViolation": false, - "valid": false, - "versionViolation": false, - "violations": [] - }, - "reasonCodes": [ - "CHANGE_ASSESSMENT", - "VALIDATION_FAILED" - ], - "reasons": [ - { - "code": "CHANGE_ASSESSMENT", - "details": {}, - "message": "No contract changes detected", - "path": null, - "severity": "INFO" - }, - { - "code": "VALIDATION_FAILED", - "details": {}, - "message": "Property name cannot be empty or whitespace-only", - "path": "schema", - "severity": "ERROR" - } - ], - "requiredVersionBump": "none", - "schemaVersion": "1", - "validation": { - "issues": [ - { - "code": "VALIDATION_FAILED", - "details": {}, - "message": "Property name cannot be empty or whitespace-only", - "path": "schema", - "severity": "ERROR" - } - ], - "valid": false - } -} diff --git a/tests/fixtures/history_golden/v1/review_multi_deploy.json b/tests/fixtures/history_golden/v1/review_multi_deploy.json deleted file mode 100644 index 02d2e165..00000000 --- a/tests/fixtures/history_golden/v1/review_multi_deploy.json +++ /dev/null @@ -1,132 +0,0 @@ -{ - "fixtureVersion": "history-v1", - "files": { - ".semapact/history/change_set_decisions/c2f4ee02-90e5-532a-ac8a-edd43e49f156/14295d1b-b52d-5d8b-8dad-2121529f099c.json": { - "checksum": "sha256:d77324f66e332dfdd6cb3ff081e65987a6eaa469d5b84bab2cb425d55e23dd62", - "topLevelKeys": ["change_set_id", "decision_id"] - }, - ".semapact/history/change_sets/c2f4ee02-90e5-532a-ac8a-edd43e49f156.json": { - "checksum": "sha256:66861c622ee7f42f4340a761d17c16a006bf738ad682c39359c552376ce95005", - "topLevelKeys": ["actor_reference", "base_revision_ref", "candidate_revision_ref", "change_set_id", "changes", "context", "contract_id", "source"] - }, - ".semapact/history/contract_revisions/290bd707-017f-56ca-a379-c8879b9ede13.json": { - "checksum": "sha256:e70f5404098efaf4cec40f9a021adbba17cad53ed0c57db6984126fdcb84373e", - "topLevelKeys": ["content_fingerprint", "contract", "revision_id"] - }, - ".semapact/history/contract_revisions/352d83a9-4ee6-508a-981f-4bc9819263c9.json": { - "checksum": "sha256:bf1e3e1836fff7827bc014caccfefd36a01aaf480ecd17eae41c9a8245e5eb9e", - "topLevelKeys": ["content_fingerprint", "contract", "revision_id"] - }, - ".semapact/history/contract_revisions/fb437fde-4baa-5dad-a382-c4df381ba412.json": { - "checksum": "sha256:be036b579fe23c3d47afb96978c1e05416564b5ad0e30df034803c2c86bbf9c2", - "topLevelKeys": ["content_fingerprint", "contract", "revision_id"] - }, - ".semapact/history/decisions/14295d1b-b52d-5d8b-8dad-2121529f099c.json": { - "checksum": "sha256:5c0ddb33910feb477c32252d21c318e273e031185f52f4295aa0074dd78bf485", - "topLevelKeys": ["breaking", "changes", "context", "contract_id", "decision", "decision_id", "evidence", "policy", "reasons", "required_version_bump", "validation"] - }, - ".semapact/history/deployment_authorizations/87ea9883-2583-51d7-8c25-fe4d1c643dc0.json": { - "checksum": "sha256:7a2ea0240736846da3a701253ad913f8a048066e30a20f7680d5385170a687c7", - "topLevelKeys": ["allowed", "applied_release_id", "contract_ops_authorization_id", "deployment_authorization_id", "deployment_plan_id"] - }, - ".semapact/history/deployment_authorizations/a1a210ef-25d0-5d00-a105-7e7907d63acb.json": { - "checksum": "sha256:915e95cdac4b9eac879221c1ffeef9ab95a8538ed0efbaf1c8f446afc77090cb", - "topLevelKeys": ["allowed", "applied_release_id", "contract_ops_authorization_id", "deployment_authorization_id", "deployment_plan_id"] - }, - ".semapact/history/deployment_plans/c684e277-793b-5628-ae09-0b67316306a8.json": { - "checksum": "sha256:f6c254152dbfe16ff017f1cc43f40390e5dc59452df730e9701cb81868a98deb", - "topLevelKeys": ["actions", "applied_release_id", "contract_id", "deployment_plan_id", "plan_version", "release_plan_id", "released_revision_ref", "selected_version", "target"] - }, - ".semapact/history/deployment_plans/fecc2ad3-b5fe-58a1-99a5-3fb30e28a6c3.json": { - "checksum": "sha256:446646a122bfc65c085a084f4bbb52d6f1ac9f6c94a1724061c2f19308c58f2c", - "topLevelKeys": ["actions", "applied_release_id", "contract_id", "deployment_plan_id", "plan_version", "release_plan_id", "released_revision_ref", "selected_version", "target"] - }, - ".semapact/history/deployment_previews/6eb94c8f-68ad-546a-8eb4-b1fb27ec21fa.json": { - "checksum": "sha256:8a42d5eeaacfed1c35fffd9da860bfb3b7ee6294d2887ac665ce0c6b2f7f500d", - "topLevelKeys": ["deployment_plan_id", "deployment_preview_id", "observation_fingerprint", "operations", "platform", "preview_version", "runtime_target", "source_identifier"] - }, - ".semapact/history/deployment_previews/778b10a8-6133-572e-b89b-6d76be994154.json": { - "checksum": "sha256:ca3f88358aaea04096290f9cb3d6a4e39218a63aec986dd5b19f93ca8ecd9d9b", - "topLevelKeys": ["deployment_plan_id", "deployment_preview_id", "observation_fingerprint", "operations", "platform", "preview_version", "runtime_target", "source_identifier"] - }, - ".semapact/history/deployment_records/61edafd1-bba0-5e2f-8fa1-f48ff7eeaafb.json": { - "checksum": "sha256:155b3dd66eca9cb519ccd3f5b3e0a6f3a4d4c430428d96ef0cddf5b6c3392aef", - "topLevelKeys": ["actor_reference", "completed_at", "deployment_authorization_id", "deployment_plan_id", "deployment_preview_id", "deployment_record_id", "external_reference", "platform", "release_record_id", "runtime_target", "source_reference", "started_at", "status"] - }, - ".semapact/history/deployment_records/c1cbc2d8-f340-5944-827a-c5bbc8d73835.json": { - "checksum": "sha256:17f423508bb8500f00f85c0ae4a9f9d1900e78c9cfff7a2b936e6e1f14357af1", - "topLevelKeys": ["actor_reference", "completed_at", "deployment_authorization_id", "deployment_plan_id", "deployment_preview_id", "deployment_record_id", "external_reference", "platform", "release_record_id", "runtime_target", "source_reference", "started_at", "status"] - }, - ".semapact/history/release_plans/45a429c1-9ff4-54da-897a-b02c7ee6027f.json": { - "checksum": "sha256:e1f6e06acd06a1b0ff719ddce44430f293b03d65e5586607f13907d6b617b237", - "topLevelKeys": ["change_set_id", "contract_id", "decision_id", "preconditions", "release_plan_id", "release_revision_ref", "required_version_bump"] - }, - ".semapact/history/release_records/de7c4ed0-d54a-5a7a-b82e-c626eab39d23.json": { - "checksum": "sha256:961815c36c9f21e9ca1a277bb256bdc6571bcd94187a2712f333fa2bf6834b9d", - "topLevelKeys": ["actual_version_bump", "applied_release_id", "authority_reference", "authorization_id", "change_set_id", "contract_id", "contract_version", "decision_id", "release_plan_id", "release_record_id", "released_revision_id", "required_version_bump", "review_evidence_action", "review_evidence_reference", "version_authority", "version_resolution_id"] - }, - ".semapact/history/runtime_observations/38695939-4fb5-583c-acfa-1a21397d89c8.json": { - "checksum": "sha256:dcad511a463004815942e9404e2d6ad5edd7efeb95afa14926f2b08b2c0bc389", - "topLevelKeys": ["observation", "observation_record_id"] - }, - ".semapact/history/runtime_observations/e259c151-972f-532a-99d1-a14dcd78b166.json": { - "checksum": "sha256:d2af366ab491fdd0bfc123940f17f90257a456264343d017218c5791d2e1a6ad", - "topLevelKeys": ["observation", "observation_record_id"] - }, - ".semapact/history/runtime_reconciliations/1e53fecd-750f-556e-97a8-060dc05e3cc2.json": { - "checksum": "sha256:87f4076f1428aa835565ced57aedd8371ff4d01de144ce6c34d395b2bd6d1a97", - "topLevelKeys": ["deployment_record_id", "observation_record_id", "release_record_id", "result", "runtime_reconciliation_record_id", "status"] - }, - ".semapact/history/runtime_reconciliations/eeaf42b7-6d15-5b80-b1c3-ce51da73293f.json": { - "checksum": "sha256:041909fb90b3c4ad24e093883d4dfed0c18bb445fbb17e9a6c5c1a47e6b6b42e", - "topLevelKeys": ["deployment_record_id", "observation_record_id", "release_record_id", "result", "runtime_reconciliation_record_id", "status"] - } - }, - "evolution": { - "contractId": "orders-product", - "proposals": [ - { - "changeSetId": "c2f4ee02-90e5-532a-ac8a-edd43e49f156", - "decisionId": "14295d1b-b52d-5d8b-8dad-2121529f099c", - "decision": "REVIEW", - "baseRevisionId": "fb437fde-4baa-5dad-a382-c4df381ba412", - "candidateRevisionId": "352d83a9-4ee6-508a-981f-4bc9819263c9", - "releases": [ - { - "releaseRecordId": "de7c4ed0-d54a-5a7a-b82e-c626eab39d23", - "contractVersion": "1.3.0", - "releasedRevisionId": "290bd707-017f-56ca-a379-c8879b9ede13", - "deployments": [ - { - "deploymentRecordId": "61edafd1-bba0-5e2f-8fa1-f48ff7eeaafb", - "runtimeTarget": "main.stage", - "status": "FAILED", - "runtime": [] - }, - { - "deploymentRecordId": "c1cbc2d8-f340-5944-827a-c5bbc8d73835", - "runtimeTarget": "main.prod", - "status": "SUCCEEDED", - "runtime": [ - { - "reconciliationRecordId": "1e53fecd-750f-556e-97a8-060dc05e3cc2", - "observationRecordId": "e259c151-972f-532a-99d1-a14dcd78b166", - "status": "IN_SYNC" - }, - { - "reconciliationRecordId": "eeaf42b7-6d15-5b80-b1c3-ce51da73293f", - "observationRecordId": "38695939-4fb5-583c-acfa-1a21397d89c8", - "status": "DRIFT" - } - ] - } - ] - } - ] - } - ], - "unlinkedReleaseIds": [], - "unlinkedRuntimeIds": [], - "brokenReferences": [] - } -} diff --git a/tests/interfaces/test_cli_outcomes.py b/tests/interfaces/test_cli_outcomes.py index bad45dcf..9762f67d 100644 --- a/tests/interfaces/test_cli_outcomes.py +++ b/tests/interfaces/test_cli_outcomes.py @@ -5,8 +5,7 @@ import pytest -from semapact.change_context import ChangeContext -from semapact.core.release import RequiredBump +from semapact.versioning import RequiredBump from semapact.exceptions import ( GovernanceBlockedError, GovernanceReviewRequiredError, @@ -44,7 +43,6 @@ def _make_dummy_decision( decision_id="test-dec-1", decision=decision, contract_id="test-contract", - context=ChangeContext(effective_date="2026-08-29"), breaking=breaking, required_version_bump=bump, validation=ValidationOutcome(valid=True), diff --git a/tests/interfaces/test_cli_subprocess_outcomes.py b/tests/interfaces/test_cli_subprocess_outcomes.py index 48ebce13..a0e7f991 100644 --- a/tests/interfaces/test_cli_subprocess_outcomes.py +++ b/tests/interfaces/test_cli_subprocess_outcomes.py @@ -142,8 +142,6 @@ def test_subprocess_analyze_release_classify_allow(active_contract_yaml: Path): str(active_contract_yaml), "--candidate", str(active_contract_yaml), - "--effective-date", - "2026-08-29", ) assert res.returncode == 0 payload = json.loads(res.stdout) @@ -163,8 +161,6 @@ def test_subprocess_analyze_release_classify_review( str(active_contract_yaml), "--candidate", str(review_candidate_yaml), - "--effective-date", - "2026-08-29", ) assert res.returncode == 0 payload = json.loads(res.stdout) @@ -184,8 +180,6 @@ def test_subprocess_analyze_release_classify_block( str(retired_contract_yaml), "--candidate", str(review_candidate_yaml), - "--effective-date", - "2026-08-29", ) assert res.returncode == 0 payload = json.loads(res.stdout) @@ -198,77 +192,8 @@ def test_subprocess_analyze_release_classify_block( # 2. PROPOSE Commands: ALLOW & REVIEW -> 0, BLOCK -> 3 (GOVERNANCE_BLOCKED) # ============================================================================== -def test_subprocess_propose_release_prepare_review( - active_contract_yaml: Path, review_candidate_yaml: Path, tmp_path: Path -): - out_yaml = tmp_path / "prepared.yaml" - res = _run_cli( - "release", - "prepare", - "--base", - str(active_contract_yaml), - "--candidate", - str(review_candidate_yaml), - "--release-tag", - "v1.1.0", - "--output", - str(out_yaml), - "--effective-date", - "2026-08-29", - ) - assert res.returncode == 0 - payload = json.loads(res.stdout) - assert payload["actualBump"] == "minor" - assert "exitCode" not in payload -def test_subprocess_propose_release_prepare_no_bump_validation_failed( - active_contract_yaml: Path, tmp_path: Path -): - out_yaml = tmp_path / "prepared.yaml" - res = _run_cli( - "release", - "prepare", - "--base", - str(active_contract_yaml), - "--candidate", - str(active_contract_yaml), - "--release-tag", - "v1.0.0", - "--output", - str(out_yaml), - "--effective-date", - "2026-08-29", - ) - # Attempting to prepare a release candidate when no bump is required returns VALIDATION_FAILED (2) - assert res.returncode == 2 - assert "Contract changes do not require a release version bump" in res.stderr - - - -def test_subprocess_propose_release_prepare_block( - retired_contract_yaml: Path, review_candidate_yaml: Path, tmp_path: Path -): - out_yaml = tmp_path / "prepared.yaml" - res = _run_cli( - "release", - "prepare", - "--base", - str(retired_contract_yaml), - "--candidate", - str(review_candidate_yaml), - "--release-tag", - "v2.0.0", - "--output", - str(out_yaml), - "--effective-date", - "2026-08-29", - ) - # PROPOSE with BLOCK decision must exit with 3 (GOVERNANCE_BLOCKED) - assert res.returncode == 3 - assert "Governance decision BLOCKED" in res.stderr - assert "Traceback (most recent call last)" not in res.stderr - def test_subprocess_propose_merge_block( retired_contract_yaml: Path, review_candidate_yaml: Path, tmp_path: Path @@ -362,8 +287,6 @@ def test_subprocess_validation_failed_on_invalid_contract( str(invalid_contract_yaml), "--candidate", str(active_contract_yaml), - "--effective-date", - "2026-08-29", ) assert res.returncode == 2 assert "❌" in res.stderr @@ -382,8 +305,6 @@ def test_subprocess_runtime_error_on_missing_file(active_contract_yaml: Path): "/nonexistent/path/contract.yaml", "--candidate", str(active_contract_yaml), - "--effective-date", - "2026-08-29", ) # File not found error is a runtime execution failure assert res.returncode == 5 diff --git a/tests/interfaces/test_deployment_cmd.py b/tests/interfaces/test_deployment_cmd.py index 7074c7be..b274741e 100644 --- a/tests/interfaces/test_deployment_cmd.py +++ b/tests/interfaces/test_deployment_cmd.py @@ -1,162 +1,106 @@ from __future__ import annotations -import json import sys -from open_data_contract_standard.model import OpenDataContractStandard, SchemaObject, SchemaProperty +from open_data_contract_standard.model import Server import pytest -from semapact.contractops import AppliedContractRelease -from semapact.deployment import ( - DeploymentAdapter, - DeploymentTarget, - build_deployment_plan, -) -from semapact.contractops.integrity import compute_applied_release_id from semapact.exceptions import ValidationError from semapact.interfaces import cli from semapact.interfaces.commands import deployment_cmd from semapact.interfaces.commands.deployment_cmd import DeploymentCommandResult from semapact.interfaces.outcomes import ProcessOutcome -from semapact.reconciliation import ReconciliationResult - -SOURCE_REFERENCE = "https://workspace.example" -def _release() -> AppliedContractRelease: - contract = OpenDataContractStandard( - apiVersion="v3.1.0", - kind="DataContract", - id="orders-product", - name="Orders", - version="1.2.0", - status="active", - schema=[ - SchemaObject( - name="orders", - physicalName="orders_runtime", - properties=[ - SchemaProperty( - name="id", - physicalName="order_id", - type="integer", - physicalType="BIGINT", - required=True, - ) - ], - ) - ], - ) - released_contract_json = json.dumps( - contract.model_dump(mode="json", by_alias=True, exclude_none=True), - sort_keys=True, - separators=(",", ":"), - ) - fields = { - "contract_id": "orders-product", - "decision_id": "decision:test", - "change_set_id": "change-set:test", - "release_plan_id": "release-plan:test", - "version_resolution_id": "version-resolution:test", - "release_revision_ref": "rev:released", - "selected_version": "1.2.0", - "authorization_id": "authorization:test", - "released_contract_json": released_contract_json, - } - return AppliedContractRelease( - applied_release_id=compute_applied_release_id(**fields), - **fields, - ) +SOURCE_REFERENCE = "https://workspace.example" -def test_deployment_parser_exposes_four_explicit_phases() -> None: +def test_deployment_parser_exposes_candidate_and_finalized_release_modes() -> None: parser = cli._build_parser() - plan = parser.parse_args( + candidate = parser.parse_args( [ "deployment", - "plan", - "--release", - "release.json", - "--platform", - "databricks", - "--runtime", - "main.silver", - "--source-reference", - SOURCE_REFERENCE, + "assess", + "--base", + "base.yaml", + "--candidate", + "candidate.yaml", + "--base-revision-ref", + "git:base", + "--candidate-revision-ref", + "git:candidate", + "--server", + "development", + "--bundle-out", + "candidate.bundle.json", + "--output", + "json", ] ) - preview = parser.parse_args( - ["deployment", "preview", "--plan", "plan.json"] - ) - execute = parser.parse_args( + release = parser.parse_args( [ "deployment", - "execute", - "--plan", - "plan.json", - "--preview", - "preview.json", - "--authorization", - "authorization.json", + "assess", + "--release", + "contract-release.json", + "--server", + "production", + "--bundle-out", + "release.bundle.json", ] ) - verify = parser.parse_args( - ["deployment", "verify", "--plan", "plan.json", "--output", "json"] - ) - - assert plan.deployment_command == "plan" - assert plan.source_reference == SOURCE_REFERENCE - assert preview.deployment_command == "preview" - assert not hasattr(preview, "warehouse_id") - assert execute.deployment_command == "execute" - assert execute.warehouse_id is None - assert verify.deployment_command == "verify" - assert verify.output == "json" - - -def test_plan_command_outputs_canonical_deployment_plan(tmp_path) -> None: - release_path = tmp_path / "release.json" - release = _release() - release_path.write_text(release.model_dump_json(), encoding="utf-8") - args = cli._build_parser().parse_args( + deploy = parser.parse_args( [ "deployment", - "plan", - "--release", - str(release_path), - "--platform", - "databricks", - "--runtime", - "main.silver", - "--source-reference", - SOURCE_REFERENCE, - "--server", - "production", + "deploy", + "--bundle", + "bundle.json", + "--warehouse-id", + "warehouse-1", + "--operational-history", + "sqlite:///history.db", + "--output", + "json", ] ) - result = deployment_cmd.run_deployment_plan(args) - payload = json.loads(result.output) + assert candidate.deployment_command == "assess" + assert candidate.release_id is None + assert candidate.server == "development" + assert candidate.bundle_out == "candidate.bundle.json" + + assert release.deployment_command == "assess" + assert release.release == "contract-release.json" + assert release.release_id is None + assert release.repository_root == "." + assert release.server == "production" - assert result.outcome is ProcessOutcome.SUCCESS - assert payload["applied_release_id"] == release.applied_release_id - assert payload["target"] == { - "platform": "databricks", - "runtime_target": "main.silver", - "source_reference": SOURCE_REFERENCE, - "server_name": "production", - } - assert payload["actions"][0]["governed_asset"] == "orders" - assert payload["actions"][0]["physical_name"] == "orders_runtime" + assert deploy.deployment_command == "deploy" + assert deploy.bundle == "bundle.json" + assert deploy.warehouse_id == "warehouse-1" + assert deploy.operational_history == "sqlite:///history.db" + assert deploy.output == "json" -def test_invalid_artifact_is_a_validation_failure(tmp_path) -> None: - invalid = tmp_path / "invalid.json" - invalid.write_text("{}", encoding="utf-8") +@pytest.mark.parametrize( + "unsupported_args", + [ + ["deployment", "plan"], + ["deployment", "preview"], + ["deployment", "execute"], + ["deployment", "verify"], + ["deployment", "approve"], + ["deployment", "assess", "--release"], + ], +) +def test_only_bundle_deployment_cli_surface_is_public(unsupported_args) -> None: + parser = cli._build_parser() - with pytest.raises(ValidationError, match="Invalid DeploymentPlan artifact"): - deployment_cmd._load_model(str(invalid), deployment_cmd.DeploymentPlan) + with pytest.raises(SystemExit) as exc: + parser.parse_args(unsupported_args) + + assert exc.value.code == 2 @pytest.mark.parametrize( @@ -167,7 +111,7 @@ def test_invalid_artifact_is_a_validation_failure(tmp_path) -> None: (ProcessOutcome.RUNTIME_INDETERMINATE, 7), ], ) -def test_main_preserves_verification_outcome_semantics( +def test_main_preserves_bundle_deploy_outcome_semantics( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], outcome: ProcessOutcome, @@ -175,81 +119,151 @@ def test_main_preserves_verification_outcome_semantics( ) -> None: monkeypatch.setattr( deployment_cmd, - "run_deployment_verify", - lambda args: DeploymentCommandResult(output="verification", outcome=outcome), + "run_deployment_deploy", + lambda args: DeploymentCommandResult(output="deployment", outcome=outcome), ) monkeypatch.setattr( sys, "argv", - ["semapact", "deployment", "verify", "--plan", "plan.json"], + ["semapact", "deployment", "deploy", "--bundle", "bundle.json"], ) assert cli.main() == expected_exit - assert capsys.readouterr().out.strip() == "verification" + assert capsys.readouterr().out.strip() == "deployment" + + +def test_candidate_assessment_requires_candidate_arguments() -> None: + args = type( + "Args", + (), + { + "base": None, + "candidate": None, + "base_revision_ref": None, + "candidate_revision_ref": None, + }, + )() + + with pytest.raises(ValidationError, match="Candidate deployment assessment"): + deployment_cmd._require_candidate_args(args) + + +def test_release_assessment_rejects_candidate_arguments() -> None: + args = type( + "Args", + (), + { + "base": "base.yaml", + "candidate": None, + "base_revision_ref": None, + "candidate_revision_ref": None, + }, + )() + + with pytest.raises(ValidationError, match="cannot be combined"): + deployment_cmd._reject_candidate_args_for_release(args) + + +def test_assessment_source_reference_prefers_contract_server_host() -> None: + server = Server.model_validate( + { + "server": "production", + "type": "databricks", + "host": SOURCE_REFERENCE, + "catalog": "main", + "schema": "silver", + } + ) + assert ( + deployment_cmd._assessment_source_reference(server, None) + == SOURCE_REFERENCE + ) -def test_verify_command_uses_unified_deployment_adapter( - tmp_path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - plan = build_deployment_plan( - _release(), - DeploymentTarget( - platform="databricks", - runtime_target="main.silver", - source_reference=SOURCE_REFERENCE, - ), +def test_assessment_source_reference_requires_cli_fallback_without_server() -> None: + assert ( + deployment_cmd._assessment_source_reference(None, SOURCE_REFERENCE) + == SOURCE_REFERENCE ) - plan_path = tmp_path / "plan.json" - plan_path.write_text(plan.model_dump_json(), encoding="utf-8") - class _Adapter(DeploymentAdapter): - key = "databricks" + with pytest.raises(ValidationError, match="provide --source-reference"): + deployment_cmd._assessment_source_reference(None, None) + - def __init__(self) -> None: - self.verify_calls = 0 +def test_assessment_source_reference_cannot_override_contract_host() -> None: + server = Server.model_validate( + { + "server": "production", + "type": "databricks", + "host": SOURCE_REFERENCE, + "catalog": "main", + "schema": "silver", + } + ) - def validate(self, plan): - raise AssertionError("CLI should call the service entrypoint") + with pytest.raises(ValidationError, match="cannot override"): + deployment_cmd._assessment_source_reference( + server, + "https://other-workspace.example", + ) - def preview(self, plan): - raise AssertionError("preview should not be called") - def execute(self, plan, preview, authorization): - raise AssertionError("execute should not be called") +def test_operational_history_cli_override_takes_precedence( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from semapact.core.config import config_manager - def verify(self, plan): - self.verify_calls += 1 - return ReconciliationResult( - contract_id=plan.contract_id, - contract_version=plan.selected_version, - observation_source_identifier=SOURCE_REFERENCE, - observation_fingerprint="obs-v2:sha256:test", - ) + monkeypatch.setattr( + config_manager, + "get", + lambda *args, **kwargs: { + "backend": "sqlite", + "path": ".semapact/from-config.db", + }, + ) - adapter = _Adapter() + assert ( + deployment_cmd._resolve_operational_history_uri( + "sqlite:///explicit.db" + ) + == "sqlite:///explicit.db" + ) - import semapact.platforms.runtime_registry as runtime_registry + +def test_operational_history_falls_back_to_typed_config( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from semapact.core.config import config_manager monkeypatch.setattr( - runtime_registry, - "create_deployment_adapter", - lambda platform: adapter, + config_manager, + "get", + lambda *args, **kwargs: { + "backend": "sqlite", + "path": ".semapact/operational.db", + }, ) - args = cli._build_parser().parse_args( - [ - "deployment", - "verify", - "--plan", - str(plan_path), - "--output", - "json", - ] + assert ( + deployment_cmd._resolve_operational_history_uri(None) + == "sqlite:///.semapact/operational.db" + ) + + +def test_invalid_operational_history_config_fails_closed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from semapact.core.config import config_manager + + monkeypatch.setattr( + config_manager, + "get", + lambda *args, **kwargs: { + "backend": "git", + "path": ".semapact/history", + }, ) - result = deployment_cmd.run_deployment_verify(args) - assert result.outcome is ProcessOutcome.SUCCESS - assert adapter.verify_calls == 1 - assert json.loads(result.output)["status"] == "IN_SYNC" + with pytest.raises(ValidationError, match="history.operational"): + deployment_cmd._resolve_operational_history_uri(None) diff --git a/tests/interfaces/test_reconcile_cmd.py b/tests/interfaces/test_reconcile_cmd.py index 0ef576d0..48581c0a 100644 --- a/tests/interfaces/test_reconcile_cmd.py +++ b/tests/interfaces/test_reconcile_cmd.py @@ -18,7 +18,7 @@ RuntimeDriftStatus, RuntimeReasonCode, ) -from semapact.services.reconciliation_service import RuntimeReconciliation +from semapact.application.models.reconciliation import RuntimeReconciliation def _analysis() -> RuntimeReconciliation: diff --git a/tests/interfaces/test_release_plan_cmd.py b/tests/interfaces/test_release_plan_cmd.py index f372cc01..a9a18ffc 100644 --- a/tests/interfaces/test_release_plan_cmd.py +++ b/tests/interfaces/test_release_plan_cmd.py @@ -22,8 +22,6 @@ def test_release_plan_parser_requires_explicit_revision_refs() -> None: "git:base-123", "--candidate-revision-ref", "git:candidate-456", - "--effective-date", - "2026-09-11", ] ) @@ -49,8 +47,6 @@ def test_release_plan_parser_accepts_git_authority_reference() -> None: "git:candidate", "--authority-reference", "v2.0.0", - "--effective-date", - "2026-09-11", ] ) @@ -83,8 +79,6 @@ def test_main_routes_release_plan_to_release_command_adapter( "git:base", "--candidate-revision-ref", "git:candidate", - "--effective-date", - "2026-09-11", ], ) @@ -92,11 +86,92 @@ def test_main_routes_release_plan_to_release_command_adapter( assert json.loads(capsys.readouterr().out) == expected -def test_legacy_release_helpers_are_labeled_as_compatibility_paths() -> None: - help_text = cli._build_parser().format_help() - release_parser = cli._build_parser()._subparsers._group_actions[0].choices["release"] - release_help = release_parser.format_help() +@pytest.mark.parametrize( + "unsupported_command", + ["prepare", "create-pr"], +) +def test_removed_release_commands_are_not_registered(unsupported_command: str) -> None: + parser = cli._build_parser() - assert "release" in help_text - assert "Compatibility helper" in release_help - assert "Compatibility Git workflow" in release_help + with pytest.raises(SystemExit) as exc: + parser.parse_args(["release", unsupported_command]) + + assert exc.value.code == 2 + + +def test_release_parser_exposes_assess_approve_finalize() -> None: + parser = cli._build_parser() + + assess = parser.parse_args( + [ + "release", + "assess", + "--base", + "base.yaml", + "--candidate", + "candidate.yaml", + "--base-revision-ref", + "git:base", + "--candidate-revision-ref", + "git:candidate", + "--bundle-out", + "release.bundle.json", + ] + ) + approve = parser.parse_args( + [ + "release", + "approve", + "--bundle", + "release.bundle.json", + "--actor-reference", + "github-environment:contract-release", + "--recorded-at", + "2026-09-20T10:00:00+10:00", + ] + ) + finalize = parser.parse_args( + [ + "release", + "finalize", + "--bundle", + "release.bundle.json", + "--output-contract", + "contract.yaml", + ] + ) + + assert assess.release_command == "assess" + assert assess.bundle_out == "release.bundle.json" + assert approve.release_command == "approve" + assert approve.repository_root == "." + assert finalize.release_command == "finalize" + assert finalize.output_contract == "contract.yaml" + + +def test_main_routes_release_assess_to_release_command_adapter( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + expected = {"bundle_digest": "sha256:test"} + monkeypatch.setattr(release_cmd, "run_release_assess", lambda args: expected) + monkeypatch.setattr( + sys, + "argv", + [ + "semapact", + "release", + "assess", + "--base", + "base.yaml", + "--candidate", + "candidate.yaml", + "--base-revision-ref", + "git:base", + "--candidate-revision-ref", + "git:candidate", + ], + ) + + assert cli.main() == 0 + assert json.loads(capsys.readouterr().out) == expected diff --git a/tests/interfaces/test_release_workflow_cmd.py b/tests/interfaces/test_release_workflow_cmd.py new file mode 100644 index 00000000..7825f19a --- /dev/null +++ b/tests/interfaces/test_release_workflow_cmd.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +from types import SimpleNamespace + +from open_data_contract_standard.model import ( + OpenDataContractStandard, + SchemaObject, + SchemaProperty, +) + +from semapact.interfaces.commands.release_cmd import ( + run_release_approve, + run_release_assess, + run_release_finalize, +) +from semapact.platforms.git import GitWorkingTreeHistoryRepository +from semapact.utils.yaml_utils import dump_yaml + + +def _contract(*, include_created_at: bool = False) -> OpenDataContractStandard: + properties = [ + SchemaProperty( + name="id", + logicalType="string", + physicalType="varchar(255)", + required=True, + ) + ] + if include_created_at: + properties.append( + SchemaProperty( + name="created_at", + logicalType="timestamp", + physicalType="timestamp", + required=False, + ) + ) + return OpenDataContractStandard( + apiVersion="v3.1.0", + kind="DataContract", + id="orders-product", + name="Orders", + version="1.2.3", + status="active", + schema=[SchemaObject(name="orders", properties=properties)], + ) + + +def test_release_finalize_materializes_selected_version_and_history(tmp_path) -> None: + base = tmp_path / "base.yaml" + candidate = tmp_path / "contract.yaml" + bundle = tmp_path / "release.bundle.json" + released = tmp_path / "released.yaml" + release_artifact = tmp_path / "contract-release.json" + approval_artifact = tmp_path / "approval.json" + dump_yaml(_contract(), base) + dump_yaml(_contract(include_created_at=True), candidate) + + run_release_assess( + SimpleNamespace( + base=str(base), + candidate=str(candidate), + base_revision_ref="git:base", + candidate_revision_ref="git:candidate", + authority_reference=None, + runtime_context="auto", + bundle_out=str(bundle), + ) + ) + run_release_approve( + SimpleNamespace( + bundle=str(bundle), + actor_reference="github-environment:contract-release", + recorded_at="2026-09-20T10:00:00+10:00", + comment=None, + repository_root=str(tmp_path), + approval_out=str(approval_artifact), + ) + ) + result = run_release_finalize( + SimpleNamespace( + bundle=str(bundle), + approval=str(approval_artifact), + output_contract=str(released), + release_out=str(release_artifact), + repository_root=str(tmp_path), + ) + ) + + materialized = OpenDataContractStandard.from_file(str(released)) + assert str(materialized.version) == "1.3.0" + + repository = GitWorkingTreeHistoryRepository(tmp_path) + record = repository.get_contract_release(result["contractReleaseId"]) + assert record.contract_version == "1.3.0" + assert record.source_revision_ref == "git:candidate" + assert result["sourceRevisionRef"] == "git:candidate" + assert record.released_contract_json + assert approval_artifact.is_file() + assert release_artifact.is_file() + assert release_artifact.read_text(encoding="utf-8").strip() diff --git a/tests/test_application_architecture.py b/tests/test_application_architecture.py index df72b707..d2a97823 100644 --- a/tests/test_application_architecture.py +++ b/tests/test_application_architecture.py @@ -8,17 +8,6 @@ from semapact.application.services.reconciliation import ReconciliationService from semapact.application.services.release_planning import ReleasePlanningService from semapact.application.services.version_authority import VersionAuthorityService -from semapact.services import ( - DeploymentService as LegacyDeploymentService, - GovernanceAnalysis as LegacyGovernanceAnalysis, - GovernanceProposal as LegacyGovernanceProposal, - GovernanceService as LegacyGovernanceService, - ReconciliationService as LegacyReconciliationService, - ReleasePlanningResult as LegacyReleasePlanningResult, - ReleasePlanningService as LegacyReleasePlanningService, - RuntimeReconciliation as LegacyRuntimeReconciliation, - VersionAuthorityService as LegacyVersionAuthorityService, -) def test_application_result_models_have_explicit_model_ownership() -> None: @@ -28,13 +17,9 @@ def test_application_result_models_have_explicit_model_ownership() -> None: assert ReleasePlanningResult.__module__ == "semapact.application.models.release" -def test_legacy_services_package_is_compatibility_only() -> None: - assert LegacyGovernanceAnalysis is GovernanceAnalysis - assert LegacyGovernanceProposal is GovernanceProposal - assert LegacyRuntimeReconciliation is RuntimeReconciliation - assert LegacyReleasePlanningResult is ReleasePlanningResult - assert LegacyGovernanceService is GovernanceService - assert LegacyReconciliationService is ReconciliationService - assert LegacyDeploymentService is DeploymentService - assert LegacyReleasePlanningService is ReleasePlanningService - assert LegacyVersionAuthorityService is VersionAuthorityService +def test_application_services_have_single_canonical_package() -> None: + assert DeploymentService.__module__ == "semapact.application.services.deployment" + assert GovernanceService.__module__ == "semapact.application.services.governance" + assert ReconciliationService.__module__ == "semapact.application.services.reconciliation" + assert ReleasePlanningService.__module__ == "semapact.application.services.release_planning" + assert VersionAuthorityService.__module__ == "semapact.application.services.version_authority" diff --git a/tests/test_architecture_consolidation.py b/tests/test_architecture_consolidation.py index 6a27e5ef..2f1a1b7b 100644 --- a/tests/test_architecture_consolidation.py +++ b/tests/test_architecture_consolidation.py @@ -4,51 +4,41 @@ import uuid from datetime import date -import pytest from open_data_contract_standard.model import ( OpenDataContractStandard, SchemaObject, SchemaProperty, ) -from semapact.change_context import ChangeContext +from semapact.application.services.release_workflow import ( + ReleaseFinalizer, + ReleaseWorkflowService, +) from semapact.contractops import ( - AuthorizationReason, - ReviewAuthorizationEvidence, - ReviewEvidenceAction, + VersionAuthority, VersionAuthorityConfig, - apply_contract_release, - authorize_contract_operation, build_change_set_from_decision, build_release_plan, resolve_release_version, ) -from semapact.core.release import ( - classify_contract_change as legacy_classify_contract_change, - normalize_semver as legacy_normalize_semver, -) from semapact.deployment import ( DeploymentTarget, - authorize_deployment, - build_deployment_plan, -) -from semapact.exceptions import ReleaseValidationError -from semapact.governance import DecisionResult, evaluate_governance_decision -from semapact.governance.change_classification import classify_contract_change -from semapact.governance.gate import GovernanceOperation -from semapact.observation import RuntimeAssetSpec as ObservationRuntimeAssetSpec -from semapact.reconciliation.binding import ( - runtime_asset_specs_from_contract as legacy_runtime_asset_specs_from_contract, + build_contract_release_deployment_source, + build_deployment_plan_from_source, ) +from semapact.governance import evaluate_governance_decision from semapact.runtime import RuntimeAssetSpec, runtime_asset_specs_from_contract from semapact.utils.deterministic import canonical_compact_json, deterministic_uuid5 from semapact.versioning import normalize_semver -CONTEXT = ChangeContext(effective_date=date(2026, 9, 10)) -def _contract(*, include_created_at: bool = False) -> OpenDataContractStandard: +def _contract( + *, + name: str = "Orders", + include_created_at: bool = False, +) -> OpenDataContractStandard: properties = [ SchemaProperty( name="id", @@ -70,155 +60,84 @@ def _contract(*, include_created_at: bool = False) -> OpenDataContractStandard: apiVersion="v3.1.0", kind="DataContract", id="orders-product", - name="Orders", + name=name, version="1.0.0", status="active", schema=[SchemaObject(name="orders", properties=properties)], ) -def _review_release_chain(): - base = _contract() - candidate = _contract(include_created_at=True) - decision = evaluate_governance_decision(base, candidate, context=CONTEXT) - assert decision.decision is DecisionResult.REVIEW - - change_set = build_change_set_from_decision( - decision, +def _finalized_allow_release(): + workflow = ReleaseWorkflowService() + bundle = workflow.assess( + _contract(name="Orders old"), + _contract(name="Orders new"), base_revision_ref="rev:base", candidate_revision_ref="rev:candidate", ) - release_plan = build_release_plan(change_set, decision) - version_resolution = resolve_release_version( - release_plan, - current_version="1.0.0", - config=VersionAuthorityConfig(), - ) - apply_evidence = ReviewAuthorizationEvidence( - evidence_reference="approval:apply", - decision_id=decision.decision_id, - change_set_id=change_set.change_set_id, - release_plan_id=release_plan.release_plan_id, - version_resolution_id=version_resolution.version_resolution_id, - operation=GovernanceOperation.APPLY, - action=ReviewEvidenceAction.APPROVE, - ) - apply_authorization = authorize_contract_operation( - decision, - change_set, - release_plan, - version_resolution, - GovernanceOperation.APPLY, - evidence=apply_evidence, - ) - release = apply_contract_release( - candidate, - candidate_revision_ref="rev:candidate", - decision=decision, - change_set=change_set, - release_plan=release_plan, - version_resolution=version_resolution, - authorization=apply_authorization, - ) - return decision, change_set, release_plan, version_resolution, release + return ReleaseFinalizer().finalize(bundle) -def test_deterministic_helper_preserves_existing_compact_uuid_formula() -> None: +def test_deterministic_helper_uses_canonical_compact_uuid_formula() -> None: namespace = uuid.UUID("3ea0f6d8-28ca-4bb4-94f5-ea1f0f48cb84") payload = {"z": "é", "a": [2, 1], "nested": {"b": True}} - - legacy_json = json.dumps( + encoded = json.dumps( payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False, ) - legacy_id = str(uuid.uuid5(namespace, legacy_json)) - - assert canonical_compact_json(payload) == legacy_json - assert deterministic_uuid5(namespace, payload) == legacy_id + assert canonical_compact_json(payload) == encoded + assert deterministic_uuid5(namespace, payload) == str(uuid.uuid5(namespace, encoded)) -def test_legacy_release_imports_delegate_to_canonical_owners() -> None: - assert legacy_normalize_semver is normalize_semver - assert legacy_classify_contract_change is classify_contract_change - assert legacy_normalize_semver("v1.2.3") == "1.2.3" - - -def test_reconciliation_asset_projection_remains_compatible_but_neutral() -> None: - assert legacy_runtime_asset_specs_from_contract is runtime_asset_specs_from_contract - assert ObservationRuntimeAssetSpec is RuntimeAssetSpec +def test_runtime_asset_projection_has_one_canonical_owner() -> None: specs = runtime_asset_specs_from_contract(_contract()) - assert specs == (RuntimeAssetSpec(governed_asset="orders", physical_name="orders"),) + + assert specs == ( + RuntimeAssetSpec(governed_asset="orders", physical_name="orders"), + ) -def test_review_deploy_authorization_is_bound_to_exact_deployment_plan() -> None: - decision, change_set, release_plan, version_resolution, release = _review_release_chain() - production_plan = build_deployment_plan( - release, +def test_finalized_release_projects_to_target_specific_canonical_plan() -> None: + release = _finalized_allow_release() + source = build_contract_release_deployment_source(release) + production = build_deployment_plan_from_source( + source, DeploymentTarget( platform="databricks", runtime_target="main.production", source_reference="https://production-workspace.example", ), ) - - evidence = ReviewAuthorizationEvidence( - evidence_reference="approval:deploy-production", - decision_id=decision.decision_id, - change_set_id=change_set.change_set_id, - release_plan_id=release_plan.release_plan_id, - version_resolution_id=version_resolution.version_resolution_id, - operation=GovernanceOperation.DEPLOY, - action=ReviewEvidenceAction.APPROVE, - scope_reference=production_plan.deployment_plan_id, - ) - contractops_authorization = authorize_contract_operation( - decision, - change_set, - release_plan, - version_resolution, - GovernanceOperation.DEPLOY, - evidence=evidence, - ) - - assert contractops_authorization.allowed is True - assert contractops_authorization.reason is AuthorizationReason.ALLOWED_BY_REVIEW - assert contractops_authorization.scope_reference == production_plan.deployment_plan_id - - deployment_authorization = authorize_deployment( - production_plan, - release, - contractops_authorization, - ) - assert deployment_authorization.allowed is True - assert deployment_authorization.deployment_plan_id == production_plan.deployment_plan_id - - staging_plan = build_deployment_plan( - release, + staging = build_deployment_plan_from_source( + source, DeploymentTarget( platform="databricks", runtime_target="main.staging", source_reference="https://staging-workspace.example", ), ) - with pytest.raises(ReleaseValidationError, match="not scoped to this DeploymentPlan"): - authorize_deployment(staging_plan, release, contractops_authorization) + assert production.source_snapshot_id == source.source_snapshot_id + assert staging.source_snapshot_id == source.source_snapshot_id + assert production.deployment_plan_id != staging.deployment_plan_id -def test_same_runtime_namespace_on_another_source_requires_distinct_plan() -> None: - _, _, _, _, release = _review_release_chain() - workspace_a = build_deployment_plan( - release, + +def test_same_runtime_namespace_on_another_source_has_distinct_plan_identity() -> None: + release = _finalized_allow_release() + source = build_contract_release_deployment_source(release) + workspace_a = build_deployment_plan_from_source( + source, DeploymentTarget( platform="databricks", runtime_target="main.production", source_reference="https://workspace-a.example", ), ) - workspace_b = build_deployment_plan( - release, + workspace_b = build_deployment_plan_from_source( + source, DeploymentTarget( platform="databricks", runtime_target="main.production", @@ -229,33 +148,35 @@ def test_same_runtime_namespace_on_another_source_requires_distinct_plan() -> No assert workspace_a.deployment_plan_id != workspace_b.deployment_plan_id -def test_publish_authorization_cannot_be_reused_for_runtime_deploy() -> None: - decision, change_set, release_plan, version_resolution, release = _review_release_chain() - plan = build_deployment_plan( - release, - DeploymentTarget( - platform="databricks", - runtime_target="main.production", - source_reference="https://production-workspace.example", - ), +def test_version_authorities_preserve_required_bump() -> None: + base = _contract() + candidate = _contract(include_created_at=True) + decision = evaluate_governance_decision(base, candidate) + change_set = build_change_set_from_decision( + decision, + base_revision_ref="rev:base", + candidate_revision_ref="rev:candidate", ) - evidence = ReviewAuthorizationEvidence( - evidence_reference="approval:publish", - decision_id=decision.decision_id, - change_set_id=change_set.change_set_id, - release_plan_id=release_plan.release_plan_id, - version_resolution_id=version_resolution.version_resolution_id, - operation=GovernanceOperation.PUBLISH, - action=ReviewEvidenceAction.APPROVE, + release_plan = build_release_plan(change_set, decision) + + semapact_resolution = resolve_release_version( + release_plan, + current_version="1.0.0", + config=VersionAuthorityConfig(authority=VersionAuthority.SEMAPACT), ) - publish_authorization = authorize_contract_operation( - decision, - change_set, + git_resolution = resolve_release_version( release_plan, - version_resolution, - GovernanceOperation.PUBLISH, - evidence=evidence, + current_version="1.0.0", + config=VersionAuthorityConfig( + authority=VersionAuthority.GIT, + tag_pattern="v{version}", + ), + authority_reference="v1.2.0", ) - with pytest.raises(ReleaseValidationError, match="DEPLOY authorization"): - authorize_deployment(plan, release, publish_authorization) + assert decision.required_version_bump == "minor" + assert semapact_resolution.required_version_bump == "minor" + assert git_resolution.required_version_bump == "minor" + assert semapact_resolution.selected_version == "1.1.0" + assert git_resolution.selected_version == "1.2.0" + assert normalize_semver("v1.2.3") == "1.2.3" diff --git a/tests/test_change_context.py b/tests/test_change_context.py index f4f8629b..a9f76dfa 100644 --- a/tests/test_change_context.py +++ b/tests/test_change_context.py @@ -13,7 +13,8 @@ ) from pydantic import ValidationError as PydanticValidationError -from semapact.governance import ChangeContext, evaluate_governance_decision +from semapact.change_context import ChangeContext +from semapact.governance import evaluate_governance_decision from semapact.lifecycle.merge_engine import ContractMergeEngine @@ -82,45 +83,16 @@ def test_change_context_is_immutable_and_serializes_effective_date() -> None: context.effective_date = date(2026, 8, 14) # type: ignore[misc] -def test_same_inputs_and_context_produce_identical_decision() -> None: +def test_governance_decision_is_independent_of_business_effective_date() -> None: base = _contract() candidate = _contract(include_legacy_property=False) - context = ChangeContext(effective_date=date(2026, 8, 13)) - first = evaluate_governance_decision(base, candidate, context=context) - second = evaluate_governance_decision(base, candidate, context=context) + first = evaluate_governance_decision(base, candidate) + second = evaluate_governance_decision(base, candidate) assert first == second assert first.decision_id == second.decision_id - assert first.model_dump(mode="json") == second.model_dump(mode="json") - assert first.context == context - - -def test_governance_evaluation_rejects_missing_upstream_context() -> None: - base = _contract() - candidate = _contract(include_legacy_property=False) - - with pytest.raises(TypeError): - evaluate_governance_decision(base, candidate) # type: ignore[call-arg] - - -def test_effective_date_is_part_of_decision_identity_when_explicit() -> None: - base = _contract() - candidate = _contract(include_legacy_property=False) - - first = evaluate_governance_decision( - base, - candidate, - context=ChangeContext(effective_date=date(2026, 8, 13)), - ) - second = evaluate_governance_decision( - base, - candidate, - context=ChangeContext(effective_date=date(2026, 8, 14)), - ) - - assert first.decision == second.decision - assert first.decision_id != second.decision_id + assert "context" not in first.model_dump(mode="json") def test_auto_deprecation_uses_explicit_effective_date() -> None: diff --git a/tests/test_change_set.py b/tests/test_change_set.py index 87c1fd5a..904fcf0f 100644 --- a/tests/test_change_set.py +++ b/tests/test_change_set.py @@ -1,7 +1,5 @@ from __future__ import annotations -from datetime import date - import pytest from open_data_contract_standard.model import ( OpenDataContractStandard, @@ -10,7 +8,6 @@ ) from pydantic import ValidationError as PydanticValidationError -from semapact.change_context import ChangeContext from semapact.contractops import build_change_set, build_change_set_from_decision from semapact.governance import evaluate_governance_decision from semapact.lifecycle.changes import ( @@ -58,7 +55,6 @@ def _contract(physical_type: str) -> OpenDataContractStandard: def test_changeset_identity_is_deterministic_and_change_order_independent() -> None: - context = ChangeContext(effective_date=date(2026, 9, 9)) id_change = _change("id", before="STRING", after="BIGINT") amount_change = _change("amount", before="DECIMAL(10,2)", after="DECIMAL(18,2)") @@ -67,14 +63,12 @@ def test_changeset_identity_is_deterministic_and_change_order_independent() -> N base_revision_ref="git:abc123", candidate_revision_ref="git:def456", changes=(id_change, amount_change), - context=context, ) second = build_change_set( contract_id="orders-product", base_revision_ref="git:abc123", candidate_revision_ref="git:def456", changes=(amount_change, id_change), - context=context, ) assert first == second @@ -83,7 +77,7 @@ def test_changeset_identity_is_deterministic_and_change_order_independent() -> N assert first.model_dump(mode="json") == second.model_dump(mode="json") -def test_changeset_identity_changes_with_revision_or_governance_context() -> None: +def test_changeset_identity_changes_with_exact_revision() -> None: change = _change("id", before="STRING", after="BIGINT") base_args = { "contract_id": "orders-product", @@ -94,31 +88,21 @@ def test_changeset_identity_changes_with_revision_or_governance_context() -> Non first = build_change_set( **base_args, - context=ChangeContext(effective_date=date(2026, 9, 9)), ) different_candidate = build_change_set( **{**base_args, "candidate_revision_ref": "git:ghi789"}, - context=ChangeContext(effective_date=date(2026, 9, 9)), - ) - different_context = build_change_set( - **base_args, - context=ChangeContext(effective_date=date(2026, 9, 10)), ) - assert first.change_set_id != different_candidate.change_set_id - assert first.change_set_id != different_context.change_set_id def test_changeset_identity_covers_provenance_without_affecting_governance() -> None: change = _change("id", before="STRING", after="BIGINT") - context = ChangeContext(effective_date=date(2026, 9, 9)) first = build_change_set( contract_id="orders-product", base_revision_ref="git:abc123", candidate_revision_ref="git:def456", changes=(change,), - context=context, source="cli", actor_reference="user:alice", ) @@ -127,14 +111,12 @@ def test_changeset_identity_covers_provenance_without_affecting_governance() -> base_revision_ref="git:abc123", candidate_revision_ref="git:def456", changes=(change,), - context=context, source="api", actor_reference="service:ci", ) assert first.change_set_id != second.change_set_id assert first.changes == second.changes - assert first.context == second.context assert first.source == "cli" assert second.source == "api" assert first.actor_reference == "user:alice" @@ -147,18 +129,16 @@ def test_changeset_is_immutable() -> None: base_revision_ref="git:abc123", candidate_revision_ref="git:def456", changes=(), - context=ChangeContext(effective_date=date(2026, 9, 9)), ) with pytest.raises(PydanticValidationError): change_set.contract_id = "other" # type: ignore[misc] -def test_changeset_from_decision_reuses_authoritative_changes_and_context() -> None: +def test_changeset_from_decision_reuses_authoritative_changes_without_date_context() -> None: base = _contract("STRING") candidate = _contract("BIGINT") - context = ChangeContext(effective_date=date(2026, 9, 9)) - decision = evaluate_governance_decision(base, candidate, context=context) + decision = evaluate_governance_decision(base, candidate) change_set = build_change_set_from_decision( decision, @@ -167,7 +147,8 @@ def test_changeset_from_decision_reuses_authoritative_changes_and_context() -> N ) assert change_set.contract_id == decision.contract_id - assert change_set.context == decision.context + assert not hasattr(change_set, "context") + assert not hasattr(decision, "context") assert change_set.changes == decision.changes assert change_set.base_revision_ref == "git:abc123" assert change_set.candidate_revision_ref == "git:def456" diff --git a/tests/test_changeset_history.py b/tests/test_changeset_history.py index 1abd3157..b52cb9cd 100644 --- a/tests/test_changeset_history.py +++ b/tests/test_changeset_history.py @@ -46,13 +46,12 @@ def _contract(*, name: str) -> OpenDataContractStandard: ) -def _proposal(*, effective_date: str = "2026-09-12"): +def _proposal(*, candidate_name: str = "Orders new"): base_revision = build_contract_revision(_contract(name="Orders old")) - candidate_revision = build_contract_revision(_contract(name="Orders new")) + candidate_revision = build_contract_revision(_contract(name=candidate_name)) proposal = GovernanceService().evaluate_proposal( base_revision.contract, candidate_revision.contract, - effective_date=effective_date, base_revision_ref=base_revision.revision_id, candidate_revision_ref=candidate_revision.revision_id, source="test", @@ -108,12 +107,12 @@ def test_records_exact_proposal_chain_without_recomputing_domain_artifacts( assert link.decision_id == proposal.decision.decision_id -def test_changeset_context_round_trips_and_different_contexts_do_not_overwrite( +def test_repeated_proposal_evaluation_reuses_same_history_identity( tmp_path: Path, ) -> None: service, backend = _service(tmp_path) - first, first_base, first_candidate = _proposal(effective_date="2026-09-12") - second, second_base, second_candidate = _proposal(effective_date="2026-09-13") + first, first_base, first_candidate = _proposal() + second, second_base, second_candidate = _proposal() service.record_proposal( first, @@ -126,10 +125,10 @@ def test_changeset_context_round_trips_and_different_contexts_do_not_overwrite( candidate_revision=second_candidate, ) - assert first.change_set.change_set_id != second.change_set.change_set_id - assert backend.get_change_set(first.change_set.change_set_id).context == first.change_set.context - assert backend.get_change_set(second.change_set.change_set_id).context == second.change_set.context - assert len(backend.list_change_sets("orders-product")) == 2 + assert first.change_set.change_set_id == second.change_set.change_set_id + assert first.decision.decision_id == second.decision.decision_id + assert len(backend.list_change_sets("orders-product")) == 1 + assert len(backend.list_decisions("orders-product")) == 1 def test_rejects_changeset_that_does_not_reference_exact_contract_revisions( @@ -143,7 +142,6 @@ def test_rejects_changeset_that_does_not_reference_exact_contract_revisions( base_revision_ref="git:base", candidate_revision_ref=original.candidate_revision_ref, changes=original.changes, - context=original.context, source=original.source, actor_reference=original.actor_reference, ) @@ -162,13 +160,13 @@ def test_rejects_decision_that_is_not_the_outcome_of_the_changeset( ) -> None: service, _ = _service(tmp_path) proposal, base_revision, candidate_revision = _proposal() - other, _, _ = _proposal(effective_date="2026-09-13") + other, _, _ = _proposal(candidate_name="Orders other") broken_proposal = proposal.__class__( change_set=proposal.change_set, decision=other.decision, ) - with pytest.raises(ValueError, match="context"): + with pytest.raises(ValueError, match="changes"): service.record_proposal( broken_proposal, base_revision=base_revision, diff --git a/tests/test_cicd_workflow_regression.py b/tests/test_cicd_workflow_regression.py new file mode 100644 index 00000000..b040783d --- /dev/null +++ b/tests/test_cicd_workflow_regression.py @@ -0,0 +1,516 @@ +from __future__ import annotations + +import json +from datetime import datetime, timezone + +from open_data_contract_standard.model import ( + OpenDataContractStandard, + SchemaObject, + SchemaProperty, + Server, +) + +from semapact.deployment import ( + DeploymentAdapter, + DeploymentPreview, + NativeOperation, + NativeOperationKind, +) +from semapact.deployment.models import compute_deployment_preview_id +from semapact.interfaces import cli +from semapact.observation import ObservedPlatformState, with_observed_state_fingerprint +from semapact.reconciliation import ReconciliationResult +from semapact.utils.yaml_utils import dump_yaml + + +SOURCE_REFERENCE = "https://workspace.example" + + +class _PreviewAdapter(DeploymentAdapter): + key = "databricks" + + def validate(self, plan) -> None: + pass + + def preview(self, plan) -> DeploymentPreview: + observation = with_observed_state_fingerprint( + ObservedPlatformState( + platform="databricks", + source_identifier=plan.target.source_reference, + assets=(), + captured_at=datetime(2026, 9, 21, tzinfo=timezone.utc), + ) + ) + assert observation.fingerprint is not None + operations = ( + NativeOperation( + kind=NativeOperationKind.CREATE, + governed_asset="orders", + statement="CREATE orders", + ), + ) + return DeploymentPreview( + deployment_preview_id=compute_deployment_preview_id( + deployment_plan_id=plan.deployment_plan_id, + platform="databricks", + runtime_target=plan.target.runtime_target, + source_identifier=observation.source_identifier, + observation_fingerprint=observation.fingerprint, + operations=operations, + ), + deployment_plan_id=plan.deployment_plan_id, + platform="databricks", + runtime_target=plan.target.runtime_target, + source_identifier=observation.source_identifier, + observation_fingerprint=observation.fingerprint, + operations=operations, + ) + + def apply(self, plan, preview) -> None: + raise AssertionError("CI assessment must not mutate runtime") + + def verify(self, plan) -> ReconciliationResult: + raise AssertionError("CI assessment must not verify runtime") + + +class _ExecutionAdapter(DeploymentAdapter): + key = "databricks" + + def __init__(self) -> None: + self.preview_calls = 0 + self.apply_calls = 0 + self.verify_calls = 0 + self.metadata_calls = 0 + + def validate(self, plan) -> None: + pass + + def preview(self, plan) -> DeploymentPreview: + self.preview_calls += 1 + observation = with_observed_state_fingerprint( + ObservedPlatformState( + platform="databricks", + source_identifier=plan.target.source_reference, + assets=(), + captured_at=datetime(2026, 9, 21, 1, tzinfo=timezone.utc), + ) + ) + assert observation.fingerprint is not None + operations = ( + NativeOperation( + kind=NativeOperationKind.NO_OP, + governed_asset="orders", + ), + ) + return DeploymentPreview( + deployment_preview_id=compute_deployment_preview_id( + deployment_plan_id=plan.deployment_plan_id, + platform="databricks", + runtime_target=plan.target.runtime_target, + source_identifier=observation.source_identifier, + observation_fingerprint=observation.fingerprint, + operations=operations, + ), + deployment_plan_id=plan.deployment_plan_id, + platform="databricks", + runtime_target=plan.target.runtime_target, + source_identifier=observation.source_identifier, + observation_fingerprint=observation.fingerprint, + operations=operations, + ) + + def apply(self, plan, preview) -> None: + self.apply_calls += 1 + + def verify(self, plan) -> ReconciliationResult: + self.verify_calls += 1 + return ReconciliationResult( + contract_id=plan.contract_id, + contract_version=plan.contract_version, + observation_source_identifier=plan.target.source_reference, + observation_fingerprint="obs-v1:sha256:verified", + ) + + def project_release_metadata(self, plan, metadata) -> None: + self.metadata_calls += 1 + + +def _server(name: str, schema_name: str) -> Server: + return Server.model_validate( + { + "server": name, + "type": "databricks", + "host": SOURCE_REFERENCE, + "catalog": "main", + "schema": schema_name, + } + ) + + +def _contract( + *, + contract_id: str = "orders-product", + include_created_at: bool = False, +) -> OpenDataContractStandard: + properties = [ + SchemaProperty( + name="id", + logicalType="string", + physicalType="varchar(255)", + required=True, + ) + ] + if include_created_at: + properties.append( + SchemaProperty( + name="created_at", + logicalType="timestamp", + physicalType="timestamp", + required=False, + ) + ) + return OpenDataContractStandard( + apiVersion="v3.1.0", + kind="DataContract", + id=contract_id, + name=contract_id, + version="1.2.3", + status="active", + servers=[ + _server("development", "development"), + _server("production", "production"), + ], + schema=[SchemaObject(name="orders", properties=properties)], + ) + + +def _run_cli(monkeypatch, capsys, *args: str) -> tuple[int, str]: + monkeypatch.setattr("sys.argv", ["semapact", *args]) + code = cli.main() + output = capsys.readouterr().out + return code, output + + +def test_data_product_sample_command_chain_executes_end_to_end( + tmp_path, + monkeypatch, + capsys, +) -> None: + """Exercise the same public artifact handoff used by the sample GitHub pipeline.""" + base_path = dump_yaml(_contract(), tmp_path / "base.yaml") + candidate_path = dump_yaml( + _contract(include_created_at=True), + tmp_path / "contract.yaml", + ) + candidate_bundle = tmp_path / "candidate.deployment.bundle.json" + release_bundle = tmp_path / "release.bundle.json" + released_contract = tmp_path / "released.yaml" + approval_artifact = tmp_path / "release.approval.json" + release_artifact = tmp_path / "contract-release.json" + production_bundle = tmp_path / "production.deployment.bundle.json" + + execution_adapter = _ExecutionAdapter() + + def _adapter_factory(platform: str, **kwargs): + assert platform == "databricks" + if kwargs.get("execution_config") is None: + return _PreviewAdapter() + return execution_adapter + + monkeypatch.setattr( + "semapact.platforms.runtime_registry.create_deployment_adapter", + _adapter_factory, + ) + monkeypatch.setattr( + "semapact.core.config.config_manager.get", + lambda *args, **kwargs: None, + ) + + # Pull-request candidate assessment: read-only and artifact-producing. + code, _ = _run_cli( + monkeypatch, + capsys, + "deployment", + "assess", + "--base", + str(base_path), + "--candidate", + str(candidate_path), + "--base-revision-ref", + "git:base", + "--candidate-revision-ref", + "git:candidate", + "--server", + "development", + "--bundle-out", + str(candidate_bundle), + "--output", + "json", + ) + assert code == 0 + assert candidate_bundle.is_file() + + # Main-branch formal release assessment. + code, release_output = _run_cli( + monkeypatch, + capsys, + "release", + "assess", + "--base", + str(base_path), + "--candidate", + str(candidate_path), + "--base-revision-ref", + "git:base", + "--candidate-revision-ref", + "git:candidate", + "--bundle-out", + str(release_bundle), + ) + assert code == 0 + assert json.loads(release_output)["decision"]["decision"] == "REVIEW" + assert release_bundle.is_file() + + # Protected release environment records approval for the exact bundle. + code, _ = _run_cli( + monkeypatch, + capsys, + "release", + "approve", + "--bundle", + str(release_bundle), + "--actor-reference", + "github-environment:contract-release/run:1", + "--recorded-at", + "2026-09-21T04:00:00Z", + "--approval-out", + str(approval_artifact), + "--repository-root", + str(tmp_path), + ) + assert code == 0 + assert approval_artifact.is_file() + + # Finalization materializes ODCS and emits the immutable ContractRelease artifact. + code, finalize_output = _run_cli( + monkeypatch, + capsys, + "release", + "finalize", + "--bundle", + str(release_bundle), + "--approval", + str(approval_artifact), + "--output-contract", + str(released_contract), + "--release-out", + str(release_artifact), + "--repository-root", + str(tmp_path), + ) + assert code == 0 + finalized = json.loads(finalize_output) + assert finalized["contractVersion"] == "1.3.0" + assert release_artifact.is_file() + assert released_contract.is_file() + + # Production assessment consumes the ContractRelease artifact directly. + code, _ = _run_cli( + monkeypatch, + capsys, + "deployment", + "assess", + "--release", + str(release_artifact), + "--server", + "production", + "--bundle-out", + str(production_bundle), + ) + assert code == 0 + assert production_bundle.is_file() + + # Protected production environment consumes the exact target bundle. + code, deploy_output = _run_cli( + monkeypatch, + capsys, + "deployment", + "deploy", + "--bundle", + str(production_bundle), + "--warehouse-id", + "warehouse-1", + "--output", + "json", + ) + assert code == 0 + deployed = json.loads(deploy_output) + assert deployed["status"] == "IN_SYNC" + assert deployed["contract_release_id"] == finalized["contractReleaseId"] + assert execution_adapter.preview_calls == 1 + assert execution_adapter.apply_calls == 1 + assert execution_adapter.verify_calls == 1 + assert execution_adapter.metadata_calls == 1 + + + +def test_central_repo_sample_fans_out_distinct_release_artifacts( + tmp_path, + monkeypatch, + capsys, +) -> None: + """Exercise central-repo classification, release fan-out, and deployment fan-out.""" + base_root = tmp_path / "base" / "contracts" + candidate_root = tmp_path / "candidate" / "contracts" + paths = ("a/b.yaml", "a_b.yaml") + contract_ids = ("orders-nested", "orders-flat") + + for relative_path, contract_id in zip(paths, contract_ids, strict=True): + dump_yaml( + _contract(contract_id=contract_id), + base_root / relative_path, + ) + dump_yaml( + _contract(contract_id=contract_id, include_created_at=True), + candidate_root / relative_path, + ) + + execution_adapter = _ExecutionAdapter() + + def _adapter_factory(platform: str, **kwargs): + assert platform == "databricks" + if kwargs.get("execution_config") is None: + return _PreviewAdapter() + return execution_adapter + + monkeypatch.setattr( + "semapact.platforms.runtime_registry.create_deployment_adapter", + _adapter_factory, + ) + monkeypatch.setattr( + "semapact.core.config.config_manager.get", + lambda *args, **kwargs: None, + ) + + code, classification_output = _run_cli( + monkeypatch, + capsys, + "release", + "classify-repo", + "--base-root", + str(base_root), + "--candidate-root", + str(candidate_root), + ) + assert code == 0 + classification = json.loads(classification_output) + changed = [ + item for item in classification["contracts"] if item["status"] == "changed" + ] + assert {item["contractRepoPath"] for item in changed} == set(paths) + artifact_keys = {item["artifactKey"] for item in changed} + assert len(artifact_keys) == 2 + assert all(len(key) == 64 for key in artifact_keys) + + finalized_ids: set[str] = set() + for item in changed: + relative_path = item["contractRepoPath"] + artifact_key = item["artifactKey"] + base_path = base_root / relative_path + candidate_path = candidate_root / relative_path + release_bundle = tmp_path / "release-bundles" / f"{artifact_key}.json" + approval_artifact = tmp_path / "finalized" / f"{artifact_key}.approval.json" + release_artifact = tmp_path / "finalized" / f"{artifact_key}.json" + deployment_bundle = tmp_path / "deploy" / f"{artifact_key}.json" + + code, _ = _run_cli( + monkeypatch, + capsys, + "release", + "assess", + "--base", + str(base_path), + "--candidate", + str(candidate_path), + "--base-revision-ref", + f"git:base:{relative_path}", + "--candidate-revision-ref", + f"git:candidate:{relative_path}", + "--bundle-out", + str(release_bundle), + ) + assert code == 0 + + code, _ = _run_cli( + monkeypatch, + capsys, + "release", + "approve", + "--bundle", + str(release_bundle), + "--actor-reference", + "github-environment:contract-release/run:central", + "--recorded-at", + "2026-09-21T04:00:00Z", + "--approval-out", + str(approval_artifact), + "--repository-root", + str(tmp_path), + ) + assert code == 0 + assert approval_artifact.is_file() + + code, finalize_output = _run_cli( + monkeypatch, + capsys, + "release", + "finalize", + "--bundle", + str(release_bundle), + "--approval", + str(approval_artifact), + "--output-contract", + str(candidate_path), + "--release-out", + str(release_artifact), + "--repository-root", + str(tmp_path), + ) + assert code == 0 + finalized = json.loads(finalize_output) + finalized_ids.add(finalized["contractReleaseId"]) + + code, _ = _run_cli( + monkeypatch, + capsys, + "deployment", + "assess", + "--release", + str(release_artifact), + "--server", + "production", + "--bundle-out", + str(deployment_bundle), + ) + assert code == 0 + + code, deploy_output = _run_cli( + monkeypatch, + capsys, + "deployment", + "deploy", + "--bundle", + str(deployment_bundle), + "--warehouse-id", + "warehouse-1", + "--output", + "json", + ) + assert code == 0 + assert json.loads(deploy_output)["status"] == "IN_SYNC" + + assert len(finalized_ids) == 2 + assert execution_adapter.preview_calls == 2 + assert execution_adapter.apply_calls == 2 + assert execution_adapter.verify_calls == 2 + assert execution_adapter.metadata_calls == 2 diff --git a/tests/test_contractops_artifact_rehydration.py b/tests/test_contractops_artifact_rehydration.py index 64d1690b..2f319be6 100644 --- a/tests/test_contractops_artifact_rehydration.py +++ b/tests/test_contractops_artifact_rehydration.py @@ -1,7 +1,6 @@ from __future__ import annotations import json -from datetime import date import pytest from open_data_contract_standard.model import ( @@ -11,27 +10,25 @@ ) from pydantic import ValidationError as PydanticValidationError -from semapact.change_context import ChangeContext +from semapact.application.models.release import ( + build_contract_release, + build_release_bundle, +) from semapact.contractops import ( - AppliedContractRelease, ChangeSet, - ContractOpsAuthorization, - PublicationResult, + ContractRelease, ReleasePlan, + ReleaseSnapshot, VersionAuthorityConfig, VersionResolution, - apply_contract_release, - authorize_contract_operation, build_change_set_from_decision, build_release_plan, - publish_contract_release, + build_release_snapshot, resolve_release_version, ) from semapact.governance import evaluate_governance_decision -from semapact.governance.gate import GovernanceOperation -CONTEXT = ChangeContext(effective_date=date(2026, 9, 11)) def _contract(*, name: str) -> OpenDataContractStandard: @@ -58,15 +55,10 @@ def _contract(*, name: str) -> OpenDataContractStandard: ) -class _Publisher: - def publish(self, release: AppliedContractRelease) -> str: - return f"registry:{release.applied_release_id}" - - def _artifact_chain(): base = _contract(name="orders-old") candidate = _contract(name="orders-new") - decision = evaluate_governance_decision(base, candidate, context=CONTEXT) + decision = evaluate_governance_decision(base, candidate) change_set = build_change_set_from_decision( decision, base_revision_ref="git:base", @@ -80,41 +72,28 @@ def _artifact_chain(): current_version="1.0.0", config=VersionAuthorityConfig(), ) - apply_authorization = authorize_contract_operation( - decision, - change_set, - release_plan, - version_resolution, - GovernanceOperation.APPLY, - ) - applied_release = apply_contract_release( + snapshot = build_release_snapshot( candidate, candidate_revision_ref="git:candidate", decision=decision, change_set=change_set, release_plan=release_plan, version_resolution=version_resolution, - authorization=apply_authorization, ) - publish_authorization = authorize_contract_operation( - decision, - change_set, - release_plan, - version_resolution, - GovernanceOperation.PUBLISH, - ) - publication = publish_contract_release( - applied_release, - authorization=publish_authorization, - publisher=_Publisher(), + bundle = build_release_bundle( + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=version_resolution, + release_snapshot=snapshot, ) + contract_release = build_contract_release(bundle) return ( change_set, release_plan, version_resolution, - apply_authorization, - applied_release, - publication, + snapshot, + contract_release, ) @@ -131,23 +110,24 @@ def test_rehydration_rejects_content_with_stale_deterministic_identity() -> None change_set, release_plan, version_resolution, - authorization, - applied_release, - publication, + snapshot, + contract_release, ) = _artifact_chain() cases = ( (ChangeSet, change_set, "candidate_revision_ref", "git:other"), (ReleasePlan, release_plan, "release_revision_ref", "git:other"), (VersionResolution, version_resolution, "release_revision_ref", "git:other"), - (ContractOpsAuthorization, authorization, "decision_id", "decision:other"), - (AppliedContractRelease, applied_release, "decision_id", "decision:other"), - (PublicationResult, publication, "publication_reference", "registry:other"), + (ReleaseSnapshot, snapshot, "decision_id", "decision:other"), + (ContractRelease, contract_release, "source_revision_ref", "git:other"), ) for model, artifact, field, tampered_value in cases: payload = artifact.model_dump(mode="json") payload[field] = tampered_value persisted_json = json.dumps(payload, separators=(",", ":"), sort_keys=True) - with pytest.raises(PydanticValidationError, match="deterministic identity"): + with pytest.raises( + (PydanticValidationError, ValueError), + match="deterministic identity|does not match", + ): model.model_validate_json(persisted_json) diff --git a/tests/test_contractops_authorization.py b/tests/test_contractops_authorization.py index 91f41b49..c5d3f7bb 100644 --- a/tests/test_contractops_authorization.py +++ b/tests/test_contractops_authorization.py @@ -1,6 +1,5 @@ from __future__ import annotations -from datetime import date import pytest from open_data_contract_standard.model import ( @@ -9,7 +8,6 @@ SchemaProperty, ) -from semapact.change_context import ChangeContext from semapact.contractops import ( AuthorizationReason, ReleasePlan, @@ -27,7 +25,6 @@ from semapact.governance.gate import GovernanceOperation -CONTEXT = ChangeContext(effective_date=date(2026, 9, 9)) def _contract( @@ -75,7 +72,7 @@ def _release_context(kind: str): else: # pragma: no cover - test helper guard raise ValueError(kind) - decision = evaluate_governance_decision(base, candidate, context=CONTEXT) + decision = evaluate_governance_decision(base, candidate) change_set = build_change_set_from_decision( decision, base_revision_ref="rev:base", diff --git a/tests/test_contractops_execution.py b/tests/test_contractops_execution.py index 164bf309..773d3df5 100644 --- a/tests/test_contractops_execution.py +++ b/tests/test_contractops_execution.py @@ -1,6 +1,5 @@ from __future__ import annotations -from datetime import date import pytest from open_data_contract_standard.model import ( @@ -9,25 +8,17 @@ SchemaProperty, ) -from semapact.change_context import ChangeContext from semapact.contractops import ( - ReviewAuthorizationEvidence, - ReviewEvidenceAction, VersionAuthorityConfig, - apply_contract_release, - authorize_contract_operation, build_change_set_from_decision, build_release_plan, - publish_contract_release, + build_release_snapshot, resolve_release_version, ) -from semapact.exceptions import ContractOpsAuthorizationError, ReleaseValidationError -from semapact.governance import DecisionResult, evaluate_governance_decision -from semapact.governance.gate import GovernanceOperation +from semapact.exceptions import ReleaseValidationError +from semapact.governance import evaluate_governance_decision -CONTEXT = ChangeContext(effective_date=date(2026, 9, 9)) - def _contract( *, @@ -64,16 +55,13 @@ def _contract( ) -def _release_context(kind: str): +def _release_context(*, include_created_at: bool = False): base = _contract(contract_name="orders-old") - if kind == "allow": - candidate = _contract(contract_name="orders-new") - elif kind == "review": - candidate = _contract(contract_name="orders-old", include_created_at=True) - else: # pragma: no cover - test helper guard - raise ValueError(kind) - - decision = evaluate_governance_decision(base, candidate, context=CONTEXT) + candidate = _contract( + contract_name="orders-new", + include_created_at=include_created_at, + ) + decision = evaluate_governance_decision(base, candidate) change_set = build_change_set_from_decision( decision, base_revision_ref="rev:base", @@ -90,326 +78,85 @@ def _release_context(kind: str): return candidate, decision, change_set, release_plan, version_resolution -def _review_evidence( - decision, - change_set, - release_plan, - version_resolution, - operation: GovernanceOperation, -) -> ReviewAuthorizationEvidence: - return ReviewAuthorizationEvidence( - evidence_reference=f"approval:{operation.value.lower()}", - decision_id=decision.decision_id, - change_set_id=change_set.change_set_id, - release_plan_id=release_plan.release_plan_id, - version_resolution_id=version_resolution.version_resolution_id, - operation=operation, - action=ReviewEvidenceAction.APPROVE, - ) - - -def _authorization( - decision, - change_set, - release_plan, - version_resolution, - operation: GovernanceOperation, - *, - approve_review: bool = True, -): - evidence = None - if decision.decision is DecisionResult.REVIEW and approve_review: - evidence = _review_evidence( - decision, - change_set, - release_plan, - version_resolution, - operation, - ) - return authorize_contract_operation( - decision, - change_set, - release_plan, - version_resolution, - operation, - evidence=evidence, - ) - - -class RecordingPublisher: - def __init__(self) -> None: - self.calls = [] - - def publish(self, release) -> str: - self.calls.append(release) - return f"published:{release.applied_release_id}" - - -def test_apply_metadata_only_release_uses_selected_patch_without_mutating_candidate() -> None: - candidate, decision, change_set, release_plan, version_resolution = _release_context( - "allow" +def test_release_snapshot_is_deterministic_and_materializes_selected_version() -> None: + candidate, decision, change_set, release_plan, version_resolution = ( + _release_context(include_created_at=True) ) - authorization = _authorization( - decision, - change_set, - release_plan, - version_resolution, - GovernanceOperation.APPLY, - ) - - assert decision.decision is DecisionResult.ALLOW - assert release_plan.required_version_bump == "none" - assert version_resolution.selected_version == "1.0.1" - first = apply_contract_release( + first = build_release_snapshot( candidate, candidate_revision_ref="rev:candidate", decision=decision, change_set=change_set, release_plan=release_plan, version_resolution=version_resolution, - authorization=authorization, ) - second = apply_contract_release( + second = build_release_snapshot( candidate, candidate_revision_ref="rev:candidate", decision=decision, change_set=change_set, release_plan=release_plan, version_resolution=version_resolution, - authorization=authorization, ) assert first == second - assert first.applied_release_id == second.applied_release_id - assert first.selected_version == "1.0.1" - assert first.release_revision_ref == "rev:candidate" - assert first.to_contract().version == "1.0.1" + assert first.release_snapshot_id == second.release_snapshot_id + assert first.selected_version == version_resolution.selected_version + assert first.to_contract().version == version_resolution.selected_version assert first.to_contract().name == "orders-new" assert candidate.version == "1.0.0" + assert not hasattr(first, "authorization_id") -def test_authorized_review_can_apply_without_rewriting_decision() -> None: - candidate, decision, change_set, release_plan, version_resolution = _release_context( - "review" - ) - authorization = _authorization( - decision, - change_set, - release_plan, - version_resolution, - GovernanceOperation.APPLY, - ) - - release = apply_contract_release( - candidate, - candidate_revision_ref="rev:candidate", - decision=decision, - change_set=change_set, - release_plan=release_plan, - version_resolution=version_resolution, - authorization=authorization, - ) - - assert decision.decision is DecisionResult.REVIEW - assert authorization.allowed is True - assert release.selected_version == "1.1.0" - assert release.to_contract().version == "1.1.0" - assert len(release.to_contract().schema_[0].properties) == 2 - - -def test_review_without_approval_cannot_apply() -> None: - candidate, decision, change_set, release_plan, version_resolution = _release_context( - "review" - ) - authorization = _authorization( - decision, - change_set, - release_plan, - version_resolution, - GovernanceOperation.APPLY, - approve_review=False, - ) - - assert authorization.allowed is False - with pytest.raises(ContractOpsAuthorizationError, match="not authorized"): - apply_contract_release( - candidate, - candidate_revision_ref="rev:candidate", - decision=decision, - change_set=change_set, - release_plan=release_plan, - version_resolution=version_resolution, - authorization=authorization, - ) - - -def test_apply_fails_closed_for_stale_candidate_revision() -> None: - candidate, decision, change_set, release_plan, version_resolution = _release_context( - "allow" - ) - authorization = _authorization( - decision, - change_set, - release_plan, - version_resolution, - GovernanceOperation.APPLY, +def test_release_snapshot_fails_closed_for_stale_candidate_revision() -> None: + candidate, decision, change_set, release_plan, version_resolution = ( + _release_context() ) with pytest.raises(ReleaseValidationError, match="planned release revision"): - apply_contract_release( + build_release_snapshot( candidate, candidate_revision_ref="rev:stale", decision=decision, change_set=change_set, release_plan=release_plan, version_resolution=version_resolution, - authorization=authorization, ) -def test_apply_fails_closed_when_candidate_version_drifted() -> None: - candidate, decision, change_set, release_plan, version_resolution = _release_context( - "allow" - ) - authorization = _authorization( - decision, - change_set, - release_plan, - version_resolution, - GovernanceOperation.APPLY, +def test_release_snapshot_fails_closed_when_candidate_version_drifted() -> None: + candidate, decision, change_set, release_plan, version_resolution = ( + _release_context() ) drifted_candidate = candidate.model_copy(deep=True) drifted_candidate.version = "9.0.0" with pytest.raises(ReleaseValidationError, match="current_version"): - apply_contract_release( + build_release_snapshot( drifted_candidate, candidate_revision_ref="rev:candidate", decision=decision, change_set=change_set, release_plan=release_plan, version_resolution=version_resolution, - authorization=authorization, ) -def test_apply_authorization_cannot_authorize_publish() -> None: - candidate, decision, change_set, release_plan, version_resolution = _release_context( - "allow" +def test_release_snapshot_rejects_tampered_version_resolution() -> None: + candidate, decision, change_set, release_plan, version_resolution = ( + _release_context() ) - apply_authorization = _authorization( - decision, - change_set, - release_plan, - version_resolution, - GovernanceOperation.APPLY, - ) - release = apply_contract_release( - candidate, - candidate_revision_ref="rev:candidate", - decision=decision, - change_set=change_set, - release_plan=release_plan, - version_resolution=version_resolution, - authorization=apply_authorization, - ) - publisher = RecordingPublisher() - - with pytest.raises(ReleaseValidationError, match="PUBLISH authorization"): - publish_contract_release( - release, - authorization=apply_authorization, - publisher=publisher, - ) - - assert publisher.calls == [] - - -def test_publish_invokes_adapter_only_after_exact_publish_authorization() -> None: - candidate, decision, change_set, release_plan, version_resolution = _release_context( - "allow" - ) - apply_authorization = _authorization( - decision, - change_set, - release_plan, - version_resolution, - GovernanceOperation.APPLY, - ) - publish_authorization = _authorization( - decision, - change_set, - release_plan, - version_resolution, - GovernanceOperation.PUBLISH, - ) - release = apply_contract_release( - candidate, - candidate_revision_ref="rev:candidate", - decision=decision, - change_set=change_set, - release_plan=release_plan, - version_resolution=version_resolution, - authorization=apply_authorization, + drifted_resolution = version_resolution.model_copy( + update={"selected_version": "v1.0.1"} ) - publisher = RecordingPublisher() - first = publish_contract_release( - release, - authorization=publish_authorization, - publisher=publisher, - ) - second = publish_contract_release( - release, - authorization=publish_authorization, - publisher=RecordingPublisher(), - ) - - assert len(publisher.calls) == 1 - assert publisher.calls[0] == release - assert first == second - assert first.publication_id == second.publication_id - assert first.applied_release_id == release.applied_release_id - assert first.authorization_id == publish_authorization.authorization_id - assert first.publication_reference == f"published:{release.applied_release_id}" - - -def test_review_without_publish_approval_never_invokes_publisher() -> None: - candidate, decision, change_set, release_plan, version_resolution = _release_context( - "review" - ) - apply_authorization = _authorization( - decision, - change_set, - release_plan, - version_resolution, - GovernanceOperation.APPLY, - ) - denied_publish_authorization = _authorization( - decision, - change_set, - release_plan, - version_resolution, - GovernanceOperation.PUBLISH, - approve_review=False, - ) - release = apply_contract_release( - candidate, - candidate_revision_ref="rev:candidate", - decision=decision, - change_set=change_set, - release_plan=release_plan, - version_resolution=version_resolution, - authorization=apply_authorization, - ) - publisher = RecordingPublisher() - - with pytest.raises(ContractOpsAuthorizationError, match="not authorized"): - publish_contract_release( - release, - authorization=denied_publish_authorization, - publisher=publisher, + with pytest.raises(ReleaseValidationError, match="deterministic identity"): + build_release_snapshot( + candidate, + candidate_revision_ref="rev:candidate", + decision=decision, + change_set=change_set, + release_plan=release_plan, + version_resolution=drifted_resolution, ) - - assert publisher.calls == [] diff --git a/tests/test_contractops_golden_scenarios.py b/tests/test_contractops_golden_scenarios.py index b694cc2b..c7bc1bbb 100644 --- a/tests/test_contractops_golden_scenarios.py +++ b/tests/test_contractops_golden_scenarios.py @@ -1,7 +1,6 @@ from __future__ import annotations from datetime import date, datetime, timezone -from types import SimpleNamespace import pytest from open_data_contract_standard.model import ( @@ -10,53 +9,32 @@ SchemaProperty, ) -from semapact.change_context import ChangeContext +from semapact.application.services.release_workflow import ( + ReleaseFinalizer, + ReleaseWorkflowService, +) from semapact.contractops import ( - ReviewAuthorizationEvidence, - ReviewEvidenceAction, VersionAuthority, VersionAuthorityConfig, - apply_contract_release, - authorize_contract_operation, build_change_set_from_decision, build_release_plan, resolve_release_version, ) from semapact.deployment import ( DeploymentTarget, - authorize_deployment, - build_deployment_plan, - verify_deployment_convergence, -) -from semapact.exceptions import ( - ContractOpsAuthorizationError, - GovernanceBlockedError, - ReleaseValidationError, - ValidationError, + build_contract_release_deployment_source, + build_deployment_plan_from_source, ) +from semapact.exceptions import ContractOpsAuthorizationError, GovernanceBlockedError from semapact.governance import DecisionResult, evaluate_governance_decision -from semapact.governance.gate import GovernanceOperation -from semapact.observation.fingerprint import with_observed_state_fingerprint -from semapact.observation.models import ( - ObservedAsset, - ObservedAssetIdentity, - ObservedPlatformState, - ObservedProperty, - ObservedPropertyIdentity, -) -from semapact.observation.providers import RuntimeAssetBinding -from semapact.platforms.databricks.deployment import DatabricksDeploymentAdapter -from semapact.reconciliation import RuntimeDriftStatus, classify_reconciliation_status -CONTEXT = ChangeContext(effective_date=date(2026, 9, 11)) -CAPTURED_AT = datetime(2026, 9, 11, 6, 0, tzinfo=timezone.utc) def _contract( *, contract_id: str = "orders-product", - contract_name: str = "orders", + name: str = "orders", include_created_at: bool = False, ) -> OpenDataContractStandard: properties = [ @@ -80,96 +58,31 @@ def _contract( apiVersion="v3.1.0", kind="DataContract", id=contract_id, - name=contract_name, + name=name, version="1.0.0", status="active", - schema=[SchemaObject(name="orders", properties=properties)], + schema=[ + SchemaObject( + name="orders", + physicalName="orders", + properties=properties, + ) + ], ) -def _release_artifacts(candidate: OpenDataContractStandard): - base = _contract(contract_name="orders-old") - decision = evaluate_governance_decision(base, candidate, context=CONTEXT) - change_set = build_change_set_from_decision( - decision, +def _allow_chain(): + workflow = ReleaseWorkflowService() + bundle = workflow.assess( + _contract(name="orders-old"), + _contract(name="orders-new"), base_revision_ref="git:base", candidate_revision_ref="git:candidate", - source="golden-test", - actor_reference="service:ci", - ) - release_plan = build_release_plan(change_set, decision) - version_resolution = resolve_release_version( - release_plan, - current_version="1.0.0", - config=VersionAuthorityConfig(), - ) - return decision, change_set, release_plan, version_resolution - - -def _review_evidence( - decision, - change_set, - release_plan, - version_resolution, - *, - operation: GovernanceOperation, - scope_reference: str | None = None, -) -> ReviewAuthorizationEvidence: - return ReviewAuthorizationEvidence( - evidence_reference=f"approval:{operation.value.lower()}", - decision_id=decision.decision_id, - change_set_id=change_set.change_set_id, - release_plan_id=release_plan.release_plan_id, - version_resolution_id=version_resolution.version_resolution_id, - operation=operation, - action=ReviewEvidenceAction.APPROVE, - scope_reference=scope_reference, ) - - -def _apply_release(candidate: OpenDataContractStandard): - decision, change_set, release_plan, version_resolution = _release_artifacts(candidate) - evidence = None - if decision.decision is DecisionResult.REVIEW: - evidence = _review_evidence( - decision, - change_set, - release_plan, - version_resolution, - operation=GovernanceOperation.APPLY, - ) - authorization = authorize_contract_operation( - decision, - change_set, - release_plan, - version_resolution, - GovernanceOperation.APPLY, - evidence=evidence, - ) - release = apply_contract_release( - candidate, - candidate_revision_ref="git:candidate", - decision=decision, - change_set=change_set, - release_plan=release_plan, - version_resolution=version_resolution, - authorization=authorization, - ) - return decision, change_set, release_plan, version_resolution, authorization, release - - -def _allow_chain(): - candidate = _contract(contract_name="orders-new") - ( - decision, - change_set, - release_plan, - version_resolution, - apply_authorization, - release, - ) = _apply_release(candidate) - deployment_plan = build_deployment_plan( - release, + release = ReleaseFinalizer().finalize(bundle) + source = build_contract_release_deployment_source(release) + plan = build_deployment_plan_from_source( + source, DeploymentTarget( platform="databricks", runtime_target="main.silver", @@ -177,327 +90,57 @@ def _allow_chain(): server_name="production", ), ) - deploy_authorization = authorize_contract_operation( - decision, - change_set, - release_plan, - version_resolution, - GovernanceOperation.DEPLOY, - ) - deployment_authorization = authorize_deployment( - deployment_plan, - release, - deploy_authorization, - ) - return SimpleNamespace( - candidate=candidate, - decision=decision, - change_set=change_set, - release_plan=release_plan, - version_resolution=version_resolution, - apply_authorization=apply_authorization, - release=release, - deployment_plan=deployment_plan, - deploy_authorization=deploy_authorization, - deployment_authorization=deployment_authorization, - ) - - -def _observed_state( - *, - present: bool, - physical_type: str | None = "varchar(255)", - nullable: bool | None = False, -) -> ObservedPlatformState: - identity = ObservedAssetIdentity( - platform="databricks", - namespace=("main", "silver"), - asset="orders", - ) - assets = () - if present: - assets = ( - ObservedAsset( - identity=identity, - asset_type="MANAGED", - properties=( - ObservedProperty( - identity=ObservedPropertyIdentity( - asset=identity, - property="id", - ), - physical_type=physical_type, - nullable=nullable, - ), - ), - ), - ) - return with_observed_state_fingerprint( - ObservedPlatformState( - platform="databricks", - source_identifier="workspace:golden", - assets=assets, - captured_at=CAPTURED_AT, - fingerprint=None, - ) - ) - - -class _RuntimeProvider: - key = "databricks" - - def __init__(self, state: ObservedPlatformState) -> None: - self.state = state - self.observe_calls = 0 - - def resolve_bindings(self, *, runtime_target, assets): - assert runtime_target == "main.silver" - return tuple( - RuntimeAssetBinding( - governed_asset=asset.governed_asset, - observed_asset=ObservedAssetIdentity( - platform="databricks", - namespace=("main", "silver"), - asset=asset.physical_name, - ), - ) - for asset in assets - ) - - def observe(self, *, bindings): - self.observe_calls += 1 - return self.state - - -class _Statements: - def __init__(self) -> None: - self.calls: list[str] = [] - - def execute_statement(self, *, statement, warehouse_id, wait_timeout): - assert warehouse_id == "warehouse-golden" - self.calls.append(statement) - return SimpleNamespace( - statement_id="statement-golden", - status=SimpleNamespace(state="SUCCEEDED", error=None), - ) - - def get_statement(self, statement_id): - raise AssertionError(f"unexpected statement polling: {statement_id}") - - -class _Client: - def __init__(self) -> None: - self.statement_execution = _Statements() - - -def _adapter(provider: _RuntimeProvider): - client = _Client() - adapter = DatabricksDeploymentAdapter( - client=client, - runtime_provider=provider, - warehouse_id="warehouse-golden", - poll_interval_seconds=0, - ) - return adapter, client - - -def _allow_semantic_projection(chain) -> dict[str, object]: - action = chain.deployment_plan.actions[0] - return { - "decision": chain.decision.decision.value, - "requiredVersionBump": chain.decision.required_version_bump, - "changeCount": len(chain.change_set.changes), - "releasePreconditions": [ - item.value for item in chain.release_plan.preconditions - ], - "versionAuthority": chain.version_resolution.authority.value, - "selectedVersion": chain.version_resolution.selected_version, - "actualBump": chain.version_resolution.actual_bump, - "apply": { - "operation": chain.apply_authorization.operation.value, - "allowed": chain.apply_authorization.allowed, - "reason": chain.apply_authorization.reason.value, - }, - "appliedContract": { - "name": chain.release.to_contract().name, - "version": chain.release.to_contract().version, - }, - "deployment": { - "operation": chain.deploy_authorization.operation.value, - "allowed": chain.deploy_authorization.allowed, - "target": chain.deployment_plan.target.model_dump(mode="json"), - "actions": [ - { - "kind": action.kind.value, - "governedAsset": action.governed_asset, - "physicalName": action.physical_name, - } - ], - }, - } + return bundle, release, source, plan -def test_allow_chain_has_stable_cross_boundary_golden_semantics() -> None: +def test_allow_chain_has_stable_cross_boundary_semantics() -> None: first = _allow_chain() second = _allow_chain() - assert _allow_semantic_projection(first) == { - "decision": "ALLOW", - "requiredVersionBump": "none", - "changeCount": 1, - "releasePreconditions": [], - "versionAuthority": "semapact", - "selectedVersion": "1.0.1", - "actualBump": "patch", - "apply": { - "operation": "APPLY", - "allowed": True, - "reason": "allowed_by_governance", - }, - "appliedContract": {"name": "orders-new", "version": "1.0.1"}, - "deployment": { - "operation": "DEPLOY", - "allowed": True, - "target": { - "platform": "databricks", - "runtime_target": "main.silver", - "source_reference": "workspace:golden", - "server_name": "production", - }, - "actions": [ - { - "kind": "ENSURE_ASSET_STATE", - "governedAsset": "orders", - "physicalName": "orders", - } - ], - }, - } - - assert first.decision.decision_id == second.decision.decision_id - assert first.change_set.change_set_id == second.change_set.change_set_id - assert first.release_plan.release_plan_id == second.release_plan.release_plan_id - assert ( - first.version_resolution.version_resolution_id - == second.version_resolution.version_resolution_id - ) - assert first.apply_authorization.authorization_id == second.apply_authorization.authorization_id - assert first.release.applied_release_id == second.release.applied_release_id - assert ( - first.deployment_plan.deployment_plan_id - == second.deployment_plan.deployment_plan_id - ) - assert ( - first.deployment_authorization.deployment_authorization_id - == second.deployment_authorization.deployment_authorization_id - ) - assert first.candidate.version == "1.0.0" - + first_bundle, first_release, first_source, first_plan = first + second_bundle, second_release, second_source, second_plan = second -def test_review_requires_exact_apply_and_deployment_authorization() -> None: - candidate = _contract(contract_name="orders-old", include_created_at=True) - decision, change_set, release_plan, version_resolution = _release_artifacts(candidate) + assert first_bundle.decision.decision is DecisionResult.ALLOW + assert first_bundle.version_resolution.selected_version == "1.0.1" + assert first_release.contract_version == "1.0.1" + assert first_release.to_contract().name == "orders-new" + assert first_source.release_id == first_release.contract_release_id + assert first_plan.contract_version == first_release.contract_version - assert decision.decision is DecisionResult.REVIEW + assert first_bundle.bundle_digest == second_bundle.bundle_digest + assert first_release.contract_release_id == second_release.contract_release_id + assert first_source.source_snapshot_id == second_source.source_snapshot_id + assert first_plan.deployment_plan_id == second_plan.deployment_plan_id - denied_apply = authorize_contract_operation( - decision, - change_set, - release_plan, - version_resolution, - GovernanceOperation.APPLY, - ) - assert denied_apply.allowed is False - with pytest.raises(ContractOpsAuthorizationError): - apply_contract_release( - candidate, - candidate_revision_ref="git:candidate", - decision=decision, - change_set=change_set, - release_plan=release_plan, - version_resolution=version_resolution, - authorization=denied_apply, - ) - apply_evidence = _review_evidence( - decision, - change_set, - release_plan, - version_resolution, - operation=GovernanceOperation.APPLY, - ) - apply_authorization = authorize_contract_operation( - decision, - change_set, - release_plan, - version_resolution, - GovernanceOperation.APPLY, - evidence=apply_evidence, - ) - release = apply_contract_release( - candidate, +def test_review_release_requires_exact_publish_approval() -> None: + workflow = ReleaseWorkflowService() + bundle = workflow.assess( + _contract(name="orders", include_created_at=False), + _contract(name="orders", include_created_at=True), + base_revision_ref="git:base", candidate_revision_ref="git:candidate", - decision=decision, - change_set=change_set, - release_plan=release_plan, - version_resolution=version_resolution, - authorization=apply_authorization, - ) - plan = build_deployment_plan( - release, - DeploymentTarget( - platform="databricks", - runtime_target="main.silver", - source_reference="workspace:golden", - ), ) + assert bundle.decision.decision is DecisionResult.REVIEW - unscoped_deploy_evidence = _review_evidence( - decision, - change_set, - release_plan, - version_resolution, - operation=GovernanceOperation.DEPLOY, - ) - unscoped_deploy = authorize_contract_operation( - decision, - change_set, - release_plan, - version_resolution, - GovernanceOperation.DEPLOY, - evidence=unscoped_deploy_evidence, - ) - with pytest.raises(ReleaseValidationError, match="scoped"): - authorize_deployment(plan, release, unscoped_deploy) + with pytest.raises(ContractOpsAuthorizationError, match="requires approval"): + ReleaseFinalizer().finalize(bundle) - scoped_deploy_evidence = _review_evidence( - decision, - change_set, - release_plan, - version_resolution, - operation=GovernanceOperation.DEPLOY, - scope_reference=plan.deployment_plan_id, + approval = workflow.approve( + bundle, + actor_reference="human:reviewer", + recorded_at=datetime(2026, 9, 11, 8, tzinfo=timezone.utc), ) - scoped_deploy = authorize_contract_operation( - decision, - change_set, - release_plan, - version_resolution, - GovernanceOperation.DEPLOY, - evidence=scoped_deploy_evidence, - ) - deployment_authorization = authorize_deployment(plan, release, scoped_deploy) + release = ReleaseFinalizer().finalize(bundle, approval=approval) - assert decision.decision is DecisionResult.REVIEW - assert apply_authorization.allowed is True - assert scoped_deploy.allowed is True - assert deployment_authorization.allowed is True + assert release.contract_version == bundle.version_resolution.selected_version + assert release.release_snapshot_id == bundle.release_snapshot.release_snapshot_id def test_block_cannot_enter_release_chain() -> None: base = _contract() candidate = _contract(contract_id="other-product") - decision = evaluate_governance_decision(base, candidate, context=CONTEXT) + decision = evaluate_governance_decision(base, candidate) change_set = build_change_set_from_decision( decision, base_revision_ref="git:base", @@ -510,8 +153,15 @@ def test_block_cannot_enter_release_chain() -> None: def test_version_authorities_preserve_same_required_bump() -> None: - candidate = _contract(contract_name="orders-old", include_created_at=True) - decision, _, release_plan, _ = _release_artifacts(candidate) + base = _contract() + candidate = _contract(include_created_at=True) + decision = evaluate_governance_decision(base, candidate) + change_set = build_change_set_from_decision( + decision, + base_revision_ref="git:base", + candidate_revision_ref="git:candidate", + ) + release_plan = build_release_plan(change_set, decision) semapact_resolution = resolve_release_version( release_plan, @@ -533,82 +183,3 @@ def test_version_authorities_preserve_same_required_bump() -> None: assert git_resolution.required_version_bump == "minor" assert semapact_resolution.selected_version == "1.1.0" assert git_resolution.selected_version == "1.2.0" - assert semapact_resolution.authority_reference is None - assert git_resolution.authority_reference == "v1.2.0" - - -def test_publish_authorization_cannot_authorize_runtime_deployment() -> None: - chain = _allow_chain() - publish_authorization = authorize_contract_operation( - chain.decision, - chain.change_set, - chain.release_plan, - chain.version_resolution, - GovernanceOperation.PUBLISH, - ) - - with pytest.raises(ReleaseValidationError, match="DEPLOY authorization"): - authorize_deployment( - chain.deployment_plan, - chain.release, - publish_authorization, - ) - - -def test_execute_success_is_separate_from_runtime_convergence() -> None: - chain = _allow_chain() - missing_state = _observed_state(present=False) - provider = _RuntimeProvider(missing_state) - adapter, client = _adapter(provider) - - first_preview = adapter.preview(chain.deployment_plan) - second_preview = adapter.preview(chain.deployment_plan) - assert first_preview == second_preview - assert client.statement_execution.calls == [] - - adapter.execute( - chain.deployment_plan, - first_preview, - chain.deployment_authorization, - ) - assert len(client.statement_execution.calls) == 1 - - statement_count = len(client.statement_execution.calls) - drift_result = verify_deployment_convergence(chain.deployment_plan, provider) - assert classify_reconciliation_status(drift_result) is RuntimeDriftStatus.DRIFT - assert len(client.statement_execution.calls) == statement_count - - provider.state = _observed_state(present=True) - in_sync_result = verify_deployment_convergence(chain.deployment_plan, provider) - assert classify_reconciliation_status(in_sync_result) is RuntimeDriftStatus.IN_SYNC - assert len(client.statement_execution.calls) == statement_count - - provider.state = _observed_state( - present=True, - physical_type=None, - nullable=None, - ) - indeterminate_result = verify_deployment_convergence(chain.deployment_plan, provider) - assert ( - classify_reconciliation_status(indeterminate_result) - is RuntimeDriftStatus.INDETERMINATE - ) - assert len(client.statement_execution.calls) == statement_count - - -def test_stale_preview_fails_before_native_mutation() -> None: - chain = _allow_chain() - missing_state = _observed_state(present=False) - provider = _RuntimeProvider(missing_state) - adapter, client = _adapter(provider) - preview = adapter.preview(chain.deployment_plan) - - provider.state = _observed_state(present=True) - with pytest.raises(ValidationError, match="no longer equals|Runtime state changed"): - adapter.execute( - chain.deployment_plan, - preview, - chain.deployment_authorization, - ) - - assert client.statement_execution.calls == [] diff --git a/tests/test_databricks_convergence_e2e.py b/tests/test_databricks_convergence_e2e.py index 29f80bf6..968cc19f 100644 --- a/tests/test_databricks_convergence_e2e.py +++ b/tests/test_databricks_convergence_e2e.py @@ -3,7 +3,7 @@ import argparse import json import re -from datetime import date, datetime, timedelta, timezone +from datetime import datetime, timedelta, timezone from pathlib import Path from types import SimpleNamespace @@ -15,28 +15,15 @@ ) from semapact.application.services.deployment import DeploymentService -from semapact.change_context import ChangeContext -from semapact.contractops import ( - apply_contract_release, - authorize_contract_operation, - build_change_set_from_decision, - build_release_plan, - resolve_release_version, - VersionAuthorityConfig, -) +from semapact.application.services.deployment_workflow import DeploymentWorkflowService from semapact.deployment import ( DeploymentPlan, DeploymentPreview, DeploymentTarget, - authorize_deployment, -) -from semapact.deployment.models import ( - NativeOperationKind, - compute_deployment_authorization_id, + build_candidate_deployment_source, ) -from semapact.exceptions import ContractOpsAuthorizationError, ValidationError -from semapact.governance import DecisionResult, evaluate_governance_decision -from semapact.governance.gate import GovernanceOperation +from semapact.deployment.models import NativeOperationKind +from semapact.exceptions import ValidationError from semapact.interfaces.commands import deployment_cmd from semapact.observation.fingerprint import with_observed_state_fingerprint from semapact.observation.models import ( @@ -51,7 +38,6 @@ from semapact.reconciliation import RuntimeDriftStatus, classify_reconciliation_status -_CONTEXT = ChangeContext(effective_date=date(2026, 9, 19)) _SOURCE = "workspace:golden" _RUNTIME_TARGET = "main.silver" _CAPTURED_AT = datetime(2026, 9, 19, 10, 0, tzinfo=timezone.utc) @@ -95,53 +81,6 @@ def _contract( ) -def _release_context(*properties: SchemaProperty): - base = _contract(*properties, name="orders-old") - candidate = _contract(*properties, name="orders-new") - decision = evaluate_governance_decision(base, candidate, context=_CONTEXT) - assert decision.decision is DecisionResult.ALLOW - - change_set = build_change_set_from_decision( - decision, - base_revision_ref="git:base", - candidate_revision_ref="git:candidate", - source="databricks-convergence-golden", - actor_reference="service:ci", - ) - release_plan = build_release_plan(change_set, decision) - version_resolution = resolve_release_version( - release_plan, - current_version="1.0.0", - config=VersionAuthorityConfig(), - ) - apply_authorization = authorize_contract_operation( - decision, - change_set, - release_plan, - version_resolution, - GovernanceOperation.APPLY, - ) - release = apply_contract_release( - candidate, - candidate_revision_ref="git:candidate", - decision=decision, - change_set=change_set, - release_plan=release_plan, - version_resolution=version_resolution, - authorization=apply_authorization, - ) - deploy_authorization = authorize_contract_operation( - decision, - change_set, - release_plan, - version_resolution, - GovernanceOperation.DEPLOY, - ) - return SimpleNamespace( - release=release, - deploy_authorization=deploy_authorization, - ) - def _target(*, source_reference: str = _SOURCE) -> DeploymentTarget: return DeploymentTarget( @@ -316,20 +255,17 @@ def _workflow( workspace: _StatefulWorkspace, *properties: SchemaProperty, ): - release_context = _release_context(*properties) - service = DeploymentService() - plan = service.plan(release_context.release, _target()) - authorization = authorize_deployment( - plan, - release_context.release, - release_context.deploy_authorization, + contract = _contract(*properties, name="orders") + source = build_candidate_deployment_source( + contract, + revision_ref="git:candidate", ) + service = DeploymentService() + plan = service.plan(source, _target()) return SimpleNamespace( service=service, plan=plan, - authorization=authorization, adapter=_adapter(workspace), - release_context=release_context, ) @@ -343,7 +279,75 @@ def _assert_in_sync(workflow) -> None: assert result.unverified_paths == () -def test_missing_table_create_execute_and_fresh_verify_converge() -> None: +def test_bundle_ci_to_cd_create_and_fresh_verify_converge() -> None: + workspace = _StatefulWorkspace(present=False) + adapter = _adapter(workspace) + base = _contract( + _property("id", "integer", required=True), + name="orders-old", + ) + candidate = _contract( + _property("id", "integer", required=True), + name="orders-new", + ) + service = DeploymentWorkflowService() + + bundle = service.assess( + base, + candidate, + base_revision_ref="git:base", + candidate_revision_ref="git:candidate", + target=_target(), + adapter=adapter, + ) + assert bundle.review_preview.operations[0].kind is NativeOperationKind.CREATE + + result = service.deploy(bundle, adapter=adapter) + + assert result.status is RuntimeDriftStatus.IN_SYNC + assert result.fresh_preview.operations[0].kind is NativeOperationKind.CREATE + assert workspace.statements == [ + "CREATE TABLE `main`.`silver`.`orders` " + "(`id` INT NOT NULL) USING DELTA" + ] + + +def test_bundle_cd_replans_against_runtime_changed_after_ci() -> None: + workspace = _StatefulWorkspace(present=False) + adapter = _adapter(workspace) + base = _contract( + _property("id", "integer", required=True), + name="orders-old", + ) + candidate = _contract( + _property("id", "integer", required=True), + name="orders-new", + ) + service = DeploymentWorkflowService() + + bundle = service.assess( + base, + candidate, + base_revision_ref="git:base", + candidate_revision_ref="git:candidate", + target=_target(), + adapter=adapter, + ) + assert bundle.review_preview.operations[0].kind is NativeOperationKind.CREATE + + # Another actor converges the table after CI but before CD. + workspace.present = True + workspace.columns = [("id", "INT", False)] + + result = service.deploy(bundle, adapter=adapter) + + assert result.status is RuntimeDriftStatus.IN_SYNC + assert result.review_preview_changed is True + assert result.fresh_preview.operations[0].kind is NativeOperationKind.NO_OP + assert workspace.statements == [] + + +def test_missing_table_create_apply_and_fresh_verify_converge() -> None: workspace = _StatefulWorkspace(present=False) workflow = _workflow( workspace, @@ -356,10 +360,9 @@ def test_missing_table_create_execute_and_fresh_verify_converge() -> None: ) assert preview.operations[0].kind is NativeOperationKind.CREATE - workflow.service.execute( + workflow.service.apply( workflow.plan, preview, - workflow.authorization, adapter=workflow.adapter, ) @@ -371,7 +374,7 @@ def test_missing_table_create_execute_and_fresh_verify_converge() -> None: _assert_in_sync(workflow) -def test_missing_nullable_column_alter_execute_and_fresh_verify_converge() -> None: +def test_missing_nullable_column_alter_apply_and_fresh_verify_converge() -> None: workspace = _StatefulWorkspace( present=True, columns=(("id", "INT", False),), @@ -388,10 +391,9 @@ def test_missing_nullable_column_alter_execute_and_fresh_verify_converge() -> No ) assert preview.operations[0].kind is NativeOperationKind.ALTER - workflow.service.execute( + workflow.service.apply( workflow.plan, preview, - workflow.authorization, adapter=workflow.adapter, ) @@ -406,7 +408,7 @@ def test_missing_nullable_column_alter_execute_and_fresh_verify_converge() -> No _assert_in_sync(workflow) -def test_compliant_table_no_op_execute_and_verify_converge() -> None: +def test_compliant_table_no_op_apply_and_verify_converge() -> None: workspace = _StatefulWorkspace( present=True, columns=(("id", "INT", False),), @@ -422,10 +424,9 @@ def test_compliant_table_no_op_execute_and_verify_converge() -> None: ) assert preview.operations[0].kind is NativeOperationKind.NO_OP - workflow.service.execute( + workflow.service.apply( workflow.plan, preview, - workflow.authorization, adapter=workflow.adapter, ) @@ -433,7 +434,7 @@ def test_compliant_table_no_op_execute_and_verify_converge() -> None: _assert_in_sync(workflow) -def test_stale_runtime_between_preview_and_execute_fails_closed() -> None: +def test_stale_runtime_between_preview_and_apply_fails_closed() -> None: workspace = _StatefulWorkspace(present=False) workflow = _workflow( workspace, @@ -448,12 +449,11 @@ def test_stale_runtime_between_preview_and_execute_fails_closed() -> None: workspace.columns = [("id", "INT", False)] with pytest.raises(ValidationError, match="Runtime state changed"): - workflow.service.execute( - workflow.plan, - preview, - workflow.authorization, - adapter=workflow.adapter, - ) + workflow.service.apply( + workflow.plan, + preview, + adapter=workflow.adapter, + ) assert workspace.statements == [] @@ -479,7 +479,7 @@ def test_external_asset_requiring_mutation_fails_closed() -> None: assert workspace.statements == [] -def test_wrong_workspace_source_fails_preview_and_execute_closed() -> None: +def test_wrong_workspace_source_fails_preview_and_apply_closed() -> None: wrong_workspace = _StatefulWorkspace( present=False, source_identifier="workspace:other", @@ -507,51 +507,16 @@ def test_wrong_workspace_source_fails_preview_and_execute_closed() -> None: workspace.source_identifier = "workspace:other" with pytest.raises(ValidationError, match="Runtime source changed"): - workflow.service.execute( - workflow.plan, - preview, - workflow.authorization, - adapter=workflow.adapter, - ) - - assert workspace.statements == [] - - -def test_denied_deployment_authorization_fails_before_native_mutation() -> None: - workspace = _StatefulWorkspace(present=False) - workflow = _workflow( - workspace, - _property("id", "integer", required=True), - ) - preview = workflow.service.preview( + workflow.service.apply( workflow.plan, + preview, adapter=workflow.adapter, ) - denied = workflow.authorization.model_copy( - update={ - "allowed": False, - "deployment_authorization_id": compute_deployment_authorization_id( - contract_ops_authorization_id=( - workflow.authorization.contract_ops_authorization_id - ), - deployment_plan_id=workflow.plan.deployment_plan_id, - applied_release_id=workflow.plan.applied_release_id, - allowed=False, - ), - } - ) - - with pytest.raises(ContractOpsAuthorizationError, match="not allowed"): - workflow.service.execute( - workflow.plan, - preview, - denied, - adapter=workflow.adapter, - ) assert workspace.statements == [] + def test_databricks_integer_target_alias_does_not_create_false_drift() -> None: workspace = _StatefulWorkspace( present=True, @@ -569,86 +534,3 @@ def test_databricks_integer_target_alias_does_not_create_false_drift() -> None: assert preview.operations[0].kind is NativeOperationKind.NO_OP _assert_in_sync(workflow) - - -def test_cli_round_trip_matches_application_service_and_converges( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - release_context = _release_context( - _property("id", "integer", required=True), - ) - workspace = _StatefulWorkspace(present=False) - adapter = _adapter(workspace) - service = DeploymentService() - - expected_plan = service.plan(release_context.release, _target()) - expected_preview = service.preview(expected_plan, adapter=adapter) - authorization = authorize_deployment( - expected_plan, - release_context.release, - release_context.deploy_authorization, - ) - - release_path = tmp_path / "release.json" - plan_path = tmp_path / "plan.json" - preview_path = tmp_path / "preview.json" - authorization_path = tmp_path / "authorization.json" - release_path.write_text( - release_context.release.model_dump_json(), - encoding="utf-8", - ) - - plan_result = deployment_cmd.run_deployment_plan( - argparse.Namespace( - release=str(release_path), - platform="databricks", - runtime=_RUNTIME_TARGET, - source_reference=_SOURCE, - server="production", - ) - ) - cli_plan = DeploymentPlan.model_validate_json(plan_result.output) - assert cli_plan == expected_plan - plan_path.write_text(cli_plan.model_dump_json(), encoding="utf-8") - - import semapact.platforms.runtime_registry as runtime_registry - - monkeypatch.setattr( - runtime_registry, - "create_deployment_adapter", - lambda platform, **kwargs: adapter, - ) - - preview_result = deployment_cmd.run_deployment_preview( - argparse.Namespace(plan=str(plan_path)) - ) - cli_preview = DeploymentPreview.model_validate_json(preview_result.output) - assert cli_preview == expected_preview - preview_path.write_text(cli_preview.model_dump_json(), encoding="utf-8") - authorization_path.write_text( - authorization.model_dump_json(), - encoding="utf-8", - ) - - execute_result = deployment_cmd.run_deployment_execute( - argparse.Namespace( - plan=str(plan_path), - preview=str(preview_path), - authorization=str(authorization_path), - warehouse_id="warehouse-golden", - ) - ) - assert json.loads(execute_result.output)["providerExecution"] == "SUCCEEDED" - - verify_result = deployment_cmd.run_deployment_verify( - argparse.Namespace( - plan=str(plan_path), - output="json", - ) - ) - assert json.loads(verify_result.output)["status"] == "IN_SYNC" - assert workspace.statements == [ - "CREATE TABLE `main`.`silver`.`orders` " - "(`id` INT NOT NULL) USING DELTA" - ] diff --git a/tests/test_delta_import_integration.py b/tests/test_delta_import_integration.py index 85e67d4e..162c9410 100644 --- a/tests/test_delta_import_integration.py +++ b/tests/test_delta_import_integration.py @@ -12,7 +12,7 @@ _extract_delta_relationships, ) from semapact.interfaces.commands.utils import _split_discovered_delta_tables -from semapact.services import GovernanceService +from semapact.application.services.governance import GovernanceService from semapact.utils.storage_adapter import LocalStorageAdapter @@ -146,7 +146,7 @@ def test_local_delta_import_preserves_governance_regression_path(tmp_path: Path) prop for prop in merged_orders.properties or [] if prop.name == "legacy_col" ) - assert analysis.decision.context == analysis.context + assert not hasattr(analysis.decision, "context") assert analysis.decision.decision == DecisionResult.REVIEW assert analysis.decision.required_version_bump == "minor" assert _custom_property_value(legacy, "lifecycleStatus") == "deprecated" diff --git a/tests/test_deployment_databricks.py b/tests/test_deployment_databricks.py index 8e0f5662..b8d06ca4 100644 --- a/tests/test_deployment_databricks.py +++ b/tests/test_deployment_databricks.py @@ -7,19 +7,18 @@ import pytest from open_data_contract_standard.model import SchemaObject, SchemaProperty +from semapact.deployment import RuntimeReleaseMetadata from semapact.deployment.models import ( DeploymentAction, DeploymentActionKind, - DeploymentAuthorization, DeploymentPlan, DeploymentTarget, NativeOperation, NativeOperationKind, - compute_deployment_authorization_id, compute_deployment_plan_id, compute_deployment_preview_id, ) -from semapact.exceptions import ContractOpsAuthorizationError, ValidationError +from semapact.exceptions import ValidationError from semapact.observation.fingerprint import with_observed_state_fingerprint from semapact.observation.models import ( ObservedAsset, @@ -71,41 +70,24 @@ def _plan(*properties: SchemaProperty, source_reference: str = SOURCE_REFERENCE) source_reference=source_reference, ) plan_id = compute_deployment_plan_id( - applied_release_id="applied:test", + source_snapshot_id="applied:test", contract_id="orders-product", - release_plan_id="release-plan:test", - released_revision_ref="rev:released", - selected_version="1.2.0", + revision_ref="rev:released", + contract_version="1.2.0", target=target, actions=(action,), ) return DeploymentPlan( deployment_plan_id=plan_id, - applied_release_id="applied:test", + source_snapshot_id="applied:test", contract_id="orders-product", - release_plan_id="release-plan:test", - released_revision_ref="rev:released", - selected_version="1.2.0", + revision_ref="rev:released", + contract_version="1.2.0", target=target, actions=(action,), ) -def _authorization(plan: DeploymentPlan, allowed: bool = True) -> DeploymentAuthorization: - authorization_id = compute_deployment_authorization_id( - contract_ops_authorization_id="contractops-auth:test", - deployment_plan_id=plan.deployment_plan_id, - applied_release_id=plan.applied_release_id, - allowed=allowed, - ) - return DeploymentAuthorization( - deployment_authorization_id=authorization_id, - contract_ops_authorization_id="contractops-auth:test", - deployment_plan_id=plan.deployment_plan_id, - applied_release_id=plan.applied_release_id, - allowed=allowed, - ) - def _state( *columns: tuple[str, str, bool], @@ -283,36 +265,34 @@ def test_preview_rejects_cross_source_runtime_evidence() -> None: adapter.preview(plan) -def test_execute_fails_closed_for_denied_stale_and_cross_source() -> None: +def test_apply_fails_closed_for_stale_and_cross_source() -> None: plan = _plan(_property("id", "BIGINT", required=True)) before = _state(("id", "bigint", False)) adapter, provider, _ = _adapter(before) preview = adapter.preview(plan) - with pytest.raises(ContractOpsAuthorizationError, match="not allowed"): - adapter.execute(plan, preview, _authorization(plan, False)) provider.state = _state( ("id", "bigint", False), ("other", "string", True), ) with pytest.raises(ValidationError, match="Runtime state changed"): - adapter.execute(plan, preview, _authorization(plan)) + adapter.apply(plan, preview) provider.state = before.model_copy(update={"source_identifier": "workspace-b"}) with pytest.raises(ValidationError, match="Runtime source changed"): - adapter.execute(plan, preview, _authorization(plan)) + adapter.apply(plan, preview) -def test_execute_rejects_tampered_plan_and_forged_native_command() -> None: +def test_apply_rejects_tampered_plan_and_forged_native_command() -> None: plan = _plan(_property("id", "BIGINT", required=True)) current = _state(("id", "bigint", False)) adapter, _, client = _adapter(current) preview = adapter.preview(plan) - tampered = plan.model_copy(update={"selected_version": "9.9.9"}) + tampered = plan.model_copy(update={"contract_version": "9.9.9"}) with pytest.raises(ValueError, match="DeploymentPlan deterministic identity"): - adapter.execute(tampered, preview, _authorization(plan)) + adapter.apply(tampered, preview) forged_operations = ( NativeOperation( @@ -333,11 +313,11 @@ def test_execute_rejects_tampered_plan_and_forged_native_command() -> None: update={"deployment_preview_id": forged_id, "operations": forged_operations} ) with pytest.raises(ValidationError, match="no longer equals"): - adapter.execute(plan, forged, _authorization(plan)) + adapter.apply(plan, forged) assert client.statement_execution.calls == [] -def test_execute_runs_exact_preview_statement() -> None: +def test_apply_runs_exact_preview_statement() -> None: plan = _plan( _property("id", "BIGINT", required=True), _property("note", "STRING"), @@ -346,25 +326,66 @@ def test_execute_runs_exact_preview_statement() -> None: adapter, _, client = _adapter(current) preview = adapter.preview(plan) - adapter.execute(plan, preview, _authorization(plan)) + adapter.apply(plan, preview) assert client.statement_execution.calls == [ "ALTER TABLE `main`.`silver`.`orders` ADD COLUMNS (`note` STRING)" ] -def test_no_op_execute_does_not_require_warehouse() -> None: +def test_release_metadata_projects_version_and_provenance_as_uc_tags() -> None: + plan = _plan(_property("id", "BIGINT", required=True)) + current = _state(("id", "bigint", False)) + adapter, _, client = _adapter(current) + + adapter.project_release_metadata( + plan, + RuntimeReleaseMetadata( + contract_id="orders-product", + contract_version="1.2.0", + contract_release_id="release-record-1", + source_revision_ref="rev:released", + ), + ) + + assert client.statement_execution.calls == [ + "ALTER TABLE `main`.`silver`.`orders` SET TAGS " + "('semapact_contract_id' = 'orders-product', " + "'semapact_contract_version' = '1.2.0', " + "'semapact_release_id' = 'release-record-1', " + "'semapact_source_revision' = 'rev:released')" + ] + + +def test_release_metadata_projection_requires_warehouse() -> None: + plan = _plan(_property("id", "BIGINT", required=True)) + current = _state(("id", "bigint", False)) + adapter, _, _ = _adapter(current, warehouse_id=None) + + with pytest.raises(ValidationError, match="warehouse_id"): + adapter.project_release_metadata( + plan, + RuntimeReleaseMetadata( + contract_id="orders-product", + contract_version="1.2.0", + contract_release_id="release-record-1", + source_revision_ref="rev:released", + ), + ) + + +def test_no_op_apply_does_not_require_warehouse() -> None: plan = _plan(_property("id", "BIGINT", required=True)) current = _state(("id", "bigint", False)) adapter, _, client = _adapter(current, warehouse_id=None) preview = adapter.preview(plan) assert preview.operations[0].kind is NativeOperationKind.NO_OP - adapter.execute(plan, preview, _authorization(plan)) + adapter.apply(plan, preview) assert client.statement_execution.calls == [] -def test_mutation_execute_without_warehouse_fails_closed() -> None: +def test_mutation_apply_without_warehouse_fails_closed() -> None: plan = _plan( _property("id", "BIGINT", required=True), _property("note", "STRING"), @@ -374,7 +395,7 @@ def test_mutation_execute_without_warehouse_fails_closed() -> None: preview = adapter.preview(plan) with pytest.raises(ValidationError, match="warehouse_id"): - adapter.execute(plan, preview, _authorization(plan)) + adapter.apply(plan, preview) assert client.statement_execution.calls == [] diff --git a/tests/test_deployment_history.py b/tests/test_deployment_history.py deleted file mode 100644 index 0279e699..00000000 --- a/tests/test_deployment_history.py +++ /dev/null @@ -1,285 +0,0 @@ -from __future__ import annotations - -from datetime import datetime, timedelta, timezone -from pathlib import Path - -import pytest - -from semapact.application.services.deployment_history import DeploymentHistoryService -from semapact.contractops import VersionAuthority -from semapact.deployment import ( - DeploymentAuthorization, - DeploymentPlan, - DeploymentPreview, - DeploymentTarget, -) -from semapact.deployment.models import ( - compute_deployment_authorization_id, - compute_deployment_plan_id, - compute_deployment_preview_id, -) -from semapact.history import ( - DeploymentStatus, - HistoryCorruptionError, - ReleaseRecord, -) -from semapact.history.integrity import compute_release_record_id -from semapact.platforms.git import GitWorkingTreeHistoryRepository - - -def _release_record() -> ReleaseRecord: - fields = { - "contract_id": "orders-product", - "contract_version": "1.3.0", - "decision_id": "decision-1", - "change_set_id": "changeset-1", - "release_plan_id": "release-plan-1", - "version_resolution_id": "version-resolution-1", - "authorization_id": "apply-authorization-1", - "applied_release_id": "applied-release-1", - "released_revision_id": "released-revision-1", - "required_version_bump": "minor", - "actual_version_bump": "minor", - "version_authority": VersionAuthority.SEMAPACT, - "authority_reference": None, - "review_evidence_reference": None, - "review_evidence_action": None, - } - release_record_id = compute_release_record_id( - **{ - **fields, - "version_authority": fields["version_authority"].value, - } - ) - return ReleaseRecord(release_record_id=release_record_id, **fields) - - -def _deployment_artifacts(*, preview_runtime_target: str = "catalog.schema", allowed: bool = True): - target = DeploymentTarget( - platform="databricks", - runtime_target="catalog.schema", - source_reference="workspace:test", - ) - plan_id = compute_deployment_plan_id( - applied_release_id="applied-release-1", - contract_id="orders-product", - release_plan_id="release-plan-1", - released_revision_ref="candidate-revision-1", - selected_version="1.3.0", - target=target, - actions=(), - ) - plan = DeploymentPlan( - deployment_plan_id=plan_id, - applied_release_id="applied-release-1", - contract_id="orders-product", - release_plan_id="release-plan-1", - released_revision_ref="candidate-revision-1", - selected_version="1.3.0", - target=target, - actions=(), - ) - - preview_id = compute_deployment_preview_id( - deployment_plan_id=plan.deployment_plan_id, - platform="databricks", - runtime_target=preview_runtime_target, - source_identifier="workspace:test", - observation_fingerprint="observation-1", - operations=(), - ) - preview = DeploymentPreview( - deployment_preview_id=preview_id, - deployment_plan_id=plan.deployment_plan_id, - platform="databricks", - runtime_target=preview_runtime_target, - source_identifier="workspace:test", - observation_fingerprint="observation-1", - operations=(), - ) - - authorization_id = compute_deployment_authorization_id( - contract_ops_authorization_id="deploy-authorization-1", - deployment_plan_id=plan.deployment_plan_id, - applied_release_id=plan.applied_release_id, - allowed=allowed, - ) - authorization = DeploymentAuthorization( - deployment_authorization_id=authorization_id, - contract_ops_authorization_id="deploy-authorization-1", - deployment_plan_id=plan.deployment_plan_id, - applied_release_id=plan.applied_release_id, - allowed=allowed, - ) - return plan, preview, authorization - - -def _service(tmp_path: Path): - backend = GitWorkingTreeHistoryRepository(tmp_path) - backend.put_release_record(_release_record()) - service = DeploymentHistoryService( - releases=backend, - deployment_plans=backend, - deployment_previews=backend, - deployment_authorizations=backend, - deployment_records=backend, - ) - return service, backend - - -def test_records_terminal_deployment_occurrence(tmp_path: Path) -> None: - service, backend = _service(tmp_path) - plan, preview, authorization = _deployment_artifacts() - started = datetime(2026, 9, 13, 0, 0, tzinfo=timezone.utc) - completed = started + timedelta(seconds=12) - - record = service.record_execution( - plan=plan, - preview=preview, - authorization=authorization, - status=DeploymentStatus.SUCCEEDED, - started_at=started, - completed_at=completed, - actor_reference="agent:deploy", - external_reference="pipeline:42", - ) - - assert backend.get_deployment_plan(plan.deployment_plan_id) == plan - assert backend.get_deployment_preview(preview.deployment_preview_id) == preview - assert ( - backend.get_deployment_authorization(authorization.deployment_authorization_id) - == authorization - ) - assert backend.get_deployment_record(record.deployment_record_id) == record - assert backend.list_deployment_records_for_release(record.release_record_id) == ( - record, - ) - assert backend.list_deployment_records_for_plan(plan.deployment_plan_id) == (record,) - assert record.platform == "databricks" - assert record.runtime_target == "catalog.schema" - assert record.source_reference == "workspace:test" - assert record.status is DeploymentStatus.SUCCEEDED - - -def test_exact_deployment_history_write_is_idempotent(tmp_path: Path) -> None: - service, backend = _service(tmp_path) - plan, preview, authorization = _deployment_artifacts() - started = datetime(2026, 9, 13, 0, 0, tzinfo=timezone.utc) - completed = started + timedelta(seconds=1) - - first = service.record_execution( - plan=plan, - preview=preview, - authorization=authorization, - status=DeploymentStatus.SUCCEEDED, - started_at=started, - completed_at=completed, - ) - second = service.record_execution( - plan=plan, - preview=preview, - authorization=authorization, - status=DeploymentStatus.SUCCEEDED, - started_at=started, - completed_at=completed, - ) - - assert second == first - assert backend.list_deployment_records_for_plan(plan.deployment_plan_id) == (first,) - - -def test_failed_attempt_remains_after_later_success(tmp_path: Path) -> None: - service, backend = _service(tmp_path) - plan, preview, authorization = _deployment_artifacts() - started = datetime(2026, 9, 13, 0, 0, tzinfo=timezone.utc) - - failed = service.record_execution( - plan=plan, - preview=preview, - authorization=authorization, - status=DeploymentStatus.FAILED, - started_at=started, - completed_at=started + timedelta(seconds=2), - external_reference="pipeline:42", - ) - succeeded = service.record_execution( - plan=plan, - preview=preview, - authorization=authorization, - status=DeploymentStatus.SUCCEEDED, - started_at=started + timedelta(minutes=5), - completed_at=started + timedelta(minutes=5, seconds=3), - external_reference="pipeline:43", - ) - - records = backend.list_deployment_records_for_plan(plan.deployment_plan_id) - assert set(records) == {failed, succeeded} - assert {record.status for record in records} == { - DeploymentStatus.FAILED, - DeploymentStatus.SUCCEEDED, - } - - -def test_rejects_preview_for_different_runtime_target(tmp_path: Path) -> None: - service, _ = _service(tmp_path) - plan, preview, authorization = _deployment_artifacts( - preview_runtime_target="other.schema" - ) - - with pytest.raises(ValueError, match="runtime target"): - service.record_execution( - plan=plan, - preview=preview, - authorization=authorization, - status=DeploymentStatus.FAILED, - started_at=datetime(2026, 9, 13, tzinfo=timezone.utc), - completed_at=datetime(2026, 9, 13, 0, 0, 1, tzinfo=timezone.utc), - ) - - -def test_rejects_denied_deployment_authorization(tmp_path: Path) -> None: - service, _ = _service(tmp_path) - plan, preview, authorization = _deployment_artifacts(allowed=False) - - with pytest.raises(ValueError, match="allowed"): - service.record_execution( - plan=plan, - preview=preview, - authorization=authorization, - status=DeploymentStatus.FAILED, - started_at=datetime(2026, 9, 13, tzinfo=timezone.utc), - completed_at=datetime(2026, 9, 13, 0, 0, 1, tzinfo=timezone.utc), - ) - - -def test_rejects_naive_execution_timestamps(tmp_path: Path) -> None: - service, _ = _service(tmp_path) - plan, preview, authorization = _deployment_artifacts() - - with pytest.raises(ValueError, match="timezone-aware"): - service.record_execution( - plan=plan, - preview=preview, - authorization=authorization, - status=DeploymentStatus.SUCCEEDED, - started_at=datetime(2026, 9, 13), - completed_at=datetime(2026, 9, 13, 0, 0, 1), - ) - - -def test_tampered_deployment_record_identity_fails_closed(tmp_path: Path) -> None: - service, backend = _service(tmp_path) - plan, preview, authorization = _deployment_artifacts() - started = datetime(2026, 9, 13, tzinfo=timezone.utc) - record = service.record_execution( - plan=plan, - preview=preview, - authorization=authorization, - status=DeploymentStatus.SUCCEEDED, - started_at=started, - completed_at=started + timedelta(seconds=1), - ) - - tampered = record.model_copy(update={"status": DeploymentStatus.FAILED}) - with pytest.raises(HistoryCorruptionError, match="invalid"): - backend.put_deployment_record(tampered) diff --git a/tests/test_deployment_orchestrator.py b/tests/test_deployment_orchestrator.py index 20b5de40..5532033e 100644 --- a/tests/test_deployment_orchestrator.py +++ b/tests/test_deployment_orchestrator.py @@ -10,12 +10,10 @@ from semapact.deployment.models import ( DeploymentAction, DeploymentActionKind, - DeploymentAuthorization, DeploymentPlan, DeploymentTarget, NativeOperation, NativeOperationKind, - compute_deployment_authorization_id, compute_deployment_plan_id, ) from semapact.deployment.orchestrator import DeploymentOrchestrator @@ -145,18 +143,22 @@ def _plan() -> DeploymentPlan: runtime_target="main", source_reference="source-1", ) - kwargs = dict( - applied_release_id="release-1", + identity_kwargs = dict( + source_snapshot_id="release-1", contract_id="contract-1", - release_plan_id="release-plan-1", - released_revision_ref="revision-1", - selected_version="1.0.0", + revision_ref="revision-1", + contract_version="1.0.0", target=target, actions=(action,), ) return DeploymentPlan( - deployment_plan_id=compute_deployment_plan_id(**kwargs), - **kwargs, + deployment_plan_id=compute_deployment_plan_id(**identity_kwargs), + source_snapshot_id="release-1", + contract_id="contract-1", + revision_ref="revision-1", + contract_version="1.0.0", + target=target, + actions=(action,), ) @@ -171,7 +173,9 @@ def _observation() -> ObservedPlatformState: ) -def test_generic_orchestrator_owns_observe_preview_freshness_and_execute() -> None: + + +def test_generic_orchestrator_owns_observe_preview_freshness_and_apply() -> None: plan = _plan() runtime_provider = _RuntimeProvider(_observation()) executor = _Executor() @@ -190,21 +194,7 @@ def test_generic_orchestrator_owns_observe_preview_freshness_and_execute() -> No assert preview.operations[0].kind is NativeOperationKind.CREATE assert preview.operations[0].statement == "CREATE_ASSET orders" - authorization_id = compute_deployment_authorization_id( - contract_ops_authorization_id="contract-auth-1", - deployment_plan_id=plan.deployment_plan_id, - applied_release_id=plan.applied_release_id, - allowed=True, - ) - authorization = DeploymentAuthorization( - deployment_authorization_id=authorization_id, - contract_ops_authorization_id="contract-auth-1", - deployment_plan_id=plan.deployment_plan_id, - applied_release_id=plan.applied_release_id, - allowed=True, - ) - - orchestrator.execute(plan, preview, authorization) + orchestrator.apply(plan, preview) assert runtime_provider.observe_calls == 2 assert executor.operations == [preview.operations[0]] diff --git a/tests/test_deployment_plan.py b/tests/test_deployment_plan.py index f044f8ed..46632d18 100644 --- a/tests/test_deployment_plan.py +++ b/tests/test_deployment_plan.py @@ -10,14 +10,18 @@ ) from pydantic import ValidationError as PydanticValidationError -from semapact.contractops import AppliedContractRelease -from semapact.contractops.integrity import compute_applied_release_id +from semapact.contractops import ContractRelease +from semapact.contractops.integrity import compute_contract_release_id from semapact.deployment import ( DeploymentAction, DeploymentActionKind, + DeploymentPlan, DeploymentTarget, - build_deployment_plan, + build_candidate_deployment_source, + build_contract_release_deployment_source, + build_deployment_plan_from_source, ) +from semapact.odcs.serialization import canonical_contract_json def _schema( @@ -39,38 +43,41 @@ def _schema( ) -def _release( +def _contract( *, schemas: list[SchemaObject] | None = None, -) -> AppliedContractRelease: - contract = OpenDataContractStandard( + version: str = "1.2.0", +) -> OpenDataContractStandard: + return OpenDataContractStandard( apiVersion="v3.1.0", kind="DataContract", id="orders-product", name="Orders", - version="1.2.0", + version=version, status="active", schema=schemas or [_schema("orders")], ) - released_contract_json = json.dumps( - contract.model_dump(mode="json", by_alias=True, exclude_none=True), - sort_keys=True, - separators=(",", ":"), - ensure_ascii=False, - ) + + +def _release( + *, + schemas: list[SchemaObject] | None = None, +) -> ContractRelease: + contract = _contract(schemas=schemas) + released_contract_json = canonical_contract_json(contract) fields = { "contract_id": "orders-product", + "contract_version": "1.2.0", "decision_id": "decision:test", "change_set_id": "change-set:test", "release_plan_id": "release-plan:test", "version_resolution_id": "version-resolution:test", - "release_revision_ref": "rev:released", - "selected_version": "1.2.0", - "authorization_id": "authorization:test", + "release_snapshot_id": "release-snapshot:test", + "source_revision_ref": "rev:released", "released_contract_json": released_contract_json, } - return AppliedContractRelease( - applied_release_id=compute_applied_release_id(**fields), + return ContractRelease( + contract_release_id=compute_contract_release_id(**fields), **fields, ) @@ -84,78 +91,89 @@ def _target() -> DeploymentTarget: ) -def test_same_exact_release_and_target_produce_same_plan() -> None: - release = _release(schemas=[_schema("zeta"), _schema("alpha")]) +def test_candidate_source_produces_canonical_v1_plan() -> None: + source = build_candidate_deployment_source( + _contract(), + revision_ref="rev:candidate", + ) - first = build_deployment_plan(release, _target()) - second = build_deployment_plan(release, _target()) + plan = build_deployment_plan_from_source(source, _target()) - assert first == second - assert first.deployment_plan_id == second.deployment_plan_id - assert first.model_dump(mode="json") == second.model_dump(mode="json") - assert [action.governed_asset for action in first.actions] == ["alpha", "zeta"] + assert plan.plan_version == "1" + assert plan.source_snapshot_id == source.source_snapshot_id + assert plan.contract_id == source.contract_id + assert plan.contract_version == source.contract_version + assert not hasattr(plan, "release_id") + assert not hasattr(plan, "release_plan_id") -def test_plan_preserves_exact_applied_release_provenance() -> None: +def test_finalized_release_source_produces_canonical_v1_plan() -> None: release = _release() + source = build_contract_release_deployment_source(release) + + plan = build_deployment_plan_from_source(source, _target()) + + assert plan.plan_version == "1" + assert source.release_id == release.contract_release_id + assert plan.source_snapshot_id == source.source_snapshot_id + assert plan.contract_version == release.contract_version - plan = build_deployment_plan(release, _target()) - assert plan.applied_release_id == release.applied_release_id - assert plan.contract_id == release.contract_id - assert plan.release_plan_id == release.release_plan_id - assert plan.released_revision_ref == release.release_revision_ref - assert plan.selected_version == release.selected_version - assert plan.plan_version == "2" - assert plan.target.source_reference == "https://workspace.example" +def test_same_exact_source_and_target_produce_same_plan() -> None: + source = build_contract_release_deployment_source( + _release(schemas=[_schema("zeta"), _schema("alpha")]) + ) + + first = build_deployment_plan_from_source(source, _target()) + second = build_deployment_plan_from_source(source, _target()) + + assert first == second + assert first.deployment_plan_id == second.deployment_plan_id + assert [action.governed_asset for action in first.actions] == ["alpha", "zeta"] def test_actions_are_provider_neutral_ensure_state_intents() -> None: - plan = build_deployment_plan( - _release(schemas=[_schema("orders"), _schema("customers")]), - _target(), + source = build_candidate_deployment_source( + _contract(schemas=[_schema("orders"), _schema("customers")]), + revision_ref="rev:candidate", ) + plan = build_deployment_plan_from_source(source, _target()) assert plan.actions assert all( action.kind is DeploymentActionKind.ENSURE_ASSET_STATE for action in plan.actions ) - assert {action.kind.value for action in plan.actions} == {"ENSURE_ASSET_STATE"} def test_physical_name_is_binding_hint_not_governed_identity() -> None: - plan = build_deployment_plan( - _release(schemas=[_schema("Orders", physical_name="prod_orders_v2")]), - _target(), + source = build_candidate_deployment_source( + _contract(schemas=[_schema("Orders", physical_name="prod_orders_v2")]), + revision_ref="rev:candidate", ) + plan = build_deployment_plan_from_source(source, _target()) action = plan.actions[0] assert action.governed_asset == "orders" assert action.physical_name == "prod_orders_v2" - desired = SchemaObject.model_validate_json(action.desired_state_json) - assert desired.name == "Orders" - assert desired.physicalName == "prod_orders_v2" - def test_missing_physical_name_falls_back_to_governed_schema_name() -> None: - plan = build_deployment_plan( - _release(schemas=[_schema("Orders")]), - _target(), + source = build_candidate_deployment_source( + _contract(schemas=[_schema("Orders")]), + revision_ref="rev:candidate", ) + plan = build_deployment_plan_from_source(source, _target()) - action = plan.actions[0] - assert action.governed_asset == "orders" - assert action.physical_name == "Orders" + assert plan.actions[0].physical_name == "Orders" -def test_target_is_explicit_and_changes_plan_identity() -> None: - release = _release() +def test_target_changes_plan_identity() -> None: + source = build_contract_release_deployment_source(_release()) - production = build_deployment_plan(release, _target()) - staging = build_deployment_plan( - release, + production = build_deployment_plan_from_source(source, _target()) + staging = build_deployment_plan_from_source( + source, DeploymentTarget( platform="databricks", runtime_target="main.staging", @@ -165,42 +183,6 @@ def test_target_is_explicit_and_changes_plan_identity() -> None: ) assert production.deployment_plan_id != staging.deployment_plan_id - assert production.target.platform == "databricks" - assert staging.target.runtime_target == "main.staging" - - -def test_runtime_source_changes_plan_identity_for_same_runtime_target() -> None: - release = _release() - first = build_deployment_plan(release, _target()) - second = build_deployment_plan( - release, - DeploymentTarget( - platform="databricks", - runtime_target="main.analytics", - source_reference="https://other-workspace.example", - server_name="production", - ), - ) - - assert first.deployment_plan_id != second.deployment_plan_id - - -def test_schema_order_is_canonicalized_within_each_exact_release() -> None: - first_release = _release( - schemas=[_schema("zeta"), _schema("alpha")], - ) - second_release = _release( - schemas=[_schema("alpha"), _schema("zeta")], - ) - - first = build_deployment_plan(first_release, _target()) - second = build_deployment_plan(second_release, _target()) - - assert [action.governed_asset for action in first.actions] == ["alpha", "zeta"] - assert [action.governed_asset for action in second.actions] == ["alpha", "zeta"] - # Exact release snapshot identity remains authoritative; two releases with - # differently serialized source schema order remain distinct authorities. - assert first.deployment_plan_id != second.deployment_plan_id def test_desired_state_identity_mismatch_fails_closed() -> None: @@ -220,8 +202,11 @@ def test_desired_state_identity_mismatch_fails_closed() -> None: ) -def test_deployment_models_are_immutable() -> None: - plan = build_deployment_plan(_release(), _target()) +def test_canonical_deployment_plan_rejects_release_provenance_fields() -> None: + source = build_contract_release_deployment_source(_release()) + plan = build_deployment_plan_from_source(source, _target()) + payload = plan.model_dump(mode="json") + payload["release_id"] = "unexpected" with pytest.raises(PydanticValidationError): - setattr(plan, "selected_version", "9.9.9") + DeploymentPlan.model_validate(payload) diff --git a/tests/test_deployment_service.py b/tests/test_deployment_service.py index 6cb90a3b..cbbb0153 100644 --- a/tests/test_deployment_service.py +++ b/tests/test_deployment_service.py @@ -3,13 +3,15 @@ from datetime import datetime, timezone import pytest -from open_data_contract_standard.model import SchemaObject, SchemaProperty +from open_data_contract_standard.model import ( + SchemaObject, + SchemaProperty, +) from semapact.deployment import ( DeploymentAction, DeploymentAdapter, DeploymentActionKind, - DeploymentAuthorization, DeploymentPlan, DeploymentPreview, DeploymentTarget, @@ -17,14 +19,13 @@ NativeOperationKind, ) from semapact.deployment.models import ( - compute_deployment_authorization_id, compute_deployment_plan_id, compute_deployment_preview_id, ) from semapact.exceptions import ValidationError from semapact.observation import ObservedPlatformState, with_observed_state_fingerprint from semapact.reconciliation import ReconciliationResult -from semapact.services.deployment_service import DeploymentService +from semapact.application.services.deployment import DeploymentService CAPTURED_AT = datetime(2026, 9, 10, 10, 0, tzinfo=timezone.utc) SOURCE_REFERENCE = "https://adb.example" @@ -56,9 +57,10 @@ def verify(self, plan: DeploymentPlan) -> ReconciliationResult: self.verify_calls += 1 return self.verification_result - def execute(self, plan, preview, authorization) -> None: + def apply(self, plan, preview) -> None: self.execute_calls += 1 - self.executed = (plan, preview, authorization) + self.executed = (plan, preview, None) + def _plan() -> DeploymentPlan: @@ -87,21 +89,19 @@ def _plan() -> DeploymentPlan: source_reference=SOURCE_REFERENCE, ) plan_id = compute_deployment_plan_id( - applied_release_id="release-1", + source_snapshot_id="release-1", contract_id="orders-contract", - release_plan_id="release-plan-1", - released_revision_ref="abc123", - selected_version="1.2.3", + revision_ref="abc123", + contract_version="1.2.3", target=target, actions=(action,), ) return DeploymentPlan( deployment_plan_id=plan_id, - applied_release_id="release-1", + source_snapshot_id="release-1", contract_id="orders-contract", - release_plan_id="release-plan-1", - released_revision_ref="abc123", - selected_version="1.2.3", + revision_ref="abc123", + contract_version="1.2.3", target=target, actions=(action,), ) @@ -150,7 +150,7 @@ def _verification(plan: DeploymentPlan) -> ReconciliationResult: assert observation.fingerprint is not None return ReconciliationResult( contract_id=plan.contract_id, - contract_version=plan.selected_version, + contract_version=plan.contract_version, observation_source_identifier=observation.source_identifier, observation_fingerprint=observation.fingerprint, ) @@ -163,21 +163,6 @@ def _adapter(plan: DeploymentPlan) -> FakeDeploymentAdapter: ) -def _authorization(plan: DeploymentPlan) -> DeploymentAuthorization: - authorization_id = compute_deployment_authorization_id( - contract_ops_authorization_id="contractops-auth-1", - deployment_plan_id=plan.deployment_plan_id, - applied_release_id=plan.applied_release_id, - allowed=True, - ) - return DeploymentAuthorization( - deployment_authorization_id=authorization_id, - contract_ops_authorization_id="contractops-auth-1", - deployment_plan_id=plan.deployment_plan_id, - applied_release_id=plan.applied_release_id, - allowed=True, - ) - def test_preview_delegates_to_unified_adapter_entrypoint() -> None: plan = _plan() @@ -190,21 +175,20 @@ def test_preview_delegates_to_unified_adapter_entrypoint() -> None: assert adapter.execute_calls == 0 -def test_execute_delegates_exact_canonical_artifacts() -> None: +def test_apply_delegates_exact_plan_and_preview_without_internal_authorization() -> None: plan = _plan() preview = _preview(plan) - authorization = _authorization(plan) adapter = _adapter(plan) - DeploymentService().execute( + DeploymentService().apply( plan, preview, - authorization, adapter=adapter, ) assert adapter.execute_calls == 1 - assert adapter.executed == (plan, preview, authorization) + assert adapter.executed == (plan, preview, None) + def test_verify_delegates_to_same_adapter_entrypoint() -> None: @@ -217,7 +201,7 @@ def test_verify_delegates_to_same_adapter_entrypoint() -> None: assert adapter.verify_calls == 1 -@pytest.mark.parametrize("operation", ["preview", "verify", "execute"]) +@pytest.mark.parametrize("operation", ["preview", "verify", "apply"]) def test_service_rejects_adapter_platform_mismatch(operation: str) -> None: plan = _plan() adapter = _adapter(plan) @@ -229,12 +213,7 @@ def test_service_rejects_adapter_platform_mismatch(operation: str) -> None: elif operation == "verify": DeploymentService().verify(plan, adapter=adapter) else: - DeploymentService().execute( - plan, - _preview(plan), - _authorization(plan), - adapter=adapter, - ) + DeploymentService().apply(plan, _preview(plan), adapter=adapter) assert adapter.preview_calls == 0 assert adapter.verify_calls == 0 diff --git a/tests/test_deployment_verification.py b/tests/test_deployment_verification.py index c01a2bad..bf7c859c 100644 --- a/tests/test_deployment_verification.py +++ b/tests/test_deployment_verification.py @@ -82,21 +82,19 @@ def _plan() -> DeploymentPlan: source_reference=SOURCE_REFERENCE, ) plan_id = compute_deployment_plan_id( - applied_release_id="release-1", + source_snapshot_id="release-1", contract_id="orders-contract", - release_plan_id="release-plan-1", - released_revision_ref="abc123", - selected_version="1.2.3", + revision_ref="abc123", + contract_version="1.2.3", target=target, actions=(action,), ) return DeploymentPlan( deployment_plan_id=plan_id, - applied_release_id="release-1", + source_snapshot_id="release-1", contract_id="orders-contract", - release_plan_id="release-plan-1", - released_revision_ref="abc123", - selected_version="1.2.3", + revision_ref="abc123", + contract_version="1.2.3", target=target, actions=(action,), ) @@ -194,7 +192,7 @@ def test_runtime_source_mismatch_fails_closed() -> None: def test_tampered_deployment_plan_identity_fails_closed() -> None: - plan = _plan().model_copy(update={"selected_version": "9.9.9"}) + plan = _plan().model_copy(update={"contract_version": "9.9.9"}) with pytest.raises(ValueError, match="deterministic identity"): verify_deployment_convergence(plan, FakeRuntimeProvider(_observation())) diff --git a/tests/test_deployment_workflow_service.py b/tests/test_deployment_workflow_service.py new file mode 100644 index 00000000..ccc1ef02 --- /dev/null +++ b/tests/test_deployment_workflow_service.py @@ -0,0 +1,461 @@ +from __future__ import annotations + +from datetime import datetime, timezone + +import pytest +from open_data_contract_standard.model import ( + OpenDataContractStandard, + SchemaObject, + SchemaProperty, +) +from pydantic import ValidationError as PydanticValidationError + +from semapact.application.services.deployment_workflow import DeploymentWorkflowService +from semapact.application.services.release_workflow import ReleaseFinalizer, ReleaseWorkflowService +from semapact.deployment import ( + DeploymentAdapter, + DeploymentPreview, + DeploymentTarget, + NativeOperation, + NativeOperationKind, +) +from semapact.deployment.models import compute_deployment_preview_id +from semapact.governance import DecisionResult +from semapact.observation import ObservedPlatformState, with_observed_state_fingerprint +from semapact.reconciliation import ReconciliationResult, RuntimeDriftStatus + + +class _PreviewAdapter(DeploymentAdapter): + key = "fake" + + def __init__(self) -> None: + self.preview_calls = 0 + + def validate(self, plan) -> None: + pass + + def preview(self, plan) -> DeploymentPreview: + self.preview_calls += 1 + observation = with_observed_state_fingerprint( + ObservedPlatformState( + platform="fake", + source_identifier=plan.target.source_reference, + assets=(), + captured_at=datetime(2026, 9, 20, tzinfo=timezone.utc), + ) + ) + assert observation.fingerprint is not None + operations = ( + NativeOperation( + kind=NativeOperationKind.CREATE, + governed_asset="orders", + statement="CREATE orders", + ), + ) + return DeploymentPreview( + deployment_preview_id=compute_deployment_preview_id( + deployment_plan_id=plan.deployment_plan_id, + platform="fake", + runtime_target=plan.target.runtime_target, + source_identifier=observation.source_identifier, + observation_fingerprint=observation.fingerprint, + operations=operations, + ), + deployment_plan_id=plan.deployment_plan_id, + platform="fake", + runtime_target=plan.target.runtime_target, + source_identifier=observation.source_identifier, + observation_fingerprint=observation.fingerprint, + operations=operations, + ) + + def apply(self, plan, preview) -> None: + raise AssertionError("CI bundle construction must not apply") + + def execute(self, plan, preview, authorization) -> None: + raise AssertionError("CI bundle construction must not execute") + + def verify(self, plan) -> ReconciliationResult: + raise AssertionError("CI bundle construction must not verify deployment") + + +class _ExecutionAdapter(DeploymentAdapter): + key = "fake" + + def __init__(self) -> None: + self.preview_calls = 0 + self.execute_calls = 0 + self.verify_calls = 0 + self.executed_preview = None + self.metadata_calls = [] + + def validate(self, plan) -> None: + pass + + def preview(self, plan) -> DeploymentPreview: + self.preview_calls += 1 + observation = with_observed_state_fingerprint( + ObservedPlatformState( + platform="fake", + source_identifier=plan.target.source_reference, + assets=(), + captured_at=datetime(2026, 9, 20, 1, tzinfo=timezone.utc), + ) + ) + assert observation.fingerprint is not None + operations = ( + NativeOperation( + kind=NativeOperationKind.NO_OP, + governed_asset="orders", + ), + ) + return DeploymentPreview( + deployment_preview_id=compute_deployment_preview_id( + deployment_plan_id=plan.deployment_plan_id, + platform="fake", + runtime_target=plan.target.runtime_target, + source_identifier=observation.source_identifier, + observation_fingerprint=observation.fingerprint, + operations=operations, + ), + deployment_plan_id=plan.deployment_plan_id, + platform="fake", + runtime_target=plan.target.runtime_target, + source_identifier=observation.source_identifier, + observation_fingerprint=observation.fingerprint, + operations=operations, + ) + + def apply(self, plan, preview) -> None: + self.execute_calls += 1 + self.executed_preview = preview + + def execute(self, plan, preview, authorization) -> None: + assert authorization.allowed is True + assert authorization.source_snapshot_id == plan.source_snapshot_id + self.apply(plan, preview) + + def project_release_metadata(self, plan, metadata) -> None: + self.metadata_calls.append((plan, metadata)) + + def verify(self, plan) -> ReconciliationResult: + self.verify_calls += 1 + return ReconciliationResult( + contract_id=plan.contract_id, + contract_version=plan.contract_version, + observation_source_identifier=plan.target.source_reference, + observation_fingerprint="obs-v2:sha256:verified", + ) + + +class _OperationalSink: + def __init__(self) -> None: + self.events = [] + + def record_deployment(self, event) -> None: + self.events.append(event) + + +class _FailingExecutionAdapter(_ExecutionAdapter): + def apply(self, plan, preview) -> None: + raise RuntimeError("provider mutation failed") + + +def _contract( + *, + name: str, + include_created_at: bool = False, +) -> OpenDataContractStandard: + properties = [ + SchemaProperty( + name="id", + logicalType="string", + physicalType="varchar(255)", + required=True, + ) + ] + if include_created_at: + properties.append( + SchemaProperty( + name="created_at", + logicalType="timestamp", + physicalType="timestamp", + required=False, + ) + ) + return OpenDataContractStandard( + apiVersion="v3.1.0", + kind="DataContract", + id="orders-product", + name=name, + version="1.2.3", + status="active", + schema=[SchemaObject(name="orders", properties=properties)], + ) + + +def _target(name: str = "sales") -> DeploymentTarget: + return DeploymentTarget( + platform="fake", + runtime_target=f"main.{name}", + source_reference=f"runtime:{name}", + ) + + +def _finalized_release(): + workflow = ReleaseWorkflowService() + bundle = workflow.assess( + _contract(name="Orders"), + _contract(name="Orders", include_created_at=True), + base_revision_ref="git:base", + candidate_revision_ref="git:candidate", + ) + approval = workflow.approve( + bundle, + actor_reference="human:reviewer", + recorded_at=datetime(2026, 9, 20, 2, tzinfo=timezone.utc), + ) + return ReleaseFinalizer().finalize( + bundle, + approval=approval, + ) + + +def test_candidate_assessment_does_not_calculate_release_version() -> None: + bundle = DeploymentWorkflowService().assess( + _contract(name="Orders old"), + _contract(name="Orders new"), + base_revision_ref="git:base", + candidate_revision_ref="git:candidate", + target=_target(), + adapter=_PreviewAdapter(), + ) + + assert bundle.is_release is False + assert bundle.contract_release is None + assert bundle.decision is not None + assert bundle.change_set is not None + assert bundle.deployment_source.source_kind == "candidate" + assert bundle.deployment_source.contract_version == "1.2.3" + assert bundle.deployment_plan.plan_version == "1" + + +def test_finalized_release_assessment_binds_exact_release_record(tmp_path) -> None: + release = _finalized_release() + bundle = DeploymentWorkflowService().assess_release( + release, + target=_target(), + adapter=_PreviewAdapter(), + ) + + assert bundle.is_release is True + assert bundle.decision is None + assert bundle.change_set is None + assert bundle.contract_release == release + assert bundle.deployment_source.release_id == release.contract_release_id + assert bundle.deployment_source.contract_version == release.contract_version + assert not hasattr(bundle.deployment_plan, "release_id") + + +def test_same_candidate_inputs_produce_same_bundle_digest() -> None: + service = DeploymentWorkflowService() + kwargs = dict( + base_revision_ref="git:base", + candidate_revision_ref="git:candidate", + target=_target(), + ) + first = service.assess( + _contract(name="Orders old"), + _contract(name="Orders new"), + adapter=_PreviewAdapter(), + **kwargs, + ) + second = service.assess( + _contract(name="Orders old"), + _contract(name="Orders new"), + adapter=_PreviewAdapter(), + **kwargs, + ) + assert first == second + assert first.bundle_digest == second.bundle_digest + + +def test_bundle_rehydration_fails_closed_when_digest_is_tampered() -> None: + bundle = DeploymentWorkflowService().assess( + _contract(name="Orders old"), + _contract(name="Orders new"), + base_revision_ref="git:base", + candidate_revision_ref="git:candidate", + target=_target(), + adapter=_PreviewAdapter(), + ) + payload = bundle.model_dump(mode="json") + payload["bundle_digest"] = "sha256:" + ("0" * 64) + + with pytest.raises(PydanticValidationError, match="digest does not match"): + type(bundle).model_validate(payload) + + +def test_candidate_bundle_rejects_change_set_from_other_revision() -> None: + service = DeploymentWorkflowService() + first = service.assess( + _contract(name="Orders"), + _contract(name="Orders"), + base_revision_ref="git:base", + candidate_revision_ref="git:candidate-a", + target=_target(), + adapter=_PreviewAdapter(), + ) + second = service.assess( + _contract(name="Orders"), + _contract(name="Orders"), + base_revision_ref="git:base", + candidate_revision_ref="git:candidate-b", + target=_target(), + adapter=_PreviewAdapter(), + ) + assert second.change_set is not None + + payload = first.model_dump(mode="json") + payload["change_set"] = second.change_set.model_dump(mode="json") + + with pytest.raises(PydanticValidationError, match="candidate revision"): + type(first).model_validate(payload) + + +def test_candidate_bundle_rejects_decision_from_different_change_set() -> None: + service = DeploymentWorkflowService() + first = service.assess( + _contract(name="Orders"), + _contract(name="Orders"), + base_revision_ref="git:base", + candidate_revision_ref="git:candidate", + target=_target(), + adapter=_PreviewAdapter(), + ) + second = service.assess( + _contract(name="Orders"), + _contract(name="Orders", include_created_at=True), + base_revision_ref="git:base", + candidate_revision_ref="git:candidate", + target=_target(), + adapter=_PreviewAdapter(), + ) + assert second.decision is not None + + payload = first.model_dump(mode="json") + payload["decision"] = second.decision.model_dump(mode="json") + + with pytest.raises(PydanticValidationError, match="decision/change-set changes mismatch"): + type(first).model_validate(payload) + + +def test_candidate_review_deployment_needs_no_release_approval() -> None: + service = DeploymentWorkflowService() + bundle = service.assess( + _contract(name="Orders"), + _contract(name="Orders", include_created_at=True), + base_revision_ref="git:base", + candidate_revision_ref="git:candidate", + target=_target(), + adapter=_PreviewAdapter(), + ) + assert bundle.decision is not None + assert bundle.decision.decision is DecisionResult.REVIEW + + result = service.deploy(bundle, adapter=_ExecutionAdapter()) + + assert result.contract_release_id is None + assert result.status is RuntimeDriftStatus.IN_SYNC + + +def test_cd_uses_fresh_preview_not_ci_review_preview() -> None: + service = DeploymentWorkflowService() + bundle = service.assess( + _contract(name="Orders old"), + _contract(name="Orders new"), + base_revision_ref="git:base", + candidate_revision_ref="git:candidate", + target=_target(), + adapter=_PreviewAdapter(), + ) + adapter = _ExecutionAdapter() + + result = service.deploy(bundle, adapter=adapter) + + assert adapter.preview_calls == 1 + assert adapter.execute_calls == 1 + assert adapter.verify_calls == 1 + assert adapter.executed_preview is result.fresh_preview + assert result.fresh_preview.operations[0].kind is NativeOperationKind.NO_OP + assert result.review_preview_changed is True + + +def test_configured_operational_history_records_failed_candidate_deployment() -> None: + service = DeploymentWorkflowService() + bundle = service.assess( + _contract(name="Orders old"), + _contract(name="Orders new"), + base_revision_ref="git:base", + candidate_revision_ref="git:candidate", + target=_target(), + adapter=_PreviewAdapter(), + ) + sink = _OperationalSink() + + with pytest.raises(RuntimeError, match="provider mutation failed"): + service.deploy( + bundle, + adapter=_FailingExecutionAdapter(), + operational_history=sink, + ) + + assert len(sink.events) == 1 + event = sink.events[0] + assert event.status == "FAILED" + assert event.contract_release_id is None + assert not hasattr(event, "deployment_authorization_id") + assert event.reconciliation_status is None + + +def test_one_finalized_release_fans_out_to_multiple_targets(tmp_path) -> None: + release = _finalized_release() + service = DeploymentWorkflowService() + + dev = service.assess_release( + release, + target=_target("dev"), + adapter=_PreviewAdapter(), + ) + prod = service.assess_release( + release, + target=_target("prod"), + adapter=_PreviewAdapter(), + ) + + assert dev.contract_release == prod.contract_release == release + assert dev.bundle_digest != prod.bundle_digest + assert dev.deployment_plan.deployment_plan_id != prod.deployment_plan.deployment_plan_id + assert dev.deployment_source.release_id == prod.deployment_source.release_id + + +def test_release_deployment_projects_finalized_version_after_in_sync(tmp_path) -> None: + release = _finalized_release() + bundle = DeploymentWorkflowService().assess_release( + release, + target=_target(), + adapter=_PreviewAdapter(), + ) + adapter = _ExecutionAdapter() + result = DeploymentWorkflowService().deploy( + bundle, + adapter=adapter, + ) + + assert result.status is RuntimeDriftStatus.IN_SYNC + assert result.contract_release_id == release.contract_release_id + assert len(adapter.metadata_calls) == 1 + plan, metadata = adapter.metadata_calls[0] + assert metadata.contract_version == release.contract_version + assert metadata.contract_release_id == release.contract_release_id + assert not hasattr(plan, "release_id") diff --git a/tests/test_evolution_chain.py b/tests/test_evolution_chain.py deleted file mode 100644 index 47bd700a..00000000 --- a/tests/test_evolution_chain.py +++ /dev/null @@ -1,297 +0,0 @@ -from __future__ import annotations - -from datetime import datetime, timedelta, timezone -from pathlib import Path - -from open_data_contract_standard.model import ( - OpenDataContractStandard, - SchemaObject, - SchemaProperty, -) - -from semapact.application.services.evolution import EvolutionChainService -from semapact.application.services.governance import GovernanceService -from semapact.application.services.history import ProposalHistoryService -from semapact.application.services.runtime_history import RuntimeHistoryService -from semapact.contractops import VersionAuthority, build_change_set -from semapact.history import DeploymentRecord, DeploymentStatus, ReleaseRecord -from semapact.history.integrity import compute_deployment_record_id, compute_release_record_id -from semapact.observation import ( - ObservedAsset, - ObservedAssetIdentity, - ObservedPlatformState, - with_observed_state_fingerprint, -) -from semapact.platforms.git import GitWorkingTreeHistoryRepository -from semapact.reconciliation import ReconciliationResult -from semapact.revision import build_contract_revision - - -def _contract(*, version: str = "1.2.3", include_note: bool = False) -> OpenDataContractStandard: - properties = [ - SchemaProperty( - name="id", - logicalType="string", - physicalType="varchar(255)", - required=True, - ) - ] - if include_note: - properties.append( - SchemaProperty( - name="note", - logicalType="string", - physicalType="varchar(255)", - required=False, - ) - ) - return OpenDataContractStandard( - apiVersion="v3.1.0", - kind="DataContract", - id="orders-product", - name="Orders", - version=version, - status="active", - schema=[SchemaObject(name="orders", properties=properties)], - ) - - -def _record_proposal(backend: GitWorkingTreeHistoryRepository): - base_revision = build_contract_revision(_contract()) - candidate_revision = build_contract_revision(_contract(include_note=True)) - proposal = GovernanceService().evaluate_proposal( - base_revision.contract, - candidate_revision.contract, - effective_date="2026-09-13", - base_revision_ref=base_revision.revision_id, - candidate_revision_ref=candidate_revision.revision_id, - source="test", - actor_reference="actor:test", - ) - ProposalHistoryService( - revisions=backend, - change_sets=backend, - decisions=backend, - decision_links=backend, - ).record_proposal( - proposal, - base_revision=base_revision, - candidate_revision=candidate_revision, - ) - return proposal, base_revision, candidate_revision - - -def _release_record(proposal, released_revision_id: str) -> ReleaseRecord: - fields = dict( - contract_id="orders-product", - contract_version="1.3.0", - decision_id=proposal.decision.decision_id, - change_set_id=proposal.change_set.change_set_id, - release_plan_id="release-plan-1", - version_resolution_id="version-resolution-1", - authorization_id="apply-authorization-1", - applied_release_id="applied-release-1", - released_revision_id=released_revision_id, - required_version_bump="minor", - actual_version_bump="minor", - version_authority=VersionAuthority.SEMAPACT, - authority_reference=None, - review_evidence_reference=None, - review_evidence_action=None, - ) - release_record_id = compute_release_record_id( - **{**fields, "version_authority": fields["version_authority"].value} - ) - return ReleaseRecord(release_record_id=release_record_id, **fields) - - -def _deployment_record(release_record_id: str) -> DeploymentRecord: - started = datetime(2026, 9, 13, 1, tzinfo=timezone.utc) - completed = started + timedelta(seconds=2) - fields = dict( - release_record_id=release_record_id, - deployment_plan_id="deployment-plan-1", - deployment_preview_id="deployment-preview-1", - deployment_authorization_id="deployment-authorization-1", - platform="databricks", - runtime_target="catalog.schema", - source_reference="workspace:test", - status=DeploymentStatus.SUCCEEDED, - started_at=started, - completed_at=completed, - actor_reference=None, - external_reference="pipeline:1", - ) - deployment_record_id = compute_deployment_record_id( - release_record_id=release_record_id, - deployment_plan_id=fields["deployment_plan_id"], - deployment_preview_id=fields["deployment_preview_id"], - deployment_authorization_id=fields["deployment_authorization_id"], - platform=fields["platform"], - runtime_target=fields["runtime_target"], - source_reference=fields["source_reference"], - status=fields["status"].value, - started_at=started.isoformat(), - completed_at=completed.isoformat(), - actor_reference=None, - external_reference=fields["external_reference"], - ) - return DeploymentRecord(deployment_record_id=deployment_record_id, **fields) - - -def _observation(at: datetime) -> ObservedPlatformState: - return with_observed_state_fingerprint( - ObservedPlatformState( - platform="databricks", - source_identifier="workspace:test", - assets=( - ObservedAsset( - identity=ObservedAssetIdentity( - platform="databricks", - namespace=("catalog", "schema"), - asset="orders", - ), - asset_type="TABLE", - ), - ), - captured_at=at, - ) - ) - - -def _result(observation: ObservedPlatformState) -> ReconciliationResult: - assert observation.fingerprint is not None - return ReconciliationResult( - contract_id="orders-product", - contract_version="1.3.0", - observation_source_identifier=observation.source_identifier, - observation_fingerprint=observation.fingerprint, - ) - - -def _service(backend: GitWorkingTreeHistoryRepository) -> EvolutionChainService: - return EvolutionChainService( - revisions=backend, - change_sets=backend, - decisions=backend, - decision_links=backend, - releases=backend, - deployments=backend, - observations=backend, - runtime_reconciliations=backend, - ) - - -def test_reconstructs_complete_governance_evolution_chain(tmp_path: Path) -> None: - backend = GitWorkingTreeHistoryRepository(tmp_path) - proposal, base_revision, candidate_revision = _record_proposal(backend) - - released_revision = build_contract_revision( - _contract(version="1.3.0", include_note=True) - ) - backend.put_revision(released_revision) - release = _release_record(proposal, released_revision.revision_id) - backend.put_release_record(release) - deployment = _deployment_record(release.release_record_id) - backend.put_deployment_record(deployment) - - observation = _observation(datetime(2026, 9, 13, 2, tzinfo=timezone.utc)) - runtime = RuntimeHistoryService( - observations=backend, - reconciliations=backend, - releases=backend, - deployments=backend, - ).record_reconciliation( - observation, - _result(observation), - release_record_id=release.release_record_id, - deployment_record_id=deployment.deployment_record_id, - ) - - chain = _service(backend).reconstruct("orders-product") - - assert chain.broken_references == () - assert chain.unlinked_releases == () - assert chain.unlinked_runtime == () - assert len(chain.proposals) == 1 - path = chain.proposals[0] - assert path.base_revision == base_revision - assert path.candidate_revision == candidate_revision - assert path.decision == proposal.decision - assert len(path.releases) == 1 - release_path = path.releases[0] - assert release_path.release == release - assert release_path.released_revision == released_revision - assert len(release_path.deployments) == 1 - deployment_path = release_path.deployments[0] - assert deployment_path.deployment == deployment - assert tuple(item.reconciliation for item in deployment_path.runtime) == (runtime,) - assert deployment_path.runtime[0].observation is not None - assert _service(backend).reconstruct("orders-product") == chain - - -def test_incomplete_changeset_history_does_not_fabricate_downstream_stages(tmp_path: Path) -> None: - backend = GitWorkingTreeHistoryRepository(tmp_path) - proposal, base_revision, candidate_revision = _record_proposal(backend) - - isolated = build_change_set( - contract_id=proposal.change_set.contract_id, - base_revision_ref=base_revision.revision_id, - candidate_revision_ref=candidate_revision.revision_id, - changes=proposal.change_set.changes, - context=proposal.change_set.context, - source="isolated", - ) - backend.put_change_set(isolated) - - chain = _service(backend).reconstruct("orders-product") - isolated_paths = [item for item in chain.proposals if item.change_set == isolated] - - assert len(isolated_paths) == 1 - assert isolated_paths[0].decision is None - assert isolated_paths[0].releases == () - - -def test_broken_revision_reference_is_reported_explicitly(tmp_path: Path) -> None: - backend = GitWorkingTreeHistoryRepository(tmp_path) - proposal, _, candidate_revision = _record_proposal(backend) - broken = build_change_set( - contract_id="orders-product", - base_revision_ref="missing-revision", - candidate_revision_ref=candidate_revision.revision_id, - changes=proposal.change_set.changes, - context=proposal.change_set.context, - source="broken", - ) - backend.put_change_set(broken) - - chain = _service(backend).reconstruct("orders-product") - path = next(item for item in chain.proposals if item.change_set == broken) - - assert path.base_revision is None - assert any( - item.source_id == broken.change_set_id - and item.reference_field == "base_revision_ref" - and item.target_id == "missing-revision" - and item.reason == "NOT_FOUND" - for item in chain.broken_references - ) - - -def test_contract_level_runtime_evidence_remains_unlinked(tmp_path: Path) -> None: - backend = GitWorkingTreeHistoryRepository(tmp_path) - observation = _observation(datetime(2026, 9, 13, 3, tzinfo=timezone.utc)) - runtime = RuntimeHistoryService( - observations=backend, - reconciliations=backend, - releases=backend, - deployments=backend, - ).record_reconciliation(observation, _result(observation)) - - chain = _service(backend).reconstruct("orders-product") - - assert chain.proposals == () - assert chain.unlinked_releases == () - assert tuple(item.reconciliation for item in chain.unlinked_runtime) == (runtime,) - assert chain.unlinked_runtime[0].observation is not None - assert chain.broken_references == () diff --git a/tests/test_fail_closed_governance_integration.py b/tests/test_fail_closed_governance_integration.py index 0d34c5a4..0fe29c69 100644 --- a/tests/test_fail_closed_governance_integration.py +++ b/tests/test_fail_closed_governance_integration.py @@ -15,18 +15,15 @@ from semapact.core.lifecycle_cli import apply_lifecycle from semapact.devops.ci_cd import evaluate_ci_gate -from semapact.devops.release_workflow import ( - build_batch_release_manifest, -) from semapact.exceptions import GovernanceBlockedError, GovernanceReviewRequiredError +from semapact.change_context import ChangeContext from semapact.governance import ( - ChangeContext, GovernanceOperation, evaluate_governance_decision, ) from semapact.interfaces.commands.merge_cmd import run_merge from semapact.interfaces.commands.plan_cmd import run_plan -from semapact.interfaces.commands.release_cmd import run_release_classify, run_release_prepare +from semapact.interfaces.commands.release_cmd import run_release_classify from semapact.orchestrator.pipeline import ContractPipeline from semapact.utils.schema_utils import contract_to_dict from semapact.utils.yaml_utils import dump_yaml @@ -110,18 +107,6 @@ def test_injected_block_decision_prevents_side_effects(tmp_path, monkeypatch): apply_lifecycle(lifecycle_args, is_promote=True, context=TEST_CONTEXT) assert not (tmp_path / "lifecycle_out.yaml").exists() - # 3. release prepare: raises GovernanceBlockedError and does NOT write candidate - release_prep_args = SimpleNamespace( - base=str(base_path), - candidate=str(candidate_path), - release_tag="v2.0.0", - output=str(tmp_path / "release_out.yaml"), - runtime_context="auto", - effective_date=TEST_EFFECTIVE_DATE, - ) - with pytest.raises(GovernanceBlockedError): - run_release_prepare(release_prep_args) - assert not (tmp_path / "release_out.yaml").exists() # 4. pipeline run: writes decision manifest FIRST, then raises GovernanceBlockedError without generating GE/merged contract pipeline = ContractPipeline() @@ -272,127 +257,7 @@ def mutating_hook(name, **kwargs): assert written_data["id"] == "contract-1", "Written contract must not be mutated by plugin hook" -def test_release_create_pr_block_prevents_all_side_effects(tmp_path): - """Verify create_release_pull_request with a BLOCK decision does not dump YAML, commit/push, or create PR.""" - base = _make_contract(contract_id="contract-a", version="1.0.0", status="active") - candidate = _make_contract(contract_id="contract-a", version="1.0.0", status="active") - from open_data_contract_standard.model import DataQuality - candidate.schema_[0].properties[0].quality = [DataQuality(type="invalid_type")] - - from semapact.devops.pr_creator import GitHubConfig - from semapact.devops.release_workflow import create_release_pull_request - - config = GitHubConfig(owner="org", repo="repo", token="fake") - repo_dir = tmp_path / "repo" - repo_dir.mkdir() - contract_repo_path = "contracts/my_contract.yaml" - with mock.patch("semapact.devops.pr_creator.PullRequestCreator.create_update_pr") as mock_pr: - with pytest.raises(GovernanceBlockedError) as exc_info: - create_release_pull_request( - config=config, - repo_path=str(repo_dir), - contract_repo_path=contract_repo_path, - base_contract=base, - candidate_contract=candidate, - release_tag="v1.0.1", - source_branch="release/v1.0.1", - target_branch="main", - context=TEST_CONTEXT, - ) - - assert exc_info.value.operation == GovernanceOperation.PROPOSE - assert not (repo_dir / contract_repo_path).exists(), "Candidate YAML must not be dumped when governance gate BLOCKS" - assert mock_pr.call_count == 0, "Pull request creator must not be called when governance gate BLOCKS" - - -def test_batch_release_manifest_evaluates_governance_once_per_task(tmp_path, monkeypatch): - """Verify build_batch_release_manifest and create_release_pull_requests_from_manifest evaluate underlying policy exactly ONCE per task per phase without duplicate evaluations.""" - base_dir = tmp_path / "base_root" - cand_dir = tmp_path / "cand_root" - repo_dir = tmp_path / "repo" - base_dir.mkdir() - cand_dir.mkdir() - repo_dir.mkdir() - - for i in range(2): - base_c = _make_contract(contract_id=f"c{i}", version="1.0.0") - cand_c = _make_contract(contract_id=f"c{i}", version="1.0.0") - cand_c.schema_[0].properties.append( - SchemaProperty(name=f"new_{i}", logicalType="string", physicalType="varchar(10)", required=False) - ) - dump_yaml(contract_to_dict(base_c), base_dir / f"c{i}.yaml") - dump_yaml(contract_to_dict(cand_c), cand_dir / f"c{i}.yaml") - - policy_eval_calls = 0 - from semapact.lifecycle.policy import evaluate_merge_policy - orig_eval_policy = evaluate_merge_policy - - def counted_policy_eval(*args, **kwargs): - nonlocal policy_eval_calls - policy_eval_calls += 1 - return orig_eval_policy(*args, **kwargs) - - monkeypatch.setattr("semapact.governance.evaluator.evaluate_merge_policy", counted_policy_eval) - - # 1. Build manifest (2 tasks created, exactly 2 policy evaluations) - build = build_batch_release_manifest( - base_root=str(base_dir), - candidate_root=str(cand_dir), - context=TEST_CONTEXT, - ) - assert len(build.tasks) == 2 - assert policy_eval_calls == 2, f"Expected 2 policy evaluations during build, got {policy_eval_calls}" - - # 2. Run PR creation from manifest tasks (exactly 2 more policy evaluations: 1 per task in evaluate_governance_decision, 0 in apply_release_candidate) - monkeypatch.setattr("semapact.devops.pr_creator.PullRequestCreator.create_update_pr", lambda *a, **k: {}) - from semapact.devops.pr_creator import GitHubConfig - from semapact.devops.release_workflow import create_release_pull_requests_from_manifest - - config = GitHubConfig(owner="org", repo="repo", token="fake") - create_release_pull_requests_from_manifest( - config=config, - repo_path=str(repo_dir), - tasks=build.tasks, - ) - - assert policy_eval_calls == 4, f"Expected exactly 4 total policy evaluations across build+create (1 per task per phase), got {policy_eval_calls}" - - -def test_batch_release_manifest_skips_blocked_contracts(tmp_path): - """Batch release manifest moves contracts with BLOCK decisions to skipped list.""" - base_dir = tmp_path / "base_root" - cand_dir = tmp_path / "cand_root" - base_dir.mkdir() - cand_dir.mkdir() - - base_contract = _make_contract(contract_id="contract-1", version="1.0.0") - cand_valid = _make_contract(contract_id="contract-1", version="1.0.0") - cand_valid.schema_[0].properties.append( - SchemaProperty(name="new_field", logicalType="string", physicalType="varchar(50)", required=False) - ) - dump_yaml(contract_to_dict(base_contract), base_dir / "valid.yaml") - dump_yaml(contract_to_dict(cand_valid), cand_dir / "valid.yaml") - - base_blocked = _make_contract(contract_id="contract-blocked", version="1.0.0") - cand_blocked = _make_contract(contract_id="contract-blocked", version="2.0.0") # Version Mismatch -> BLOCK - dump_yaml(contract_to_dict(base_blocked), base_dir / "blocked.yaml") - dump_yaml(contract_to_dict(cand_blocked), cand_dir / "blocked.yaml") - - build = build_batch_release_manifest( - base_root=str(base_dir), - candidate_root=str(cand_dir), - context=TEST_CONTEXT, - ) - - # Valid additive change produces a task - task_paths = [t.contract_path for t in build.tasks] - assert "valid.yaml" in task_paths - assert "blocked.yaml" not in task_paths - - # Blocked contract is in skipped list - skipped_paths = [s.contract_repo_path for s in build.skipped] - assert "blocked.yaml" in skipped_paths def test_breaking_change_review_behavior(tmp_path, capsys): @@ -426,24 +291,11 @@ def test_breaking_change_review_behavior(tmp_path, capsys): base=str(base_path), candidate=str(cand_path), runtime_context="auto", - effective_date=TEST_EFFECTIVE_DATE, ) class_res = run_release_classify(classify_args) assert class_res["requiredBump"] == "minor" assert class_res["governanceDecision"]["decision"] == "REVIEW" - # 2. PROPOSE (release prepare): Allowed to produce candidate YAML for review - prep_args = SimpleNamespace( - base=str(base_path), - candidate=str(cand_path), - release_tag="v1.1.0", - output=str(tmp_path / "review_candidate.yaml"), - runtime_context="auto", - effective_date=TEST_EFFECTIVE_DATE, - ) - prep_res = run_release_prepare(prep_args) - assert (tmp_path / "review_candidate.yaml").exists() - assert prep_res["governanceDecision"]["decision"] == "REVIEW" # 3. APPLY (lifecycle deprecate property): Deprecating property in active contract -> REVIEW -> Raises GovernanceReviewRequiredError lifecycle_args = SimpleNamespace( @@ -459,7 +311,7 @@ def test_breaking_change_review_behavior(tmp_path, capsys): assert "Governance decision REVIEW required" in str(exc.value) # 4. CI (evaluate_ci_gate & pipeline): Returns allowed=False / raises GovernanceReviewRequiredError - decision = evaluate_governance_decision(base, candidate, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, candidate) ci_res = evaluate_ci_gate(decision) assert ci_res.allowed is False assert ci_res.reason == "review_required" @@ -514,7 +366,7 @@ def test_prepare_ci_cd_artifacts_enforces_ci_gate_directly(tmp_path): from open_data_contract_standard.model import DataQuality candidate.schema_[0].properties[0].quality = [DataQuality(type="invalid_quality_type_xyz")] - decision = evaluate_governance_decision(base, candidate, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, candidate) assert decision.decision.value == "BLOCK" merged_out = tmp_path / "direct_merged.yaml" @@ -544,36 +396,13 @@ def test_ci_cd_adapter_preserves_allowed_reason(): """Verify that evaluate_ci_gate returns 'allowed' reason when allowed is True.""" base = _make_contract(status="active") candidate = _make_contract(status="active") - decision = evaluate_governance_decision(base, candidate, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, candidate) ci_dec = evaluate_ci_gate(decision) assert ci_dec.allowed is True assert ci_dec.reason == "allowed" -def test_release_prepare_includes_breaking_changes(tmp_path): - """Verify run_release_prepare returns breaking changes when breaking policy violations exist.""" - base = _make_contract(status="active") - candidate = _make_contract(status="active") - # Removing a property from active schema creates a breaking policy change - candidate.schema_[0].properties = [] - - base_path = dump_yaml(contract_to_dict(base), tmp_path / "base.yaml") - cand_path = dump_yaml(contract_to_dict(candidate), tmp_path / "cand.yaml") - - prep_args = SimpleNamespace( - base=str(base_path), - candidate=str(cand_path), - release_tag="v2.0.0", - output=str(tmp_path / "prep_out.yaml"), - runtime_context="auto", - effective_date=TEST_EFFECTIVE_DATE, - ) - - res = run_release_prepare(prep_args) - assert len(res["breakingChanges"]) > 0 - assert any("removed" in str(bc.get("message", "")).lower() or "breaking" in str(bc.get("message", "")).lower() for bc in res["breakingChanges"]) - def test_classify_repo_sets_blocked_status(tmp_path): """Verify classify_contracts_in_repo sets status='blocked' when a contract generates a BLOCK governance decision.""" @@ -587,11 +416,10 @@ def test_classify_repo_sets_blocked_status(tmp_path): dump_yaml(contract_to_dict(base_blocked), base_dir / "blocked.yaml") dump_yaml(contract_to_dict(cand_blocked), cand_dir / "blocked.yaml") - from semapact.devops.release_workflow import classify_contracts_in_repo + from semapact.application.services.repository_classification import classify_contracts_in_repo changes = classify_contracts_in_repo( base_root=str(base_dir), candidate_root=str(cand_dir), - context=TEST_CONTEXT, ) blocked_change = next(c for c in changes if c.contract_repo_path == "blocked.yaml") assert blocked_change.status == "blocked" @@ -604,6 +432,3 @@ def test_apply_release_candidate_requires_opendatacontractstandard_runtime(): base = _make_contract() candidate_dict = contract_to_dict(base) - from semapact.core.release import apply_release_candidate - with pytest.raises(TypeError, match="candidate_contract must be OpenDataContractStandard"): - apply_release_candidate(base, candidate_dict, "v1.0.1", required_bump="patch") # type: ignore[arg-type] diff --git a/tests/test_governance_analysis_boundary.py b/tests/test_governance_analysis_boundary.py index 82ebf0f2..2f33d5f1 100644 --- a/tests/test_governance_analysis_boundary.py +++ b/tests/test_governance_analysis_boundary.py @@ -22,7 +22,7 @@ import semapact.governance.evaluator as evaluator_module from semapact.change_context import ChangeContext from semapact.governance.evaluator import evaluate_governance_decision -from semapact.services import GovernanceService +from semapact.application.services.governance import GovernanceService TEST_CONTEXT = ChangeContext(effective_date=date(2026, 1, 1)) @@ -33,7 +33,6 @@ "semapact.importers", "semapact.interfaces", "semapact.orchestrator", - "semapact.services", ) _FORBIDDEN_SIDE_EFFECT_IMPORT_ROOTS = { "git", @@ -157,8 +156,8 @@ def test_governance_decision_evaluation_is_pure_and_deterministic( _block_external_side_effects(monkeypatch) - first = evaluate_governance_decision(base, candidate, context=TEST_CONTEXT) - second = evaluate_governance_decision(base, candidate, context=TEST_CONTEXT) + first = evaluate_governance_decision(base, candidate) + second = evaluate_governance_decision(base, candidate) assert first == second assert first.decision_id == second.decision_id @@ -180,9 +179,8 @@ def test_governance_service_evaluate_preserves_analysis_boundary( decision = GovernanceService().evaluate( base, candidate, - effective_date=TEST_CONTEXT.effective_date, ) - assert decision.context == TEST_CONTEXT + assert not hasattr(decision, "context") assert base == base_before assert candidate == candidate_before diff --git a/tests/test_governance_change_integration.py b/tests/test_governance_change_integration.py index dc636150..488b08d5 100644 --- a/tests/test_governance_change_integration.py +++ b/tests/test_governance_change_integration.py @@ -2,7 +2,6 @@ from __future__ import annotations -from datetime import date from typing import Any from open_data_contract_standard.model import ( AuthoritativeDefinition, @@ -17,8 +16,7 @@ Server, ) -from semapact.change_context import ChangeContext -from semapact.core.release import classify_contract_change +from semapact.lifecycle.change_classification import classify_contract_change from semapact.governance import DecisionResult, evaluate_governance_decision from semapact.governance_codes import GovernanceReasonCode from semapact.lifecycle.changes import ( @@ -29,7 +27,6 @@ from semapact.lifecycle.policy import evaluate_merge_policy -TEST_CONTEXT = ChangeContext(effective_date=date(2026, 8, 15)) def _make_active_contract(**kwargs: Any) -> OpenDataContractStandard: @@ -448,7 +445,7 @@ def test_evaluator_attaches_merge_conflict_evidence(self) -> None: ] decision = evaluate_governance_decision( - base, cand, context=TEST_CONTEXT, merge_conflicts=conflicts + base, cand, merge_conflicts=conflicts ) assert decision.evidence.has_changes is True @@ -478,7 +475,7 @@ def test_unmatched_merge_conflict_remains_in_reasons(self) -> None: ] decision = evaluate_governance_decision( - base, cand, context=TEST_CONTEXT, merge_conflicts=conflicts + base, cand, merge_conflicts=conflicts ) assert any( @@ -496,7 +493,7 @@ def test_retired_base_mutation_blocks_even_with_validation_error(self) -> None: SchemaProperty(name="id", logicalType="string", physicalType="text") ) - decision = evaluate_governance_decision(base, cand, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, cand) assert decision.decision == DecisionResult.BLOCK assert any( @@ -514,7 +511,7 @@ def test_retired_base_unhandled_or_metadata_mutation_blocks(self) -> None: cand = _make_active_contract(status="retired") cand.price = Pricing(priceAmount=999.0, priceCurrency="USD") - decision = evaluate_governance_decision(base, cand, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, cand) assert decision.decision == DecisionResult.BLOCK assert any( diff --git a/tests/test_governance_decision.py b/tests/test_governance_decision.py index 466a0445..93f44b6e 100644 --- a/tests/test_governance_decision.py +++ b/tests/test_governance_decision.py @@ -13,8 +13,8 @@ SchemaProperty, ) +from semapact.change_context import ChangeContext from semapact.governance import ( - ChangeContext, DecisionResult, GovernanceDecision, GovernanceReason, @@ -38,7 +38,6 @@ def _evaluate( return evaluate_governance_decision( base, candidate, - context=TEST_CONTEXT, **kwargs, ) @@ -263,7 +262,6 @@ def test_governance_decision_allow_invariants_validator(): decision_id="id1", decision=DecisionResult.ALLOW, contract_id="c1", - context=TEST_CONTEXT, breaking=False, required_version_bump="none", validation=val, @@ -277,7 +275,6 @@ def test_governance_decision_allow_invariants_validator(): decision_id="id2", decision=DecisionResult.ALLOW, contract_id="c1", - context=TEST_CONTEXT, breaking=True, required_version_bump="none", validation=val, @@ -291,7 +288,6 @@ def test_governance_decision_allow_invariants_validator(): decision_id="id3", decision=DecisionResult.ALLOW, contract_id="c1", - context=TEST_CONTEXT, breaking=False, required_version_bump="minor", validation=val, @@ -348,10 +344,17 @@ def test_governance_decision_serialization_and_deserialization(): assert decision == reconstructed assert reconstructed.decision == DecisionResult.REVIEW assert reconstructed.required_version_bump == "minor" - assert reconstructed.context == TEST_CONTEXT + assert not hasattr(reconstructed, "context") assert all(isinstance(reason["code"], str) for reason in dumped_json["reasons"]) +def test_governance_decision_has_no_execution_date_context(): + decision = _evaluate(_make_contract(), _make_contract()) + + assert "context" not in decision.model_dump(mode="json") + assert not hasattr(decision, "context") + + def test_governance_decision_input_immutability(): """Input contracts are untouched after evaluate_governance_decision.""" base = _make_contract() diff --git a/tests/test_governance_gate.py b/tests/test_governance_gate.py index 0dd17394..582a2d4d 100644 --- a/tests/test_governance_gate.py +++ b/tests/test_governance_gate.py @@ -9,7 +9,6 @@ from semapact.exceptions import GovernanceBlockedError, GovernanceReviewRequiredError from semapact.governance import ( - ChangeContext, ChangeEvidence, DecisionResult, GovernanceDecision, @@ -25,7 +24,6 @@ ) -TEST_CONTEXT = ChangeContext(effective_date=date(2026, 1, 1)) def _make_decision(result: DecisionResult) -> GovernanceDecision: @@ -58,7 +56,6 @@ def _make_decision(result: DecisionResult) -> GovernanceDecision: decision_id="dec-123", decision=result, contract_id="contract-test", - context=TEST_CONTEXT, breaking=breaking, required_version_bump=bump, validation=ValidationOutcome(valid=val_valid), diff --git a/tests/test_governance_golden_scenarios.py b/tests/test_governance_golden_scenarios.py index 56c22e04..f21f690f 100644 --- a/tests/test_governance_golden_scenarios.py +++ b/tests/test_governance_golden_scenarios.py @@ -15,8 +15,7 @@ import yaml from open_data_contract_standard.model import OpenDataContractStandard -from semapact.change_context import ChangeContext -from semapact.core.release import RequiredBump +from semapact.versioning import RequiredBump from semapact.governance import ( DecisionResult, evaluate_governance_decision, @@ -27,7 +26,6 @@ from semapact.lifecycle.merge_engine import MergeConflict FIXTURES_DIR = Path(__file__).parent / "fixtures" / "governance_scenarios" -TEST_CONTEXT = ChangeContext(effective_date=date(2026, 1, 1)) @dataclass(frozen=True) @@ -344,17 +342,14 @@ def _load_contract_from_yaml(yaml_path: Path) -> OpenDataContractStandard: @pytest.mark.parametrize("scenario", SCENARIO_MATRIX, ids=lambda s: s.name) def test_governance_golden_scenarios(scenario: GovernanceGoldenScenario) -> None: - """Evaluate golden scenario, verify domain invariants, and compare byte-exact public JSON.""" + """Evaluate each canonical governance scenario and verify deterministic public projection.""" scenario_dir = FIXTURES_DIR / scenario.name assert scenario_dir.exists(), f"Scenario directory missing: {scenario_dir}" base_path = scenario_dir / "base.yaml" cand_path = scenario_dir / "candidate.yaml" - expected_path = scenario_dir / "expected.json" - assert base_path.exists(), f"base.yaml missing for {scenario.name}" assert cand_path.exists(), f"candidate.yaml missing for {scenario.name}" - assert expected_path.exists(), f"expected.json missing for {scenario.name}" base_contract = _load_contract_from_yaml(base_path) cand_contract = _load_contract_from_yaml(cand_path) @@ -363,7 +358,6 @@ def test_governance_golden_scenarios(scenario: GovernanceGoldenScenario) -> None decision = evaluate_governance_decision( base_contract, cand_contract, - context=TEST_CONTEXT, merge_conflicts=scenario.merge_conflicts, ) @@ -394,16 +388,21 @@ def test_governance_golden_scenarios(scenario: GovernanceGoldenScenario) -> None f"[{scenario.name}] validation.valid mismatch: got {decision.validation.valid}, expected {scenario.expected_validation_valid}" ) - # 3. Public projection and read-only byte-exact golden comparison (Phase 3 & 4) + # 3. Public projection remains deterministic and free of execution-time context. public_decision = to_public_governance_decision(decision) - actual_json = serialize_public_governance_decision(public_decision, indent=2) + "\n" - expected_json = expected_path.read_text(encoding="utf-8") - - assert actual_json == expected_json, ( - f"[{scenario.name}] Golden JSON mismatch:\n" - f"--- Actual ---\n{actual_json}\n" - f"--- Expected ---\n{expected_json}" + first_json = serialize_public_governance_decision(public_decision, indent=2) + second_json = serialize_public_governance_decision( + to_public_governance_decision( + evaluate_governance_decision( + base_contract, + cand_contract, + merge_conflicts=scenario.merge_conflicts, + ) + ), + indent=2, ) + assert first_json == second_json + assert '"context"' not in first_json # ============================================================================== @@ -417,7 +416,7 @@ def test_lifecycle_draft_entity_skips_breaking_checks() -> None: base = _load_contract_from_yaml(scenario_dir / "base.yaml") cand = _load_contract_from_yaml(scenario_dir / "candidate.yaml") - decision = evaluate_governance_decision(base, cand, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, cand) assert decision.decision == DecisionResult.REVIEW assert decision.breaking is False @@ -431,7 +430,7 @@ def test_lifecycle_deprecated_entity_skips_breaking_checks() -> None: base = _load_contract_from_yaml(scenario_dir / "base.yaml") cand = _load_contract_from_yaml(scenario_dir / "candidate.yaml") - decision = evaluate_governance_decision(base, cand, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, cand) assert decision.decision == DecisionResult.REVIEW assert decision.breaking is False @@ -445,7 +444,7 @@ def test_lifecycle_active_to_retired_transition_reviewable() -> None: base = _load_contract_from_yaml(scenario_dir / "base.yaml") cand = _load_contract_from_yaml(scenario_dir / "candidate.yaml") - decision = evaluate_governance_decision(base, cand, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, cand) assert decision.decision == DecisionResult.REVIEW assert decision.policy.retired_violation is False @@ -461,7 +460,7 @@ def test_lifecycle_retired_contract_mutation_blocked() -> None: base = _load_contract_from_yaml(scenario_dir / "base.yaml") cand = _load_contract_from_yaml(scenario_dir / "candidate.yaml") - decision = evaluate_governance_decision(base, cand, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, cand) assert decision.decision == DecisionResult.BLOCK assert decision.policy.retired_violation is True @@ -482,7 +481,7 @@ def test_physical_name_identity_stability() -> None: base = _load_contract_from_yaml(scenario_dir / "base.yaml") cand = _load_contract_from_yaml(scenario_dir / "candidate.yaml") - decision = evaluate_governance_decision(base, cand, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, cand) assert decision.decision == DecisionResult.REVIEW assert decision.breaking is False @@ -532,13 +531,13 @@ def test_repeated_evaluation_determinism() -> None: base = _load_contract_from_yaml(scenario_dir / "base.yaml") cand = _load_contract_from_yaml(scenario_dir / "candidate.yaml") - first_dec = evaluate_governance_decision(base, cand, context=TEST_CONTEXT) + first_dec = evaluate_governance_decision(base, cand) first_json = serialize_public_governance_decision( to_public_governance_decision(first_dec), indent=2 ) for _ in range(10): - subsequent_dec = evaluate_governance_decision(base, cand, context=TEST_CONTEXT) + subsequent_dec = evaluate_governance_decision(base, cand) subsequent_json = serialize_public_governance_decision( to_public_governance_decision(subsequent_dec), indent=2 ) @@ -603,8 +602,8 @@ def test_non_semantic_key_ordering_determinism() -> None: base_1 = OpenDataContractStandard.model_validate(base_dict_1) base_2 = OpenDataContractStandard.model_validate(base_dict_2) - dec_1 = evaluate_governance_decision(base_1, base_1, context=TEST_CONTEXT) - dec_2 = evaluate_governance_decision(base_2, base_2, context=TEST_CONTEXT) + dec_1 = evaluate_governance_decision(base_1, base_1) + dec_2 = evaluate_governance_decision(base_2, base_2) assert dec_1.decision_id == dec_2.decision_id assert to_public_governance_decision(dec_1).to_canonical_json( @@ -632,10 +631,10 @@ def test_merge_conflict_ordering_determinism() -> None: ) dec_forward = evaluate_governance_decision( - base, base, context=TEST_CONTEXT, merge_conflicts=(c1, c2) + base, base, merge_conflicts=(c1, c2) ) dec_reverse = evaluate_governance_decision( - base, base, context=TEST_CONTEXT, merge_conflicts=(c2, c1) + base, base, merge_conflicts=(c2, c1) ) assert dec_forward.decision_id == dec_reverse.decision_id @@ -665,12 +664,20 @@ def test_fixtures_are_read_only() -> None: b = _load_contract_from_yaml(s_dir / "base.yaml") c = _load_contract_from_yaml(s_dir / "candidate.yaml") d = evaluate_governance_decision( - b, c, context=TEST_CONTEXT, merge_conflicts=scenario.merge_conflicts + b, c, merge_conflicts=scenario.merge_conflicts ) pub = to_public_governance_decision(d) - actual = serialize_public_governance_decision(pub, indent=2) + "\n" - expected = (s_dir / "expected.json").read_text(encoding="utf-8") - assert actual == expected + first = serialize_public_governance_decision(pub, indent=2) + second = serialize_public_governance_decision( + to_public_governance_decision( + evaluate_governance_decision( + b, c, merge_conflicts=scenario.merge_conflicts + ) + ), + indent=2, + ) + assert first == second + assert '"context"' not in first hashes_after: dict[str, str] = {} for f in FIXTURES_DIR.rglob("*"): diff --git a/tests/test_governance_proposal.py b/tests/test_governance_proposal.py index 90ab2690..7b1f9fb7 100644 --- a/tests/test_governance_proposal.py +++ b/tests/test_governance_proposal.py @@ -11,7 +11,6 @@ import semapact.application.services.governance as governance_service_module from semapact.application.services.governance import GovernanceService -from semapact.change_context import ChangeContext from semapact.governance.models import GovernanceDecision from semapact.lifecycle.merge_engine import MergeConflict @@ -51,7 +50,6 @@ def counted_evaluator( base_contract: OpenDataContractStandard, candidate_contract: OpenDataContractStandard, *, - context: ChangeContext, merge_conflicts: Sequence[MergeConflict] = (), ) -> GovernanceDecision: nonlocal calls @@ -59,7 +57,6 @@ def counted_evaluator( return original( base_contract, candidate_contract, - context=context, merge_conflicts=merge_conflicts, ) @@ -72,7 +69,6 @@ def counted_evaluator( proposal = GovernanceService().evaluate_proposal( base, candidate, - effective_date="2026-09-09", base_revision_ref="git:abc123", candidate_revision_ref="git:def456", source="api", @@ -81,7 +77,8 @@ def counted_evaluator( assert calls == 1 assert proposal.change_set.changes == proposal.decision.changes - assert proposal.change_set.context == proposal.decision.context + assert not hasattr(proposal.change_set, "context") + assert not hasattr(proposal.decision, "context") assert proposal.change_set.base_revision_ref == "git:abc123" assert proposal.change_set.candidate_revision_ref == "git:def456" assert proposal.change_set.source == "api" diff --git a/tests/test_governance_reason_codes.py b/tests/test_governance_reason_codes.py index 68c4d963..c6013e14 100644 --- a/tests/test_governance_reason_codes.py +++ b/tests/test_governance_reason_codes.py @@ -11,7 +11,6 @@ ) from semapact.governance import ( - ChangeContext, GOVERNANCE_REASON_REGISTRY, GovernanceReasonCode, GovernanceSeverity, @@ -20,7 +19,6 @@ from semapact.lifecycle.policy import evaluate_merge_policy -TEST_CONTEXT = ChangeContext(effective_date=date(2026, 1, 1)) ISSUE_78_PUBLIC_CODES = { GovernanceReasonCode.CONTRACT_ID_CHANGED, @@ -194,8 +192,8 @@ def test_reason_code_serialization_is_stable_and_message_is_descriptive_only(): candidate = _contract() candidate.schema_[0].properties = [] - first = evaluate_governance_decision(base, candidate, context=TEST_CONTEXT) - second = evaluate_governance_decision(base, candidate, context=TEST_CONTEXT) + first = evaluate_governance_decision(base, candidate) + second = evaluate_governance_decision(base, candidate) first_json = first.model_dump(mode="json") second_json = second.model_dump(mode="json") diff --git a/tests/test_governance_service.py b/tests/test_governance_service.py index b45a4208..0ecc1fd7 100644 --- a/tests/test_governance_service.py +++ b/tests/test_governance_service.py @@ -12,7 +12,7 @@ SchemaProperty, ) -from semapact.services import GovernanceService +from semapact.application.services.governance import GovernanceService def _cp(key: str, value: str) -> CustomProperty: @@ -62,14 +62,10 @@ def _custom_property_value(entity: object, key: str) -> str | None: return None -def test_governance_service_constructs_context_from_request_value() -> None: - decision = GovernanceService().evaluate( - _contract(), - _contract(), - effective_date="2026-08-13", - ) +def test_governance_service_evaluation_has_no_business_date_context() -> None: + decision = GovernanceService().evaluate(_contract(), _contract()) - assert decision.context.effective_date == date(2026, 8, 13) + assert not hasattr(decision, "context") def test_governance_service_reuses_context_for_merge_and_evaluation() -> None: @@ -88,8 +84,8 @@ def test_governance_service_reuses_context_for_merge_and_evaluation() -> None: if prop.name == "legacy_col" ) - assert analysis.decision.context == analysis.context assert analysis.context.effective_date == date(2026, 8, 13) + assert not hasattr(analysis.decision, "context") assert _custom_property_value(merged_property, "deprecationDate") == "2026-08-13" diff --git a/tests/test_history_golden_scenarios.py b/tests/test_history_golden_scenarios.py deleted file mode 100644 index 0ed8ee99..00000000 --- a/tests/test_history_golden_scenarios.py +++ /dev/null @@ -1,647 +0,0 @@ -from __future__ import annotations - -import json -from datetime import datetime, timedelta, timezone -from pathlib import Path -from types import SimpleNamespace - -import pytest -from open_data_contract_standard.model import ( - OpenDataContractStandard, - SchemaObject, - SchemaProperty, -) - -from semapact.application.services.deployment_history import DeploymentHistoryService -from semapact.application.services.evolution import EvolutionChainService -from semapact.application.services.governance import GovernanceService -from semapact.application.services.history import ProposalHistoryService -from semapact.application.services.history_integrity import HistoryIntegrityService -from semapact.application.services.release_history import ReleaseHistoryService -from semapact.application.services.runtime_history import RuntimeHistoryService -from semapact.contractops import ( - ReviewAuthorizationEvidence, - ReviewEvidenceAction, - VersionAuthorityConfig, - apply_contract_release, - authorize_contract_operation, - build_change_set, - build_release_plan, - resolve_release_version, -) -from semapact.deployment import ( - DeploymentAuthorization, - DeploymentPreview, - DeploymentTarget, - authorize_deployment, - build_deployment_plan, -) -from semapact.deployment.models import compute_deployment_preview_id -from semapact.governance import DecisionResult -from semapact.governance.gate import GovernanceOperation -from semapact.history import DeploymentStatus, HistoryConflictError -from semapact.observation import ( - ObservedAsset, - ObservedAssetIdentity, - ObservedPlatformState, - with_observed_state_fingerprint, -) -from semapact.platforms.git import GitWorkingTreeHistoryRepository -from semapact.reconciliation import ( - ReconciliationDifference, - ReconciliationDifferenceType, - ReconciliationResult, - ReconciliationSubject, - RuntimeDriftStatus, - RuntimeReasonCode, -) -from semapact.revision import build_contract_revision - - -GOLDEN_FIXTURE = ( - Path(__file__).parent - / "fixtures" - / "history_golden" - / "v1" - / "review_multi_deploy.json" -) -CONTRACT_ID = "orders-product" -CURRENT_VERSION = "1.2.3" -EFFECTIVE_DATE = "2026-09-15" - - -def _contract( - *, - name: str = "Orders", - version: str = CURRENT_VERSION, - include_note: bool = False, -) -> OpenDataContractStandard: - properties = [ - SchemaProperty( - name="id", - logicalType="string", - physicalType="varchar(255)", - required=True, - ) - ] - if include_note: - properties.append( - SchemaProperty( - name="note", - logicalType="string", - physicalType="varchar(255)", - required=False, - ) - ) - return OpenDataContractStandard( - apiVersion="v3.1.0", - kind="DataContract", - id=CONTRACT_ID, - name=name, - version=version, - status="active", - schema=[SchemaObject(name="orders", properties=properties)], - ) - - -def _proposal_history(backend: GitWorkingTreeHistoryRepository) -> ProposalHistoryService: - return ProposalHistoryService( - revisions=backend, - change_sets=backend, - decisions=backend, - decision_links=backend, - ) - - -def _release_history(backend: GitWorkingTreeHistoryRepository) -> ReleaseHistoryService: - return ReleaseHistoryService( - revisions=backend, - change_sets=backend, - decisions=backend, - decision_links=backend, - release_plans=backend, - release_records=backend, - ) - - -def _deployment_history(backend: GitWorkingTreeHistoryRepository) -> DeploymentHistoryService: - return DeploymentHistoryService( - releases=backend, - deployment_plans=backend, - deployment_previews=backend, - deployment_authorizations=backend, - deployment_records=backend, - ) - - -def _runtime_history(backend: GitWorkingTreeHistoryRepository) -> RuntimeHistoryService: - return RuntimeHistoryService( - observations=backend, - reconciliations=backend, - releases=backend, - deployments=backend, - ) - - -def _evolution(backend: GitWorkingTreeHistoryRepository) -> EvolutionChainService: - return EvolutionChainService( - revisions=backend, - change_sets=backend, - decisions=backend, - decision_links=backend, - releases=backend, - deployments=backend, - observations=backend, - runtime_reconciliations=backend, - ) - - -def _record_release( - root: Path, - *, - base: OpenDataContractStandard, - candidate: OpenDataContractStandard, - source: str, -): - backend = GitWorkingTreeHistoryRepository(root) - base_revision = build_contract_revision(base) - candidate_revision = build_contract_revision(candidate) - proposal = GovernanceService().evaluate_proposal( - base_revision.contract, - candidate_revision.contract, - effective_date=EFFECTIVE_DATE, - base_revision_ref=base_revision.revision_id, - candidate_revision_ref=candidate_revision.revision_id, - source=source, - actor_reference="actor:golden", - ) - _proposal_history(backend).record_proposal( - proposal, - base_revision=base_revision, - candidate_revision=candidate_revision, - ) - - release_plan = build_release_plan(proposal.change_set, proposal.decision) - version_resolution = resolve_release_version( - release_plan, - current_version=CURRENT_VERSION, - config=VersionAuthorityConfig(), - ) - evidence = None - if proposal.decision.decision is DecisionResult.REVIEW: - evidence = ReviewAuthorizationEvidence( - evidence_reference="review:apply:golden", - decision_id=proposal.decision.decision_id, - change_set_id=proposal.change_set.change_set_id, - release_plan_id=release_plan.release_plan_id, - version_resolution_id=version_resolution.version_resolution_id, - operation=GovernanceOperation.APPLY, - action=ReviewEvidenceAction.APPROVE, - ) - authorization = authorize_contract_operation( - proposal.decision, - proposal.change_set, - release_plan, - version_resolution, - GovernanceOperation.APPLY, - evidence=evidence, - ) - applied_release = apply_contract_release( - candidate_revision.contract, - candidate_revision_ref=candidate_revision.revision_id, - decision=proposal.decision, - change_set=proposal.change_set, - release_plan=release_plan, - version_resolution=version_resolution, - authorization=authorization, - ) - release_record = _release_history(backend).record_release( - release_plan=release_plan, - version_resolution=version_resolution, - authorization=authorization, - applied_release=applied_release, - ) - return SimpleNamespace( - backend=backend, - base_revision=base_revision, - candidate_revision=candidate_revision, - proposal=proposal, - release_plan=release_plan, - version_resolution=version_resolution, - authorization=authorization, - applied_release=applied_release, - release_record=release_record, - ) - - -def _deployment_preview(plan, *, label: str) -> DeploymentPreview: - observation_fingerprint = f"preview-observation:{label}" - preview_id = compute_deployment_preview_id( - deployment_plan_id=plan.deployment_plan_id, - platform=plan.target.platform, - runtime_target=plan.target.runtime_target, - source_identifier=plan.target.source_reference, - observation_fingerprint=observation_fingerprint, - operations=(), - ) - return DeploymentPreview( - deployment_preview_id=preview_id, - deployment_plan_id=plan.deployment_plan_id, - platform=plan.target.platform, - runtime_target=plan.target.runtime_target, - source_identifier=plan.target.source_reference, - observation_fingerprint=observation_fingerprint, - operations=(), - ) - - -def _record_deployment( - bundle, - *, - label: str, - runtime_target: str, - source_reference: str, - status: DeploymentStatus, - started_at: datetime, -): - plan = build_deployment_plan( - bundle.applied_release, - DeploymentTarget( - platform="databricks", - runtime_target=runtime_target, - source_reference=source_reference, - server_name=label, - ), - ) - deploy_evidence = ReviewAuthorizationEvidence( - evidence_reference=f"review:deploy:{label}", - decision_id=bundle.proposal.decision.decision_id, - change_set_id=bundle.proposal.change_set.change_set_id, - release_plan_id=bundle.release_plan.release_plan_id, - version_resolution_id=bundle.version_resolution.version_resolution_id, - operation=GovernanceOperation.DEPLOY, - action=ReviewEvidenceAction.APPROVE, - scope_reference=plan.deployment_plan_id, - ) - contractops_authorization = authorize_contract_operation( - bundle.proposal.decision, - bundle.proposal.change_set, - bundle.release_plan, - bundle.version_resolution, - GovernanceOperation.DEPLOY, - evidence=deploy_evidence, - ) - deployment_authorization: DeploymentAuthorization = authorize_deployment( - plan, - bundle.applied_release, - contractops_authorization, - ) - preview = _deployment_preview(plan, label=label) - record = _deployment_history(bundle.backend).record_execution( - plan=plan, - preview=preview, - authorization=deployment_authorization, - status=status, - started_at=started_at, - completed_at=started_at + timedelta(seconds=5), - actor_reference="agent:golden-deploy", - external_reference=f"run:{label}", - ) - return SimpleNamespace( - plan=plan, - preview=preview, - authorization=deployment_authorization, - record=record, - ) - - -def _observation( - *, - captured_at: datetime, - source_reference: str, - namespace: tuple[str, str], - asset_type: str, -) -> ObservedPlatformState: - return with_observed_state_fingerprint( - ObservedPlatformState( - platform="databricks", - source_identifier=source_reference, - assets=( - ObservedAsset( - identity=ObservedAssetIdentity( - platform="databricks", - namespace=namespace, - asset="orders", - ), - asset_type=asset_type, - ), - ), - captured_at=captured_at, - ) - ) - - -def _reconciliation_result( - observation: ObservedPlatformState, - *, - version: str, - drift: bool, -) -> ReconciliationResult: - differences = () - if drift: - differences = ( - ReconciliationDifference( - difference_type=ReconciliationDifferenceType.MISMATCH, - subject=ReconciliationSubject.PHYSICAL_TYPE, - reason_code=RuntimeReasonCode.RUNTIME_PHYSICAL_TYPE_CHANGED, - path="orders.id.physical_type", - asset_identity="orders", - property_identity="id", - expected="STRING", - observed="BIGINT", - ), - ) - assert observation.fingerprint is not None - return ReconciliationResult( - contract_id=CONTRACT_ID, - contract_version=version, - observation_source_identifier=observation.source_identifier, - observation_fingerprint=observation.fingerprint, - differences=differences, - ) - - -def _record_runtime(bundle, deployment, *, captured_at: datetime, drift: bool): - namespace = tuple(deployment.record.runtime_target.split(".")) - assert len(namespace) == 2 - observation = _observation( - captured_at=captured_at, - source_reference=deployment.record.source_reference, - namespace=(namespace[0], namespace[1]), - asset_type="VIEW" if drift else "TABLE", - ) - record = _runtime_history(bundle.backend).record_reconciliation( - observation, - _reconciliation_result( - observation, - version=bundle.release_record.contract_version, - drift=drift, - ), - release_record_id=bundle.release_record.release_record_id, - deployment_record_id=deployment.record.deployment_record_id, - ) - return SimpleNamespace(observation=observation, record=record) - - -def _evolution_projection(chain) -> dict[str, object]: - proposals = [] - for proposal in chain.proposals: - releases = [] - for release in proposal.releases: - deployments = [] - for deployment in release.deployments: - deployments.append( - { - "deploymentRecordId": deployment.deployment.deployment_record_id, - "runtimeTarget": deployment.deployment.runtime_target, - "status": deployment.deployment.status.value, - "runtime": [ - { - "reconciliationRecordId": item.reconciliation.runtime_reconciliation_record_id, - "observationRecordId": item.reconciliation.observation_record_id, - "status": item.reconciliation.status.value, - } - for item in deployment.runtime - ], - } - ) - releases.append( - { - "releaseRecordId": release.release.release_record_id, - "contractVersion": release.release.contract_version, - "releasedRevisionId": release.release.released_revision_id, - "deployments": deployments, - } - ) - proposals.append( - { - "changeSetId": proposal.change_set.change_set_id, - "decisionId": proposal.decision.decision_id if proposal.decision else None, - "decision": proposal.decision.decision.value if proposal.decision else None, - "baseRevisionId": ( - proposal.base_revision.revision_id if proposal.base_revision else None - ), - "candidateRevisionId": ( - proposal.candidate_revision.revision_id - if proposal.candidate_revision - else None - ), - "releases": releases, - } - ) - return { - "contractId": chain.contract_id, - "proposals": proposals, - "unlinkedReleaseIds": [ - item.release.release_record_id for item in chain.unlinked_releases - ], - "unlinkedRuntimeIds": [ - item.reconciliation.runtime_reconciliation_record_id - for item in chain.unlinked_runtime - ], - "brokenReferences": [ - { - "sourceKind": item.source_kind, - "sourceId": item.source_id, - "referenceField": item.reference_field, - "targetKind": item.target_kind, - "targetId": item.target_id, - "reason": item.reason, - } - for item in chain.broken_references - ], - } - - -def _history_files(root: Path) -> dict[str, object]: - """Snapshot persisted schema and exact canonical content through its checksum seal.""" - history_root = root / ".semapact" / "history" - files: dict[str, object] = {} - for path in sorted(history_root.rglob("*.json"), key=Path.as_posix): - relative = path.relative_to(root).as_posix() - content = json.loads(path.read_text(encoding="utf-8")) - checksum_path = path.with_name(f"{path.name}.sha256") - files[relative] = { - "checksum": checksum_path.read_text(encoding="utf-8").strip(), - "topLevelKeys": sorted(content), - } - return files - - -def _build_review_multi_deploy(root: Path): - bundle = _record_release( - root, - base=_contract(), - candidate=_contract(include_note=True), - source="golden-review", - ) - assert bundle.proposal.decision.decision is DecisionResult.REVIEW - - start = datetime(2026, 9, 15, 0, 0, tzinfo=timezone.utc) - production = _record_deployment( - bundle, - label="production", - runtime_target="main.prod", - source_reference="workspace:prod", - status=DeploymentStatus.SUCCEEDED, - started_at=start, - ) - staging = _record_deployment( - bundle, - label="staging", - runtime_target="main.stage", - source_reference="workspace:stage", - status=DeploymentStatus.FAILED, - started_at=start + timedelta(minutes=10), - ) - sync = _record_runtime( - bundle, - production, - captured_at=start + timedelta(hours=1), - drift=False, - ) - drift = _record_runtime( - bundle, - production, - captured_at=start + timedelta(hours=2), - drift=True, - ) - assert sync.record.status is RuntimeDriftStatus.IN_SYNC - assert drift.record.status is RuntimeDriftStatus.DRIFT - - reloaded = GitWorkingTreeHistoryRepository(root) - chain = _evolution(reloaded).reconstruct(CONTRACT_ID) - assert _evolution(reloaded).reconstruct(CONTRACT_ID) == chain - integrity = HistoryIntegrityService( - storage=reloaded, - evolution=_evolution(reloaded), - ).check_contract(CONTRACT_ID) - assert integrity.valid is True - - manifest = { - "fixtureVersion": "history-v1", - "files": _history_files(root), - "evolution": _evolution_projection(chain), - } - return SimpleNamespace( - manifest=manifest, - backend=reloaded, - bundle=bundle, - production=production, - staging=staging, - sync=sync, - drift=drift, - chain=chain, - ) - - -def test_review_multi_deploy_history_matches_checked_in_v1_fixture(tmp_path: Path) -> None: - first = _build_review_multi_deploy(tmp_path / "first") - second = _build_review_multi_deploy(tmp_path / "second") - - assert first.manifest == second.manifest - expected = json.loads(GOLDEN_FIXTURE.read_text(encoding="utf-8")) - if first.manifest != expected: - pytest.fail( - "history golden fixture mismatch; update only for an intentional persisted " - "schema/identity change.\nACTUAL:\n" - + json.dumps(first.manifest, indent=2, sort_keys=True) - ) - - -def test_allow_release_reconstructs_without_review_evidence(tmp_path: Path) -> None: - bundle = _record_release( - tmp_path, - base=_contract(name="Orders old"), - candidate=_contract(name="Orders new"), - source="golden-allow", - ) - - assert bundle.proposal.decision.decision is DecisionResult.ALLOW - assert bundle.authorization.evidence_reference is None - chain = _evolution(GitWorkingTreeHistoryRepository(tmp_path)).reconstruct(CONTRACT_ID) - assert len(chain.proposals) == 1 - assert chain.proposals[0].decision is not None - assert chain.proposals[0].decision.decision is DecisionResult.ALLOW - assert len(chain.proposals[0].releases) == 1 - assert chain.proposals[0].releases[0].release == bundle.release_record - - -def test_blocked_proposal_persists_without_release(tmp_path: Path) -> None: - backend = GitWorkingTreeHistoryRepository(tmp_path) - base_revision = build_contract_revision(_contract(version=CURRENT_VERSION)) - candidate_revision = build_contract_revision(_contract(version="2.0.0")) - proposal = GovernanceService().evaluate_proposal( - base_revision.contract, - candidate_revision.contract, - effective_date=EFFECTIVE_DATE, - base_revision_ref=base_revision.revision_id, - candidate_revision_ref=candidate_revision.revision_id, - source="golden-block", - actor_reference="actor:golden", - ) - assert proposal.decision.decision is DecisionResult.BLOCK - _proposal_history(backend).record_proposal( - proposal, - base_revision=base_revision, - candidate_revision=candidate_revision, - ) - - chain = _evolution(backend).reconstruct(CONTRACT_ID) - assert len(chain.proposals) == 1 - assert chain.proposals[0].decision is not None - assert chain.proposals[0].decision.decision is DecisionResult.BLOCK - assert chain.proposals[0].releases == () - assert backend.list_release_records(CONTRACT_ID) == () - - -def test_golden_history_rejects_conflicting_overwrite(tmp_path: Path) -> None: - scenario = _build_review_multi_deploy(tmp_path) - decision = scenario.bundle.proposal.decision - conflicting = decision.model_copy(update={"contract_id": "other-contract"}) - - with pytest.raises(HistoryConflictError, match="different content"): - scenario.backend.put_decision(conflicting) - - assert scenario.backend.get_decision(decision.decision_id) == decision - - -def test_broken_reference_is_explicit_and_does_not_fabricate_release(tmp_path: Path) -> None: - bundle = _record_release( - tmp_path, - base=_contract(), - candidate=_contract(include_note=True), - source="golden-broken-base", - ) - broken = build_change_set( - contract_id=CONTRACT_ID, - base_revision_ref="missing-revision", - candidate_revision_ref=bundle.candidate_revision.revision_id, - changes=bundle.proposal.change_set.changes, - context=bundle.proposal.change_set.context, - source="golden-broken", - ) - bundle.backend.put_change_set(broken) - - chain = _evolution(bundle.backend).reconstruct(CONTRACT_ID) - broken_path = next(item for item in chain.proposals if item.change_set == broken) - assert broken_path.base_revision is None - assert broken_path.decision is None - assert broken_path.releases == () - assert any( - item.source_id == broken.change_set_id - and item.reference_field == "base_revision_ref" - and item.target_id == "missing-revision" - and item.reason == "NOT_FOUND" - for item in chain.broken_references - ) diff --git a/tests/test_history_integrity.py b/tests/test_history_integrity.py index e9f6cb46..cc2cb849 100644 --- a/tests/test_history_integrity.py +++ b/tests/test_history_integrity.py @@ -1,6 +1,5 @@ from __future__ import annotations -from datetime import date from pathlib import Path import pytest @@ -12,7 +11,6 @@ from semapact.application.services.evolution import EvolutionChainService from semapact.application.services.history_integrity import HistoryIntegrityService -from semapact.change_context import ChangeContext from semapact.contractops import build_change_set_from_decision from semapact.governance import GovernanceDecision, evaluate_governance_decision from semapact.history import ( @@ -24,7 +22,6 @@ from semapact.platforms.git import GitWorkingTreeHistoryRepository -CONTEXT = ChangeContext(effective_date=date(2026, 9, 15)) def _contract(*, name: str) -> OpenDataContractStandard: @@ -55,7 +52,6 @@ def _decision() -> GovernanceDecision: return evaluate_governance_decision( _contract(name="orders-base"), _contract(name="orders-candidate"), - context=CONTEXT, ) @@ -76,9 +72,6 @@ def _evolution_service(backend: GitWorkingTreeHistoryRepository) -> EvolutionCha decisions=backend, decision_links=backend, releases=backend, - deployments=backend, - observations=backend, - runtime_reconciliations=backend, ) diff --git a/tests/test_history_repository.py b/tests/test_history_repository.py index c69ac4f4..803c9806 100644 --- a/tests/test_history_repository.py +++ b/tests/test_history_repository.py @@ -1,6 +1,5 @@ from __future__ import annotations -from datetime import date from pathlib import Path import pytest @@ -10,7 +9,6 @@ SchemaProperty, ) -from semapact.change_context import ChangeContext from semapact.contractops import build_change_set_from_decision from semapact.governance import GovernanceDecision, evaluate_governance_decision from semapact.history import ( @@ -23,7 +21,6 @@ from semapact.platforms.git import GitWorkingTreeHistoryRepository -CONTEXT = ChangeContext(effective_date=date(2026, 9, 12)) def _contract(*, name: str) -> OpenDataContractStandard: @@ -54,7 +51,6 @@ def _decision(*, candidate_name: str) -> GovernanceDecision: return evaluate_governance_decision( _contract(name="orders-base"), _contract(name=candidate_name), - context=CONTEXT, ) @@ -78,7 +74,7 @@ def test_artifacts_round_trip_through_segregated_repository_ports( assert decisions.get_decision(decision.decision_id) == decision assert change_sets.get_change_set(change_set.change_set_id) == change_set - assert change_sets.get_change_set(change_set.change_set_id).context == CONTEXT + assert not hasattr(change_sets.get_change_set(change_set.change_set_id), "context") def test_identical_writes_are_idempotent(tmp_path: Path) -> None: diff --git a/tests/test_identity_integration.py b/tests/test_identity_integration.py index 238c4856..480285e8 100644 --- a/tests/test_identity_integration.py +++ b/tests/test_identity_integration.py @@ -3,9 +3,9 @@ import pytest from open_data_contract_standard.model import SchemaObject, SchemaProperty -from semapact.core.release import classify_contract_change +from semapact.lifecycle.change_classification import classify_contract_change from semapact.exceptions import ValidationError -from semapact.governance import ChangeContext +from semapact.change_context import ChangeContext from semapact.lifecycle.merge_engine import ContractMergeEngine from semapact.lifecycle.policy import evaluate_merge_policy diff --git a/tests/test_lifecycle_effective_scope_matrix.py b/tests/test_lifecycle_effective_scope_matrix.py index 30f9da8a..7e51c656 100644 --- a/tests/test_lifecycle_effective_scope_matrix.py +++ b/tests/test_lifecycle_effective_scope_matrix.py @@ -121,17 +121,6 @@ def _cp(key: str, value: str) -> CustomProperty: LifecycleStatus.DEPRECATED, False, ), - # 9. Proposed contract alias: interpreted as draft - ( - "proposed", - None, - None, - None, - LifecycleStatus.DRAFT, - LifecycleStatus.DRAFT, - LifecycleStatus.DRAFT, - False, - ), ], ) def test_lifecycle_effective_scope_matrix( diff --git a/tests/test_lifecycle_fail_closed_governance.py b/tests/test_lifecycle_fail_closed_governance.py index 86ce0310..2eaac1e0 100644 --- a/tests/test_lifecycle_fail_closed_governance.py +++ b/tests/test_lifecycle_fail_closed_governance.py @@ -12,9 +12,8 @@ from semapact.governance.evaluator import evaluate_governance_decision from semapact.governance.models import DecisionResult from semapact.governance_codes import GovernanceReasonCode -from semapact.lifecycle.helpers import allows_breaking_changes from semapact.lifecycle.merge_engine import ContractMergeEngine -from semapact.services.governance_service import GovernanceService +from semapact.application.services.governance import GovernanceService TEST_CONTEXT = ChangeContext(effective_date=date(2026, 8, 14)) @@ -62,7 +61,7 @@ def test_invalid_root_lifecycle_evaluates_to_block(): candidate = base.model_copy(deep=True) candidate.status = "invalid_status_xyz" - decision = evaluate_governance_decision(base, candidate, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, candidate) assert decision.decision == DecisionResult.BLOCK assert any( r.code == GovernanceReasonCode.VALIDATION_FAILED @@ -78,7 +77,7 @@ def test_invalid_schema_lifecycle_evaluates_to_block(): _cp("lifecycleStatus", "unknown_schema_status") ] - decision = evaluate_governance_decision(base, candidate, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, candidate) assert decision.decision == DecisionResult.BLOCK assert any( r.code == GovernanceReasonCode.VALIDATION_FAILED @@ -94,7 +93,7 @@ def test_invalid_property_lifecycle_evaluates_to_block(): _cp("lifecycleStatus", "invalid_prop_status") ] - decision = evaluate_governance_decision(base, candidate, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, candidate) assert decision.decision == DecisionResult.BLOCK assert any( r.code == GovernanceReasonCode.VALIDATION_FAILED @@ -110,7 +109,7 @@ def test_invalid_nested_property_lifecycle_evaluates_to_block(): _cp("lifecycleStatus", "garbage_nested_status") ] - decision = evaluate_governance_decision(base, candidate, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, candidate) assert decision.decision == DecisionResult.BLOCK assert any( r.code == GovernanceReasonCode.VALIDATION_FAILED @@ -193,18 +192,18 @@ def test_merge_engine_detects_conflict_when_source_status_missing(): assert any(c.rule == "physical_type_change" for c in result.conflicts) -def test_merge_engine_auto_deprecates_when_source_status_proposed_or_missing(): +def test_merge_engine_auto_deprecates_when_source_status_draft(): """Active governed target + source missing an active property -> auto-deprecation must occur.""" engine = ContractMergeEngine() target = _base_active_contract() # status: active - # Source has status="proposed" and is missing "customer" property + # Source is draft and is missing "customer" property source = OpenDataContractStandard( apiVersion="v3.1.0", kind="DataContract", id="orders_contract", version="1.0.0", - status="proposed", + status="draft", schema=[ SchemaObject( name="orders", @@ -235,20 +234,3 @@ def test_merge_engine_auto_deprecates_when_source_status_proposed_or_missing(): assert cp_map.get("lifecycleStatus") == "deprecated" assert cp_map.get("semapact.removed") == "true" - -def test_allows_breaking_changes_compatibility_wrapper(): - active_contract = _base_active_contract() - draft_contract = active_contract.model_copy(deep=True) - draft_contract.status = "draft" - - assert allows_breaking_changes(active_contract) is True - assert allows_breaking_changes(draft_contract) is False - - active_prop = SchemaProperty( - name="col", customProperties=[_cp("lifecycleStatus", "active")] - ) - deprecated_prop = SchemaProperty( - name="col", customProperties=[_cp("lifecycleStatus", "deprecated")] - ) - assert allows_breaking_changes(active_prop) is True - assert allows_breaking_changes(deprecated_prop) is False diff --git a/tests/test_lifecycle_status_resolver.py b/tests/test_lifecycle_status_resolver.py index 91ec9083..b8959053 100644 --- a/tests/test_lifecycle_status_resolver.py +++ b/tests/test_lifecycle_status_resolver.py @@ -28,12 +28,10 @@ def _cp(key: str, value: str) -> CustomProperty: return CustomProperty(property=key, value=value) -def test_normalize_status_valid_and_aliases(): +def test_normalize_status_valid_values(): assert normalize_status("draft") is LifecycleStatus.DRAFT assert normalize_status("DRAFT") is LifecycleStatus.DRAFT assert normalize_status(" draft ") is LifecycleStatus.DRAFT - assert normalize_status("proposed") is LifecycleStatus.DRAFT - assert normalize_status("PROPOSED") is LifecycleStatus.DRAFT assert normalize_status("active") is LifecycleStatus.ACTIVE assert normalize_status("Active") is LifecycleStatus.ACTIVE assert normalize_status("deprecated") is LifecycleStatus.DEPRECATED @@ -54,6 +52,9 @@ def test_normalize_status_invalid_inputs(): with pytest.raises(ValueError, match="Unknown lifecycle status: 'invalid_val'"): normalize_status("invalid_val") + with pytest.raises(ValueError, match="Unknown lifecycle status: 'proposed'"): + normalize_status("proposed") + def test_lifecycle_from_custom_properties(): assert lifecycle_from_custom_properties(None) is None @@ -92,7 +93,7 @@ def test_resolve_contract_lifecycle_precedence(): ) assert resolve_contract_lifecycle(c1) is LifecycleStatus.ACTIVE - # 2. Legacy customProperties fallback when root status is missing + # Root customProperties do not define contract lifecycle. c2 = OpenDataContractStandard( apiVersion="v3.1.0", kind="DataContract", @@ -100,9 +101,9 @@ def test_resolve_contract_lifecycle_precedence(): version="1.0.0", customProperties=[_cp("lifecycleStatus", "deprecated")], ) - assert resolve_contract_lifecycle(c2) is LifecycleStatus.DEPRECATED + assert resolve_contract_lifecycle(c2) is LifecycleStatus.DRAFT - # 3. Canonical default when neither is provided + # Canonical default when status is absent c3 = OpenDataContractStandard( apiVersion="v3.1.0", kind="DataContract", diff --git a/tests/test_operational_history.py b/tests/test_operational_history.py new file mode 100644 index 00000000..a19d386b --- /dev/null +++ b/tests/test_operational_history.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import json +import sqlite3 +from datetime import datetime, timezone + +from semapact.history import ( + build_operational_deployment_event, + create_operational_history_sink, +) +from semapact.platforms.delta import DeltaOperationalHistorySink +from semapact.platforms.sqlite import SQLiteOperationalHistorySink +from semapact.reconciliation import RuntimeDriftStatus + + +def _event(): + return build_operational_deployment_event( + bundle_digest="sha256:" + ("a" * 64), + contract_release_id=None, + contract_id="orders-product", + contract_version="1.2.3", + revision_ref="git:abc123", + deployment_plan_id="plan-1", + deployment_preview_id="preview-1", + platform="databricks", + runtime_target="main.sales", + source_reference="https://workspace.example", + status="SUCCEEDED", + reconciliation_status=RuntimeDriftStatus.IN_SYNC, + started_at=datetime(2026, 9, 20, 1, 0, tzinfo=timezone.utc), + completed_at=datetime(2026, 9, 20, 1, 1, tzinfo=timezone.utc), + error_message=None, + ) + + +def test_operational_history_is_disabled_when_not_configured() -> None: + assert create_operational_history_sink(None) is None + assert create_operational_history_sink("") is None + + +def test_sqlite_operational_history_is_idempotent(tmp_path) -> None: + database = tmp_path / "operational.db" + sink = create_operational_history_sink(f"sqlite:///{database}") + + assert isinstance(sink, SQLiteOperationalHistorySink) + event = _event() + assert event.event_version == "1" + sink.record_deployment(event) + sink.record_deployment(event) + + with sqlite3.connect(database) as connection: + rows = connection.execute( + "SELECT event_id, payload_json FROM deployment_events" + ).fetchall() + + assert len(rows) == 1 + assert rows[0][0] == event.event_id + assert json.loads(rows[0][1])["deployment_plan_id"] == "plan-1" + + +def test_delta_operational_history_adapter_is_lazy() -> None: + sink = create_operational_history_sink("delta:///tmp/semapact-history") + + assert isinstance(sink, DeltaOperationalHistorySink) diff --git a/tests/test_operational_history_config.py b/tests/test_operational_history_config.py new file mode 100644 index 00000000..3ae13fc1 --- /dev/null +++ b/tests/test_operational_history_config.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import json +from pathlib import Path +import pytest +from pydantic import ValidationError as PydanticValidationError + +from semapact.core.config_schema import ( + DeltaOperationalHistoryConfig, + SQLiteOperationalHistoryConfig, + SemaPactConfigSchema, + operational_history_uri_from_config, + parse_operational_history_config, + published_config_json_schema, +) + + +def test_operational_history_config_is_disabled_when_omitted() -> None: + assert parse_operational_history_config(None) is None + assert operational_history_uri_from_config(None) is None + + +def test_sqlite_operational_history_config_is_typed() -> None: + config = parse_operational_history_config( + { + "backend": "sqlite", + "path": ".semapact/operational.db", + } + ) + + assert isinstance(config, SQLiteOperationalHistoryConfig) + assert config.as_uri() == "sqlite:///.semapact/operational.db" + + +def test_delta_operational_history_config_is_typed() -> None: + config = parse_operational_history_config( + { + "backend": "delta", + "table_uri": "s3://governance/semapact/history", + } + ) + + assert isinstance(config, DeltaOperationalHistoryConfig) + assert ( + config.as_uri() + == "delta:///s3://governance/semapact/history" + ) + + +@pytest.mark.parametrize( + "payload", + [ + {"backend": "sqlite"}, + {"backend": "delta"}, + {"backend": "git", "path": ".semapact/history"}, + {"backend": "sqlite", "path": "", "extra": "forbidden"}, + ], +) +def test_invalid_operational_history_config_fails_closed(payload) -> None: + with pytest.raises(PydanticValidationError): + parse_operational_history_config(payload) + + +def test_root_config_schema_exposes_operational_backend_discriminator() -> None: + schema = SemaPactConfigSchema.model_json_schema() + + assert "history" in schema["properties"] + rendered = str(schema) + assert "sqlite" in rendered + assert "delta" in rendered + assert "discriminator" in rendered + + +def test_published_config_json_schema_matches_pydantic_source_of_truth() -> None: + schema = json.loads( + Path("schemas/semapact-config.schema.json").read_text(encoding="utf-8") + ) + + assert schema == published_config_json_schema() diff --git a/tests/test_pipeline_manifest.py b/tests/test_pipeline_manifest.py index 75d5b400..af905745 100644 --- a/tests/test_pipeline_manifest.py +++ b/tests/test_pipeline_manifest.py @@ -13,7 +13,8 @@ ) from semapact.exceptions import GovernanceReviewRequiredError -from semapact.governance import ChangeContext, enforce_governance_gate +from semapact.change_context import ChangeContext +from semapact.governance import enforce_governance_gate from semapact.governance.evaluator import evaluate_governance_decision from semapact.lifecycle.merge_engine import MergeResult from semapact.orchestrator.pipeline import ContractPipeline, _build_manifest_payload @@ -69,7 +70,6 @@ def test_manifest_breaking_changes_project_authoritative_decision() -> None: decision = evaluate_governance_decision( base, candidate, - context=TEST_CONTEXT, ) decision_payload = decision.model_dump(mode="json") manifest = _build_manifest_payload(decision) @@ -92,7 +92,6 @@ def test_manifest_non_breaking_change_has_no_breaking_projection() -> None: decision = evaluate_governance_decision( base, candidate, - context=TEST_CONTEXT, ) manifest = _build_manifest_payload(decision) diff --git a/tests/test_public_governance_decision.py b/tests/test_public_governance_decision.py index 4d74e830..f807084a 100644 --- a/tests/test_public_governance_decision.py +++ b/tests/test_public_governance_decision.py @@ -2,8 +2,6 @@ from __future__ import annotations -from datetime import date -from pathlib import Path import pytest from pydantic import ValidationError as PydanticValidationError from open_data_contract_standard.model import ( @@ -13,9 +11,7 @@ SchemaProperty, ) -from semapact.change_context import ChangeContext from semapact.governance import ( - PublicChangeContextV1, PublicChangeEvidenceV1, PublicGovernanceChangeEvidenceV1, PublicGovernanceChangeV1, @@ -29,10 +25,6 @@ ) -FIXTURES_DIR = Path(__file__).parent / "fixtures" / "governance_decisions" -TEST_CONTEXT = ChangeContext(effective_date=date(2026, 1, 1)) - - def _get_schemas(contract: OpenDataContractStandard) -> list[SchemaObject]: return getattr(contract, "schema_", getattr(contract, "schema", [])) or [] @@ -80,7 +72,7 @@ def test_public_decision_allow_scenario(): base = _make_contract() candidate = _make_contract() - decision = evaluate_governance_decision(base, candidate, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, candidate) public_dec = to_public_governance_decision(decision) assert isinstance(public_dec, PublicGovernanceDecisionV1) @@ -89,7 +81,6 @@ def test_public_decision_allow_scenario(): assert public_dec.contract_id == "my-data-contract" assert public_dec.breaking is False assert public_dec.required_version_bump == "none" - assert public_dec.context.effective_date == "2026-01-01" assert public_dec.validation.valid is True assert public_dec.policy.valid is True assert public_dec.policy.id_violation is False @@ -108,7 +99,7 @@ def test_public_decision_allow_scenario(): assert dumped["schemaVersion"] == "1" assert dumped["decisionId"] == decision.decision_id assert dumped["contractId"] == "my-data-contract" - assert dumped["context"] == {"effectiveDate": "2026-01-01"} + assert "context" not in dumped assert dumped["requiredVersionBump"] == "none" assert dumped["evidence"] == {"hasChanges": False, "mergeConflictsCount": 0} assert dumped["changes"] == [] @@ -123,7 +114,7 @@ def test_public_decision_review_scenario(): CustomProperty(property="lifecycleStatus", value="deprecated") ] - decision = evaluate_governance_decision(base, cand, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, cand) public_dec = to_public_governance_decision(decision) assert public_dec.schema_version == "1" @@ -152,7 +143,7 @@ def test_public_decision_breaking_review_scenario(): cand = _make_contract() _get_schemas(cand)[0].properties[1].physicalType = "decimal(8,2)" - decision = evaluate_governance_decision(base, cand, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, cand) public_dec = to_public_governance_decision(decision) assert public_dec.schema_version == "1" @@ -174,7 +165,7 @@ def test_public_decision_block_retired_scenario(): cand = _make_contract(status="retired") _get_schemas(cand)[0].properties[1].description = "New description" - decision = evaluate_governance_decision(base, cand, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, cand) public_dec = to_public_governance_decision(decision) assert public_dec.decision == "BLOCK" @@ -194,7 +185,7 @@ def test_public_decision_block_validation_scenario(): SchemaProperty(name="", logicalType="string", physicalType="", required=True) ) - decision = evaluate_governance_decision(base, cand, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, cand) public_dec = to_public_governance_decision(decision) assert public_dec.decision == "BLOCK" @@ -208,7 +199,6 @@ def test_public_decision_block_validation_scenario(): def test_public_models_direct_instantiation_and_helpers(): """Verify direct instantiation of public models and helper structures.""" - context = PublicChangeContextV1(effective_date="2026-08-28") reason = PublicGovernanceReasonV1( code="TEST_CODE", severity="WARNING", @@ -239,7 +229,6 @@ def test_public_models_direct_instantiation_and_helpers(): decision_id="00000000-0000-0000-0000-000000000000", decision="REVIEW", contract_id="orders", - context=context, breaking=True, required_version_bump="major", reason_codes=("DECIMAL_PRECISION_REDUCED",), @@ -250,7 +239,7 @@ def test_public_models_direct_instantiation_and_helpers(): changes=(change,), ) - assert decision.context.effective_date == "2026-08-28" + assert not hasattr(decision, "context") assert decision.changes[0].evidence[0].source == "MERGE_CONFLICT" assert decision.changes[0].field == "physicalType" assert decision.validation.issues[0].details == {"key": "val"} @@ -264,7 +253,6 @@ def test_public_literals_strict_validation(): "decisionId": "test", "decision": "BANANA", # Invalid decision "contractId": "orders", - "context": {"effectiveDate": "2026-01-01"}, "breaking": False, "requiredVersionBump": "none", "reasonCodes": [], @@ -281,7 +269,6 @@ def test_public_literals_strict_validation(): "decisionId": "test", "decision": "ALLOW", "contractId": "orders", - "context": {"effectiveDate": "2026-01-01"}, "breaking": False, "requiredVersionBump": "huge", # Invalid bump "reasonCodes": [], @@ -309,7 +296,6 @@ def test_public_literals_strict_validation(): "decisionId": "test", "decision": "ALLOW", "contractId": "orders", - "context": {"effectiveDate": "2026-01-01"}, "breaking": False, "requiredVersionBump": "patch", # Invalid bump for v1 "reasonCodes": [], @@ -324,7 +310,7 @@ def test_public_literals_strict_validation(): def test_public_decision_immutability_and_extra_forbid(): """Public governance models are frozen and reject extra fields.""" base = _make_contract() - decision = evaluate_governance_decision(base, base, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, base) public_dec = to_public_governance_decision(decision) # Immutability @@ -338,7 +324,6 @@ def test_public_decision_immutability_and_extra_forbid(): "decisionId": "test", "decision": "ALLOW", "contractId": "orders", - "context": {"effectiveDate": "2026-01-01"}, "breaking": False, "requiredVersionBump": "none", "reasonCodes": [], @@ -374,7 +359,6 @@ def test_public_decision_byte_level_determinism(): decision_id="11111111-1111-1111-1111-111111111111", decision="REVIEW", contract_id="orders", - context=PublicChangeContextV1(effective_date="2026-01-01"), breaking=True, required_version_bump="major", reason_codes=("Z_CODE", "A_CODE"), @@ -397,7 +381,6 @@ def test_public_decision_byte_level_determinism(): decision_id="11111111-1111-1111-1111-111111111111", decision="REVIEW", contract_id="orders", - context=PublicChangeContextV1(effective_date="2026-01-01"), breaking=True, required_version_bump="major", reason_codes=("Z_CODE", "A_CODE"), @@ -421,7 +404,7 @@ def test_public_decision_roundtrip_deserialization(): cand = _make_contract() _get_schemas(cand)[0].properties[1].physicalType = "decimal(8,2)" - decision = evaluate_governance_decision(base, cand, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, cand) public_dec = to_public_governance_decision(decision) json_str = serialize_public_governance_decision(public_dec) @@ -432,54 +415,3 @@ def test_public_decision_roundtrip_deserialization(): assert restored.decision == "REVIEW" assert restored.schema_version == "1" - -def test_golden_fixtures_match(): - """Verify golden JSON fixtures match projected decisions with exact string equality (read-only assertion).""" - # 1. ALLOW clean - base = _make_contract() - allow_dec = evaluate_governance_decision(base, base, context=TEST_CONTEXT) - allow_pub = to_public_governance_decision(allow_dec) - allow_json_str = serialize_public_governance_decision(allow_pub, indent=2) + "\n" - expected_allow = (FIXTURES_DIR / "allow_clean_decision.json").read_text(encoding="utf-8") - assert allow_json_str == expected_allow - - # 2. BREAKING review decimal - cand_breaking = _make_contract() - _get_schemas(cand_breaking)[0].properties[1].physicalType = "decimal(8,2)" - breaking_dec = evaluate_governance_decision(base, cand_breaking, context=TEST_CONTEXT) - breaking_pub = to_public_governance_decision(breaking_dec) - breaking_json_str = serialize_public_governance_decision(breaking_pub, indent=2) + "\n" - expected_breaking = (FIXTURES_DIR / "breaking_review_decision.json").read_text(encoding="utf-8") - assert breaking_json_str == expected_breaking - - # 3. REVIEW deprecate property - cand_deprecate = _make_contract() - _get_schemas(cand_deprecate)[0].properties[1].customProperties = [ - CustomProperty(property="lifecycleStatus", value="deprecated") - ] - review_dec = evaluate_governance_decision(base, cand_deprecate, context=TEST_CONTEXT) - review_pub = to_public_governance_decision(review_dec) - review_json_str = serialize_public_governance_decision(review_pub, indent=2) + "\n" - expected_review = (FIXTURES_DIR / "review_deprecate_decision.json").read_text(encoding="utf-8") - assert review_json_str == expected_review - - # 4. BLOCK retired contract - base_retired = _make_contract(status="retired") - cand_retired = _make_contract(status="retired") - _get_schemas(cand_retired)[0].properties[1].description = "Retired update" - retired_dec = evaluate_governance_decision(base_retired, cand_retired, context=TEST_CONTEXT) - retired_pub = to_public_governance_decision(retired_dec) - retired_json_str = serialize_public_governance_decision(retired_pub, indent=2) + "\n" - expected_retired = (FIXTURES_DIR / "block_retired_decision.json").read_text(encoding="utf-8") - assert retired_json_str == expected_retired - - # 5. BLOCK invalid validation - cand_invalid = _make_contract() - _get_schemas(cand_invalid)[0].properties.append( - SchemaProperty(name="", logicalType="string", physicalType="", required=True) - ) - invalid_dec = evaluate_governance_decision(base, cand_invalid, context=TEST_CONTEXT) - invalid_pub = to_public_governance_decision(invalid_dec) - invalid_json_str = serialize_public_governance_decision(invalid_pub, indent=2) + "\n" - expected_invalid = (FIXTURES_DIR / "block_validation_decision.json").read_text(encoding="utf-8") - assert invalid_json_str == expected_invalid diff --git a/tests/test_reconciliation_service.py b/tests/test_reconciliation_service.py index 6d5f937f..93456660 100644 --- a/tests/test_reconciliation_service.py +++ b/tests/test_reconciliation_service.py @@ -16,7 +16,7 @@ with_observed_state_fingerprint, ) from semapact.reconciliation import RuntimeDriftStatus -from semapact.services.reconciliation_service import ReconciliationService +from semapact.application.services.reconciliation import ReconciliationService class _Loader: diff --git a/tests/test_release_history.py b/tests/test_release_history.py deleted file mode 100644 index 45424ec0..00000000 --- a/tests/test_release_history.py +++ /dev/null @@ -1,302 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import pytest -from open_data_contract_standard.model import ( - OpenDataContractStandard, - SchemaObject, - SchemaProperty, -) - -from semapact.application.services.governance import GovernanceService -from semapact.application.services.history import ProposalHistoryService -from semapact.application.services.release_history import ReleaseHistoryService -from semapact.contractops import ( - ReviewAuthorizationEvidence, - ReviewEvidenceAction, - VersionAuthorityConfig, - apply_contract_release, - authorize_contract_operation, - build_release_plan, - resolve_release_version, -) -from semapact.governance.gate import GovernanceOperation -from semapact.history import HistoryConflictError -from semapact.platforms.git import GitWorkingTreeHistoryRepository -from semapact.revision import build_contract_revision - - -def _contract(*, include_note: bool = False) -> OpenDataContractStandard: - properties = [ - SchemaProperty( - name="id", - logicalType="string", - physicalType="varchar(255)", - required=True, - ) - ] - if include_note: - properties.append( - SchemaProperty( - name="note", - logicalType="string", - physicalType="varchar(255)", - required=False, - ) - ) - - return OpenDataContractStandard( - apiVersion="v3.1.0", - kind="DataContract", - id="orders-product", - name="Orders", - version="1.2.3", - status="active", - schema=[ - SchemaObject( - name="orders", - properties=properties, - ) - ], - ) - - -def _services(tmp_path: Path): - backend = GitWorkingTreeHistoryRepository(tmp_path) - proposal_history = ProposalHistoryService( - revisions=backend, - change_sets=backend, - decisions=backend, - decision_links=backend, - ) - release_history = ReleaseHistoryService( - revisions=backend, - change_sets=backend, - decisions=backend, - decision_links=backend, - release_plans=backend, - release_records=backend, - ) - return proposal_history, release_history, backend - - -def _release_bundle(*, source: str): - base_revision = build_contract_revision(_contract()) - candidate_revision = build_contract_revision(_contract(include_note=True)) - proposal = GovernanceService().evaluate_proposal( - base_revision.contract, - candidate_revision.contract, - effective_date="2026-09-12", - base_revision_ref=base_revision.revision_id, - candidate_revision_ref=candidate_revision.revision_id, - source=source, - actor_reference=f"actor:{source}", - ) - release_plan = build_release_plan(proposal.change_set, proposal.decision) - version_resolution = resolve_release_version( - release_plan, - current_version="1.2.3", - config=VersionAuthorityConfig(), - ) - apply_review = ReviewAuthorizationEvidence( - evidence_reference=f"review:{source}", - decision_id=proposal.decision.decision_id, - change_set_id=proposal.change_set.change_set_id, - release_plan_id=release_plan.release_plan_id, - version_resolution_id=version_resolution.version_resolution_id, - operation=GovernanceOperation.APPLY, - action=ReviewEvidenceAction.APPROVE, - ) - authorization = authorize_contract_operation( - proposal.decision, - proposal.change_set, - release_plan, - version_resolution, - GovernanceOperation.APPLY, - evidence=apply_review, - ) - applied_release = apply_contract_release( - candidate_revision.contract, - candidate_revision_ref=candidate_revision.revision_id, - decision=proposal.decision, - change_set=proposal.change_set, - release_plan=release_plan, - version_resolution=version_resolution, - authorization=authorization, - ) - return ( - proposal, - base_revision, - candidate_revision, - release_plan, - version_resolution, - authorization, - applied_release, - ) - - -def _record_bundle(tmp_path: Path, *, source: str = "test"): - proposal_history, release_history, backend = _services(tmp_path) - ( - proposal, - base_revision, - candidate_revision, - release_plan, - version_resolution, - authorization, - applied_release, - ) = _release_bundle(source=source) - proposal_history.record_proposal( - proposal, - base_revision=base_revision, - candidate_revision=candidate_revision, - ) - record = release_history.record_release( - release_plan=release_plan, - version_resolution=version_resolution, - authorization=authorization, - applied_release=applied_release, - ) - return ( - record, - proposal, - candidate_revision, - release_plan, - version_resolution, - authorization, - applied_release, - release_history, - backend, - ) - - -def test_records_release_against_exact_plan_and_released_revision(tmp_path: Path) -> None: - ( - record, - proposal, - candidate_revision, - release_plan, - version_resolution, - authorization, - applied_release, - _, - backend, - ) = _record_bundle(tmp_path) - - assert backend.get_release_plan(release_plan.release_plan_id) == release_plan - assert backend.get_release_record(record.release_record_id) == record - assert ( - backend.get_release_record_by_version( - "orders-product", - version_resolution.selected_version, - ) - == record - ) - released_revision = backend.get_revision(record.released_revision_id) - assert released_revision.revision_id != candidate_revision.revision_id - assert str(released_revision.contract.version) == version_resolution.selected_version - assert record.decision_id == proposal.decision.decision_id - assert record.change_set_id == proposal.change_set.change_set_id - assert record.version_resolution_id == version_resolution.version_resolution_id - assert record.authorization_id == authorization.authorization_id - assert record.applied_release_id == applied_release.applied_release_id - assert record.required_version_bump == version_resolution.required_version_bump - assert record.actual_version_bump == version_resolution.actual_bump - assert record.version_authority == version_resolution.authority - assert record.review_evidence_reference == authorization.evidence_reference - assert record.review_evidence_action == authorization.evidence_action - - -def test_exact_release_history_write_is_idempotent(tmp_path: Path) -> None: - ( - first, - _, - _, - release_plan, - version_resolution, - authorization, - applied_release, - release_history, - backend, - ) = _record_bundle(tmp_path) - - second = release_history.record_release( - release_plan=release_plan, - version_resolution=version_resolution, - authorization=authorization, - applied_release=applied_release, - ) - - assert second == first - assert backend.list_release_records("orders-product") == (first,) - - -def test_conflicting_release_for_same_contract_version_fails_closed(tmp_path: Path) -> None: - _record_bundle(tmp_path, source="first") - proposal_history, release_history, _ = _services(tmp_path) - ( - proposal, - base_revision, - candidate_revision, - release_plan, - version_resolution, - authorization, - applied_release, - ) = _release_bundle(source="second") - proposal_history.record_proposal( - proposal, - base_revision=base_revision, - candidate_revision=candidate_revision, - ) - - with pytest.raises(HistoryConflictError, match="version"): - release_history.record_release( - release_plan=release_plan, - version_resolution=version_resolution, - authorization=authorization, - applied_release=applied_release, - ) - - -def test_release_history_rejects_wrong_apply_authorization(tmp_path: Path) -> None: - proposal_history, release_history, _ = _services(tmp_path) - ( - proposal, - base_revision, - candidate_revision, - release_plan, - version_resolution, - _, - applied_release, - ) = _release_bundle(source="release") - proposal_history.record_proposal( - proposal, - base_revision=base_revision, - candidate_revision=candidate_revision, - ) - publish_review = ReviewAuthorizationEvidence( - evidence_reference="review:publish", - decision_id=proposal.decision.decision_id, - change_set_id=proposal.change_set.change_set_id, - release_plan_id=release_plan.release_plan_id, - version_resolution_id=version_resolution.version_resolution_id, - operation=GovernanceOperation.PUBLISH, - action=ReviewEvidenceAction.APPROVE, - ) - publish_authorization = authorize_contract_operation( - proposal.decision, - proposal.change_set, - release_plan, - version_resolution, - GovernanceOperation.PUBLISH, - evidence=publish_review, - ) - - with pytest.raises(ValueError, match="APPLY"): - release_history.record_release( - release_plan=release_plan, - version_resolution=version_resolution, - authorization=publish_authorization, - applied_release=applied_release, - ) diff --git a/tests/test_release_plan.py b/tests/test_release_plan.py index 4c0b943d..4a0f9959 100644 --- a/tests/test_release_plan.py +++ b/tests/test_release_plan.py @@ -1,7 +1,5 @@ from __future__ import annotations -from datetime import date - import pytest from open_data_contract_standard.model import ( OpenDataContractStandard, @@ -10,7 +8,6 @@ ) from pydantic import ValidationError as PydanticValidationError -from semapact.change_context import ChangeContext from semapact.contractops import ( ReleasePrecondition, build_change_set_from_decision, @@ -20,9 +17,6 @@ from semapact.governance import DecisionResult, evaluate_governance_decision -CONTEXT = ChangeContext(effective_date=date(2026, 9, 9)) - - def _contract( *, contract_id: str = "orders-product", @@ -64,7 +58,7 @@ def _proposal( base_revision_ref: str = "rev:base", candidate_revision_ref: str = "rev:candidate", ): - decision = evaluate_governance_decision(base, candidate, context=CONTEXT) + decision = evaluate_governance_decision(base, candidate) change_set = build_change_set_from_decision( decision, base_revision_ref=base_revision_ref, @@ -146,7 +140,6 @@ def test_release_plan_fails_closed_for_mismatched_proposal_artifacts() -> None: review_change_set, review_decision = _proposal(base, review_candidate) metadata_change_set, _ = _proposal(base, metadata_candidate) - assert review_change_set.context == metadata_change_set.context assert review_change_set.contract_id == metadata_change_set.contract_id assert review_change_set.changes != metadata_change_set.changes diff --git a/tests/test_release_planning_service.py b/tests/test_release_planning_service.py index 0211bcdb..296cdb83 100644 --- a/tests/test_release_planning_service.py +++ b/tests/test_release_planning_service.py @@ -7,7 +7,8 @@ ) from semapact.contractops import VersionAuthorityConfig, resolve_release_version -from semapact.services import GovernanceService, ReleasePlanningService +from semapact.application.services.governance import GovernanceService +from semapact.application.services.release_planning import ReleasePlanningService class CountingGovernanceService: @@ -69,7 +70,6 @@ def test_release_planning_composes_one_decision_into_exact_m2_artifacts() -> Non result = service.plan( _contract(name="Orders old"), _contract(name="Orders new"), - effective_date="2026-09-11", base_revision_ref="git:base-123", candidate_revision_ref="git:candidate-456", ) @@ -98,14 +98,12 @@ def test_release_planning_is_deterministic_for_same_exact_inputs() -> None: first = service.plan( base, candidate, - effective_date="2026-09-11", base_revision_ref="git:base-123", candidate_revision_ref="git:candidate-456", ) second = service.plan( base, candidate, - effective_date="2026-09-11", base_revision_ref="git:base-123", candidate_revision_ref="git:candidate-456", ) diff --git a/tests/test_release_workflow_service.py b/tests/test_release_workflow_service.py new file mode 100644 index 00000000..40bb73d0 --- /dev/null +++ b/tests/test_release_workflow_service.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +from datetime import datetime, timezone + +import pytest +from open_data_contract_standard.model import ( + OpenDataContractStandard, + SchemaObject, + SchemaProperty, +) +from pydantic import ValidationError as PydanticValidationError + +from semapact.approval import build_approval_record +from semapact.application.models.release import ReleaseBundle +from semapact.application.services.release_approval import ReleaseApprovalResolver +from semapact.application.services.release_workflow import ReleaseFinalizer, ReleaseWorkflowService +from semapact.exceptions import ContractOpsAuthorizationError +from semapact.contractops import ReviewEvidenceAction +from semapact.governance import DecisionResult, GovernanceOperation +from semapact.platforms.git import GitWorkingTreeHistoryRepository + + +def _contract(*, include_created_at: bool = False) -> OpenDataContractStandard: + properties = [ + SchemaProperty( + name="id", + logicalType="string", + physicalType="varchar(255)", + required=True, + ) + ] + if include_created_at: + properties.append( + SchemaProperty( + name="created_at", + logicalType="timestamp", + physicalType="timestamp", + required=False, + ) + ) + return OpenDataContractStandard( + apiVersion="v3.1.0", + kind="DataContract", + id="orders-product", + name="Orders", + version="1.2.3", + status="active", + schema=[SchemaObject(name="orders", properties=properties)], + ) + + +def _bundle() -> ReleaseBundle: + return ReleaseWorkflowService().assess( + _contract(), + _contract(include_created_at=True), + base_revision_ref="git:base", + candidate_revision_ref="git:candidate", + ) + + +def test_release_assess_is_target_neutral_and_resolves_version_once() -> None: + bundle = _bundle() + + assert bundle.decision.decision is DecisionResult.REVIEW + assert bundle.version_resolution.current_version == "1.2.3" + assert bundle.version_resolution.selected_version == "1.3.0" + assert bundle.release_snapshot.selected_version == "1.3.0" + assert "target" not in bundle.model_dump(mode="json") + + +def test_release_bundle_digest_is_deterministic_and_tamper_evident() -> None: + first = _bundle() + second = _bundle() + assert first == second + assert first.bundle_digest == second.bundle_digest + + payload = first.model_dump(mode="json") + payload["bundle_digest"] = "sha256:" + ("0" * 64) + with pytest.raises(PydanticValidationError, match="digest does not match"): + ReleaseBundle.model_validate(payload) + + +def test_release_approval_binds_publish_scope_and_exact_bundle() -> None: + bundle = _bundle() + approval = ReleaseWorkflowService().approve( + bundle, + actor_reference="github:user:reviewer", + recorded_at=datetime(2026, 9, 20, 2, tzinfo=timezone.utc), + ) + + assert approval.operation is GovernanceOperation.PUBLISH + assert approval.scope_reference == bundle.release_snapshot.release_snapshot_id + assert approval.evidence_references == (bundle.bundle_digest,) + + +def test_review_release_finalize_requires_exact_approval(tmp_path) -> None: + bundle = _bundle() + repository = GitWorkingTreeHistoryRepository(tmp_path) + workflow = ReleaseWorkflowService() + finalizer = ReleaseFinalizer() + + with pytest.raises(ContractOpsAuthorizationError, match="requires approval"): + finalizer.finalize(bundle) + + approval = workflow.approve( + bundle, + actor_reference="github:user:reviewer", + recorded_at=datetime(2026, 9, 20, 2, tzinfo=timezone.utc), + ) + repository.put_approval_record(approval) + record = finalizer.finalize( + bundle, + approval=approval, + ) + repository.put_contract_release(record) + + assert record.contract_version == "1.3.0" + assert record.release_snapshot_id == bundle.release_snapshot.release_snapshot_id + assert record.source_revision_ref == "git:candidate" + assert ( + repository.get_contract_release(record.contract_release_id) + == record + ) + + +def test_release_approval_resolver_finds_only_exact_bundle(tmp_path) -> None: + bundle = _bundle() + repository = GitWorkingTreeHistoryRepository(tmp_path) + approval = ReleaseWorkflowService().approve( + bundle, + actor_reference="github:user:reviewer", + recorded_at=datetime(2026, 9, 20, 2, tzinfo=timezone.utc), + ) + repository.put_approval_record(approval) + + assert ReleaseApprovalResolver(repository).resolve(bundle) == approval + + tampered = bundle.model_copy( + update={"bundle_digest": "sha256:" + ("f" * 64)} + ) + assert ReleaseApprovalResolver(repository).resolve(tampered) is None + + + +def test_release_approval_resolver_fails_closed_on_exact_conflict(tmp_path) -> None: + bundle = _bundle() + repository = GitWorkingTreeHistoryRepository(tmp_path) + approval = ReleaseWorkflowService().approve( + bundle, + actor_reference="github:user:reviewer", + recorded_at=datetime(2026, 9, 20, 2, tzinfo=timezone.utc), + ) + rejection = build_approval_record( + decision_id=bundle.decision.decision_id, + change_set_id=bundle.change_set.change_set_id, + release_plan_id=bundle.release_plan.release_plan_id, + version_resolution_id=bundle.version_resolution.version_resolution_id, + operation=GovernanceOperation.PUBLISH, + action=ReviewEvidenceAction.REJECT, + actor_reference="github:user:owner", + recorded_at=datetime(2026, 9, 20, 3, tzinfo=timezone.utc), + scope_reference=bundle.release_snapshot.release_snapshot_id, + evidence_references=(bundle.bundle_digest,), + ) + repository.put_approval_record(approval) + repository.put_approval_record(rejection) + + resolver = ReleaseApprovalResolver(repository) + assert resolver.has_conflict(bundle) is True + assert resolver.resolve(bundle) is None diff --git a/tests/test_retired_immutability.py b/tests/test_retired_immutability.py index a2fcdb21..15220270 100644 --- a/tests/test_retired_immutability.py +++ b/tests/test_retired_immutability.py @@ -4,7 +4,7 @@ import json from datetime import date from pathlib import Path -from unittest.mock import MagicMock, patch +from unittest.mock import patch import pytest from open_data_contract_standard.model import ( @@ -19,7 +19,6 @@ from semapact.change_context import ChangeContext from semapact.core.loader import ContractLoader -from semapact.devops.release_workflow import build_batch_release_manifest, create_release_pull_request from semapact.exceptions import GovernanceBlockedError from semapact.governance import ( DecisionResult, @@ -35,12 +34,9 @@ run_lifecycle_promote, ) from semapact.interfaces.commands.merge_cmd import run_merge -from semapact.interfaces.commands.release_cmd import ( - run_release_classify, - run_release_prepare, -) +from semapact.interfaces.commands.release_cmd import run_release_classify from semapact.orchestrator.pipeline import ContractPipeline -from semapact.services import GovernanceService +from semapact.application.services.governance import GovernanceService from semapact.utils.schema_utils import contract_to_dict from semapact.utils.yaml_utils import dump_yaml @@ -106,7 +102,7 @@ def test_retired_unchanged_allows(self) -> None: base = _make_retired_contract() candidate = _make_retired_contract() - decision = evaluate_governance_decision(base, candidate, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, candidate) assert decision.decision == DecisionResult.ALLOW assert decision.policy.retired_violation is False @@ -122,7 +118,7 @@ def test_retired_descriptive_metadata_change_blocks(self) -> None: candidate.description = Description(usage="Updated description on retired contract") candidate.tags = ["legacy", "orders", "updated"] - decision = evaluate_governance_decision(base, candidate, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, candidate) assert decision.decision == DecisionResult.BLOCK assert decision.policy.retired_violation is True @@ -131,25 +127,6 @@ def test_retired_descriptive_metadata_change_blocks(self) -> None: for r in decision.reasons ) - def test_retired_via_custom_properties_effective_lifecycle_blocks(self) -> None: - """Effective lifecycle from customProperties.lifecycleStatus=retired blocks candidate mutation.""" - base = _make_retired_contract() - base.status = None - base.customProperties = [ - CustomProperty(property="lifecycleStatus", value="retired") - ] - - candidate = base.model_copy(deep=True) - candidate.description = Description(usage="Mutated candidate description") - - decision = evaluate_governance_decision(base, candidate, context=TEST_CONTEXT) - - assert decision.decision == DecisionResult.BLOCK - assert decision.policy.retired_violation is True - assert any( - r.code == GovernanceReasonCode.RETIRED_CONTRACT_MODIFIED - for r in decision.reasons - ) def test_retired_schema_change_blocks(self) -> None: """Adding a new schema object to a retired contract produces BLOCK.""" @@ -163,7 +140,7 @@ def test_retired_schema_change_blocks(self) -> None: ) ) - decision = evaluate_governance_decision(base, candidate, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, candidate) assert decision.decision == DecisionResult.BLOCK assert decision.policy.retired_violation is True @@ -182,7 +159,7 @@ def test_retired_property_change_blocks(self) -> None: SchemaProperty(name="customer_id", logicalType="string", required=False) ) - decision = evaluate_governance_decision(base, candidate, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, candidate) assert decision.decision == DecisionResult.BLOCK assert decision.policy.retired_violation is True @@ -206,7 +183,7 @@ def test_retired_quality_rule_change_blocks(self) -> None: ) ] - decision = evaluate_governance_decision(base, candidate, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, candidate) assert decision.decision == DecisionResult.BLOCK assert decision.policy.retired_violation is True @@ -229,7 +206,7 @@ def test_retired_relationship_change_blocks(self) -> None: ) ] - decision = evaluate_governance_decision(base, candidate, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, candidate) assert decision.decision == DecisionResult.BLOCK assert decision.policy.retired_violation is True @@ -246,7 +223,7 @@ def test_retired_custom_properties_change_blocks(self) -> None: CustomProperty(property="costCenter", value="finance") ] - decision = evaluate_governance_decision(base, candidate, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, candidate) assert decision.decision == DecisionResult.BLOCK assert decision.policy.retired_violation is True @@ -263,7 +240,7 @@ def test_retired_reactivation_blocks(self) -> None: candidate = _make_retired_contract() candidate.status = target_status - decision = evaluate_governance_decision(base, candidate, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, candidate) assert decision.decision == DecisionResult.BLOCK assert decision.policy.retired_violation is True @@ -278,7 +255,7 @@ def test_retired_version_change_blocks(self) -> None: candidate = _make_retired_contract() candidate.version = "2.0.0" - decision = evaluate_governance_decision(base, candidate, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, candidate) assert decision.decision == DecisionResult.BLOCK assert decision.policy.retired_violation is True @@ -292,7 +269,7 @@ def test_active_to_retired_transition_reviews_not_mutation(self) -> None: base = _make_active_contract() candidate = _make_retired_contract() - decision = evaluate_governance_decision(base, candidate, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, candidate) assert decision.decision == DecisionResult.REVIEW assert decision.policy.retired_violation is False @@ -312,7 +289,7 @@ def test_draft_and_deprecated_to_retired_transition_reviews(self) -> None: base.status = start_status candidate = _make_retired_contract() - decision = evaluate_governance_decision(base, candidate, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, candidate) assert decision.decision == DecisionResult.REVIEW assert decision.policy.retired_violation is False @@ -333,7 +310,7 @@ def test_retired_invalid_mutated_candidate_blocks_cleanly(self) -> None: candidate.status = "invalid_status_value" candidate.description = Description(usage="Mutated invalid contract") - decision = evaluate_governance_decision(base, candidate, context=TEST_CONTEXT) + decision = evaluate_governance_decision(base, candidate) assert decision.decision == DecisionResult.BLOCK assert decision.validation.valid is False @@ -492,72 +469,6 @@ def test_lifecycle_deprecate_property_blocks_retired_contract(self, tmp_path: Pa ) assert contract_path.read_text(encoding="utf-8") == original_content - def test_release_prepare_blocks_retired_contract(self, tmp_path: Path) -> None: - """run_release_prepare blocks retired contract modification and writes no output.""" - base_retired = _make_retired_contract() - candidate = _make_retired_contract() - candidate.description = Description(usage="Candidate with metadata updates") - - base_path = dump_yaml(contract_to_dict(base_retired), tmp_path / "base.yaml") - cand_path = dump_yaml(contract_to_dict(candidate), tmp_path / "cand.yaml") - out_path = tmp_path / "promoted.yaml" - - args = argparse.Namespace( - base=str(base_path), - candidate=str(cand_path), - output=str(out_path), - release_tag="orders/v1.1.0", - runtime_context="auto", - effective_date="2026-08-14", - ) - - with pytest.raises(GovernanceBlockedError) as exc_info: - run_release_prepare(args) - - assert exc_info.value.operation == GovernanceOperation.PROPOSE - assert exc_info.value.decision is not None - assert any( - r.code == GovernanceReasonCode.RETIRED_CONTRACT_MODIFIED - for r in exc_info.value.decision.reasons - ) - assert not out_path.exists() - - def test_release_create_pr_blocks_retired_contract(self, tmp_path: Path) -> None: - """create_release_pull_request blocks retired contract mutation before Git/file mutations.""" - base_retired = _make_retired_contract() - candidate = _make_retired_contract() - candidate.description = Description(usage="Modified retired contract") - - repo_dir = tmp_path / "repo" - repo_dir.mkdir() - contract_file = repo_dir / "contracts" / "orders.yaml" - contract_file.parent.mkdir(parents=True) - dump_yaml(contract_to_dict(base_retired), contract_file) - - mock_config = MagicMock() - - with pytest.raises(GovernanceBlockedError) as exc_info: - create_release_pull_request( - config=mock_config, - repo_path=str(repo_dir), - contract_repo_path="contracts/orders.yaml", - base_contract=base_retired, - candidate_contract=candidate, - release_tag="orders/v1.1.0", - source_branch="release/orders-v1.1.0", - target_branch="main", - context=TEST_CONTEXT, - ) - - assert exc_info.value.operation == GovernanceOperation.PROPOSE - assert exc_info.value.decision is not None - assert any( - r.code == GovernanceReasonCode.RETIRED_CONTRACT_MODIFIED - for r in exc_info.value.decision.reasons - ) - # Verify no git push or PR creation was called - mock_config.assert_not_called() - def test_pipeline_ci_run_blocks_retired_contract(self, tmp_path: Path) -> None: """ContractPipeline.run on retired base writes audit manifest and blocks before writing artifacts.""" base_retired = _make_retired_contract() @@ -609,32 +520,6 @@ def test_pipeline_ci_run_blocks_retired_contract(self, tmp_path: Path) -> None: assert not merged_out.exists() assert not suite_out.exists() - def test_batch_release_manifest_skips_retired_contract(self, tmp_path: Path) -> None: - """build_batch_release_manifest skips retired mutated contracts via PROPOSE gate.""" - base_root = tmp_path / "base" - cand_root = tmp_path / "cand" - base_root.mkdir() - cand_root.mkdir() - - base_retired = _make_retired_contract() - cand_retired_mutated = _make_retired_contract() - cand_retired_mutated.description = Description(usage="Mutated retired description") - - dump_yaml(contract_to_dict(base_retired), base_root / "orders.yaml") - dump_yaml(contract_to_dict(cand_retired_mutated), cand_root / "orders.yaml") - - build = build_batch_release_manifest( - base_root=base_root, - candidate_root=cand_root, - context=TEST_CONTEXT, - ) - - # The retired mutated contract must be skipped, not included in tasks - assert len(build.tasks) == 0 - assert len(build.skipped) == 1 - assert build.skipped[0].contract_repo_path == "orders.yaml" - - # ============================================================================== # 3. Read-Only Matrix # ============================================================================== @@ -661,7 +546,6 @@ def test_analyze_unchanged_retired_allowed(self) -> None: decision = GovernanceService().evaluate( base_retired, candidate, - effective_date=date(2026, 8, 14), ) gate_res = evaluate_governance_gate(decision, GovernanceOperation.ANALYZE) @@ -682,7 +566,6 @@ def test_analyze_mutated_retired_returns_block_decision_without_error(self, tmp_ base=str(base_path), candidate=str(cand_path), runtime_context="auto", - effective_date="2026-08-14", ) result = run_release_classify(args) diff --git a/tests/test_runtime_history.py b/tests/test_runtime_history.py deleted file mode 100644 index 8172b4fb..00000000 --- a/tests/test_runtime_history.py +++ /dev/null @@ -1,287 +0,0 @@ -from __future__ import annotations - -from datetime import datetime, timedelta, timezone -from pathlib import Path - -import pytest - -from semapact.application.services.runtime_history import RuntimeHistoryService -from semapact.contractops import VersionAuthority -from semapact.history import DeploymentRecord, DeploymentStatus, ReleaseRecord -from semapact.history.integrity import compute_deployment_record_id, compute_release_record_id -from semapact.observation import ( - ObservedAsset, - ObservedAssetIdentity, - ObservedPlatformState, - fingerprint_observed_state, - with_observed_state_fingerprint, -) -from semapact.platforms.git import GitWorkingTreeHistoryRepository -from semapact.reconciliation import ( - ReconciliationDifference, - ReconciliationDifferenceType, - ReconciliationResult, - ReconciliationSubject, - RuntimeDriftStatus, - RuntimeReasonCode, -) - - -def _raw_observation( - at: datetime, - *, - asset_type: str = "TABLE", - source: str = "workspace:test", -) -> ObservedPlatformState: - return ObservedPlatformState( - platform="databricks", - source_identifier=source, - assets=( - ObservedAsset( - identity=ObservedAssetIdentity( - platform="databricks", - namespace=("catalog", "schema"), - asset="orders", - ), - asset_type=asset_type, - ), - ), - captured_at=at, - ) - - -def _observation(at: datetime, *, asset_type: str = "TABLE", source: str = "workspace:test") -> ObservedPlatformState: - return with_observed_state_fingerprint( - _raw_observation(at, asset_type=asset_type, source=source) - ) - - -def _result(observation: ObservedPlatformState, *, drift: bool = False) -> ReconciliationResult: - differences = () - if drift: - differences = ( - ReconciliationDifference( - difference_type=ReconciliationDifferenceType.MISMATCH, - subject=ReconciliationSubject.PHYSICAL_TYPE, - reason_code=RuntimeReasonCode.RUNTIME_PHYSICAL_TYPE_CHANGED, - path="orders.id.physical_type", - asset_identity="orders", - property_identity="id", - expected="STRING", - observed="BIGINT", - ), - ) - assert observation.fingerprint is not None - return ReconciliationResult( - contract_id="orders-product", - contract_version="1.3.0", - observation_source_identifier=observation.source_identifier, - observation_fingerprint=observation.fingerprint, - differences=differences, - ) - - -def _release() -> ReleaseRecord: - fields = dict( - contract_id="orders-product", - contract_version="1.3.0", - decision_id="decision-1", - change_set_id="changeset-1", - release_plan_id="release-plan-1", - version_resolution_id="version-resolution-1", - authorization_id="apply-authorization-1", - applied_release_id="applied-release-1", - released_revision_id="released-revision-1", - required_version_bump="minor", - actual_version_bump="minor", - version_authority=VersionAuthority.SEMAPACT, - authority_reference=None, - review_evidence_reference=None, - review_evidence_action=None, - ) - record_id = compute_release_record_id( - **{**fields, "version_authority": fields["version_authority"].value} - ) - return ReleaseRecord(release_record_id=record_id, **fields) - - -def _deployment(release_id: str, *, source: str = "workspace:test") -> DeploymentRecord: - started = datetime(2026, 9, 13, tzinfo=timezone.utc) - completed = started + timedelta(seconds=1) - fields = dict( - release_record_id=release_id, - deployment_plan_id="deployment-plan-1", - deployment_preview_id="deployment-preview-1", - deployment_authorization_id="deployment-authorization-1", - platform="databricks", - runtime_target="catalog.schema", - source_reference=source, - status=DeploymentStatus.SUCCEEDED, - started_at=started, - completed_at=completed, - actor_reference=None, - external_reference=None, - ) - record_id = compute_deployment_record_id( - release_record_id=release_id, - deployment_plan_id=fields["deployment_plan_id"], - deployment_preview_id=fields["deployment_preview_id"], - deployment_authorization_id=fields["deployment_authorization_id"], - platform=fields["platform"], - runtime_target=fields["runtime_target"], - source_reference=fields["source_reference"], - status=fields["status"].value, - started_at=started.isoformat(), - completed_at=completed.isoformat(), - actor_reference=None, - external_reference=None, - ) - return DeploymentRecord(deployment_record_id=record_id, **fields) - - -def _service(tmp_path: Path): - backend = GitWorkingTreeHistoryRepository(tmp_path) - return RuntimeHistoryService( - observations=backend, - reconciliations=backend, - releases=backend, - deployments=backend, - ), backend - - -def test_round_trip_and_idempotency(tmp_path: Path) -> None: - service, backend = _service(tmp_path) - observation = _observation(datetime(2026, 9, 13, 1, tzinfo=timezone.utc)) - result = _result(observation) - - first = service.record_reconciliation(observation, result) - second = service.record_reconciliation(observation, result) - - assert first == second - assert backend.get_runtime_observation_record(first.observation_record_id).observation == observation - assert backend.get_runtime_reconciliation_record(first.runtime_reconciliation_record_id) == first - assert first.status is RuntimeDriftStatus.IN_SYNC - - -def test_missing_fingerprint_is_materialized_before_history_persistence(tmp_path: Path) -> None: - service, backend = _service(tmp_path) - observation = _raw_observation(datetime(2026, 9, 13, 1, tzinfo=timezone.utc)) - semantic_fingerprint = fingerprint_observed_state(observation) - result = ReconciliationResult( - contract_id="orders-product", - contract_version="1.3.0", - observation_source_identifier=observation.source_identifier, - observation_fingerprint=semantic_fingerprint, - ) - - record = service.record_reconciliation(observation, result) - persisted = backend.get_runtime_observation_record(record.observation_record_id) - - assert observation.fingerprint is None - assert persisted.observation.fingerprint == semantic_fingerprint - - -def test_stale_materialized_fingerprint_is_rejected(tmp_path: Path) -> None: - service, _ = _service(tmp_path) - observation = _raw_observation(datetime(2026, 9, 13, 1, tzinfo=timezone.utc)).model_copy( - update={"fingerprint": "stale"} - ) - result = ReconciliationResult( - contract_id="orders-product", - contract_version="1.3.0", - observation_source_identifier=observation.source_identifier, - observation_fingerprint=fingerprint_observed_state(observation), - ) - - with pytest.raises(ValueError, match="canonical semantic content"): - service.record_reconciliation(observation, result) - - -def test_same_semantic_state_at_different_times_keeps_both_observations(tmp_path: Path) -> None: - service, backend = _service(tmp_path) - first_observation = _observation(datetime(2026, 9, 13, 1, tzinfo=timezone.utc)) - second_observation = _observation(datetime(2026, 9, 13, 2, tzinfo=timezone.utc)) - - first = service.record_reconciliation(first_observation, _result(first_observation)) - second = service.record_reconciliation(second_observation, _result(second_observation)) - - assert first_observation.fingerprint == second_observation.fingerprint - assert first.observation_record_id != second.observation_record_id - assert len(backend.list_runtime_observation_records("workspace:test")) == 2 - - -def test_sync_then_drift_coexist(tmp_path: Path) -> None: - service, backend = _service(tmp_path) - sync_observation = _observation(datetime(2026, 9, 13, 1, tzinfo=timezone.utc)) - drift_observation = _observation( - datetime(2026, 9, 13, 2, tzinfo=timezone.utc), - asset_type="VIEW", - ) - - sync = service.record_reconciliation(sync_observation, _result(sync_observation)) - drift = service.record_reconciliation(drift_observation, _result(drift_observation, drift=True)) - - records = backend.list_runtime_reconciliation_records_for_source("workspace:test") - assert set(records) == {sync, drift} - assert {item.status for item in records} == {RuntimeDriftStatus.IN_SYNC, RuntimeDriftStatus.DRIFT} - - -def test_exact_release_and_deployment_links_are_optional(tmp_path: Path) -> None: - service, backend = _service(tmp_path) - release = _release() - deployment = _deployment(release.release_record_id) - backend.put_release_record(release) - backend.put_deployment_record(deployment) - observation = _observation(datetime(2026, 9, 13, 1, tzinfo=timezone.utc)) - - record = service.record_reconciliation( - observation, - _result(observation), - release_record_id=release.release_record_id, - deployment_record_id=deployment.deployment_record_id, - ) - - assert record.release_record_id == release.release_record_id - assert record.deployment_record_id == deployment.deployment_record_id - - -def test_deployment_success_can_coexist_with_later_drift(tmp_path: Path) -> None: - service, backend = _service(tmp_path) - release = _release() - deployment = _deployment(release.release_record_id) - backend.put_release_record(release) - backend.put_deployment_record(deployment) - observation = _observation( - datetime(2026, 9, 13, 2, tzinfo=timezone.utc), - asset_type="VIEW", - ) - - record = service.record_reconciliation( - observation, - _result(observation, drift=True), - release_record_id=release.release_record_id, - deployment_record_id=deployment.deployment_record_id, - ) - - assert deployment.status is DeploymentStatus.SUCCEEDED - assert record.status is RuntimeDriftStatus.DRIFT - - -def test_rejects_mismatched_runtime_evidence(tmp_path: Path) -> None: - service, backend = _service(tmp_path) - observation = _observation(datetime(2026, 9, 13, 1, tzinfo=timezone.utc)) - wrong_result = _result(observation).model_copy(update={"observation_fingerprint": "wrong"}) - with pytest.raises(ValueError, match="fingerprint"): - service.record_reconciliation(observation, wrong_result) - - release = _release() - deployment = _deployment(release.release_record_id, source="workspace:other") - backend.put_release_record(release) - backend.put_deployment_record(deployment) - with pytest.raises(ValueError, match="source"): - service.record_reconciliation( - observation, - _result(observation), - release_record_id=release.release_record_id, - deployment_record_id=deployment.deployment_record_id, - ) diff --git a/tests/test_semapact_cli_readable.py b/tests/test_semapact_cli_readable.py index d2377d8b..2d1549a8 100644 --- a/tests/test_semapact_cli_readable.py +++ b/tests/test_semapact_cli_readable.py @@ -321,8 +321,6 @@ def test_cli_release_classify_outputs_per_contract_required_bump( str(base_path), "--candidate", str(candidate_path), - "--effective-date", - TEST_EFFECTIVE_DATE, ], ) @@ -336,58 +334,6 @@ def test_cli_release_classify_outputs_per_contract_required_bump( assert payload["hasChanges"] is True -def test_cli_release_prepare_outputs_promoted_contract( - sample_odcs_model, tmp_path, capsys, monkeypatch -): - base_contract = sample_odcs_model.model_copy(deep=True) - candidate_contract = sample_odcs_model.model_copy(deep=True) - assert candidate_contract.schema_ is not None - assert candidate_contract.schema_[0].properties is not None - candidate_contract.schema_[0].properties.append( - SchemaProperty( - id="new_optional_column", - name="new_optional_column", - physicalName="new_optional_column", - logicalType="string", - physicalType="STRING", - required=False, - ) - ) - - base_path = dump_yaml(base_contract, tmp_path / "base.yaml") - candidate_path = dump_yaml(candidate_contract, tmp_path / "candidate.yaml") - output_path = tmp_path / "promoted.yaml" - - monkeypatch.setattr( - "sys.argv", - [ - "semapact", - "release", - "prepare", - "--base", - str(base_path), - "--candidate", - str(candidate_path), - "--release-tag", - "orders/v1.2.0", - "--output", - str(output_path), - "--effective-date", - TEST_EFFECTIVE_DATE, - ], - ) - - exit_code = cli.main() - payload = json.loads(capsys.readouterr().out) - promoted = load_yaml(output_path) - - assert exit_code == 0 - assert payload["requiredBump"] == "minor" - assert payload["actualBump"] == "minor" - assert payload["targetVersion"] == "1.2.0" - assert promoted["version"] == "1.2.0" - assert promoted["id"] == str(base_contract.id) - def test_cli_release_classify_repo_outputs_per_contract_results( sample_odcs_model, tmp_path, capsys, monkeypatch @@ -414,8 +360,6 @@ def test_cli_release_classify_repo_outputs_per_contract_results( str(base_root), "--candidate-root", str(candidate_root), - "--effective-date", - TEST_EFFECTIVE_DATE, ], ) @@ -429,245 +373,4 @@ def test_cli_release_classify_repo_outputs_per_contract_results( assert by_path["changed.yaml"]["suggested_release_version"] is None -def test_cli_release_create_prs_outputs_batch_payload( - sample_odcs_model, tmp_path, capsys, monkeypatch -): - repo_path = tmp_path / "repo" - repo_path.mkdir() - base_contract = sample_odcs_model.model_copy(deep=True) - candidate_contract = sample_odcs_model.model_copy(deep=True) - assert candidate_contract.description is not None - candidate_contract.description.usage = "Updated descriptive text only" - - base_path = dump_yaml(base_contract, tmp_path / "base.yaml") - candidate_path = dump_yaml(candidate_contract, tmp_path / "candidate.yaml") - manifest_path = tmp_path / "manifest.json" - manifest_path.write_text( - json.dumps( - [ - { - "base": str(base_path), - "candidate": str(candidate_path), - "contract_path": "contracts/orders.yaml", - "release_tag": "orders/v1.1.1", - "source_branch": "release/orders-v1.1.1", - "target_branch": "release", - "effective_date": TEST_EFFECTIVE_DATE, - } - ] - ), - encoding="utf-8", - ) - - captured_kwargs: dict[str, Any] = {} - - def _fake_create_prs_from_manifest(**kwargs): - captured_kwargs.update(kwargs) - return [ - { - "promotion": { - "contractId": str(base_contract.id), - "targetVersion": "1.1.1", - }, - "pullRequest": {"pullRequestId": 88}, - } - ] - - monkeypatch.setattr( - "semapact.devops.release_workflow.create_release_pull_requests_from_manifest", - _fake_create_prs_from_manifest, - ) - - # Mock the config manager with fallback values that should be overridden - config_vals = { - "git.provider": "github", - "git.organization": "fallback-org", - "git.project": "fallback-proj", - "git.repository_id": "fallback-repo", - "git.pat_token": "fallback-token", - } - monkeypatch.setattr( - "semapact.core.config.config_manager.get", - lambda key, *args, **kwargs: config_vals.get(key, kwargs.get("default")), - ) - - monkeypatch.setattr( - "sys.argv", - [ - "semapact", - "release", - "create-prs", - "--manifest", - str(manifest_path), - "--repo-path", - str(repo_path), - "--git-provider", - "azure", - "--organization", - "org", - "--project", - "proj", - "--repository-id", - "repo", - "--pat-token", - "token", - ], - ) - - exit_code = cli.main() - payload = json.loads(capsys.readouterr().out) - - assert exit_code == 0 - assert payload["results"][0]["pullRequest"]["pullRequestId"] == 88 - assert payload["tasks"][0]["release_tag"] == "orders/v1.1.1" - assert payload["tasks"][0]["effective_date"] == TEST_EFFECTIVE_DATE - # Verify that CLI arguments overrode the config fallbacks - assert type(captured_kwargs["config"]).__name__ == "AzureDevOpsConfig" - assert captured_kwargs["config"].organization == "org" - assert captured_kwargs["config"].project == "proj" - assert captured_kwargs["config"].repository_id == "repo" - assert captured_kwargs["config"].pat_token == "token" - - -def test_cli_release_create_prs_uses_config_fallback( - sample_odcs_model, tmp_path, capsys, monkeypatch -): - repo_path = tmp_path / "repo" - repo_path.mkdir() - base_contract = sample_odcs_model.model_copy(deep=True) - candidate_contract = sample_odcs_model.model_copy(deep=True) - assert candidate_contract.description is not None - candidate_contract.description.usage = "Updated descriptive text only" - - base_path = dump_yaml(base_contract, tmp_path / "base.yaml") - candidate_path = dump_yaml(candidate_contract, tmp_path / "candidate.yaml") - manifest_path = tmp_path / "manifest.json" - manifest_path.write_text( - json.dumps( - [ - { - "base": str(base_path), - "candidate": str(candidate_path), - "contract_path": "contracts/orders.yaml", - "release_tag": "orders/v1.1.1", - "source_branch": "release/orders-v1.1.1", - "target_branch": "release", - "effective_date": TEST_EFFECTIVE_DATE, - } - ] - ), - encoding="utf-8", - ) - - captured_kwargs: dict[str, Any] = {} - - def _fake_create_prs_from_manifest(**kwargs): - captured_kwargs.update(kwargs) - return [ - { - "promotion": { - "contractId": str(base_contract.id), - "targetVersion": "1.1.1", - }, - "pullRequest": {"pullRequestId": 88}, - } - ] - - monkeypatch.setattr( - "semapact.devops.release_workflow.create_release_pull_requests_from_manifest", - _fake_create_prs_from_manifest, - ) - - # Mock the config manager - config_vals = { - "git.provider": "github", - "git.github_owner": "fallback-owner", - "git.github_repo": "fallback-repo", - "git.github_token": "fallback-token", - } - monkeypatch.setattr( - "semapact.core.config.config_manager.get", - lambda key, *args, **kwargs: config_vals.get(key, kwargs.get("default")), - ) - - monkeypatch.setattr( - "sys.argv", - [ - "semapact", - "release", - "create-prs", - "--manifest", - str(manifest_path), - "--repo-path", - str(repo_path), - # Notice we are omitting --organization, --project, --github-owner, etc. - ], - ) - - exit_code = cli.main() - json.loads(capsys.readouterr().out) - - assert exit_code == 0 - assert type(captured_kwargs["config"]).__name__ == "GitHubConfig" - assert captured_kwargs["config"].owner == "fallback-owner" - assert captured_kwargs["config"].repo == "fallback-repo" - assert captured_kwargs["config"].token == "fallback-token" - - -def test_cli_release_build_manifest_writes_json_array_and_summary( - sample_odcs_model, tmp_path, capsys, monkeypatch -): - base_root = tmp_path / "base" - candidate_root = tmp_path / "candidate" - output_path = tmp_path / "release_manifest.json" - - docs_only = sample_odcs_model.model_copy(deep=True) - assert docs_only.description is not None - docs_only.description.usage = "Updated descriptive text only" - - additive = sample_odcs_model.model_copy(deep=True) - assert additive.schema_ is not None - assert additive.schema_[0].properties is not None - additive.schema_[0].properties.append( - SchemaProperty( - name="new_optional_column", - logicalType="string", - physicalType="STRING", - required=False, - ) - ) - - dump_yaml(sample_odcs_model, base_root / "orders.yaml") - dump_yaml(docs_only, candidate_root / "orders.yaml") - dump_yaml(sample_odcs_model, base_root / "payments.yaml") - dump_yaml(additive, candidate_root / "payments.yaml") - - monkeypatch.setattr( - "sys.argv", - [ - "semapact", - "release", - "build-manifest", - "--base-root", - str(base_root), - "--candidate-root", - str(candidate_root), - "--output", - str(output_path), - "--effective-date", - TEST_EFFECTIVE_DATE, - ], - ) - - exit_code = cli.main() - payload = json.loads(capsys.readouterr().out) - manifest = json.loads(output_path.read_text(encoding="utf-8")) - - assert exit_code == 0 - assert payload["output"] == str(output_path.resolve()) - assert payload["tasks"][0]["contract_path"] == "payments.yaml" - assert payload["tasks"][0]["effective_date"] == TEST_EFFECTIVE_DATE - assert payload["skipped"][0]["contract_repo_path"] == "orders.yaml" - assert manifest[0]["release_tag"].endswith("/v1.2.0") - assert manifest[0]["effective_date"] == TEST_EFFECTIVE_DATE diff --git a/tests/test_semapact_devops_examples_readable.py b/tests/test_semapact_devops_examples_readable.py index 8687bab2..6c24e9cf 100644 --- a/tests/test_semapact_devops_examples_readable.py +++ b/tests/test_semapact_devops_examples_readable.py @@ -1,42 +1,68 @@ from __future__ import annotations -import json from pathlib import Path +import yaml -def test_release_manifest_example_is_valid_json_array(): - manifest_path = Path("examples/release/release-manifest.example.json") - payload = json.loads(manifest_path.read_text(encoding="utf-8")) - assert isinstance(payload, list) - assert payload - assert { - "base", - "candidate", - "contract_path", - "release_tag", - "source_branch", - "target_branch", - } <= set(payload[0]) - - -def test_ci_shell_examples_reference_release_commands(): - pr_example = Path("examples/ci/pr-check.example.sh").read_text(encoding="utf-8") - release_example = Path("examples/ci/release.example.sh").read_text(encoding="utf-8") +def test_pr_validation_examples_use_repository_classification() -> None: + shell = Path("examples/ci/pr-check.example.sh").read_text(encoding="utf-8") + azure = Path("examples/azure-devops/semapact-pr-validation.yml").read_text( + encoding="utf-8" + ) - assert "release classify-repo" in pr_example - assert "release build-manifest" in release_example - assert "release create-prs" in release_example + assert "release classify-repo" in shell + assert "release classify-repo" in azure + assert "build-manifest" not in shell + assert "create-prs" not in shell + assert "build-manifest" not in azure + assert "create-prs" not in azure -def test_azure_devops_examples_reference_release_commands(): - pr_pipeline = Path("examples/azure-devops/semapact-pr-validation.yml").read_text( +def test_data_product_github_example_uses_bundle_driven_ci_cd() -> None: + workflow = Path("examples/github/data-product-ci-cd.yml").read_text( encoding="utf-8" ) - release_pipeline = Path("examples/azure-devops/semapact-release.yml").read_text( + + assert "release assess" in workflow + assert "release approve" in workflow + assert "release finalize" in workflow + assert "deployment assess" in workflow + assert "--release " in workflow + assert "--release-id" not in workflow + assert "deployment deploy" in workflow + assert "environment: contract-release" in workflow + assert "environment: production" in workflow + assert "--operational-history" not in workflow + + +def test_central_contract_repo_github_example_fans_out_and_commits_ledger_once() -> None: + workflow = Path("examples/github/central-contract-repo-ci-cd.yml").read_text( encoding="utf-8" ) - assert "release classify-repo" in pr_pipeline - assert "release build-manifest" in release_pipeline - assert "release create-prs" in release_pipeline + assert "release classify-repo" in workflow + assert "fromJSON(needs.detect-contracts.outputs.matrix)" in workflow + assert "artifact: .artifactKey" in workflow + assert 'gsub("[^A-Za-z0-9_-]"; "_")' not in workflow + assert "release assess" in workflow + assert "release approve" in workflow + assert "release finalize" in workflow + assert "finalize-releases:" in workflow + assert "deployment assess" in workflow + assert "--release " in workflow + assert "--release-id" not in workflow + assert "deployment deploy" in workflow + assert "semapact-finalized-releases" in workflow + assert "--operational-history" not in workflow + + +def test_bundle_driven_github_examples_are_valid_yaml() -> None: + for path in ( + Path("examples/github/data-product-ci-cd.yml"), + Path("examples/github/central-contract-repo-ci-cd.yml"), + ): + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + assert isinstance(payload, dict) + assert isinstance(payload.get("jobs"), dict) + assert payload["jobs"] diff --git a/tests/test_semapact_devops_readable.py b/tests/test_semapact_devops_readable.py index 6e4e949c..ba61e5c2 100644 --- a/tests/test_semapact_devops_readable.py +++ b/tests/test_semapact_devops_readable.py @@ -10,9 +10,7 @@ from semapact.devops.audit import build_audit_metadata from semapact.devops.ci_cd import evaluate_ci_gate, write_ci_summary from semapact.devops.pr_creator import AzureDevOpsConfig, PullRequestCreator -from semapact.governance import ChangeContext -TEST_CONTEXT = ChangeContext(effective_date=date(2026, 8, 13)) def _creator() -> PullRequestCreator: @@ -56,7 +54,6 @@ def test_ci_gate_allows_only_when_validation_and_policy_are_valid(): validation=val, policy=pol, evidence=evi, - context=TEST_CONTEXT, ) dec_review = GovernanceDecision( decision_id="id2", @@ -67,7 +64,6 @@ def test_ci_gate_allows_only_when_validation_and_policy_are_valid(): validation=val, policy=pol, evidence=evi, - context=TEST_CONTEXT, ) dec_block = GovernanceDecision( decision_id="id3", @@ -78,7 +74,6 @@ def test_ci_gate_allows_only_when_validation_and_policy_are_valid(): validation=ValidationOutcome(valid=False), policy=PolicyOutcome(valid=False), evidence=evi, - context=TEST_CONTEXT, ) res_allow = evaluate_ci_gate(dec_allow) diff --git a/tests/test_semapact_devops_release_workflow_readable.py b/tests/test_semapact_devops_release_workflow_readable.py deleted file mode 100644 index 1ad9cc32..00000000 --- a/tests/test_semapact_devops_release_workflow_readable.py +++ /dev/null @@ -1,331 +0,0 @@ -from __future__ import annotations - -from datetime import date -import json - -from semapact.core.release import prepare_release_candidate -from semapact.devops.pr_creator import AzureDevOpsConfig -from semapact.devops.release_workflow import ( - build_batch_release_manifest, - BatchReleaseTask, - build_release_pr_plan, - classify_contracts_in_repo, - create_release_pull_request, - create_release_pull_requests_from_manifest, -) -from semapact.governance import ChangeContext -from semapact.interfaces import cli -from semapact.utils.yaml_utils import dump_yaml, load_yaml - - -TEST_CONTEXT = ChangeContext(effective_date=date(2026, 1, 1)) - - -def test_build_release_pr_plan_uses_per_contract_defaults(sample_odcs_model): - base = sample_odcs_model.model_copy(deep=True) - candidate = sample_odcs_model.model_copy(deep=True) - assert candidate.schema_ is not None - assert candidate.schema_[0].properties is not None - candidate.schema_[0].properties.append( - candidate.schema_[0] - .properties[0] - .model_copy(update={"name": "new_optional_column", "id": "new_optional_column"}) - ) - - promotion = prepare_release_candidate(base, candidate, "orders/v1.2.0") - plan = build_release_pr_plan( - promotion=promotion, - contract_repo_path="contracts/orders.yaml", - source_branch="release/orders-v1.1.1", - target_branch="release", - ) - - assert plan.contract_id == str(base.id) - assert plan.target_version == "1.2.0" - assert plan.commit_message == f"release({base.id}): prepare 1.2.0" - assert "current version" in plan.description - - -def test_create_release_pull_request_writes_contract_and_calls_pr_creator( - sample_odcs_model, tmp_path, monkeypatch -): - repo_path = tmp_path / "repo" - repo_path.mkdir() - base = sample_odcs_model.model_copy(deep=True) - candidate = sample_odcs_model.model_copy(deep=True) - assert candidate.schema_ is not None - assert candidate.schema_[0].properties is not None - candidate.schema_[0].properties.append( - candidate.schema_[0] - .properties[0] - .model_copy(update={"name": "new_optional_column", "id": "new_optional_column"}) - ) - - captured: dict[str, object] = {} - - def fake_create_update_pr(self, **kwargs): # noqa: ANN001 - captured["kwargs"] = kwargs - return {"pullRequestId": 42} - - monkeypatch.setattr( - "semapact.devops.release_workflow.PullRequestCreator.create_update_pr", - fake_create_update_pr, - ) - - payload = create_release_pull_request( - config=AzureDevOpsConfig( - organization="org", - project="proj", - repository_id="repo", - pat_token="token", - ), - repo_path=str(repo_path), - contract_repo_path="contracts/orders.yaml", - base_contract=base, - candidate_contract=candidate, - release_tag="orders/v1.2.0", - source_branch="release/orders-v1.2.0", - target_branch="release", - context=TEST_CONTEXT, - push=True, - ) - - written = load_yaml(repo_path / "contracts/orders.yaml") - assert payload["pullRequest"]["pullRequestId"] == 42 - assert payload["promotion"]["targetVersion"] == "1.2.0" - assert written["version"] == "1.2.0" - assert written["id"] == str(base.id) - assert captured["kwargs"]["paths"] == ["contracts/orders.yaml"] # type: ignore[index] - assert captured["kwargs"]["push"] is True # type: ignore[index] - - -def test_cli_release_create_pr_outputs_plan_and_pr_payload( - sample_odcs_model, tmp_path, capsys, monkeypatch -): - repo_path = tmp_path / "repo" - repo_path.mkdir() - base = sample_odcs_model.model_copy(deep=True) - candidate = sample_odcs_model.model_copy(deep=True) - assert candidate.schema_ is not None - assert candidate.schema_[0].properties is not None - candidate.schema_[0].properties.append( - candidate.schema_[0] - .properties[0] - .model_copy(update={"name": "new_optional_column", "id": "new_optional_column"}) - ) - - base_path = dump_yaml(base, tmp_path / "base.yaml") - candidate_path = dump_yaml(candidate, tmp_path / "candidate.yaml") - - monkeypatch.setattr( - "semapact.devops.release_workflow.create_release_pull_request", - lambda **kwargs: { - "promotion": {"contractId": str(base.id), "targetVersion": "1.2.0"}, - "pullRequest": {"pullRequestId": 77}, - }, - ) - - monkeypatch.setattr( - "sys.argv", - [ - "semapact", - "release", - "create-pr", - "--base", - str(base_path), - "--candidate", - str(candidate_path), - "--release-tag", - "orders/v1.2.0", - "--repo-path", - str(repo_path), - "--contract-path", - "contracts/orders.yaml", - "--source-branch", - "release/orders-v1.2.0", - "--target-branch", - "release", - "--effective-date", - "2026-01-01", - "--organization", - "org", - "--project", - "proj", - "--repository-id", - "repo", - "--pat-token", - "token", - ], - ) - - exit_code = cli.main() - payload = json.loads(capsys.readouterr().out) - - assert exit_code == 0 - assert payload["pullRequest"]["pullRequestId"] == 77 - assert payload["promotion"]["targetVersion"] == "1.2.0" - - -def test_classify_contracts_in_repo_reports_changed_added_removed_and_unchanged( - sample_odcs_model, tmp_path -): - base_root = tmp_path / "base" - candidate_root = tmp_path / "candidate" - - unchanged = sample_odcs_model.model_copy(deep=True) - changed = sample_odcs_model.model_copy(deep=True) - assert changed.description is not None - changed.description.usage = "Updated descriptive text only" - added = sample_odcs_model.model_copy(deep=True) - added.id = "new-contract" - added.version = "1.0.0" - - dump_yaml(unchanged, base_root / "unchanged.yaml") - dump_yaml(unchanged, candidate_root / "unchanged.yaml") - dump_yaml(sample_odcs_model, base_root / "changed.yaml") - dump_yaml(changed, candidate_root / "changed.yaml") - dump_yaml(sample_odcs_model, base_root / "removed.yaml") - dump_yaml(added, candidate_root / "added.yaml") - - results = classify_contracts_in_repo( - base_root=base_root, - candidate_root=candidate_root, - context=TEST_CONTEXT, - ) - by_path = {item.contract_repo_path: item for item in results} - - assert by_path["unchanged.yaml"].status == "unchanged" - assert by_path["changed.yaml"].status == "changed" - assert by_path["changed.yaml"].required_bump == "none" - assert by_path["changed.yaml"].suggested_release_version is None - assert by_path["added.yaml"].status == "added" - assert by_path["removed.yaml"].status == "removed" - - -def test_create_release_pull_requests_from_manifest_runs_each_contract( - sample_odcs_model, tmp_path, monkeypatch -): - repo_path = tmp_path / "repo" - repo_path.mkdir() - base_a = sample_odcs_model.model_copy(deep=True) - candidate_a = sample_odcs_model.model_copy(deep=True) - assert candidate_a.schema_ is not None - assert candidate_a.schema_[0].properties is not None - candidate_a.schema_[0].properties.append( - candidate_a.schema_[0] - .properties[0] - .model_copy(update={"name": "new_optional_column", "id": "new_optional_column"}) - ) - - base_b = sample_odcs_model.model_copy(deep=True) - base_b.id = "payments" - candidate_b = base_b.model_copy(deep=True) - assert candidate_b.schema_ is not None - assert candidate_b.schema_[0].properties is not None - candidate_b.schema_[0].properties.append( - candidate_b.schema_[0] - .properties[0] - .model_copy(update={"name": "new_optional_column", "id": "new_optional_column"}) - ) - - base_a_path = dump_yaml(base_a, tmp_path / "base-a.yaml") - candidate_a_path = dump_yaml(candidate_a, tmp_path / "candidate-a.yaml") - base_b_path = dump_yaml(base_b, tmp_path / "base-b.yaml") - candidate_b_path = dump_yaml(candidate_b, tmp_path / "candidate-b.yaml") - - calls: list[str] = [] - - def fake_create_update_pr(self, **kwargs): # noqa: ANN001 - calls.append(kwargs["paths"][0]) - return {"pullRequestId": len(calls)} - - monkeypatch.setattr( - "semapact.devops.release_workflow.PullRequestCreator.create_update_pr", - fake_create_update_pr, - ) - - results = create_release_pull_requests_from_manifest( - config=AzureDevOpsConfig( - organization="org", - project="proj", - repository_id="repo", - pat_token="token", - ), - repo_path=str(repo_path), - tasks=[ - BatchReleaseTask( - base=str(base_a_path), - candidate=str(candidate_a_path), - contract_path="contracts/orders.yaml", - release_tag="orders/v1.2.0", - source_branch="release/orders-v1.2.0", - target_branch="release", - effective_date="2026-01-01", - ), - BatchReleaseTask( - base=str(base_b_path), - candidate=str(candidate_b_path), - contract_path="contracts/payments.yaml", - release_tag="payments/v1.2.0", - source_branch="release/payments-v1.2.0", - target_branch="release", - effective_date="2026-01-01", - ), - ], - push=False, - ) - - assert len(results) == 2 - assert calls == ["contracts/orders.yaml", "contracts/payments.yaml"] - - -def test_build_batch_release_manifest_generates_editable_tasks_and_skips_manual_cases( - sample_odcs_model, tmp_path -): - base_root = tmp_path / "base" - candidate_root = tmp_path / "candidate" - - unchanged = sample_odcs_model.model_copy(deep=True) - docs_only = sample_odcs_model.model_copy(deep=True) - assert docs_only.description is not None - docs_only.description.usage = "Updated descriptive text only" - - additive = sample_odcs_model.model_copy(deep=True) - assert additive.schema_ is not None - assert additive.schema_[0].properties is not None - additive.schema_[0].properties.append( - additive.schema_[0] - .properties[0] - .model_copy(update={"name": "new_optional_column", "id": "new_optional_column"}) - ) - - added = sample_odcs_model.model_copy(deep=True) - added.id = "new-contract" - added.version = "1.0.0" - - dump_yaml(unchanged, base_root / "unchanged.yaml") - dump_yaml(unchanged, candidate_root / "unchanged.yaml") - dump_yaml(sample_odcs_model, base_root / "docs.yaml") - dump_yaml(docs_only, candidate_root / "docs.yaml") - dump_yaml(sample_odcs_model, base_root / "additive.yaml") - dump_yaml(additive, candidate_root / "additive.yaml") - dump_yaml(sample_odcs_model, base_root / "removed.yaml") - dump_yaml(added, candidate_root / "added.yaml") - - build = build_batch_release_manifest( - base_root=base_root, - candidate_root=candidate_root, - context=TEST_CONTEXT, - ) - tasks_by_path = {task.contract_path: task for task in build.tasks} - skipped_by_path = {item.contract_repo_path: item for item in build.skipped} - - assert tasks_by_path["additive.yaml"].release_tag.endswith("/v1.2.0") - assert tasks_by_path["additive.yaml"].target_branch == "release" - assert tasks_by_path["additive.yaml"].effective_date == "2026-01-01" - assert "docs.yaml" in skipped_by_path - assert skipped_by_path["docs.yaml"].status == "changed" - assert "unchanged.yaml" in skipped_by_path - assert skipped_by_path["unchanged.yaml"].status == "unchanged" - assert skipped_by_path["added.yaml"].status == "added" - assert skipped_by_path["removed.yaml"].status == "removed" diff --git a/tests/test_semapact_editor_contract_readable.py b/tests/test_semapact_editor_contract_readable.py index 049a06da..cd2b8f0f 100644 --- a/tests/test_semapact_editor_contract_readable.py +++ b/tests/test_semapact_editor_contract_readable.py @@ -6,13 +6,11 @@ SchemaProperty, ) -from semapact.core.editor_contract import ( +from semapact.core.editor_semantics import ( contract_description_part, contract_tags, + field_declared_lifecycle_status, field_examples_text, - field_lifecycle_status, -) -from semapact.core.editor_semantics import ( contract_api_version, contract_data_product, contract_domain, @@ -96,8 +94,8 @@ def test_field_helpers_support_odcs_model_and_ui_working_copy(): "customProperties": [{"property": "lifecycleStatus", "value": "deprecated"}], } - assert field_lifecycle_status(field_model) == "active" - assert field_lifecycle_status(field_dict) == "deprecated" + assert field_declared_lifecycle_status(field_model) == "active" + assert field_declared_lifecycle_status(field_dict) == "deprecated" assert field_examples_text(field_model) == "a\nb" assert field_examples_text(field_dict) == "c\nd" diff --git a/tests/test_semapact_lifecycle_cli.py b/tests/test_semapact_lifecycle_cli.py index 659c4086..4bf154f7 100644 --- a/tests/test_semapact_lifecycle_cli.py +++ b/tests/test_semapact_lifecycle_cli.py @@ -6,7 +6,7 @@ from semapact.core.lifecycle_cli import apply_lifecycle from semapact.exceptions import GovernanceReviewRequiredError -from semapact.governance import ChangeContext +from semapact.change_context import ChangeContext from semapact.utils.yaml_utils import load_yaml Args = namedtuple( diff --git a/tests/test_semapact_lifecycle_merge_engine_readable.py b/tests/test_semapact_lifecycle_merge_engine_readable.py index e41e3c30..5267636f 100644 --- a/tests/test_semapact_lifecycle_merge_engine_readable.py +++ b/tests/test_semapact_lifecycle_merge_engine_readable.py @@ -12,7 +12,7 @@ ) import semapact.lifecycle.merge_engine as merge_engine -from semapact.governance import ChangeContext +from semapact.change_context import ChangeContext from semapact.lifecycle.merge_engine import ContractMergeEngine diff --git a/tests/test_semapact_merge_engine.py b/tests/test_semapact_merge_engine.py index 6f6b37b7..d1c2de5f 100644 --- a/tests/test_semapact_merge_engine.py +++ b/tests/test_semapact_merge_engine.py @@ -11,7 +11,7 @@ ) import semapact.lifecycle.merge_engine as merge_engine -from semapact.governance import ChangeContext +from semapact.change_context import ChangeContext from semapact.lifecycle.merge_engine import ContractMergeEngine @@ -353,7 +353,6 @@ def test_merge_engine_rejects_retired_contract_modification(): decision = evaluate_governance_decision( existing, result.contract, - context=TEST_CONTEXT, merge_conflicts=result.conflicts, ) assert decision.decision == DecisionResult.BLOCK diff --git a/tests/test_semapact_orchestrator_pipeline_readable.py b/tests/test_semapact_orchestrator_pipeline_readable.py index ad1e9f50..38a774d8 100644 --- a/tests/test_semapact_orchestrator_pipeline_readable.py +++ b/tests/test_semapact_orchestrator_pipeline_readable.py @@ -14,7 +14,8 @@ from semapact.lifecycle.merge_engine import MergeConflict, MergeResult from semapact.lifecycle.policy import BreakingChange, PolicyEvaluation from semapact.exceptions import GovernanceBlockedError -from semapact.governance import ChangeContext, GovernanceGateResult +from semapact.change_context import ChangeContext +from semapact.governance import GovernanceGateResult from semapact.orchestrator.pipeline import ContractPipeline from deltalake import write_deltalake @@ -137,7 +138,6 @@ def test_pipeline_prepare_ci_cd_artifacts_writes_manifest_and_outputs( decision = evaluate_governance_decision( merged_contract, merged_contract, - context=TEST_CONTEXT, ) artifacts = pipeline.prepare_ci_cd_artifacts( diff --git a/tests/test_semapact_release_workflow_readable.py b/tests/test_semapact_release_workflow_readable.py deleted file mode 100644 index a5e498f3..00000000 --- a/tests/test_semapact_release_workflow_readable.py +++ /dev/null @@ -1,131 +0,0 @@ -from __future__ import annotations - - -import pytest -from open_data_contract_standard.model import CustomProperty, SchemaProperty - -from semapact.core.release import ( - classify_contract_change, - classify_version_bump, - parse_release_tag_version, - prepare_release_candidate, - suggest_release_version, -) - - -def test_release_classification_returns_none_for_description_only_change( - sample_odcs_model, -): - base = sample_odcs_model.model_copy(deep=True) - candidate = sample_odcs_model.model_copy(deep=True) - assert candidate.description is not None - candidate.description.usage = "Updated descriptive text only" - - result = classify_contract_change(base, candidate) - - assert result.has_changes is True - assert result.required_bump == "none" - - -def test_release_classification_returns_minor_for_added_property(sample_odcs_model): - base = sample_odcs_model.model_copy(deep=True) - candidate = sample_odcs_model.model_copy(deep=True) - candidate.schema_[0].properties.append( # type: ignore[index,union-attr] - SchemaProperty( - name="new_optional_column", - logicalType="string", - physicalType="STRING", - required=False, - ) - ) - - result = classify_contract_change(base, candidate) - - assert result.required_bump == "minor" - assert any("minor version bump" in reason for reason in result.reasons) - - -def test_release_classification_treats_deprecation_as_minor(sample_odcs_model): - base = sample_odcs_model.model_copy(deep=True) - candidate = sample_odcs_model.model_copy(deep=True) - candidate.schema_[0].properties[0].customProperties = [ # type: ignore[index,union-attr] - CustomProperty(property="lifecycleStatus", value="deprecated") - ] - - result = classify_contract_change(base, candidate) - - assert result.required_bump == "minor" - - -def test_release_classification_returns_major_for_breaking_change(sample_odcs_model): - base = sample_odcs_model.model_copy(deep=True) - candidate = sample_odcs_model.model_copy(deep=True) - candidate.schema_[0].properties[0].required = True # type: ignore[index,union-attr] - - result = classify_contract_change(base, candidate) - - assert result.required_bump == "major" - assert result.breaking_changes - - -def test_release_tag_helpers_parse_and_classify_versions(): - assert parse_release_tag_version("orders/v1.2.3") == "1.2.3" - assert parse_release_tag_version("v2.0.0") == "2.0.0" - assert classify_version_bump("1.0.0", "1.0.1") == "patch" - assert classify_version_bump("1.0.0", "1.1.0") == "minor" - assert classify_version_bump("1.0.0", "2.0.0") == "major" - - -def test_suggest_release_version_uses_last_released_version_not_unreleased_chain(): - assert suggest_release_version("1.2.0", "major") == "2.0.0" - assert suggest_release_version("1.2.0", "minor") == "1.3.0" - assert suggest_release_version("1.2.0", "none") == "1.2.0" - - -def test_prepare_release_candidate_applies_explicit_release_tag(sample_odcs_model): - base = sample_odcs_model.model_copy(deep=True) - candidate = sample_odcs_model.model_copy(deep=True) - candidate.schema_[0].properties.append( # type: ignore[index,union-attr] - SchemaProperty( - name="new_optional_column", - logicalType="string", - physicalType="STRING", - required=False, - ) - ) - - result = prepare_release_candidate(base, candidate, "orders/v1.2.0") - - assert result.current_version == str(base.version) - assert result.target_version == "1.2.0" - assert result.actual_bump == "minor" - assert result.contract.version == "1.2.0" - assert result.contract.id == base.id - - -def test_prepare_release_candidate_rejects_insufficient_bump(sample_odcs_model): - base = sample_odcs_model.model_copy(deep=True) - candidate = sample_odcs_model.model_copy(deep=True) - candidate.schema_[0].properties.append( # type: ignore[index,union-attr] - SchemaProperty( - name="new_optional_column", - logicalType="string", - physicalType="STRING", - required=False, - ) - ) - - with pytest.raises(ValueError, match="requires at least a minor bump"): - prepare_release_candidate(base, candidate, "orders/v1.1.1") - - -def test_prepare_release_candidate_rejects_description_only_changes(sample_odcs_model): - base = sample_odcs_model.model_copy(deep=True) - candidate = sample_odcs_model.model_copy(deep=True) - assert candidate.description is not None - candidate.description.usage = "Updated descriptive text only" - - with pytest.raises(ValueError, match="do not require a release version bump"): - prepare_release_candidate(base, candidate, "orders/v1.1.1") - - diff --git a/tests/test_version_authority_service.py b/tests/test_version_authority_service.py index 793f3f8c..29a39999 100644 --- a/tests/test_version_authority_service.py +++ b/tests/test_version_authority_service.py @@ -6,7 +6,7 @@ from semapact.contractops.integrity import compute_release_plan_id from semapact.core.config import ConfigManager from semapact.exceptions import ReleaseValidationError, ValidationError -from semapact.services import VersionAuthorityService +from semapact.application.services.version_authority import VersionAuthorityService from semapact.versioning import RequiredBump From 06b6aa7e7cf1cbf9e1af396f55591f1cf62148d2 Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Mon, 21 Sep 2026 21:45:30 +1000 Subject: [PATCH 33/35] fix(databricks): make release provenance projection idempotent (#240) * fix(databricks): make release provenance tags idempotent * test(databricks): cover idempotent release tag projection * fix(databricks): unset only existing provenance tags * test(databricks): cover partial provenance replacement * fix(databricks): normalize UC tag lookup identifiers --- semapact/platforms/databricks/deployment.py | 81 +++++++++++-- tests/test_deployment_databricks.py | 121 +++++++++++++++++++- 2 files changed, 189 insertions(+), 13 deletions(-) diff --git a/semapact/platforms/databricks/deployment.py b/semapact/platforms/databricks/deployment.py index fd013dbb..3853b8c2 100644 --- a/semapact/platforms/databricks/deployment.py +++ b/semapact/platforms/databricks/deployment.py @@ -83,6 +83,18 @@ def execute(self, operation: NativeOperation) -> None: raise ValidationError( "Databricks executable operation requires a SQL statement" ) + self._run_statement(statement) + + def query_rows(self, statement: str) -> tuple[tuple[str | None, ...], ...]: + """Execute one small metadata query and return its inline JSON rows.""" + response = self._run_statement(statement) + result = getattr(response, "result", None) + data_array = getattr(result, "data_array", None) if result is not None else None + if not data_array: + return () + return tuple(tuple(value for value in row) for row in data_array) + + def _run_statement(self, statement: str) -> Any: if self._warehouse_id is None: raise ValidationError( "Databricks deployment execution requires a SQL warehouse_id" @@ -120,6 +132,7 @@ def execute(self, operation: NativeOperation) -> None: raise RuntimeError( f"Databricks deployment statement finished with state {state}: {error}" ) + return response class DatabricksDeploymentAdapter(DeploymentOrchestrator): @@ -134,6 +147,13 @@ def __init__( poll_interval_seconds: float = 1.0, max_poll_attempts: int = 300, ) -> None: + statement_executor = DatabricksStatementExecutor( + client=client, + warehouse_id=warehouse_id, + poll_interval_seconds=poll_interval_seconds, + max_poll_attempts=max_poll_attempts, + ) + self._statement_executor = statement_executor super().__init__( runtime_provider=runtime_provider, schema_mapper=SqlSchemaMapper( @@ -143,12 +163,7 @@ def __init__( ), transition_planner=DatabricksSchemaTransitionPlanner(), transition_compiler=DatabricksTransitionCompiler(), - executor=DatabricksStatementExecutor( - client=client, - warehouse_id=warehouse_id, - poll_interval_seconds=poll_interval_seconds, - max_poll_attempts=max_poll_attempts, - ), + executor=statement_executor, ) @@ -189,7 +204,29 @@ def project_release_metadata( f"`{part}`" for part in (catalog, schema_name, action.physical_name) ) - self._executor.execute( + current_tags = self._read_release_tags( + catalog=catalog, + schema_name=schema_name, + table_name=action.physical_name, + reserved_keys=tuple(sorted(tags)), + ) + if current_tags == tags: + continue + if current_tags: + rendered_current_keys = ", ".join( + _sql_string(key) for key in sorted(current_tags) + ) + self._statement_executor.execute( + NativeOperation( + kind=NativeOperationKind.ALTER, + governed_asset=action.governed_asset, + statement=( + f"ALTER TABLE {qualified} " + f"UNSET TAGS ({rendered_current_keys})" + ), + ) + ) + self._statement_executor.execute( NativeOperation( kind=NativeOperationKind.ALTER, governed_asset=action.governed_asset, @@ -199,6 +236,36 @@ def project_release_metadata( ) ) + def _read_release_tags( + self, + *, + catalog: str, + schema_name: str, + table_name: str, + reserved_keys: tuple[str, ...], + ) -> dict[str, str]: + keys = ", ".join(_sql_string(key) for key in reserved_keys) + statement = ( + "SELECT tag_name, tag_value " + f"FROM `{catalog}`.information_schema.table_tags " + f"WHERE schema_name = {_sql_string(schema_name.casefold())} " + f"AND table_name = {_sql_string(table_name.casefold())} " + f"AND tag_name IN ({keys})" + ) + current: dict[str, str] = {} + for row in self._statement_executor.query_rows(statement): + if len(row) != 2 or row[0] is None or row[1] is None: + raise RuntimeError( + "Databricks table tag query returned an invalid row" + ) + key, value = row + if key in current: + raise RuntimeError( + f"Databricks table tag query returned duplicate key: {key}" + ) + current[key] = value + return current + def _statement_state(response: Any) -> str: status = getattr(response, "status", None) diff --git a/tests/test_deployment_databricks.py b/tests/test_deployment_databricks.py index b8d06ca4..ed642679 100644 --- a/tests/test_deployment_databricks.py +++ b/tests/test_deployment_databricks.py @@ -150,14 +150,25 @@ def observe(self, *, bindings): class _Statements: - def __init__(self) -> None: + def __init__(self, *, tags: dict[str, str] | None = None) -> None: self.calls: list[str] = [] + self.tags = dict(tags or {}) def execute_statement(self, *, statement, warehouse_id, wait_timeout): self.calls.append(statement) + result = None + if statement.startswith("SELECT tag_name, tag_value "): + result = SimpleNamespace( + data_array=[ + [key, value] + for key, value in sorted(self.tags.items()) + if key.startswith("semapact_") + ] + ) return SimpleNamespace( statement_id="s-1", status=SimpleNamespace(state="SUCCEEDED", error=None), + result=result, ) def get_statement(self, statement_id): @@ -165,12 +176,17 @@ def get_statement(self, statement_id): class _Client: - def __init__(self) -> None: - self.statement_execution = _Statements() + def __init__(self, *, tags: dict[str, str] | None = None) -> None: + self.statement_execution = _Statements(tags=tags) -def _adapter(state: ObservedPlatformState, *, warehouse_id: str | None = "warehouse-1"): - client = _Client() +def _adapter( + state: ObservedPlatformState, + *, + warehouse_id: str | None = "warehouse-1", + tags: dict[str, str] | None = None, +): + client = _Client(tags=tags) provider = _Provider(state) adapter = DatabricksDeploymentAdapter( client=client, @@ -348,7 +364,10 @@ def test_release_metadata_projects_version_and_provenance_as_uc_tags() -> None: ), ) - assert client.statement_execution.calls == [ + assert client.statement_execution.calls[0].startswith( + "SELECT tag_name, tag_value FROM `main`.information_schema.table_tags " + ) + assert client.statement_execution.calls[1:] == [ "ALTER TABLE `main`.`silver`.`orders` SET TAGS " "('semapact_contract_id' = 'orders-product', " "'semapact_contract_version' = '1.2.0', " @@ -357,6 +376,96 @@ def test_release_metadata_projects_version_and_provenance_as_uc_tags() -> None: ] +def test_release_metadata_projection_is_no_op_when_tags_already_match() -> None: + plan = _plan(_property("id", "BIGINT", required=True)) + current = _state(("id", "bigint", False)) + desired = { + "semapact_contract_id": "orders-product", + "semapact_contract_version": "1.2.0", + "semapact_release_id": "release-record-1", + "semapact_source_revision": "rev:released", + } + adapter, _, client = _adapter(current, tags=desired) + + adapter.project_release_metadata( + plan, + RuntimeReleaseMetadata( + contract_id="orders-product", + contract_version="1.2.0", + contract_release_id="release-record-1", + source_revision_ref="rev:released", + ), + ) + + assert len(client.statement_execution.calls) == 1 + assert client.statement_execution.calls[0].startswith( + "SELECT tag_name, tag_value FROM `main`.information_schema.table_tags " + ) + + +def test_release_metadata_replaces_only_reserved_stale_tags() -> None: + plan = _plan(_property("id", "BIGINT", required=True)) + current = _state(("id", "bigint", False)) + adapter, _, client = _adapter( + current, + tags={ + "semapact_contract_id": "orders-product", + "semapact_contract_version": "1.1.0", + "semapact_release_id": "release-record-old", + "semapact_source_revision": "rev:old", + "business_owner": "sales", + }, + ) + + adapter.project_release_metadata( + plan, + RuntimeReleaseMetadata( + contract_id="orders-product", + contract_version="1.2.0", + contract_release_id="release-record-1", + source_revision_ref="rev:released", + ), + ) + + assert client.statement_execution.calls[1:] == [ + "ALTER TABLE `main`.`silver`.`orders` UNSET TAGS " + "('semapact_contract_id', 'semapact_contract_version', " + "'semapact_release_id', 'semapact_source_revision')", + "ALTER TABLE `main`.`silver`.`orders` SET TAGS " + "('semapact_contract_id' = 'orders-product', " + "'semapact_contract_version' = '1.2.0', " + "'semapact_release_id' = 'release-record-1', " + "'semapact_source_revision' = 'rev:released')", + ] + + +def test_release_metadata_unsets_only_reserved_keys_that_exist() -> None: + plan = _plan(_property("id", "BIGINT", required=True)) + current = _state(("id", "bigint", False)) + adapter, _, client = _adapter( + current, + tags={ + "semapact_contract_version": "1.1.0", + "business_owner": "sales", + }, + ) + + adapter.project_release_metadata( + plan, + RuntimeReleaseMetadata( + contract_id="orders-product", + contract_version="1.2.0", + contract_release_id="release-record-1", + source_revision_ref="rev:released", + ), + ) + + assert client.statement_execution.calls[1] == ( + "ALTER TABLE `main`.`silver`.`orders` " + "UNSET TAGS ('semapact_contract_version')" + ) + + def test_release_metadata_projection_requires_warehouse() -> None: plan = _plan(_property("id", "BIGINT", required=True)) current = _state(("id", "bigint", False)) From 6bf10b8ef73c8dd827a13a5003beebdb484fe207 Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Mon, 21 Sep 2026 21:52:06 +1000 Subject: [PATCH 34/35] test(databricks): add protected live release deployment smoke (#241) * test(databricks): add opt-in live release deployment smoke * test(databricks): keep live smoke opt-in * test: register live Databricks marker * ci(databricks): add protected live smoke workflow * docs(databricks): document opt-in live smoke * test(databricks): harden live smoke target safety --- .github/workflows/live-databricks-smoke.yml | 71 ++++ docs/live_databricks_smoke.md | 155 +++++++++ pyproject.toml | 3 + ...est_databricks_release_deployment_smoke.py | 325 ++++++++++++++++++ 4 files changed, 554 insertions(+) create mode 100644 .github/workflows/live-databricks-smoke.yml create mode 100644 docs/live_databricks_smoke.md create mode 100644 tests/live/test_databricks_release_deployment_smoke.py diff --git a/.github/workflows/live-databricks-smoke.yml b/.github/workflows/live-databricks-smoke.yml new file mode 100644 index 00000000..7d5e048f --- /dev/null +++ b/.github/workflows/live-databricks-smoke.yml @@ -0,0 +1,71 @@ +name: Live Databricks Smoke + +on: + workflow_dispatch: + inputs: + catalog: + description: Unity Catalog catalog containing dedicated smoke schemas + required: true + type: string + uat_schema: + description: Dedicated UAT schema (must start with semapact_smoke_) + required: true + default: semapact_smoke_uat + type: string + prod_schema: + description: Dedicated PROD schema (must start with semapact_smoke_) + required: true + default: semapact_smoke_prod + type: string + warehouse_id: + description: SQL warehouse ID used for smoke DDL and tag queries + required: true + type: string + +permissions: + contents: read + +concurrency: + group: live-databricks-smoke + cancel-in-progress: false + +jobs: + smoke: + runs-on: ubuntu-latest + timeout-minutes: 30 + environment: databricks-smoke + env: + UV_NO_PROGRESS: "1" + SEMAPACT_RUN_LIVE_DATABRICKS: "1" + SEMAPACT_LIVE_DATABRICKS_CONFIRM: I_UNDERSTAND_THIS_CREATES_TABLES + SEMAPACT_LIVE_DATABRICKS_CATALOG: ${{ inputs.catalog }} + SEMAPACT_LIVE_DATABRICKS_UAT_SCHEMA: ${{ inputs.uat_schema }} + SEMAPACT_LIVE_DATABRICKS_PROD_SCHEMA: ${{ inputs.prod_schema }} + SEMAPACT_LIVE_DATABRICKS_WAREHOUSE_ID: ${{ inputs.warehouse_id }} + SEMAPACT_LIVE_RUN_ID: ${{ github.run_id }}_${{ github.run_attempt }} + DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} + DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} + DATABRICKS_CLIENT_ID: ${{ secrets.DATABRICKS_CLIENT_ID }} + DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_CLIENT_SECRET }} + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - name: Set up uv + uses: astral-sh/setup-uv@v7 + + - name: Install Databricks test dependencies + run: uv sync --extra databricks --group dev --frozen + + - name: Run live Databricks smoke + run: >- + uv run pytest + tests/live/test_databricks_release_deployment_smoke.py + -m live_databricks + -vv diff --git a/docs/live_databricks_smoke.md b/docs/live_databricks_smoke.md new file mode 100644 index 00000000..9efa1f09 --- /dev/null +++ b/docs/live_databricks_smoke.md @@ -0,0 +1,155 @@ +# Live Databricks smoke + +The live Databricks smoke is an opt-in production-integration check for the canonical SemaPact Data Engineer workflow. + +It is intentionally separate from normal PR CI. + +## What it proves + +The smoke runs against a real Databricks workspace and exercises: + +```text +candidate desired state +→ CREATE in UAT +→ fresh IN_SYNC + +candidate schema evolution +→ ADD nullable column in UAT +→ fresh IN_SYNC + +formal release +→ ContractRelease + +ContractRelease +→ CREATE in PROD +→ fresh IN_SYNC +→ Unity Catalog release provenance tags + +same ContractRelease again +→ NO_OP +→ fresh IN_SYNC +→ provenance projection remains idempotent +``` + +Candidate deployment must not create formal release provenance tags. + +## Safety boundary + +The smoke only creates generated tables whose names begin with: + +```text +semapact_smoke_ +``` + +Both configured schemas must also begin with: + +```text +semapact_smoke_ +``` + +The schemas must already exist. The smoke never creates or deletes catalogs or schemas. + +Each run uses a unique table name derived from the GitHub Actions run ID. Cleanup deletes only those exact generated tables. + +The test also requires the explicit confirmation value: + +```text +SEMAPACT_LIVE_DATABRICKS_CONFIRM=I_UNDERSTAND_THIS_CREATES_TABLES +``` + +## GitHub Actions + +Run: + +```text +Live Databricks Smoke +``` + +through `workflow_dispatch`. + +The workflow uses the protected GitHub Environment: + +```text +databricks-smoke +``` + +Configure the environment with Databricks unified-auth credentials. Supported standard environment secrets include: + +```text +DATABRICKS_HOST + +# PAT option +DATABRICKS_TOKEN + +# OAuth service-principal option +DATABRICKS_CLIENT_ID +DATABRICKS_CLIENT_SECRET +``` + +Do not configure both authentication approaches unless the Databricks SDK configuration intentionally requires it. + +Workflow inputs provide: + +```text +catalog +uat_schema +prod_schema +warehouse_id +``` + +Recommended dedicated schemas: + +```text +semapact_smoke_uat +semapact_smoke_prod +``` + +## Required Databricks capability + +The smoke identity must be able to: + +* use the configured catalog and smoke schemas; +* observe table metadata; +* create managed Delta tables in the smoke schemas; +* add a nullable column; +* query the catalog information schema for table tags; +* apply/remove SemaPact-owned table tags; +* delete only the generated smoke tables during cleanup; +* execute the required SQL through the configured warehouse. + +The more general least-privilege production identity design remains tracked separately in the private roadmap. + +## Local execution + +The test is skipped unless explicitly enabled. + +Example environment: + +```text +SEMAPACT_RUN_LIVE_DATABRICKS=1 +SEMAPACT_LIVE_DATABRICKS_CONFIRM=I_UNDERSTAND_THIS_CREATES_TABLES +SEMAPACT_LIVE_DATABRICKS_CATALOG= +SEMAPACT_LIVE_DATABRICKS_UAT_SCHEMA=semapact_smoke_uat +SEMAPACT_LIVE_DATABRICKS_PROD_SCHEMA=semapact_smoke_prod +SEMAPACT_LIVE_DATABRICKS_WAREHOUSE_ID= +SEMAPACT_LIVE_RUN_ID=local_001 +``` + +Then run: + +```bash +pytest tests/live/test_databricks_release_deployment_smoke.py -m live_databricks -vv +``` + +Databricks authentication is resolved by the official SDK through its normal unified-auth configuration. + +## Non-goals + +This smoke does not test: + +* destructive schema evolution; +* rename/type/nullability migration; +* customer production tables; +* performance or load; +* continuous monitoring; +* infrastructure provisioning. diff --git a/pyproject.toml b/pyproject.toml index bf8b0ddc..998f47bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -118,6 +118,9 @@ dev = [ [tool.pytest.ini_options] testpaths = ["tests"] addopts = "-q" +markers = [ + "live_databricks: opt-in tests against a real Databricks workspace", +] [tool.setuptools.packages.find] include = ["semapact*"] diff --git a/tests/live/test_databricks_release_deployment_smoke.py b/tests/live/test_databricks_release_deployment_smoke.py new file mode 100644 index 00000000..848fe17f --- /dev/null +++ b/tests/live/test_databricks_release_deployment_smoke.py @@ -0,0 +1,325 @@ +from __future__ import annotations + +from datetime import datetime, timezone +import os +import re + +import pytest +from open_data_contract_standard.model import ( + OpenDataContractStandard, + SchemaObject, + SchemaProperty, +) + +from semapact.application.services.deployment_workflow import DeploymentWorkflowService +from semapact.application.services.release_workflow import ( + ReleaseFinalizer, + ReleaseWorkflowService, +) +from semapact.deployment import DeploymentTarget, NativeOperationKind +from semapact.exceptions import ValidationError +from semapact.governance import DecisionResult +from semapact.platforms.databricks import create_databricks_workspace_client +from semapact.platforms.databricks.deployment import ( + DatabricksDeploymentExecutionConfig, + DatabricksStatementExecutor, +) +from semapact.platforms.runtime_registry import create_deployment_adapter +from semapact.reconciliation import RuntimeDriftStatus +from semapact.schema import validate_simple_sql_identifier + + +pytestmark = [ + pytest.mark.live_databricks, + pytest.mark.skipif( + os.environ.get("SEMAPACT_RUN_LIVE_DATABRICKS") != "1", + reason="live Databricks smoke is opt-in", + ), +] + +_RESERVED_TAGS = ( + "semapact_contract_id", + "semapact_contract_version", + "semapact_release_id", + "semapact_source_revision", +) +_CONFIRMATION = "I_UNDERSTAND_THIS_CREATES_TABLES" + + +def _required_env(name: str) -> str: + value = os.environ.get(name, "").strip() + if not value: + pytest.fail(f"live Databricks smoke requires {name}") + return value + + +def _safe_identifier(name: str, label: str) -> str: + try: + validate_simple_sql_identifier(name, label) + except ValidationError as exc: + pytest.fail(str(exc)) + return name + + +def _safe_schema(name: str) -> str: + _safe_identifier(name, "schema") + if not name.casefold().startswith("semapact_smoke_"): + pytest.fail( + "live Databricks smoke schemas must start with 'semapact_smoke_'" + ) + return name + + +def _safe_run_id(value: str) -> str: + normalized = re.sub(r"[^a-zA-Z0-9_]", "_", value.strip()) + if not normalized: + pytest.fail("SEMAPACT_LIVE_RUN_ID must contain an identifier") + return normalized[:48].casefold() + + +def _contract( + *, + table_name: str, + include_note: bool, +) -> OpenDataContractStandard: + properties = [ + SchemaProperty( + name="id", + physicalName="id", + logicalType="integer", + physicalType="BIGINT", + required=True, + ) + ] + if include_note: + properties.append( + SchemaProperty( + name="note", + physicalName="note", + logicalType="string", + physicalType="STRING", + required=False, + ) + ) + return OpenDataContractStandard( + apiVersion="v3.1.0", + kind="DataContract", + id=f"semapact-live-smoke-{table_name}", + name=f"SemaPact live smoke {table_name}", + version="1.0.0", + status="active", + schema=[ + SchemaObject( + name=table_name, + physicalName=table_name, + physicalType="table", + properties=properties, + ) + ], + ) + + +def _target(*, catalog: str, schema_name: str, source_reference: str) -> DeploymentTarget: + return DeploymentTarget( + platform="databricks", + runtime_target=f"{catalog}.{schema_name}", + source_reference=source_reference, + ) + + +def _adapter(*, warehouse_id: str): + return create_deployment_adapter( + "databricks", + execution_config=DatabricksDeploymentExecutionConfig( + warehouse_id=warehouse_id + ), + ) + + +def _read_release_tags( + *, + client, + warehouse_id: str, + catalog: str, + schema_name: str, + table_name: str, +) -> dict[str, str]: + executor = DatabricksStatementExecutor( + client=client, + warehouse_id=warehouse_id, + poll_interval_seconds=1, + ) + keys = ", ".join(f"'{key}'" for key in _RESERVED_TAGS) + rows = executor.query_rows( + "SELECT tag_name, tag_value " + f"FROM `{catalog}`.information_schema.table_tags " + f"WHERE schema_name = '{schema_name.casefold()}' " + f"AND table_name = '{table_name.casefold()}' " + f"AND tag_name IN ({keys})" + ) + return {str(key): str(value) for key, value in rows} + + +def _delete_if_present(client, full_name: str) -> None: + exists = client.tables.exists(full_name=full_name) + if bool(getattr(exists, "table_exists", False)): + client.tables.delete(full_name=full_name) + + +def test_live_candidate_release_and_redeployment_converge() -> None: + if os.environ.get("SEMAPACT_LIVE_DATABRICKS_CONFIRM") != _CONFIRMATION: + pytest.fail( + "set SEMAPACT_LIVE_DATABRICKS_CONFIRM=" + f"{_CONFIRMATION} to run the live smoke" + ) + + catalog = _safe_identifier( + _required_env("SEMAPACT_LIVE_DATABRICKS_CATALOG"), + "catalog", + ) + uat_schema = _safe_schema(_required_env("SEMAPACT_LIVE_DATABRICKS_UAT_SCHEMA")) + prod_schema = _safe_schema(_required_env("SEMAPACT_LIVE_DATABRICKS_PROD_SCHEMA")) + if uat_schema.casefold() == prod_schema.casefold(): + pytest.fail("live Databricks smoke requires distinct UAT and PROD schemas") + warehouse_id = _required_env("SEMAPACT_LIVE_DATABRICKS_WAREHOUSE_ID") + run_id = _safe_run_id(_required_env("SEMAPACT_LIVE_RUN_ID")) + + table_name = f"semapact_smoke_{run_id}" + client = create_databricks_workspace_client() + source_reference = str(getattr(client.config, "host", "") or "").strip() + if not source_reference: + pytest.fail("Databricks SDK did not resolve a workspace host") + + uat_full_name = f"{catalog}.{uat_schema}.{table_name}" + prod_full_name = f"{catalog}.{prod_schema}.{table_name}" + + base = _contract(table_name=table_name, include_note=False) + candidate = _contract(table_name=table_name, include_note=True) + deployment = DeploymentWorkflowService() + + try: + _delete_if_present(client, uat_full_name) + _delete_if_present(client, prod_full_name) + + uat_target = _target( + catalog=catalog, + schema_name=uat_schema, + source_reference=source_reference, + ) + + create_bundle = deployment.assess( + base, + base, + base_revision_ref=f"live:base:{run_id}", + candidate_revision_ref=f"live:base:{run_id}", + target=uat_target, + adapter=_adapter(warehouse_id=warehouse_id), + ) + assert create_bundle.is_release is False + assert create_bundle.review_preview.operations[0].kind is NativeOperationKind.CREATE + + create_result = deployment.deploy( + create_bundle, + adapter=_adapter(warehouse_id=warehouse_id), + ) + assert create_result.status is RuntimeDriftStatus.IN_SYNC + assert create_result.contract_release_id is None + + alter_bundle = deployment.assess( + base, + candidate, + base_revision_ref=f"live:base:{run_id}", + candidate_revision_ref=f"live:candidate:{run_id}", + target=uat_target, + adapter=_adapter(warehouse_id=warehouse_id), + ) + assert alter_bundle.is_release is False + assert alter_bundle.review_preview.operations[0].kind is NativeOperationKind.ALTER + + alter_result = deployment.deploy( + alter_bundle, + adapter=_adapter(warehouse_id=warehouse_id), + ) + assert alter_result.status is RuntimeDriftStatus.IN_SYNC + assert alter_result.contract_release_id is None + assert _read_release_tags( + client=client, + warehouse_id=warehouse_id, + catalog=catalog, + schema_name=uat_schema, + table_name=table_name, + ) == {} + + release_workflow = ReleaseWorkflowService() + release_bundle = release_workflow.assess( + base, + candidate, + base_revision_ref=f"live:base:{run_id}", + candidate_revision_ref=f"live:candidate:{run_id}", + ) + approval = None + if release_bundle.decision.decision is DecisionResult.REVIEW: + approval = release_workflow.approve( + release_bundle, + actor_reference="live-smoke:reviewer", + recorded_at=datetime.now(timezone.utc), + comment="Live Databricks smoke approval", + ) + release = ReleaseFinalizer().finalize( + release_bundle, + approval=approval, + ) + assert release.contract_version != base.version + + prod_target = _target( + catalog=catalog, + schema_name=prod_schema, + source_reference=source_reference, + ) + prod_bundle = deployment.assess_release( + release, + target=prod_target, + adapter=_adapter(warehouse_id=warehouse_id), + ) + assert prod_bundle.review_preview.operations[0].kind is NativeOperationKind.CREATE + + prod_result = deployment.deploy( + prod_bundle, + adapter=_adapter(warehouse_id=warehouse_id), + ) + assert prod_result.status is RuntimeDriftStatus.IN_SYNC + assert prod_result.contract_release_id == release.contract_release_id + + expected_tags = { + "semapact_contract_id": release.contract_id, + "semapact_contract_version": release.contract_version, + "semapact_release_id": release.contract_release_id, + "semapact_source_revision": release.source_revision_ref, + } + assert _read_release_tags( + client=client, + warehouse_id=warehouse_id, + catalog=catalog, + schema_name=prod_schema, + table_name=table_name, + ) == expected_tags + + repeat_result = deployment.deploy( + prod_bundle, + adapter=_adapter(warehouse_id=warehouse_id), + ) + assert repeat_result.status is RuntimeDriftStatus.IN_SYNC + assert all( + operation.kind is NativeOperationKind.NO_OP + for operation in repeat_result.fresh_preview.operations + ) + assert _read_release_tags( + client=client, + warehouse_id=warehouse_id, + catalog=catalog, + schema_name=prod_schema, + table_name=table_name, + ) == expected_tags + finally: + _delete_if_present(client, uat_full_name) + _delete_if_present(client, prod_full_name) From cfa8b670b68ade783343a68e188f24bb5dd20339 Mon Sep 17 00:00:00 2001 From: Elliot Sun Date: Thu, 24 Sep 2026 11:40:54 +1000 Subject: [PATCH 35/35] feat(databricks): unify configured client composition and UC data-product import (#242) * feat(databricks): integrate ConfigManager with workspace client and streamline Unity importer - Resolve Databricks connection hints (workspace_url/host, token, profile) through ConfigManager fallback hierarchy in create_databricks_workspace_client - Remove _databricks_env and process-wide environment variable mutations in Unity Catalog importer - Delegate table discovery and metadata retrieval directly to authenticated WorkspaceClient and pure schema mapping - Add end-to-end regression script for live Databricks schema Data Products - Update unit and live smoke tests with proper configuration isolation * chore(databricks): keep local smoke settings out of product changes * chore(databricks): keep local smoke settings out of product changes * chore(databricks): remove environment-specific live regression script * refactor(config): type Databricks connection hints * feat(databricks): centralize SemaPact connection hint resolution * refactor(databricks): keep workspace client factory configuration-neutral * refactor(databricks): resolve product config at composition boundary * refactor(unity): derive relationships from SDK metadata * refactor(unity): import UC data products through one SDK client * test(unity): cover SDK-native relationship metadata * test(unity): remove obsolete REST metadata aliases * test(databricks): keep low-level client factory configuration-neutral * test(databricks): cover configuration hint resolution * test(unity): mock the importer at the orchestration boundary * test(unity): cover schema-level data product import * test(databricks): cover runtime config composition * docs(config): publish typed Databricks connection hints * docs(config): document Databricks hint precedence * fix(databricks): restore workspace client source newlines * refactor(unity): aggregate relationship import across data product * refactor(unity): enrich relationships once per data product * test(unity): cover data-product relationship aggregation * refactor(databricks): expose explicit configured client composition * refactor(databricks): reuse configured client composition * refactor(unity): reuse configured Databricks client composition * refactor(databricks): make readiness use configured composition * test(databricks): use configured client in live smoke * test(databricks): assert configured runtime client boundary * test(databricks): cover configured client composition * test(unity): assert configured client composition boundary * test(databricks): mock configured readiness client * refactor(unity): treat only absent client as unresolved * build(databricks): align SDK floor with validated provider surface * build(databricks): sync lock metadata with SDK floor * refactor(unity): keep lifecycle status outside runtime import * test(unity): keep imported lifecycle state unpromoted * test(unity): keep CLI tests at importer boundary * fix(unity): persist non-secret workspace binding in imported server * test(unity): verify imported runtime server binding * refactor(databricks): type configured client boundary * fix(unity): surface unmapped relationship metadata * test(unity): cover unmapped relationship evidence --- docs/configuration.md | 35 ++ pyproject.toml | 2 + schemas/semapact-config.schema.json | 55 +++ semapact/core/config_schema.py | 37 +- semapact/importers/unity_importer.py | 150 ++++---- semapact/importers/unity_relationships.py | 192 +++------- semapact/platforms/databricks/client.py | 28 +- .../platforms/databricks/configuration.py | 96 +++++ semapact/platforms/databricks/readiness.py | 6 +- semapact/platforms/runtime_registry.py | 8 +- ...est_databricks_release_deployment_smoke.py | 6 +- tests/test_databricks_configuration.py | 136 +++++++ tests/test_databricks_readiness.py | 8 +- ...test_runtime_registry_databricks_config.py | 34 ++ tests/test_semapact_cli_readable.py | 70 +--- ...semapact_orchestrator_pipeline_readable.py | 99 +++-- tests/test_semapact_unity_relationships.py | 213 +++++++---- tests/test_unity_importer.py | 202 ++++++++++ tests/test_unity_relationships_internals.py | 83 +++-- uv.lock | 351 +++++++++++++++++- 20 files changed, 1344 insertions(+), 467 deletions(-) create mode 100644 semapact/platforms/databricks/configuration.py create mode 100644 tests/test_databricks_configuration.py create mode 100644 tests/test_runtime_registry_databricks_config.py create mode 100644 tests/test_unity_importer.py diff --git a/docs/configuration.md b/docs/configuration.md index 9c1b0d79..41287ceb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -9,6 +9,41 @@ SemaPact loads configuration in this order: Operational deployment history is disabled when no backend is configured. +## Databricks connection hints + +SemaPact can provide optional Databricks connection hints without replacing the +official SDK unified-authentication chain. + +Project configuration: + +```yaml +databricks: + workspace_url: https://adb-..azuredatabricks.net + profile: DEFAULT +``` + +A token is also a supported typed field when required: + +```yaml +databricks: + token: +``` + +Do not commit long-lived credentials to a repository. Prefer a Databricks +profile, workload identity/service-principal authentication, or environment +secrets for CI/CD. + +Connection hints resolve in this order: + +1. explicit caller/CLI value; +2. `SEMAPACT_DATABRICKS_WORKSPACE_URL`, `SEMAPACT_DATABRICKS_TOKEN`, or `SEMAPACT_DATABRICKS_PROFILE`; +3. local/global SemaPact `databricks` configuration; +4. if still omitted, the Databricks SDK unified-authentication chain, including its standard `DATABRICKS_*` environment variables and profile configuration. + +The low-level `create_databricks_workspace_client` factory remains +configuration-neutral. Config resolution happens at SemaPact composition +boundaries before the SDK client is constructed. + ## Operational history SQLite: diff --git a/pyproject.toml b/pyproject.toml index 998f47bf..79c03d19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,7 @@ tui = [ "textual>=8.2.7", ] databricks = [ + "databricks-sdk>=0.109.0", "datacontract-cli[databricks]>=0.12.0", "pyspark>=3.3.0", ] @@ -97,6 +98,7 @@ all = [ "openai>=1.0.0", "litellm>=1.83.14", "textual>=8.2.7", + "databricks-sdk>=0.109.0", "datacontract-cli[databricks]>=0.12.0", "pyspark>=3.3.0", "azure-identity>=1.16.0", diff --git a/schemas/semapact-config.schema.json b/schemas/semapact-config.schema.json index 0a939a73..4efc2f61 100644 --- a/schemas/semapact-config.schema.json +++ b/schemas/semapact-config.schema.json @@ -1,5 +1,49 @@ { "$defs": { + "DatabricksConfig": { + "additionalProperties": false, + "description": "Project/global Databricks connection hints.\n\nThese are composition hints only. Missing values are intentionally left to\nthe Databricks SDK unified-authentication chain.", + "properties": { + "workspace_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Workspace Url" + }, + "token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Token" + }, + "profile": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Profile" + } + }, + "title": "DatabricksConfig", + "type": "object" + }, "DeltaOperationalHistoryConfig": { "additionalProperties": false, "description": "Delta Lake operational history configuration.", @@ -84,6 +128,17 @@ "additionalProperties": true, "description": "Incremental root schema for project/global SemaPact configuration.", "properties": { + "databricks": { + "anyOf": [ + { + "$ref": "#/$defs/DatabricksConfig" + }, + { + "type": "null" + } + ], + "default": null + }, "history": { "anyOf": [ { diff --git a/semapact/core/config_schema.py b/semapact/core/config_schema.py index d0252535..21ebb9d6 100644 --- a/semapact/core/config_schema.py +++ b/semapact/core/config_schema.py @@ -1,4 +1,4 @@ -"""Typed configuration schema for optional operational deployment history.""" +"""Typed configuration schema for supported SemaPact configuration sections.""" from __future__ import annotations @@ -7,6 +7,32 @@ from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, field_validator +def _clean_optional_string(value: str | None) -> str | None: + if value is None: + return None + cleaned = value.strip() + return cleaned or None + + +class DatabricksConfig(BaseModel): + """Project/global Databricks connection hints. + + These are composition hints only. Missing values are intentionally left to + the Databricks SDK unified-authentication chain. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + workspace_url: str | None = None + token: str | None = None + profile: str | None = None + + @field_validator("workspace_url", "token", "profile") + @classmethod + def _normalize_optional_hint(cls, value: str | None) -> str | None: + return _clean_optional_string(value) + + class _OperationalHistoryConfig(BaseModel): """Shared immutable configuration contract for operational history backends.""" @@ -76,11 +102,20 @@ class SemaPactConfigSchema(BaseModel): }, ) + databricks: DatabricksConfig | None = None history: HistoryConfig | None = None + _OPERATIONAL_HISTORY_ADAPTER = TypeAdapter(OperationalHistoryConfig) +def parse_databricks_config(value: object) -> DatabricksConfig | None: + """Validate the databricks config subsection fail closed.""" + if value is None: + return None + return DatabricksConfig.model_validate(value) + + def parse_operational_history_config( value: object, ) -> OperationalHistoryConfig | None: diff --git a/semapact/importers/unity_importer.py b/semapact/importers/unity_importer.py index 679cabee..f6ed960f 100644 --- a/semapact/importers/unity_importer.py +++ b/semapact/importers/unity_importer.py @@ -1,20 +1,10 @@ -"""Single implementation for Unity Catalog contract imports. - -This module consolidates the Unity import logic that was previously duplicated -in ``semapact.interfaces.cli`` and ``semapact.orchestrator.pipeline``. - -Environment variable mutation is isolated inside a context manager so it cannot -leak into concurrent operations. -""" +"""Unity Catalog metadata import into canonical ODCS contracts.""" from __future__ import annotations -import contextlib import logging -import os -from typing import Iterator +from typing import Any -from datacontract.data_contract import DataContract from open_data_contract_standard.model import OpenDataContractStandard from semapact.importers.unity_relationships import ( @@ -24,84 +14,106 @@ LOGGER = logging.getLogger(__name__) -@contextlib.contextmanager -def _databricks_env( - workspace_url: str | None, token: str | None, profile: str | None -) -> Iterator[None]: - """Temporarily set Databricks env vars and restore them on exit. - - This context manager ensures the process-global environment is restored - even when the import raises, preventing credential leaks across calls. - """ - env_keys = ( - "DATACONTRACT_DATABRICKS_SERVER_HOSTNAME", - "DATACONTRACT_DATABRICKS_TOKEN", - "DATABRICKS_CONFIG_PROFILE", - ) - backup = {key: os.environ.get(key) for key in env_keys} - if workspace_url: - os.environ["DATACONTRACT_DATABRICKS_SERVER_HOSTNAME"] = workspace_url - if token: - os.environ["DATACONTRACT_DATABRICKS_TOKEN"] = token - if profile: - os.environ["DATABRICKS_CONFIG_PROFILE"] = profile - try: - yield - finally: - for key, value in backup.items(): - if value is None: - os.environ.pop(key, None) - else: - os.environ[key] = value - - def import_unity_contract( *, table_fqn: str, workspace_url: str | None = None, token: str | None = None, + profile: str | None = None, sql_http_path: str | None = None, extract_lineage: bool = False, + client: Any | None = None, ) -> OpenDataContractStandard: - """Import Unity Catalog metadata into ODCS using datacontract-cli. + """Import one UC table or one entire UC schema as an ODCS data product. - Runtime lineage is deliberately excluded from this importer. The legacy - lineage-related keyword arguments remain temporarily accepted so existing - callers fail with an explicit boundary error instead of a Python signature - error. They must not trigger ODCS mutation. + Authentication/configuration is resolved at this import composition boundary. + The importer itself uses only the official Databricks SDK client and never + mutates process-global environment variables. - Raises ``ValueError`` when credentials are missing or lineage enrichment is - requested through the import boundary. + Runtime lineage remains observation evidence and is not projected into ODCS. """ del sql_http_path if extract_lineage: raise ValueError( - "Unity lineage is runtime observation evidence and can no longer be " + "Unity lineage is runtime observation evidence and cannot be " "projected into ODCS during import" ) - from semapact.core.config import config_manager - - workspace_url = workspace_url or config_manager.get("databricks.workspace_url") - token = token or config_manager.get("databricks.token") - profile = config_manager.get("databricks.profile") - - if not profile and (not workspace_url or not token): + parts = tuple(part.strip() for part in table_fqn.split(".")) + if len(parts) not in {2, 3} or not all(parts): raise ValueError( - "databricks.workspace_url and databricks.token (or databricks.profile) are required for Unity Catalog imports" + "Unity import source must use catalog.schema or catalog.schema.table" ) - LOGGER.info("Importing Unity Catalog contract: %s", table_fqn) - with _databricks_env(workspace_url, token, profile): - imported = DataContract.import_from_source( - format="unity", - source=None, - unity_table_full_name=[table_fqn], - ) - return enrich_unity_contract_relationships( - imported, - table_fqn=table_fqn, + from datacontract.imports.unity_importer import ( + convert_unity_schema, + create_odcs, + ) + + from semapact.platforms.databricks.configuration import ( + create_configured_databricks_workspace_client, + ) + from semapact.platforms.databricks.discovery import discover_databricks_tables + + ws_client = ( + client + if client is not None + else create_configured_databricks_workspace_client( workspace_url=workspace_url, token=token, + profile=profile, + ) + ) + + is_schema_level = len(parts) == 2 + if is_schema_level: + catalog, schema_name = parts + tables_to_import = discover_databricks_tables( + client=ws_client, + catalog_name=catalog, + schema_name=schema_name, ) + if not tables_to_import: + raise ValueError( + f"No tables discovered in Unity Catalog schema {table_fqn}" + ) + else: + tables_to_import = (table_fqn,) + + LOGGER.info( + "Importing %d Unity Catalog table(s) from %s", + len(tables_to_import), + table_fqn, + ) + + imported = create_odcs() + imported.schema_ = [] + table_metadata: dict[str, dict[str, Any]] = {} + for table_name in tables_to_import: + table_info = ws_client.tables.get(table_name) + imported = convert_unity_schema(imported, table_info) + as_dict = getattr(table_info, "as_dict", None) + if not callable(as_dict): + raise TypeError("Databricks TableInfo must expose as_dict()") + metadata = as_dict() + if not isinstance(metadata, dict): + raise TypeError("Databricks TableInfo.as_dict() must return a dict") + table_metadata[table_name] = metadata + + resolved_host = getattr(getattr(ws_client, "config", None), "host", None) + if not isinstance(resolved_host, str) or not resolved_host.strip(): + raise ValueError("Databricks SDK did not resolve a workspace host") + for server in imported.servers or []: + if str(getattr(server, "type", "") or "").strip().casefold() == "databricks": + server.host = resolved_host.strip().rstrip("/") + + if is_schema_level: + catalog, schema_name = parts + imported.id = f"{catalog}-{schema_name}-product" + imported.name = f"{catalog.capitalize()} {schema_name.capitalize()} Data Product" + + return enrich_unity_contract_relationships( + imported, + table_metadata=table_metadata, + ) diff --git a/semapact/importers/unity_relationships.py b/semapact/importers/unity_relationships.py index 4052e0ff..f1cfc896 100644 --- a/semapact/importers/unity_relationships.py +++ b/semapact/importers/unity_relationships.py @@ -1,11 +1,7 @@ from __future__ import annotations -import json from dataclasses import dataclass -from typing import Any, Callable, Iterable, Sequence, Tuple -from urllib.error import HTTPError, URLError -from urllib.parse import quote -from urllib.request import Request, urlopen +from typing import Any, Iterable, Mapping, Sequence, Tuple from open_data_contract_standard.model import ( CustomProperty, @@ -33,147 +29,70 @@ class UnityForeignKey: def enrich_unity_contract_relationships( contract: OpenDataContractStandard, *, - table_fqn: str, - workspace_url: str, - token: str, - fetcher: Callable[[str, str, str], dict[str, Any]] | None = None, + table_metadata: Mapping[str, Mapping[str, Any]], ) -> OpenDataContractStandard: - """Best-effort Unity relationship enrichment using table metadata. + """Project SDK-returned Unity foreign keys for an entire data product. - If Unity relationship metadata is unavailable, this function does not fail import. - Instead it records fallback metadata on the contract customProperties. + Relationship extraction is best effort per table. A contract-level summary + records the total imported relationships and any table-specific failures. + No second HTTP client or bearer-token path is introduced here. """ - table_metadata_fetcher = fetcher or _fetch_unity_table_metadata - try: - metadata = table_metadata_fetcher(workspace_url, token, table_fqn) - foreign_keys = _extract_foreign_keys(metadata) - imported_count = _apply_foreign_keys( - contract, table_fqn=table_fqn, foreign_keys=foreign_keys - ) - _upsert_contract_custom_property( - contract, UNITY_RELATIONSHIPS_IMPORTED_KEY, "true" - ) - _upsert_contract_custom_property( - contract, UNITY_RELATIONSHIPS_COUNT_KEY, str(imported_count) - ) - except Exception as exc: # pragma: no cover - behavior validated via unit tests - import logging - - logging.getLogger(__name__).debug(f"Failed to fetch unity relationships: {exc}") - _upsert_contract_custom_property( - contract, UNITY_RELATIONSHIPS_IMPORTED_KEY, "false" - ) + imported_count = 0 + failures: list[str] = [] + + for table_fqn in sorted(table_metadata, key=lambda value: (value.casefold(), value)): + try: + foreign_keys = _extract_foreign_keys(table_metadata[table_fqn]) + imported_count += _apply_foreign_keys( + contract, + table_fqn=table_fqn, + foreign_keys=foreign_keys, + ) + except Exception as exc: # pragma: no cover - validated through unit tests + failures.append(f"{table_fqn}: {exc}") + + _upsert_contract_custom_property( + contract, + UNITY_RELATIONSHIPS_IMPORTED_KEY, + "false" if failures else "true", + ) + _upsert_contract_custom_property( + contract, + UNITY_RELATIONSHIPS_COUNT_KEY, + str(imported_count), + ) + if failures: _upsert_contract_custom_property( - contract, UNITY_RELATIONSHIPS_REASON_KEY, str(exc) + contract, + UNITY_RELATIONSHIPS_REASON_KEY, + "; ".join(failures), ) return contract - -def _fetch_unity_table_metadata( - workspace_url: str, token: str, table_fqn: str -) -> dict[str, Any]: - if not workspace_url or not token: - raise ValueError( - "workspace_url and token are required for Unity relationship import" - ) - - base_url = workspace_url.rstrip("/") - endpoint = f"{base_url}/api/2.1/unity-catalog/tables/{quote(table_fqn, safe='')}" - request = Request( - endpoint, - headers={ - "Authorization": f"Bearer {token}", - "Accept": "application/json", - }, - method="GET", - ) - - try: - with urlopen(request, timeout=8) as response: - payload = response.read().decode("utf-8") - except HTTPError as exc: - from semapact.exceptions import StorageError - - raise StorageError( - f"Unity table metadata request failed: HTTP {exc.code}" - ) from exc - except URLError as exc: - from semapact.exceptions import StorageError - - raise StorageError( - f"Unity table metadata request failed: {exc.reason}" - ) from exc - - parsed = json.loads(payload) - if not isinstance(parsed, dict): - raise RuntimeError("Unity table metadata response is not a JSON object") - return parsed - - -def _extract_foreign_keys(metadata: dict[str, Any]) -> list[UnityForeignKey]: - constraints = _constraint_items(metadata) +def _extract_foreign_keys(metadata: Mapping[str, Any]) -> list[UnityForeignKey]: foreign_keys: list[UnityForeignKey] = [] - for item in constraints: + for item in _constraint_items(metadata): record = _parse_constraint_record(item) if record is not None: foreign_keys.append(record) return foreign_keys -def _constraint_items(metadata: dict[str, Any]) -> list[dict[str, Any]]: - merged: list[dict[str, Any]] = [] - for key in ( - "table_constraints", - "tableConstraints", - "constraints", - "foreign_keys", - "foreignKeys", - ): - value = metadata.get(key) - if isinstance(value, list): - merged.extend(item for item in value if isinstance(item, dict)) - return merged +def _constraint_items(metadata: Mapping[str, Any]) -> list[Mapping[str, Any]]: + value = metadata.get("table_constraints") + if not isinstance(value, list): + return [] + return [item for item in value if isinstance(item, Mapping)] -def _parse_constraint_record(item: dict[str, Any]) -> UnityForeignKey | None: - constraint_type = str( - item.get("constraint_type") - or item.get("constraintType") - or item.get("type") - or item.get("kind") - or "" - ).lower() - if "foreign" not in constraint_type and "reference" not in constraint_type: +def _parse_constraint_record(item: Mapping[str, Any]) -> UnityForeignKey | None: + value = item.get("foreign_key_constraint") + if not isinstance(value, Mapping): return None - source_columns = _to_string_list( - item.get("columns") - or item.get("column_names") - or item.get("columnNames") - or item.get("from_columns") - or item.get("fromColumns") - or item.get("child_columns") - or item.get("childColumns") - or item.get("from") - ) - target_columns = _to_string_list( - item.get("referenced_columns") - or item.get("referencedColumns") - or item.get("to_columns") - or item.get("toColumns") - or item.get("parent_columns") - or item.get("parentColumns") - or item.get("to") - ) - target_table = _to_string( - item.get("referenced_table") - or item.get("referencedTable") - or item.get("to_table") - or item.get("toTable") - or item.get("parent_table") - or item.get("parentTable") - ) - + source_columns = _to_string_list(value.get("child_columns")) + target_columns = _to_string_list(value.get("parent_columns")) + target_table = _to_string(value.get("parent_table")) if not source_columns or not target_columns or not target_table: return None @@ -181,7 +100,7 @@ def _parse_constraint_record(item: dict[str, Any]) -> UnityForeignKey | None: source_columns=source_columns, target_table=target_table, target_columns=target_columns, - constraint_name=_to_string(item.get("name")), + constraint_name=_to_string(value.get("name")), ) @@ -193,14 +112,9 @@ def _to_string(value: Any) -> str | None: def _to_string_list(value: Any) -> list[str]: - if value is None: + if not isinstance(value, list): return [] - if isinstance(value, str): - parts = [part.strip() for part in value.split(",")] - return [part for part in parts if part] - if isinstance(value, list): - return [item for item in (_to_string(item) for item in value) if item] - return [] + return [item for item in (_to_string(item) for item in value) if item] def _apply_foreign_keys( @@ -211,6 +125,10 @@ def _apply_foreign_keys( ) -> int: schema_obj = _resolve_target_schema(contract, table_fqn=table_fqn) if schema_obj is None: + if foreign_keys: + raise ValueError( + f"Unity relationship metadata has no matching governed asset: {table_fqn}" + ) return 0 imported_count = 0 @@ -263,7 +181,7 @@ def _resolve_target_schema( for item in schema_items: if (item.name or "").strip().lower() == short_name: return item - return schema_items[0] + return None def _constraint_custom_props(name: str | None) -> list[CustomProperty] | None: diff --git a/semapact/platforms/databricks/client.py b/semapact/platforms/databricks/client.py index 913a71d7..5e0aaa95 100644 --- a/semapact/platforms/databricks/client.py +++ b/semapact/platforms/databricks/client.py @@ -1,13 +1,8 @@ """Databricks authenticated-client construction boundary. -This module owns construction of an initialized Databricks ``WorkspaceClient``. -Downstream platform capabilities such as observation consume the resulting -client and remain independent from the authentication mechanism used to create -it. - -SemaPact forwards only the connection/authentication hints supplied by the -caller. The Databricks SDK remains responsible for selecting and validating the -authentication mechanism, including its default/unified authentication chain. +This module owns construction of an initialized Databricks WorkspaceClient. +Connection-hint resolution belongs to the calling composition boundary; missing +hints are intentionally left to the Databricks SDK unified-authentication chain. """ from __future__ import annotations @@ -24,14 +19,14 @@ def create_databricks_workspace_client( token: str | None = None, profile: str | None = None, ) -> WorkspaceClient: - """Create a Databricks SDK client from the auth hints the caller has. + """Create a Databricks SDK client from explicit optional hints. - SemaPact does not choose an authentication provider. Non-empty values are - forwarded to ``WorkspaceClient`` and omitted values are left for the SDK to - resolve from its standard configuration/authentication chain. Calling this - function with no arguments is therefore equivalent to ``WorkspaceClient()``. + This function does not read SemaPact configuration or mutate process-global + environment variables. Callers that want project/global SemaPact settings + resolve them before crossing this boundary. - The function performs no credential logging or serialization. + Omitted values are left for the SDK to resolve from its standard unified + authentication chain. """ kwargs: dict[str, str] = {} @@ -39,8 +34,9 @@ def create_databricks_workspace_client( if host: kwargs["host"] = host.rstrip("/") - if token and token.strip(): - kwargs["token"] = token + resolved_token = _clean_optional(token) + if resolved_token: + kwargs["token"] = resolved_token selected_profile = _clean_optional(profile) if selected_profile: diff --git a/semapact/platforms/databricks/configuration.py b/semapact/platforms/databricks/configuration.py new file mode 100644 index 00000000..e1a0d327 --- /dev/null +++ b/semapact/platforms/databricks/configuration.py @@ -0,0 +1,96 @@ +"""Databricks connection-hint resolution for SemaPact composition roots.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from pydantic import ValidationError as PydanticValidationError + +from semapact.core.config import config_manager +from semapact.core.config_schema import parse_databricks_config +from semapact.exceptions import ValidationError + +if TYPE_CHECKING: + from databricks.sdk import WorkspaceClient + + +@dataclass(frozen=True) +class DatabricksConnectionHints: + """Resolved optional hints passed to the Databricks SDK client factory.""" + + workspace_url: str | None = None + token: str | None = None + profile: str | None = None + + +def resolve_databricks_connection_hints( + *, + workspace_url: str | None = None, + token: str | None = None, + profile: str | None = None, +) -> DatabricksConnectionHints: + """Resolve explicit hints, SemaPact overrides, then project/global config. + + Standard Databricks environment variables and ~/.databrickscfg are not read + here. When a hint remains absent, the official SDK resolves it through its + normal unified-authentication chain. + """ + try: + configured = parse_databricks_config( + config_manager.get("databricks", default=None) + ) + except PydanticValidationError as exc: + raise ValidationError(f"Invalid databricks configuration: {exc}") from exc + + return DatabricksConnectionHints( + workspace_url=_first_nonblank( + workspace_url, + os.environ.get("SEMAPACT_DATABRICKS_WORKSPACE_URL"), + configured.workspace_url if configured is not None else None, + ), + token=_first_nonblank( + token, + os.environ.get("SEMAPACT_DATABRICKS_TOKEN"), + configured.token if configured is not None else None, + ), + profile=_first_nonblank( + profile, + os.environ.get("SEMAPACT_DATABRICKS_PROFILE"), + configured.profile if configured is not None else None, + ), + ) + + +def create_configured_databricks_workspace_client( + *, + workspace_url: str | None = None, + token: str | None = None, + profile: str | None = None, +) -> WorkspaceClient: + """Construct a WorkspaceClient after resolving SemaPact connection hints.""" + from semapact.platforms.databricks.client import ( + create_databricks_workspace_client, + ) + + hints = resolve_databricks_connection_hints( + workspace_url=workspace_url, + token=token, + profile=profile, + ) + return create_databricks_workspace_client( + workspace_url=hints.workspace_url, + token=hints.token, + profile=hints.profile, + ) + + +def _first_nonblank(*values: str | None) -> str | None: + for value in values: + if value is None: + continue + cleaned = value.strip() + if cleaned: + return cleaned + return None diff --git a/semapact/platforms/databricks/readiness.py b/semapact/platforms/databricks/readiness.py index 76642921..3d5011a4 100644 --- a/semapact/platforms/databricks/readiness.py +++ b/semapact/platforms/databricks/readiness.py @@ -6,7 +6,9 @@ from typing import Any from semapact.application.models.readiness import ReadinessCheck, ReadinessStatus -from semapact.platforms.databricks.client import create_databricks_workspace_client +from semapact.platforms.databricks.configuration import ( + create_configured_databricks_workspace_client, +) _TERMINAL_STATES = {"SUCCEEDED", "FAILED", "CANCELED", "CLOSED"} @@ -34,7 +36,7 @@ def __init__( def run(self) -> tuple[ReadinessCheck, ...]: try: - client = create_databricks_workspace_client( + client = create_configured_databricks_workspace_client( workspace_url=self._workspace_url, ) except Exception as exc: diff --git a/semapact/platforms/runtime_registry.py b/semapact/platforms/runtime_registry.py index 03f199c7..79c479bb 100644 --- a/semapact/platforms/runtime_registry.py +++ b/semapact/platforms/runtime_registry.py @@ -196,12 +196,12 @@ def _create_databricks_client_and_provider( *, contract_server: Server | None = None, ): - from semapact.platforms.databricks import ( - DatabricksRuntimeProvider, - create_databricks_workspace_client, + from semapact.platforms.databricks import DatabricksRuntimeProvider + from semapact.platforms.databricks.configuration import ( + create_configured_databricks_workspace_client, ) - client = create_databricks_workspace_client( + client = create_configured_databricks_workspace_client( workspace_url=_clean(contract_server.host) if contract_server else None ) source_identifier = getattr(getattr(client, "config", None), "host", None) diff --git a/tests/live/test_databricks_release_deployment_smoke.py b/tests/live/test_databricks_release_deployment_smoke.py index 848fe17f..6addfb0e 100644 --- a/tests/live/test_databricks_release_deployment_smoke.py +++ b/tests/live/test_databricks_release_deployment_smoke.py @@ -19,7 +19,9 @@ from semapact.deployment import DeploymentTarget, NativeOperationKind from semapact.exceptions import ValidationError from semapact.governance import DecisionResult -from semapact.platforms.databricks import create_databricks_workspace_client +from semapact.platforms.databricks.configuration import ( + create_configured_databricks_workspace_client, +) from semapact.platforms.databricks.deployment import ( DatabricksDeploymentExecutionConfig, DatabricksStatementExecutor, @@ -185,7 +187,7 @@ def test_live_candidate_release_and_redeployment_converge() -> None: run_id = _safe_run_id(_required_env("SEMAPACT_LIVE_RUN_ID")) table_name = f"semapact_smoke_{run_id}" - client = create_databricks_workspace_client() + client = create_configured_databricks_workspace_client() source_reference = str(getattr(client.config, "host", "") or "").strip() if not source_reference: pytest.fail("Databricks SDK did not resolve a workspace host") diff --git a/tests/test_databricks_configuration.py b/tests/test_databricks_configuration.py new file mode 100644 index 00000000..cb8ef81d --- /dev/null +++ b/tests/test_databricks_configuration.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import pytest + +from semapact.exceptions import ValidationError +from semapact.platforms.databricks.configuration import ( + create_configured_databricks_workspace_client, + resolve_databricks_connection_hints, +) + + +def test_databricks_connection_hints_use_typed_config( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("SEMAPACT_DATABRICKS_WORKSPACE_URL", raising=False) + monkeypatch.delenv("SEMAPACT_DATABRICKS_TOKEN", raising=False) + monkeypatch.delenv("SEMAPACT_DATABRICKS_PROFILE", raising=False) + monkeypatch.setattr( + "semapact.platforms.databricks.configuration.config_manager.get", + lambda key, default=None: { + "workspace_url": " https://config.example/ ", + "token": " config-token ", + "profile": " config-profile ", + } + if key == "databricks" + else default, + ) + + hints = resolve_databricks_connection_hints() + + assert hints.workspace_url == "https://config.example/" + assert hints.token == "config-token" + assert hints.profile == "config-profile" + + +def test_databricks_connection_hints_precedence( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv( + "SEMAPACT_DATABRICKS_WORKSPACE_URL", + "https://env.example", + ) + monkeypatch.setenv("SEMAPACT_DATABRICKS_TOKEN", "env-token") + monkeypatch.setenv("SEMAPACT_DATABRICKS_PROFILE", "env-profile") + monkeypatch.setattr( + "semapact.platforms.databricks.configuration.config_manager.get", + lambda key, default=None: { + "workspace_url": "https://config.example", + "token": "config-token", + "profile": "config-profile", + } + if key == "databricks" + else default, + ) + + env_hints = resolve_databricks_connection_hints() + assert env_hints.workspace_url == "https://env.example" + assert env_hints.token == "env-token" + assert env_hints.profile == "env-profile" + + explicit_hints = resolve_databricks_connection_hints( + workspace_url="https://explicit.example", + token="explicit-token", + profile="explicit-profile", + ) + assert explicit_hints.workspace_url == "https://explicit.example" + assert explicit_hints.token == "explicit-token" + assert explicit_hints.profile == "explicit-profile" + + +def test_databricks_connection_hints_leave_sdk_defaults_unset( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("SEMAPACT_DATABRICKS_WORKSPACE_URL", raising=False) + monkeypatch.delenv("SEMAPACT_DATABRICKS_TOKEN", raising=False) + monkeypatch.delenv("SEMAPACT_DATABRICKS_PROFILE", raising=False) + monkeypatch.setattr( + "semapact.platforms.databricks.configuration.config_manager.get", + lambda key, default=None: default, + ) + + hints = resolve_databricks_connection_hints() + + assert hints.workspace_url is None + assert hints.token is None + assert hints.profile is None + + +def test_invalid_databricks_config_fails_closed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "semapact.platforms.databricks.configuration.config_manager.get", + lambda key, default=None: {"unknown": "value"} if key == "databricks" else default, + ) + + with pytest.raises(ValidationError, match="Invalid databricks configuration"): + resolve_databricks_connection_hints() + + +def test_configured_workspace_client_resolves_then_delegates( + monkeypatch: pytest.MonkeyPatch, +) -> None: + sentinel = object() + captured: dict[str, object] = {} + + monkeypatch.setattr( + "semapact.platforms.databricks.configuration.resolve_databricks_connection_hints", + lambda **kwargs: type( + "Hints", + (), + { + "workspace_url": "https://config.example", + "token": "config-token", + "profile": "config-profile", + }, + )(), + ) + + def fake_create_client(**kwargs): # noqa: ANN003 + captured.update(kwargs) + return sentinel + + monkeypatch.setattr( + "semapact.platforms.databricks.client.create_databricks_workspace_client", + fake_create_client, + ) + + client = create_configured_databricks_workspace_client() + + assert client is sentinel + assert captured == { + "workspace_url": "https://config.example", + "token": "config-token", + "profile": "config-profile", + } diff --git a/tests/test_databricks_readiness.py b/tests/test_databricks_readiness.py index 5defb975..0a56a221 100644 --- a/tests/test_databricks_readiness.py +++ b/tests/test_databricks_readiness.py @@ -57,7 +57,7 @@ def test_databricks_probe_checks_identity_uc_and_read_only_statement_execution( client = _Client() monkeypatch.setattr( readiness, - "create_databricks_workspace_client", + "create_configured_databricks_workspace_client", lambda **kwargs: client, ) @@ -87,7 +87,7 @@ def test_missing_warehouse_blocks_deployment_readiness( client = _Client() monkeypatch.setattr( readiness, - "create_databricks_workspace_client", + "create_configured_databricks_workspace_client", lambda **kwargs: client, ) @@ -114,7 +114,7 @@ def _raise(**kwargs): monkeypatch.setattr( readiness, - "create_databricks_workspace_client", + "create_configured_databricks_workspace_client", _raise, ) @@ -143,7 +143,7 @@ def _deny(*, full_name: str): client.schemas.get = _deny monkeypatch.setattr( readiness, - "create_databricks_workspace_client", + "create_configured_databricks_workspace_client", lambda **kwargs: client, ) diff --git a/tests/test_runtime_registry_databricks_config.py b/tests/test_runtime_registry_databricks_config.py new file mode 100644 index 00000000..544bc2fe --- /dev/null +++ b/tests/test_runtime_registry_databricks_config.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from semapact.platforms.runtime_registry import _create_databricks_client_and_provider + + +def test_runtime_registry_uses_configured_databricks_client_composition( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, object] = {} + client = SimpleNamespace(config=SimpleNamespace(host="https://config.example")) + + def fake_create_configured_client(**kwargs): # noqa: ANN003 + captured.update(kwargs) + return client + + monkeypatch.setattr( + "semapact.platforms.databricks.configuration.create_configured_databricks_workspace_client", + fake_create_configured_client, + ) + monkeypatch.setattr( + "semapact.platforms.databricks.DatabricksRuntimeProvider", + lambda **kwargs: SimpleNamespace(**kwargs), + ) + + resolved_client, provider = _create_databricks_client_and_provider() + + assert resolved_client is client + assert captured == {"workspace_url": None} + assert provider.client is client + assert provider.source_identifier == "https://config.example" diff --git a/tests/test_semapact_cli_readable.py b/tests/test_semapact_cli_readable.py index 2d1549a8..4ceabd8e 100644 --- a/tests/test_semapact_cli_readable.py +++ b/tests/test_semapact_cli_readable.py @@ -190,31 +190,13 @@ def test_cli_import_uc_runs_unity_enrichment( ): captured: dict[str, Any] = {} - def _fake_import_from_source(**kwargs): - captured["import_kwargs"] = kwargs + def _fake_unity_import(**kwargs): # noqa: ANN003 + captured.update(kwargs) return sample_unity_contract_model.model_copy(deep=True) - def _fake_enrich(contract, **kwargs): # noqa: ANN001 - captured["enrich_kwargs"] = kwargs - return contract - - monkeypatch.setattr( - "semapact.importers.unity_importer.DataContract.import_from_source", - _fake_import_from_source, - ) - monkeypatch.setattr( - "semapact.importers.unity_importer.enrich_unity_contract_relationships", - _fake_enrich, - ) - - # Mock the config manager with fallback values that should be overridden - config_vals = { - "databricks.workspace_url": "https://fallback.example", - "databricks.token": "fallback-token", - } monkeypatch.setattr( - "semapact.core.config.config_manager.get", - lambda key, *args, **kwargs: config_vals.get(key, kwargs.get("default")), + "semapact.importers.unity_importer.import_unity_contract", + _fake_unity_import, ) output_path = tmp_path / "out.yaml" @@ -239,43 +221,24 @@ def _fake_enrich(contract, **kwargs): # noqa: ANN001 exit_code = cli.main() assert exit_code == 0 - assert captured["import_kwargs"]["format"] == "unity" - assert captured["import_kwargs"]["unity_table_full_name"] == ["main.silver.orders"] - assert captured["enrich_kwargs"]["table_fqn"] == "main.silver.orders" - assert captured["enrich_kwargs"]["workspace_url"] == "https://adb.example" - assert captured["enrich_kwargs"]["token"] == "token" - + assert captured["table_fqn"] == "main.silver.orders" + assert captured["workspace_url"] == "https://adb.example" + assert captured["token"] == "token" + assert captured["sql_http_path"] is None + assert captured["extract_lineage"] is False -def test_cli_import_uc_uses_config_fallback( +def test_cli_import_uc_leaves_missing_connection_hints_to_importer( sample_unity_contract_model, tmp_path, monkeypatch ): captured: dict[str, Any] = {} - def _fake_import_from_source(**kwargs): - captured["import_kwargs"] = kwargs + def _fake_unity_import(**kwargs): # noqa: ANN003 + captured.update(kwargs) return sample_unity_contract_model.model_copy(deep=True) - def _fake_enrich(contract, **kwargs): # noqa: ANN001 - captured["enrich_kwargs"] = kwargs - return contract - - monkeypatch.setattr( - "semapact.importers.unity_importer.DataContract.import_from_source", - _fake_import_from_source, - ) - monkeypatch.setattr( - "semapact.importers.unity_importer.enrich_unity_contract_relationships", - _fake_enrich, - ) - - # Mock the config manager - config_vals = { - "databricks.workspace_url": "https://fallback.example", - "databricks.token": "fallback-token", - } monkeypatch.setattr( - "semapact.core.config.config_manager.get", - lambda key, *args, **kwargs: config_vals.get(key, kwargs.get("default")), + "semapact.importers.unity_importer.import_unity_contract", + _fake_unity_import, ) output_path = tmp_path / "out.yaml" @@ -296,9 +259,8 @@ def _fake_enrich(contract, **kwargs): # noqa: ANN001 exit_code = cli.main() assert exit_code == 0 - assert captured["enrich_kwargs"]["workspace_url"] == "https://fallback.example" - assert captured["enrich_kwargs"]["token"] == "fallback-token" - + assert captured["workspace_url"] is None + assert captured["token"] is None def test_cli_release_classify_outputs_per_contract_required_bump( sample_odcs_model, tmp_path, capsys, monkeypatch diff --git a/tests/test_semapact_orchestrator_pipeline_readable.py b/tests/test_semapact_orchestrator_pipeline_readable.py index 38a774d8..c610626c 100644 --- a/tests/test_semapact_orchestrator_pipeline_readable.py +++ b/tests/test_semapact_orchestrator_pipeline_readable.py @@ -52,59 +52,47 @@ def fake_import(format: str, source: str | None = None, **_kwargs): # noqa: ANN assert sql_contract.id == "from-importer" -def test_pipeline_import_schema_requires_uc_credentials(): - pipeline = ContractPipeline() - - with pytest.raises(Exception, match="databricks.workspace_url and databricks.token"): - pipeline.import_schema("uc", "main.silver.orders") - - -def test_pipeline_import_schema_supports_uc_when_credentials_are_given(monkeypatch): +def test_pipeline_import_schema_delegates_uc_to_unity_importer(monkeypatch): captured: dict[str, object] = {} + imported = OpenDataContractStandard.model_validate( + { + "apiVersion": "v3.1.0", + "kind": "DataContract", + "id": "uc-id", + "name": "orders", + "version": "1.0.0", + "status": "draft", + "schema": [ + { + "name": "t1", + "properties": [{"name": "id", "logicalType": "string"}], + } + ], + } + ) - def fake_import(format: str, source: str | None = None, **_kwargs): # noqa: ANN001 - return OpenDataContractStandard.model_validate( - { - "apiVersion": "v3.1.0", - "kind": "DataContract", - "id": "uc-id", - "name": "orders", - "version": "1.0.0", - "status": "draft", - "schema": [ - { - "name": "t1", - "properties": [{"name": "id", "logicalType": "string"}], - } - ], - } - ) - - def fake_enrich(contract, **kwargs): # noqa: ANN001 + def fake_unity_import(**kwargs): # noqa: ANN003 captured.update(kwargs) - return contract + return imported monkeypatch.setattr( - "semapact.importers.unity_importer.DataContract.import_from_source", - staticmethod(fake_import), - ) - monkeypatch.setattr( - "semapact.importers.unity_importer.enrich_unity_contract_relationships", - fake_enrich, + "semapact.orchestrator.pipeline.import_unity_contract", + fake_unity_import, ) - pipeline = ContractPipeline() - contract = pipeline.import_schema( + contract = ContractPipeline().import_schema( "uc", "main.silver.orders", uc_workspace_url="https://adb.example", uc_token="token", ) - assert contract.id == "uc-id" - assert captured["table_fqn"] == "main.silver.orders" - assert captured["workspace_url"] == "https://adb.example" - assert captured["token"] == "token" + assert contract is imported + assert captured == { + "table_fqn": "main.silver.orders", + "workspace_url": "https://adb.example", + "token": "token", + } def test_pipeline_import_schema_rejects_unknown_source_type(): @@ -647,9 +635,8 @@ def test_pipeline_run_executes_real_unity_workflow( ) ) - def fake_import_from_source(format, source=None, **kwargs): # noqa: ANN001 - assert format == "unity" - assert kwargs["unity_table_full_name"] == ["main.silver.orders"] + def fake_unity_import(**kwargs): # noqa: ANN003 + captured.update(kwargs) return imported_contract def fake_export_to_path( @@ -659,25 +646,21 @@ def fake_export_to_path( path.write_text('{"expectations": []}', encoding="utf-8") return path - def fake_enrich(contract, **kwargs): # noqa: ANN001 - captured.update(kwargs) - return contract - monkeypatch.setattr( - "semapact.importers.unity_importer.DataContract.import_from_source", - staticmethod(fake_import_from_source), + "semapact.orchestrator.pipeline.import_unity_contract", + fake_unity_import, ) monkeypatch.setattr( "semapact.orchestrator.pipeline.GreatExpectationsExporter.export_to_path", fake_export_to_path, ) - monkeypatch.setattr( - "semapact.importers.unity_importer.enrich_unity_contract_relationships", - fake_enrich, - ) monkeypatch.setattr( "semapact.governance.gate.evaluate_governance_gate", - lambda d, op: GovernanceGateResult(allowed=True, reason="allowed", decision_id="test-allow"), + lambda d, op: GovernanceGateResult( + allowed=True, + reason="allowed", + decision_id="test-allow", + ), ) artifacts = ContractPipeline().run( source_type="uc", @@ -703,9 +686,11 @@ def fake_enrich(contract, **kwargs): # noqa: ANN001 assert manifest["policyValid"] is True assert merged_schema.description == "Imported Unity orders table" assert "processed_at" in merged_props - assert captured["table_fqn"] == "main.silver.orders" - assert captured["workspace_url"] == "https://adb.example" - assert captured["token"] == "token" + assert captured == { + "table_fqn": "main.silver.orders", + "workspace_url": "https://adb.example", + "token": "token", + } def test_pipeline_run_blocks_root_version_change_outside_release_flow( diff --git a/tests/test_semapact_unity_relationships.py b/tests/test_semapact_unity_relationships.py index 081b309d..ae4574a8 100644 --- a/tests/test_semapact_unity_relationships.py +++ b/tests/test_semapact_unity_relationships.py @@ -1,5 +1,7 @@ from __future__ import annotations +from open_data_contract_standard.model import SchemaObject + from semapact.constants import ( UNITY_CONSTRAINT_NAME_KEY, UNITY_RELATIONSHIPS_COUNT_KEY, @@ -19,7 +21,7 @@ def _custom_props_map(contract) -> dict[str, object]: # noqa: ANN001 } -def test_unity_relationship_enrichment_imports_property_and_schema_relationships( +def test_unity_relationship_enrichment_imports_sdk_foreign_keys( sample_unity_contract_model, ): contract = sample_unity_contract_model.model_copy(deep=True) @@ -27,62 +29,55 @@ def test_unity_relationship_enrichment_imports_property_and_schema_relationships schema = contract.schema_[0] assert schema.properties is not None template = schema.properties[0] - schema.properties.append( - template.model_copy( - update={ - "id": "parent_tenant", - "name": "parent_tenant", - "physicalName": "parent_tenant", - "logicalType": "string", - "physicalType": "STRING", - "required": True, - } - ) - ) - schema.properties.append( - template.model_copy( - update={ - "id": "parent_code", - "name": "parent_code", - "physicalName": "parent_code", - "logicalType": "string", - "physicalType": "STRING", - "required": True, - } - ) + schema.properties.extend( + [ + template.model_copy( + update={ + "id": "parent_tenant", + "name": "parent_tenant", + "physicalName": "parent_tenant", + "logicalType": "string", + "physicalType": "STRING", + "required": True, + } + ), + template.model_copy( + update={ + "id": "parent_code", + "name": "parent_code", + "physicalName": "parent_code", + "logicalType": "string", + "physicalType": "STRING", + "required": True, + } + ), + ] ) - def fake_fetcher( - workspace_url: str, token: str, table_fqn: str - ) -> dict[str, object]: - assert workspace_url == "https://adb.example" - assert token == "token" - assert table_fqn == "main.silver.orders" - return { - "tableConstraints": [ - { - "constraintType": "FOREIGN_KEY", - "columns": ["id"], - "referencedTable": "main.ref.customers", - "referencedColumns": ["customer_id"], - "name": "fk_orders_customer", - }, - { - "constraintType": "FOREIGN_KEY", - "columns": ["parent_tenant", "parent_code"], - "referencedTable": "main.ref.parents", - "referencedColumns": ["tenant", "code"], - "name": "fk_orders_parent", - }, - ] - } - enriched = enrich_unity_contract_relationships( contract, - table_fqn="main.silver.orders", - workspace_url="https://adb.example", - token="token", - fetcher=fake_fetcher, + table_metadata={ + "main.silver.orders": { + "table_constraints": [ + { + "foreign_key_constraint": { + "child_columns": ["id"], + "parent_table": "main.ref.customers", + "parent_columns": ["customer_id"], + "name": "fk_orders_customer", + } + }, + { + "foreign_key_constraint": { + "child_columns": ["parent_tenant", "parent_code"], + "parent_table": "main.ref.parents", + "parent_columns": ["tenant", "code"], + "name": "fk_orders_parent", + } + }, + ] + } + }, ) assert enriched.schema_ is not None @@ -108,37 +103,115 @@ def fake_fetcher( "main.ref.parents.tenant", "main.ref.parents.code", ] - assert enriched_schema.relationships[0].customProperties is not None - schema_rel_props = { - item.property: item.value - for item in enriched_schema.relationships[0].customProperties - if item.property - } - assert schema_rel_props[UNITY_CONSTRAINT_NAME_KEY] == "fk_orders_parent" props = _custom_props_map(enriched) assert props[UNITY_RELATIONSHIPS_IMPORTED_KEY] == "true" assert props[UNITY_RELATIONSHIPS_COUNT_KEY] == "2" -def test_unity_relationship_enrichment_records_fallback_on_fetch_error( +def test_unity_relationship_enrichment_aggregates_across_data_product( sample_unity_contract_model, ): contract = sample_unity_contract_model.model_copy(deep=True) + assert contract.schema_ is not None + first = contract.schema_[0] + assert first.properties is not None + contract.schema_.append( + SchemaObject( + name="items", + physicalName="items", + physicalType="table", + properties=[first.properties[0].model_copy(deep=True)], + ) + ) + + enriched = enrich_unity_contract_relationships( + contract, + table_metadata={ + "main.silver.orders": { + "table_constraints": [ + { + "foreign_key_constraint": { + "child_columns": ["id"], + "parent_table": "main.ref.customers", + "parent_columns": ["id"], + "name": "fk_orders_customer", + } + } + ] + }, + "main.silver.items": { + "table_constraints": [ + { + "foreign_key_constraint": { + "child_columns": ["id"], + "parent_table": "main.ref.products", + "parent_columns": ["id"], + "name": "fk_items_product", + } + } + ] + }, + }, + ) - def broken_fetcher( - _workspace_url: str, _token: str, _table_fqn: str - ) -> dict[str, object]: - raise RuntimeError("metadata endpoint unavailable") + props = _custom_props_map(enriched) + assert props[UNITY_RELATIONSHIPS_IMPORTED_KEY] == "true" + assert props[UNITY_RELATIONSHIPS_COUNT_KEY] == "2" + + +def test_unity_relationship_enrichment_ignores_non_foreign_key_constraints( + sample_unity_contract_model, +): + contract = sample_unity_contract_model.model_copy(deep=True) + + enriched = enrich_unity_contract_relationships( + contract, + table_metadata={ + "main.silver.orders": { + "table_constraints": [ + { + "primary_key_constraint": { + "child_columns": ["id"], + "name": "pk_orders", + } + } + ] + } + }, + ) + + props = _custom_props_map(enriched) + assert props[UNITY_RELATIONSHIPS_IMPORTED_KEY] == "true" + assert props[UNITY_RELATIONSHIPS_COUNT_KEY] == "0" + + +def test_unity_relationship_enrichment_records_unmapped_table_failure( + sample_unity_contract_model, +): + contract = sample_unity_contract_model.model_copy(deep=True) enriched = enrich_unity_contract_relationships( contract, - table_fqn="main.silver.orders", - workspace_url="https://adb.example", - token="token", - fetcher=broken_fetcher, + table_metadata={ + "main.silver.missing": { + "table_constraints": [ + { + "foreign_key_constraint": { + "child_columns": ["id"], + "parent_table": "main.ref.customers", + "parent_columns": ["id"], + "name": "fk_missing_customer", + } + } + ] + } + }, ) props = _custom_props_map(enriched) assert props[UNITY_RELATIONSHIPS_IMPORTED_KEY] == "false" - assert "metadata endpoint unavailable" in str(props[UNITY_RELATIONSHIPS_REASON_KEY]) + assert props[UNITY_RELATIONSHIPS_COUNT_KEY] == "0" + assert "no matching governed asset" in str( + props[UNITY_RELATIONSHIPS_REASON_KEY] + ) diff --git a/tests/test_unity_importer.py b/tests/test_unity_importer.py new file mode 100644 index 00000000..8993b475 --- /dev/null +++ b/tests/test_unity_importer.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +from dataclasses import dataclass +from types import SimpleNamespace + +import pytest +from open_data_contract_standard.model import ( + OpenDataContractStandard, + SchemaObject, + SchemaProperty, +) + +from semapact.importers.unity_importer import import_unity_contract +@dataclass +class _TableInfo: + full_name: str + table_type: str = "MANAGED" + + @property + def name(self) -> str: + return self.full_name.split(".")[-1] + + def as_dict(self) -> dict[str, object]: + return { + "full_name": self.full_name, + "name": self.name, + "table_type": self.table_type, + "table_constraints": [], + } + + +class _Tables: + def __init__(self, tables: list[_TableInfo]) -> None: + self._tables = {table.full_name: table for table in tables} + self.get_calls: list[str] = [] + + def list(self, *, catalog_name: str, schema_name: str): + prefix = f"{catalog_name}.{schema_name}." + return [ + table + for name, table in self._tables.items() + if name.startswith(prefix) + ] + + def get(self, full_name: str): + self.get_calls.append(full_name) + return self._tables[full_name] + + +class _Client: + def __init__(self, tables: list[_TableInfo]) -> None: + self.tables = _Tables(tables) + self.config = SimpleNamespace(host="https://adb.example") + + +def _install_mapper(monkeypatch: pytest.MonkeyPatch) -> None: + def create_odcs(): + return OpenDataContractStandard.model_validate( + { + "apiVersion": "v3.1.0", + "kind": "DataContract", + "id": "imported", + "name": "Imported", + "version": "1.0.0", + "status": "draft", + "servers": [ + { + "server": "databricks", + "type": "databricks", + "catalog": "main", + "schema": "gold", + } + ], + "schema": [], + } + ) + + def convert_unity_schema(contract, table_info): # noqa: ANN001 + contract.schema_ = list(contract.schema_ or []) + contract.schema_.append( + SchemaObject( + name=table_info.name, + physicalName=table_info.name, + physicalType="table", + properties=[ + SchemaProperty( + name="id", + physicalName="id", + logicalType="integer", + physicalType="BIGINT", + required=True, + ) + ], + ) + ) + return contract + + monkeypatch.setattr( + "datacontract.imports.unity_importer.create_odcs", + create_odcs, + ) + monkeypatch.setattr( + "datacontract.imports.unity_importer.convert_unity_schema", + convert_unity_schema, + ) + + +def test_schema_level_import_builds_one_data_product_from_all_discovered_assets( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _install_mapper(monkeypatch) + client = _Client( + [ + _TableInfo("main.gold.orders", "MANAGED"), + _TableInfo("main.gold.customer_view", "VIEW"), + ] + ) + + contract = import_unity_contract( + table_fqn="main.gold", + client=client, + ) + + assert contract.id == "main-gold-product" + assert contract.status == "draft" + assert contract.servers is not None + assert contract.servers[0].host == "https://adb.example" + assert [schema.name for schema in contract.schema_ or []] == [ + "customer_view", + "orders", + ] + assert client.tables.get_calls == [ + "main.gold.customer_view", + "main.gold.orders", + ] + + +def test_table_level_import_fetches_only_requested_table( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _install_mapper(monkeypatch) + client = _Client( + [ + _TableInfo("main.gold.orders"), + _TableInfo("main.gold.customers"), + ] + ) + + contract = import_unity_contract( + table_fqn="main.gold.orders", + client=client, + ) + + assert [schema.name for schema in contract.schema_ or []] == ["orders"] + assert client.tables.get_calls == ["main.gold.orders"] + + +def test_import_uses_configured_workspace_client_when_not_injected( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _install_mapper(monkeypatch) + client = _Client([_TableInfo("main.gold.orders")]) + captured: dict[str, object] = {} + + def fake_create_configured_client(**kwargs): # noqa: ANN003 + captured.update(kwargs) + return client + + monkeypatch.setattr( + "semapact.platforms.databricks.configuration.create_configured_databricks_workspace_client", + fake_create_configured_client, + ) + + import_unity_contract( + table_fqn="main.gold.orders", + workspace_url="https://explicit.example", + token="explicit-token", + profile="explicit-profile", + ) + + assert captured == { + "workspace_url": "https://explicit.example", + "token": "explicit-token", + "profile": "explicit-profile", + } + + +@pytest.mark.parametrize( + "source", + [ + "", + "main", + "main.gold.orders.extra", + "main..orders", + ], +) +def test_import_rejects_ambiguous_unity_source(source: str) -> None: + with pytest.raises( + ValueError, + match="catalog.schema or catalog.schema.table", + ): + import_unity_contract(table_fqn=source, client=_Client([])) diff --git a/tests/test_unity_relationships_internals.py b/tests/test_unity_relationships_internals.py index 19f75fdd..be33037e 100644 --- a/tests/test_unity_relationships_internals.py +++ b/tests/test_unity_relationships_internals.py @@ -1,42 +1,55 @@ from __future__ import annotations -from semapact.importers.unity_relationships import _constraint_items +from semapact.importers.unity_relationships import ( + _constraint_items, + _parse_constraint_record, +) -def test_constraint_items_empty_metadata(): - assert _constraint_items({}) == [] - - -def test_constraint_items_supported_keys(): +def test_constraint_items_reads_sdk_table_constraints_only() -> None: metadata = { - "table_constraints": [{"id": 1}], - "tableConstraints": [{"id": 2}], - "constraints": [{"id": 3}], - "foreign_keys": [{"id": 4}], - "foreignKeys": [{"id": 5}], + "table_constraints": [ + {"primary_key_constraint": {"name": "pk"}}, + {"foreign_key_constraint": {"name": "fk"}}, + "not-a-mapping", + ], + "constraints": [{"id": "non-canonical"}], } - result = _constraint_items(metadata) - assert len(result) == 5 - assert {"id": 1} in result - assert {"id": 2} in result - assert {"id": 3} in result - assert {"id": 4} in result - assert {"id": 5} in result - - -def test_constraint_items_ignores_non_list_values(): - metadata = {"table_constraints": "not a list", "constraints": [{"id": 1}]} - assert _constraint_items(metadata) == [{"id": 1}] - - -def test_constraint_items_filters_non_dict_items(): - metadata = {"constraints": [{"id": 1}, "not a dict", 123, None]} - assert _constraint_items(metadata) == [{"id": 1}] - -def test_constraint_items_mixed_keys(): - metadata = {"tableConstraints": [{"id": 1}], "foreign_keys": [{"id": 2}]} - result = _constraint_items(metadata) - assert len(result) == 2 - assert {"id": 1} in result - assert {"id": 2} in result + assert _constraint_items(metadata) == [ + {"primary_key_constraint": {"name": "pk"}}, + {"foreign_key_constraint": {"name": "fk"}}, + ] + + +def test_parse_constraint_record_maps_sdk_foreign_key_shape() -> None: + record = _parse_constraint_record( + { + "foreign_key_constraint": { + "name": "fk_orders_customer", + "child_columns": ["customer_id"], + "parent_table": "main.ref.customers", + "parent_columns": ["id"], + } + } + ) + + assert record is not None + assert record.constraint_name == "fk_orders_customer" + assert record.source_columns == ["customer_id"] + assert record.target_table == "main.ref.customers" + assert record.target_columns == ["id"] + + +def test_parse_constraint_record_ignores_incomplete_foreign_key() -> None: + assert ( + _parse_constraint_record( + { + "foreign_key_constraint": { + "name": "fk_incomplete", + "child_columns": ["customer_id"], + } + } + ) + is None + ) diff --git a/uv.lock b/uv.lock index f9d41f57..025962bc 100644 --- a/uv.lock +++ b/uv.lock @@ -197,6 +197,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, ] +[[package]] +name = "antlr4-python3-runtime" +version = "4.11.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/64/c548678120cccf784f555972ed37cedcf4f026abeec30ab0340c3af4ea07/antlr4-python3-runtime-4.11.1.tar.gz", hash = "sha256:a53de701312f9bdacc5258a6872cd6c62b90d3a90ae25e494026f76267333b60", size = 116945, upload-time = "2022-09-04T21:37:45.242Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/3a/4f2e9d67277cc255a17a34c1b54e02d939ca5991bc596dee3d63e587e512/antlr4_python3_runtime-4.11.1-py3-none-any.whl", hash = "sha256:ff1954eda1ca9072c02bf500387d0c86cb549bef4dbb3b64f39468b547ec5f6b", size = 144186, upload-time = "2022-09-04T21:37:43.264Z" }, +] + [[package]] name = "anyio" version = "4.13.0" @@ -616,6 +625,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a2/ca/7e8365deec19afb2b2c7be7c1c0aa8f99633b54e90c570999acda93260fc/cryptography-48.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:db63bf618e5dea46c07de12e900fe1cdd2541e6dc9dbae772a70b7d4d4765f6a", size = 3739536, upload-time = "2026-05-04T22:59:29.61Z" }, ] +[[package]] +name = "databricks-sdk" +version = "0.109.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "protobuf" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b2/ab/067704013b24b2d1a479174571f0181b137bc2ae0ac54da4f1dc4abda808/databricks_sdk-0.109.0.tar.gz", hash = "sha256:96b90f5ee2e0fac89f351aa77348263e1e0ed5d4d867cdfaefd2e0acc01b1e81", size = 944643, upload-time = "2026-05-18T09:18:44.571Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b8/a5/e958674354b329917300ad57be27fc953516e541d05d5e723855af6f9ac9/databricks_sdk-0.109.0-py3-none-any.whl", hash = "sha256:46745a25324a5b36f01236760e44fc6e2bce3f051224e6f8aef7633477fcbec8", size = 891530, upload-time = "2026-05-18T09:18:42.272Z" }, +] + [[package]] name = "databricks-sql-connector" version = "4.2.6" @@ -666,6 +689,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/ee/0f6e4eeea7b83257af5c96523b17af07f97b5d4885662ad9e45854e15df5/datacontract_cli-0.12.4-py3-none-any.whl", hash = "sha256:536321fcde612dc92d8924856ad11489f884a73f78a46707114c9e54cfa67aa5", size = 339977, upload-time = "2026-05-21T20:26:20.569Z" }, ] +[package.optional-dependencies] +databricks = [ + { name = "databricks-sdk" }, + { name = "databricks-sql-connector" }, + { name = "pyspark" }, + { name = "soda-core-spark", extra = ["databricks"] }, + { name = "soda-core-spark-df" }, +] + [[package]] name = "datacontract-specification" version = "1.2.3" @@ -923,6 +955,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl", hash = "sha256:11ef7bb35dab8a394fde6e608221d5cf3e8499401c249bebaeaad760a1a8dec2", size = 203402, upload-time = "2026-04-29T20:42:36.842Z" }, ] +[[package]] +name = "google-auth" +version = "2.58.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "pyasn1-modules" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/ca/f398a483ce5aad18ca2f735646e45ccee2439bd94a41a4ad0cfa646bd495/google_auth-2.58.0.tar.gz", hash = "sha256:55e30cf15e737de92c5323d78cda8a83fcd57e7ffbaf900c4600039fd60a80fd", size = 380018, upload-time = "2026-09-09T20:49:38.043Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/13/477d90d09591b3938b45c4e11f4d8a51291682112cb5efcac961e815d562/google_auth-2.58.0-py3-none-any.whl", hash = "sha256:8a9c4645bb4c8e91668fb1934b95ae6a8687084232753639220ba9bf04a1610d", size = 262404, upload-time = "2026-09-09T20:49:33.951Z" }, +] + +[[package]] +name = "googleapis-common-protos" +version = "1.75.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8a/c5/4353a188e2c335aee33269e8b654af228278cca8e5f0b4b5f11e5d0e9adb/googleapis_common_protos-1.75.3.tar.gz", hash = "sha256:57c435ac2c68b108999b6db075d9053e4d7a936ba57b4a3d45667b1346f1738a", size = 153905, upload-time = "2026-09-03T22:31:21.869Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/7a/7d79170c6ce6f12e109df2b3879d6b934010cf4f99aea8de8b7e5408c174/googleapis_common_protos-1.75.3-py3-none-any.whl", hash = "sha256:a018d2bf098ca9fb6faa08d5bb780e2a2c2f73c566f069761331386c9596d3f2", size = 306984, upload-time = "2026-09-03T22:30:45.133Z" }, +] + [[package]] name = "great-expectations" version = "1.17.2" @@ -1119,6 +1176,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7d/f9/97f2ca8bb3ec6e4b1d64f983ebe98b9a192faddff67fac3d6303a537e670/importlib_metadata-8.9.0-py3-none-any.whl", hash = "sha256:e0f761b6ea91ced3b0844c14c9d955224d538105921f8e6754c00f6ca79fba7f", size = 27220, upload-time = "2026-03-20T16:56:25.07Z" }, ] +[[package]] +name = "inflect" +version = "7.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "more-itertools" }, + { name = "typeguard" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/c6/943357d44a21fd995723d07ccaddd78023eace03c1846049a2645d4324a3/inflect-7.5.0.tar.gz", hash = "sha256:faf19801c3742ed5a05a8ce388e0d8fe1a07f8d095c82201eb904f5d27ad571f", size = 73751, upload-time = "2024-12-28T17:11:18.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/eb/427ed2b20a38a4ee29f24dbe4ae2dafab198674fe9a85e3d6adf9e5f5f41/inflect-7.5.0-py3-none-any.whl", hash = "sha256:2aea70e5e70c35d8350b8097396ec155ffd68def678c7ff97f51aa69c1d92344", size = 35197, upload-time = "2024-12-28T17:11:15.931Z" }, +] + [[package]] name = "iniconfig" version = "2.3.0" @@ -1503,6 +1573,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/7f/a946aa4f8752b37102b41e64dca18a1976ac705c3a0d1dfe74d820a02552/mistune-3.2.1-py3-none-any.whl", hash = "sha256:78cdb0ba5e938053ccf63651b352508d2efa9411dc8810bfb05f2dc5140c0048", size = 53749, upload-time = "2026-05-03T14:33:20.551Z" }, ] +[[package]] +name = "more-itertools" +version = "11.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/de/1d/f4da6f02cdffe04d6362210b807146a26044c88d839208aec273bb0d9184/more_itertools-11.1.0.tar.gz", hash = "sha256:48e8f4d9e7e5878571ecf6f2b4e57634f93cd474cc8cfbd2376f2d11b396e30d", size = 145772, upload-time = "2026-05-22T14:14:29.909Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl", hash = "sha256:4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192", size = 72226, upload-time = "2026-05-22T14:14:28.824Z" }, +] + [[package]] name = "msal" version = "1.36.0" @@ -1796,6 +1875,87 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910, upload-time = "2024-06-28T14:03:41.161Z" }, ] +[[package]] +name = "opentelemetry-api" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ee/8b/aa9e2d8b8dfa7c946f7dec5d1f8f6ba8eca062f43509a06bdb5ce93d26c0/opentelemetry_api-1.44.0.tar.gz", hash = "sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a", size = 72406, upload-time = "2026-07-16T15:25:32.678Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/6f/a04e900f465ff3221ccc395522503e2d10e79fa21f2723c8e177aae1e0d1/opentelemetry_api-1.44.0-py3-none-any.whl", hash = "sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef", size = 60018, upload-time = "2026-07-16T15:25:11.657Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-proto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/09/4d717852c1cf3f854b76c7110a5d00883bc3c99288b9b0dbcbeb9e306eb6/opentelemetry_exporter_otlp_proto_common-1.44.0.tar.gz", hash = "sha256:dc87a5a5bc58f149a56d1547e4691588fa12994cdc3bc039a694ccb3375862ac", size = 20202, upload-time = "2026-07-16T15:25:37.658Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/71/65fd9d54c10b860f87c045ccee1264cab7011268895d3528818a29c1172a/opentelemetry_exporter_otlp_proto_common-1.44.0-py3-none-any.whl", hash = "sha256:9a9fe61bba73d802904bc989f1d6b4a7b1ee40f06c40e98d6f85af65aaebb694", size = 17045, upload-time = "2026-07-16T15:25:18.201Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1a/87/95e2a5aaa795b4e2260d74e16df2d5541deb2ea9de010bcd615f4dee2654/opentelemetry_exporter_otlp_proto_http-1.44.0.tar.gz", hash = "sha256:c633d7270ad6b57cd4cfbe8b0007a9e2e7c0cb50bd6c50fe2a7b245f721a09d8", size = 25806, upload-time = "2026-07-16T15:25:39.162Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/d0/fdeb1a98d8d3a6205f5f297c51b4a9bfe65126ab60339669bbe3dd54c2e2/opentelemetry_exporter_otlp_proto_http-1.44.0-py3-none-any.whl", hash = "sha256:838592fce774c1c8bb7b9a0a7facbfa82e17be5a8a4e94cef10cb84ae026bae3", size = 21850, upload-time = "2026-07-16T15:25:20.006Z" }, +] + +[[package]] +name = "opentelemetry-proto" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/64/01/40ac4ae9a149263cc52c2cee200ddd80cb6d8db1a4610abf8eabce0fe771/opentelemetry_proto-1.44.0.tar.gz", hash = "sha256:c547a79c2f8c0c515d31509154682e5921c7cfd5ca67b70e1f9266e2c3e103f3", size = 46488, upload-time = "2026-07-16T15:25:45.34Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/7c/8be563d68e93bbefa5c8affb82ddcff91b3ad858ce49957ba7b16fd3e0ab/opentelemetry_proto-1.44.0-py3-none-any.whl", hash = "sha256:898b155a0e1557afd867478fb6158e8122a46329ca0bb8dc53cc55e98f017f56", size = 72483, upload-time = "2026-07-16T15:25:28.429Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5d/77/a6592cbc7c8d9bcc9d6757a9df45e04a7c585e3e6e7a13456da522b21109/opentelemetry_sdk-1.44.0.tar.gz", hash = "sha256:cebe7f65dc12f26ead75c6064de12fd2a9052e5060c0272d402cfa203aae123b", size = 208624, upload-time = "2026-07-16T15:25:46.078Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/23/ff077e61886ee020a17ce9c8b6fa11c601c8d8345b09ea24f605445df62a/opentelemetry_sdk-1.44.0-py3-none-any.whl", hash = "sha256:df081c4c6bcfdb1211e3e86140376792643128a25f8d72d1d27675936e7e96ad", size = 137221, upload-time = "2026-07-16T15:25:29.534Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.65b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8f/73/0cbdebcb4cf545fdd328da14f5137e37d0770c3f26185e478b0d15d94f50/opentelemetry_semantic_conventions-0.65b0.tar.gz", hash = "sha256:f9b2b81e9d5b64f11bc952075e7e9c7fb0aab075c7fd1c46d597f1b919852d60", size = 148774, upload-time = "2026-07-16T15:25:46.902Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/0e/49df70d9b81fb5cbae4bbf2a49d865b09bcbcbc4eb53f5851b1027738d78/opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb", size = 204645, upload-time = "2026-07-16T15:25:30.688Z" }, +] + [[package]] name = "orderly-set" version = "5.5.0" @@ -1997,6 +2157,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, ] +[[package]] +name = "protobuf" +version = "6.33.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", size = 425739, upload-time = "2026-03-18T19:04:48.373Z" }, + { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", size = 437089, upload-time = "2026-03-18T19:04:50.381Z" }, + { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", size = 427737, upload-time = "2026-03-18T19:04:51.866Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", size = 324610, upload-time = "2026-03-18T19:04:53.096Z" }, + { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", size = 339381, upload-time = "2026-03-18T19:04:54.616Z" }, + { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" }, + { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, +] + [[package]] name = "py4j" version = "0.10.9.9" @@ -2056,6 +2231,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/be/6f79d55816d5c22557cf27533543d5d70dfe692adfbee4b99f2760674f38/pyarrow-24.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:c91d00057f23b8d353039520dc3a6c09d8608164c692e9f59a175a42b2ae0c19", size = 28131282, upload-time = "2026-04-21T10:51:16.815Z" }, ] +[[package]] +name = "pyasn1" +version = "0.6.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a4/9a/23310166d960def5897e91fe20e5b724601b02a22e84ba1f94232c0b7f67/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81", size = 151262, upload-time = "2026-07-09T01:12:33.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/3b/6163796d69c3977d1e4287bea4a6979161cbbdd170ebb430511e8e1999ce/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b", size = 84410, upload-time = "2026-07-09T01:12:32.92Z" }, +] + +[[package]] +name = "pyasn1-modules" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, +] + [[package]] name = "pybreaker" version = "1.4.1" @@ -2220,12 +2416,12 @@ wheels = [ [[package]] name = "pyspark" -version = "4.1.2" +version = "3.5.9" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "py4j" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5e/71/4dd20c69332a2a4bf7ece8a655c9da98e4bd9b6bcea235349c1a00399d57/pyspark-4.1.2.tar.gz", hash = "sha256:fa5d6159f700d0990a07f4f62df1b7449401dccee9cd7d5d6df8957530841602", size = 455428043, upload-time = "2026-05-21T14:49:21.785Z" } +sdist = { url = "https://files.pythonhosted.org/packages/95/ce/81e53e729790556e3983e95de1a7d5df91a34adfcd34b5a5ab0e0c6e9b33/pyspark-3.5.9.tar.gz", hash = "sha256:ea27adc39ddac9413b8951e45aa748cbed6c785971b81386efc41938f6243d93", size = 317858593, upload-time = "2026-07-16T08:52:04.494Z" } [[package]] name = "pytest" @@ -2605,11 +2801,62 @@ wheels = [ [[package]] name = "ruamel-yaml" -version = "0.19.1" +version = "0.17.40" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c7/3b/ebda527b56beb90cb7652cb1c7e4f91f48649fbcd8d2eb2fb6e77cd3329b/ruamel_yaml-0.19.1.tar.gz", hash = "sha256:53eb66cd27849eff968ebf8f0bf61f46cdac2da1d1f3576dd4ccee9b25c31993", size = 142709, upload-time = "2026-01-02T16:50:31.84Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b8/0c/51f6841f1d84f404f92463fc2b1ba0da357ca1e3db6b7fbda26956c3b82a/ruamel_yaml-0.19.1-py3-none-any.whl", hash = "sha256:27592957fedf6e0b62f281e96effd28043345e0e66001f97683aa9a40c667c93", size = 118102, upload-time = "2026-01-02T16:50:29.201Z" }, +dependencies = [ + { name = "ruamel-yaml-clib", marker = "python_full_version < '3.13' and platform_python_implementation == 'CPython'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d1/d6/eb2833ccba5ea36f8f4de4bcfa0d1a91eb618f832d430b70e3086821f251/ruamel.yaml-0.17.40.tar.gz", hash = "sha256:6024b986f06765d482b5b07e086cc4b4cd05dd22ddcbc758fa23d54873cf313d", size = 137672, upload-time = "2023-10-20T12:53:56.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/79/5e2cffa1c77432f11cd93a5351f30732c997a239d3a3090856a72d6d8ba7/ruamel.yaml-0.17.40-py3-none-any.whl", hash = "sha256:b16b6c3816dff0a93dca12acf5e70afd089fa5acb80604afd1ffa8b465b7722c", size = 113666, upload-time = "2023-10-20T12:53:52.628Z" }, +] + +[[package]] +name = "ruamel-yaml-clib" +version = "0.2.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ea/97/60fda20e2fb54b83a61ae14648b0817c8f5d84a3821e40bfbdae1437026a/ruamel_yaml_clib-0.2.15.tar.gz", hash = "sha256:46e4cc8c43ef6a94885f72512094e482114a8a706d3c555a34ed4b0d20200600", size = 225794, upload-time = "2025-11-16T16:12:59.761Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/80/8ce7b9af532aa94dd83360f01ce4716264db73de6bc8efd22c32341f6658/ruamel_yaml_clib-0.2.15-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c583229f336682b7212a43d2fa32c30e643d3076178fb9f7a6a14dde85a2d8bd", size = 147998, upload-time = "2025-11-16T16:13:13.241Z" }, + { url = "https://files.pythonhosted.org/packages/53/09/de9d3f6b6701ced5f276d082ad0f980edf08ca67114523d1b9264cd5e2e0/ruamel_yaml_clib-0.2.15-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56ea19c157ed8c74b6be51b5fa1c3aff6e289a041575f0556f66e5fb848bb137", size = 132743, upload-time = "2025-11-16T16:13:14.265Z" }, + { url = "https://files.pythonhosted.org/packages/0e/f7/73a9b517571e214fe5c246698ff3ed232f1ef863c8ae1667486625ec688a/ruamel_yaml_clib-0.2.15-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5fea0932358e18293407feb921d4f4457db837b67ec1837f87074667449f9401", size = 731459, upload-time = "2025-11-16T20:22:44.338Z" }, + { url = "https://files.pythonhosted.org/packages/9b/a2/0dc0013169800f1c331a6f55b1282c1f4492a6d32660a0cf7b89e6684919/ruamel_yaml_clib-0.2.15-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef71831bd61fbdb7aa0399d5c4da06bea37107ab5c79ff884cc07f2450910262", size = 749289, upload-time = "2025-11-16T16:13:15.633Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ed/3fb20a1a96b8dc645d88c4072df481fe06e0289e4d528ebbdcc044ebc8b3/ruamel_yaml_clib-0.2.15-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:617d35dc765715fa86f8c3ccdae1e4229055832c452d4ec20856136acc75053f", size = 777630, upload-time = "2025-11-16T16:13:16.898Z" }, + { url = "https://files.pythonhosted.org/packages/60/50/6842f4628bc98b7aa4733ab2378346e1441e150935ad3b9f3c3c429d9408/ruamel_yaml_clib-0.2.15-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1b45498cc81a4724a2d42273d6cfc243c0547ad7c6b87b4f774cb7bcc131c98d", size = 744368, upload-time = "2025-11-16T16:13:18.117Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b0/128ae8e19a7d794c2e36130a72b3bb650ce1dd13fb7def6cf10656437dcf/ruamel_yaml_clib-0.2.15-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:def5663361f6771b18646620fca12968aae730132e104688766cf8a3b1d65922", size = 745233, upload-time = "2025-11-16T20:22:45.833Z" }, + { url = "https://files.pythonhosted.org/packages/75/05/91130633602d6ba7ce3e07f8fc865b40d2a09efd4751c740df89eed5caf9/ruamel_yaml_clib-0.2.15-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:014181cdec565c8745b7cbc4de3bf2cc8ced05183d986e6d1200168e5bb59490", size = 770963, upload-time = "2025-11-16T16:13:19.344Z" }, + { url = "https://files.pythonhosted.org/packages/fd/4b/fd4542e7f33d7d1bc64cc9ac9ba574ce8cf145569d21f5f20133336cdc8c/ruamel_yaml_clib-0.2.15-cp311-cp311-win32.whl", hash = "sha256:d290eda8f6ada19e1771b54e5706b8f9807e6bb08e873900d5ba114ced13e02c", size = 102640, upload-time = "2025-11-16T16:13:20.498Z" }, + { url = "https://files.pythonhosted.org/packages/bb/eb/00ff6032c19c7537371e3119287999570867a0eafb0154fccc80e74bf57a/ruamel_yaml_clib-0.2.15-cp311-cp311-win_amd64.whl", hash = "sha256:bdc06ad71173b915167702f55d0f3f027fc61abd975bd308a0968c02db4a4c3e", size = 121996, upload-time = "2025-11-16T16:13:21.855Z" }, + { url = "https://files.pythonhosted.org/packages/72/4b/5fde11a0722d676e469d3d6f78c6a17591b9c7e0072ca359801c4bd17eee/ruamel_yaml_clib-0.2.15-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cb15a2e2a90c8475df45c0949793af1ff413acfb0a716b8b94e488ea95ce7cff", size = 149088, upload-time = "2025-11-16T16:13:22.836Z" }, + { url = "https://files.pythonhosted.org/packages/85/82/4d08ac65ecf0ef3b046421985e66301a242804eb9a62c93ca3437dc94ee0/ruamel_yaml_clib-0.2.15-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:64da03cbe93c1e91af133f5bec37fd24d0d4ba2418eaf970d7166b0a26a148a2", size = 134553, upload-time = "2025-11-16T16:13:24.151Z" }, + { url = "https://files.pythonhosted.org/packages/b9/cb/22366d68b280e281a932403b76da7a988108287adff2bfa5ce881200107a/ruamel_yaml_clib-0.2.15-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f6d3655e95a80325b84c4e14c080b2470fe4f33b6846f288379ce36154993fb1", size = 737468, upload-time = "2025-11-16T20:22:47.335Z" }, + { url = "https://files.pythonhosted.org/packages/71/73/81230babf8c9e33770d43ed9056f603f6f5f9665aea4177a2c30ae48e3f3/ruamel_yaml_clib-0.2.15-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:71845d377c7a47afc6592aacfea738cc8a7e876d586dfba814501d8c53c1ba60", size = 753349, upload-time = "2025-11-16T16:13:26.269Z" }, + { url = "https://files.pythonhosted.org/packages/61/62/150c841f24cda9e30f588ef396ed83f64cfdc13b92d2f925bb96df337ba9/ruamel_yaml_clib-0.2.15-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11e5499db1ccbc7f4b41f0565e4f799d863ea720e01d3e99fa0b7b5fcd7802c9", size = 788211, upload-time = "2025-11-16T16:13:27.441Z" }, + { url = "https://files.pythonhosted.org/packages/30/93/e79bd9cbecc3267499d9ead919bd61f7ddf55d793fb5ef2b1d7d92444f35/ruamel_yaml_clib-0.2.15-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4b293a37dc97e2b1e8a1aec62792d1e52027087c8eea4fc7b5abd2bdafdd6642", size = 743203, upload-time = "2025-11-16T16:13:28.671Z" }, + { url = "https://files.pythonhosted.org/packages/8d/06/1eb640065c3a27ce92d76157f8efddb184bd484ed2639b712396a20d6dce/ruamel_yaml_clib-0.2.15-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:512571ad41bba04eac7268fe33f7f4742210ca26a81fe0c75357fa682636c690", size = 747292, upload-time = "2025-11-16T20:22:48.584Z" }, + { url = "https://files.pythonhosted.org/packages/a5/21/ee353e882350beab65fcc47a91b6bdc512cace4358ee327af2962892ff16/ruamel_yaml_clib-0.2.15-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5e9f630c73a490b758bf14d859a39f375e6999aea5ddd2e2e9da89b9953486a", size = 771624, upload-time = "2025-11-16T16:13:29.853Z" }, + { url = "https://files.pythonhosted.org/packages/57/34/cc1b94057aa867c963ecf9ea92ac59198ec2ee3a8d22a126af0b4d4be712/ruamel_yaml_clib-0.2.15-cp312-cp312-win32.whl", hash = "sha256:f4421ab780c37210a07d138e56dd4b51f8642187cdfb433eb687fe8c11de0144", size = 100342, upload-time = "2025-11-16T16:13:31.067Z" }, + { url = "https://files.pythonhosted.org/packages/b3/e5/8925a4208f131b218f9a7e459c0d6fcac8324ae35da269cb437894576366/ruamel_yaml_clib-0.2.15-cp312-cp312-win_amd64.whl", hash = "sha256:2b216904750889133d9222b7b873c199d48ecbb12912aca78970f84a5aa1a4bc", size = 119013, upload-time = "2025-11-16T16:13:32.164Z" }, + { url = "https://files.pythonhosted.org/packages/17/5e/2f970ce4c573dc30c2f95825f2691c96d55560268ddc67603dc6ea2dd08e/ruamel_yaml_clib-0.2.15-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dcec721fddbb62e60c2801ba08c87010bd6b700054a09998c4d09c08147b8fb", size = 147450, upload-time = "2025-11-16T16:13:33.542Z" }, + { url = "https://files.pythonhosted.org/packages/d6/03/a1baa5b94f71383913f21b96172fb3a2eb5576a4637729adbf7cd9f797f8/ruamel_yaml_clib-0.2.15-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:65f48245279f9bb301d1276f9679b82e4c080a1ae25e679f682ac62446fac471", size = 133139, upload-time = "2025-11-16T16:13:34.587Z" }, + { url = "https://files.pythonhosted.org/packages/dc/19/40d676802390f85784235a05788fd28940923382e3f8b943d25febbb98b7/ruamel_yaml_clib-0.2.15-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:46895c17ead5e22bea5e576f1db7e41cb273e8d062c04a6a49013d9f60996c25", size = 731474, upload-time = "2025-11-16T20:22:49.934Z" }, + { url = "https://files.pythonhosted.org/packages/ce/bb/6ef5abfa43b48dd55c30d53e997f8f978722f02add61efba31380d73e42e/ruamel_yaml_clib-0.2.15-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3eb199178b08956e5be6288ee0b05b2fb0b5c1f309725ad25d9c6ea7e27f962a", size = 748047, upload-time = "2025-11-16T16:13:35.633Z" }, + { url = "https://files.pythonhosted.org/packages/ff/5d/e4f84c9c448613e12bd62e90b23aa127ea4c46b697f3d760acc32cb94f25/ruamel_yaml_clib-0.2.15-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d1032919280ebc04a80e4fb1e93f7a738129857eaec9448310e638c8bccefcf", size = 782129, upload-time = "2025-11-16T16:13:36.781Z" }, + { url = "https://files.pythonhosted.org/packages/de/4b/e98086e88f76c00c88a6bcf15eae27a1454f661a9eb72b111e6bbb69024d/ruamel_yaml_clib-0.2.15-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ab0df0648d86a7ecbd9c632e8f8d6b21bb21b5fc9d9e095c796cacf32a728d2d", size = 736848, upload-time = "2025-11-16T16:13:37.952Z" }, + { url = "https://files.pythonhosted.org/packages/0c/5c/5964fcd1fd9acc53b7a3a5d9a05ea4f95ead9495d980003a557deb9769c7/ruamel_yaml_clib-0.2.15-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:331fb180858dd8534f0e61aa243b944f25e73a4dae9962bd44c46d1761126bbf", size = 741630, upload-time = "2025-11-16T20:22:51.718Z" }, + { url = "https://files.pythonhosted.org/packages/07/1e/99660f5a30fceb58494598e7d15df883a07292346ef5696f0c0ae5dee8c6/ruamel_yaml_clib-0.2.15-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fd4c928ddf6bce586285daa6d90680b9c291cfd045fc40aad34e445d57b1bf51", size = 766619, upload-time = "2025-11-16T16:13:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/36/2f/fa0344a9327b58b54970e56a27b32416ffbcfe4dcc0700605516708579b2/ruamel_yaml_clib-0.2.15-cp313-cp313-win32.whl", hash = "sha256:bf0846d629e160223805db9fe8cc7aec16aaa11a07310c50c8c7164efa440aec", size = 100171, upload-time = "2025-11-16T16:13:40.456Z" }, + { url = "https://files.pythonhosted.org/packages/06/c4/c124fbcef0684fcf3c9b72374c2a8c35c94464d8694c50f37eef27f5a145/ruamel_yaml_clib-0.2.15-cp313-cp313-win_amd64.whl", hash = "sha256:45702dfbea1420ba3450bb3dd9a80b33f0badd57539c6aac09f42584303e0db6", size = 118845, upload-time = "2025-11-16T16:13:41.481Z" }, + { url = "https://files.pythonhosted.org/packages/3e/bd/ab8459c8bb759c14a146990bf07f632c1cbec0910d4853feeee4be2ab8bb/ruamel_yaml_clib-0.2.15-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:753faf20b3a5906faf1fc50e4ddb8c074cb9b251e00b14c18b28492f933ac8ef", size = 147248, upload-time = "2025-11-16T16:13:42.872Z" }, + { url = "https://files.pythonhosted.org/packages/69/f2/c4cec0a30f1955510fde498aac451d2e52b24afdbcb00204d3a951b772c3/ruamel_yaml_clib-0.2.15-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:480894aee0b29752560a9de46c0e5f84a82602f2bc5c6cde8db9a345319acfdf", size = 133764, upload-time = "2025-11-16T16:13:43.932Z" }, + { url = "https://files.pythonhosted.org/packages/82/c7/2480d062281385a2ea4f7cc9476712446e0c548cd74090bff92b4b49e898/ruamel_yaml_clib-0.2.15-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4d3b58ab2454b4747442ac76fab66739c72b1e2bb9bd173d7694b9f9dbc9c000", size = 730537, upload-time = "2025-11-16T20:22:52.918Z" }, + { url = "https://files.pythonhosted.org/packages/75/08/e365ee305367559f57ba6179d836ecc3d31c7d3fdff2a40ebf6c32823a1f/ruamel_yaml_clib-0.2.15-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bfd309b316228acecfa30670c3887dcedf9b7a44ea39e2101e75d2654522acd4", size = 746944, upload-time = "2025-11-16T16:13:45.338Z" }, + { url = "https://files.pythonhosted.org/packages/a1/5c/8b56b08db91e569d0a4fbfa3e492ed2026081bdd7e892f63ba1c88a2f548/ruamel_yaml_clib-0.2.15-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2812ff359ec1f30129b62372e5f22a52936fac13d5d21e70373dbca5d64bb97c", size = 778249, upload-time = "2025-11-16T16:13:46.871Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1d/70dbda370bd0e1a92942754c873bd28f513da6198127d1736fa98bb2a16f/ruamel_yaml_clib-0.2.15-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7e74ea87307303ba91073b63e67f2c667e93f05a8c63079ee5b7a5c8d0d7b043", size = 737140, upload-time = "2025-11-16T16:13:48.349Z" }, + { url = "https://files.pythonhosted.org/packages/5b/87/822d95874216922e1120afb9d3fafa795a18fdd0c444f5c4c382f6dac761/ruamel_yaml_clib-0.2.15-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:713cd68af9dfbe0bb588e144a61aad8dcc00ef92a82d2e87183ca662d242f524", size = 741070, upload-time = "2025-11-16T20:22:54.151Z" }, + { url = "https://files.pythonhosted.org/packages/b9/17/4e01a602693b572149f92c983c1f25bd608df02c3f5cf50fd1f94e124a59/ruamel_yaml_clib-0.2.15-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:542d77b72786a35563f97069b9379ce762944e67055bea293480f7734b2c7e5e", size = 765882, upload-time = "2025-11-16T16:13:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/9f/17/7999399081d39ebb79e807314de6b611e1d1374458924eb2a489c01fc5ad/ruamel_yaml_clib-0.2.15-cp314-cp314-win32.whl", hash = "sha256:424ead8cef3939d690c4b5c85ef5b52155a231ff8b252961b6516ed7cf05f6aa", size = 102567, upload-time = "2025-11-16T16:13:50.78Z" }, + { url = "https://files.pythonhosted.org/packages/d2/67/be582a7370fdc9e6846c5be4888a530dcadd055eef5b932e0e85c33c7d73/ruamel_yaml_clib-0.2.15-cp314-cp314-win_amd64.whl", hash = "sha256:ac9b8d5fa4bb7fd2917ab5027f60d4234345fd366fe39aa711d5dca090aa1467", size = 122847, upload-time = "2025-11-16T16:13:51.807Z" }, ] [[package]] @@ -2711,7 +2958,6 @@ wheels = [ [[package]] name = "semapact" -version = "0.8.1" source = { editable = "." } dependencies = [ { name = "datacontract-cli" }, @@ -2725,7 +2971,8 @@ all = [ { name = "azure-identity" }, { name = "azure-storage-file-datalake" }, { name = "boto3" }, - { name = "databricks-sql-connector" }, + { name = "databricks-sdk" }, + { name = "datacontract-cli", extra = ["databricks"] }, { name = "deltalake" }, { name = "great-expectations" }, { name = "litellm" }, @@ -2744,12 +2991,10 @@ azure = [ { name = "azure-storage-file-datalake" }, ] databricks = [ - { name = "databricks-sql-connector" }, + { name = "databricks-sdk" }, + { name = "datacontract-cli", extra = ["databricks"] }, { name = "pyspark" }, ] -dbt = [ - { name = "datacontract-cli" }, -] delta = [ { name = "deltalake" }, { name = "pandas" }, @@ -2796,10 +3041,11 @@ requires-dist = [ { name = "azure-storage-file-datalake", marker = "extra == 'azure'", specifier = ">=12.16.0" }, { name = "boto3", marker = "extra == 'all'", specifier = ">=1.34.0" }, { name = "boto3", marker = "extra == 's3'", specifier = ">=1.34.0" }, - { name = "databricks-sql-connector", marker = "extra == 'all'", specifier = ">=3.0.0" }, - { name = "databricks-sql-connector", marker = "extra == 'databricks'", specifier = ">=3.0.0" }, + { name = "databricks-sdk", marker = "extra == 'all'", specifier = ">=0.109.0" }, + { name = "databricks-sdk", marker = "extra == 'databricks'", specifier = ">=0.109.0" }, { name = "datacontract-cli", specifier = ">=0.12.0" }, - { name = "datacontract-cli", extras = ["dbt"], marker = "extra == 'dbt'", specifier = ">=0.12.0" }, + { name = "datacontract-cli", extras = ["databricks"], marker = "extra == 'all'", specifier = ">=0.12.0" }, + { name = "datacontract-cli", extras = ["databricks"], marker = "extra == 'databricks'", specifier = ">=0.12.0" }, { name = "deltalake", marker = "extra == 'all'", specifier = ">=0.18.0" }, { name = "deltalake", marker = "extra == 'delta'", specifier = ">=0.18.0" }, { name = "great-expectations", marker = "extra == 'all'", specifier = ">=1.0.0" }, @@ -2830,7 +3076,7 @@ requires-dist = [ { name = "textual", marker = "extra == 'all'", specifier = ">=8.2.7" }, { name = "textual", marker = "extra == 'tui'", specifier = ">=8.2.7" }, ] -provides-extras = ["sql", "delta", "quality", "llm", "tui", "databricks", "azure", "graph", "s3", "dbt", "all"] +provides-extras = ["sql", "delta", "quality", "llm", "tui", "databricks", "azure", "graph", "s3", "all"] [package.metadata.requires-dev] dev = [ @@ -2877,6 +3123,58 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] +[[package]] +name = "soda-core" +version = "3.5.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "antlr4-python3-runtime" }, + { name = "click" }, + { name = "inflect" }, + { name = "jinja2" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "requests" }, + { name = "ruamel-yaml" }, + { name = "sqlparse" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b5/a6/1d18be773ddf2ae415d6f03b9ee99630b2145cf4acb60b1557488239e2a1/soda_core-3.5.6.tar.gz", hash = "sha256:9cf0aa483e6c85f57e25ea5e68ef08139360dbc7f76bf7c0571c14a5427681fc", size = 144115, upload-time = "2025-09-24T10:43:43.104Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/2a/efdb55e006be131c43dbdb541275606a786f7a3347fa807cb8ce21b247cf/soda_core-3.5.6-py3-none-any.whl", hash = "sha256:cf1b78d6348b0a2e81dbc345e25207e8f8606c51ab310b294291a86fc20c756f", size = 197640, upload-time = "2025-09-24T10:43:41.875Z" }, +] + +[[package]] +name = "soda-core-spark" +version = "3.5.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "soda-core" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/05/f6d2bb6a534115bf9b47227119b3a43809af6a394e892df77609b65b6bc6/soda_core_spark-3.5.6.tar.gz", hash = "sha256:c82202129a5b22b59225d902ebd61d8ad388fb8e19c4fdac565d0cc987297f33", size = 9915, upload-time = "2025-09-24T10:44:17.853Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/87/959c30eca35975098ec13a6b82ff2761034443188935f30aa03401e18a71/soda_core_spark-3.5.6-py3-none-any.whl", hash = "sha256:7eb68beaf654018fb44b9691481f680b937348d432e8a84bf3488b0918687571", size = 10027, upload-time = "2025-09-24T10:44:15.543Z" }, +] + +[package.optional-dependencies] +databricks = [ + { name = "databricks-sql-connector" }, +] + +[[package]] +name = "soda-core-spark-df" +version = "3.5.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyspark" }, + { name = "soda-core-spark" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9f/71/64a85e503801444795aea3c73d89db5b1d82d8b65dd8ba0f1915ca431c7c/soda_core_spark_df-3.5.6.tar.gz", hash = "sha256:9031bf7e44f73bb3e9d5692c29771a41a1fd68765951021622c245b7cbe91dc4", size = 9507, upload-time = "2025-09-24T10:44:19.903Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/5b/f1ec8b380c2d38b454a882ebca388049cd4a900dcdfe7477d4111166d340/soda_core_spark_df-3.5.6-py3-none-any.whl", hash = "sha256:18d67c6ca5d716d7bfa87fe03dc68084611f722639263c6a1f9b593cac246b3e", size = 10405, upload-time = "2025-09-24T10:44:19.011Z" }, +] + [[package]] name = "sqlalchemy" version = "2.0.49" @@ -2939,6 +3237,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/4e/80705091aaf9c95e125d243f0aa871bc9f3670b4c9d963e6bad3b3dce8ff/sqlglot-30.8.0-py3-none-any.whl", hash = "sha256:af903378c331d5b72277a1b41118f07bc3e50cf4478e2d47eed12c96ee6a22a4", size = 687831, upload-time = "2026-05-13T09:04:36.336Z" }, ] +[[package]] +name = "sqlparse" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/d3/3f06a1006f2261d1342aefb3c71eed02f5d4ca5bdbecd86ebc12ad38306e/sqlparse-0.6.0.tar.gz", hash = "sha256:113c35c75365ab9cc9c7231d68c6428fb11c085fc8e9eb1ad659b7ddbf6cd2b9", size = 178477, upload-time = "2026-08-13T19:16:06.396Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/50/f00935da0ec7cbf325f8dc4f772ae46fbc7b672dd62876e73f0a94adda57/sqlparse-0.6.0-py3-none-any.whl", hash = "sha256:b861c0288ce2fa56209a9a6412d2e066ac664b3873b89c26c9d8415e8e32996f", size = 50070, upload-time = "2026-08-13T19:16:04.062Z" }, +] + [[package]] name = "textual" version = "8.2.7" @@ -3055,6 +3362,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" }, ] +[[package]] +name = "typeguard" +version = "4.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b4/de/4420db493fa8fc0856d5e5c1b159c63a323d2de2317babe36b01568928e8/typeguard-4.6.0.tar.gz", hash = "sha256:e7414f09111317de3e335de92cd397c5c0ca00b1cc1676de12e1d444a79b3f21", size = 82330, upload-time = "2026-07-26T08:40:23.207Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/eb/461d5f167b6f5c7d97696f397c82f82e3480e003fce3f0a1cd1dd26e2eb2/typeguard-4.6.0-py3-none-any.whl", hash = "sha256:79878165bb86f2cf5d41d159a0ff1792a796cf496882d2fe1b1c6c7049b9cdd7", size = 36884, upload-time = "2026-07-26T08:40:21.868Z" }, +] + [[package]] name = "typer" version = "0.24.2"