feat(control-plane): add admission invariant repair core
This commit is contained in:
@@ -0,0 +1,288 @@
|
|||||||
|
"""Unified autonomous control-plane primitives.
|
||||||
|
|
||||||
|
This is deliberately a library, not another daemon. Task API, workers and host
|
||||||
|
recovery code consume the same Admission -> Invariant -> Repair semantics.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import asdict, dataclass
|
||||||
|
from enum import Enum
|
||||||
|
import re
|
||||||
|
from typing import Callable, Iterable, Mapping, Sequence
|
||||||
|
|
||||||
|
from aios_capability_contract import (
|
||||||
|
capability_contract,
|
||||||
|
has_capabilities,
|
||||||
|
normalize_required_capabilities,
|
||||||
|
normalize_worker_capabilities,
|
||||||
|
registered_capability,
|
||||||
|
validate_capability_contract_registry,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FailureClass(str, Enum):
|
||||||
|
CAPABILITY_CONTRACT_ERROR = "CAPABILITY_CONTRACT_ERROR"
|
||||||
|
CREDENTIAL_PROJECTION_ERROR = "CREDENTIAL_PROJECTION_ERROR"
|
||||||
|
DEPENDENCY_UNAVAILABLE = "DEPENDENCY_UNAVAILABLE"
|
||||||
|
POLICY_DENIED = "POLICY_DENIED"
|
||||||
|
RESOURCE_UNAVAILABLE = "RESOURCE_UNAVAILABLE"
|
||||||
|
WORKER_CRASH = "WORKER_CRASH"
|
||||||
|
LEASE_LOST = "LEASE_LOST"
|
||||||
|
BUSINESS_PERMANENT = "BUSINESS_PERMANENT"
|
||||||
|
UNKNOWN = "UNKNOWN"
|
||||||
|
|
||||||
|
|
||||||
|
class RepairLevel(int, Enum):
|
||||||
|
AUTO_LOCAL = 0
|
||||||
|
AUTO_REPAIR_TASK = 1
|
||||||
|
OWNER_APPROVAL = 2
|
||||||
|
MANUAL = 3
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class InvariantResult:
|
||||||
|
name: str
|
||||||
|
passed: bool
|
||||||
|
detail: str = ""
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, object]:
|
||||||
|
return asdict(self)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class AdmissionDecision:
|
||||||
|
admitted: bool
|
||||||
|
state: str
|
||||||
|
invariants: tuple[InvariantResult, ...]
|
||||||
|
failure_class: FailureClass | None = None
|
||||||
|
repair_level: RepairLevel | None = None
|
||||||
|
repair_action: str | None = None
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"schema": "aios.runtime-admission.decision.v1",
|
||||||
|
"admitted": self.admitted,
|
||||||
|
"state": self.state,
|
||||||
|
"invariants": [item.as_dict() for item in self.invariants],
|
||||||
|
"failure_class": None if self.failure_class is None else self.failure_class.value,
|
||||||
|
"repair_level": None if self.repair_level is None else int(self.repair_level),
|
||||||
|
"repair_action": self.repair_action,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class RepairDecision:
|
||||||
|
failure_class: FailureClass
|
||||||
|
level: RepairLevel
|
||||||
|
action: str
|
||||||
|
consumes_business_attempt: bool
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"failure_class": self.failure_class.value,
|
||||||
|
"level": int(self.level),
|
||||||
|
"action": self.action,
|
||||||
|
"consumes_business_attempt": self.consumes_business_attempt,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def classify_failure(error: object, *, phase: str = "") -> FailureClass:
|
||||||
|
text = (str(error) + " " + str(phase)).lower()
|
||||||
|
if any(token in text for token in (
|
||||||
|
"credential", "permission denied", "token registry", "secret", "unauthorized"
|
||||||
|
)):
|
||||||
|
return FailureClass.CREDENTIAL_PROJECTION_ERROR
|
||||||
|
if any(token in text for token in (
|
||||||
|
"transport", "unavailable", "timeout", "connection", "bad gateway", "http_5"
|
||||||
|
)):
|
||||||
|
return FailureClass.DEPENDENCY_UNAVAILABLE
|
||||||
|
if any(token in text for token in (
|
||||||
|
"capability", "eligible", "matcher", "contract"
|
||||||
|
)):
|
||||||
|
return FailureClass.CAPABILITY_CONTRACT_ERROR
|
||||||
|
if any(token in text for token in (
|
||||||
|
"policy", "forbidden", "approval required", "not authorized"
|
||||||
|
)):
|
||||||
|
return FailureClass.POLICY_DENIED
|
||||||
|
if any(token in text for token in ("lease lost", "lease_expired", "heartbeat_failed")):
|
||||||
|
return FailureClass.LEASE_LOST
|
||||||
|
if any(token in text for token in ("out of memory", "disk full", "resource")):
|
||||||
|
return FailureClass.RESOURCE_UNAVAILABLE
|
||||||
|
if any(token in text for token in ("worker crash", "process exited", "signal")):
|
||||||
|
return FailureClass.WORKER_CRASH
|
||||||
|
if any(token in text for token in ("invalid release", "business permanent")):
|
||||||
|
return FailureClass.BUSINESS_PERMANENT
|
||||||
|
return FailureClass.UNKNOWN
|
||||||
|
|
||||||
|
|
||||||
|
def repair_for_failure(failure: FailureClass) -> RepairDecision:
|
||||||
|
table = {
|
||||||
|
FailureClass.CAPABILITY_CONTRACT_ERROR: RepairDecision(
|
||||||
|
failure, RepairLevel.AUTO_REPAIR_TASK,
|
||||||
|
"create_same_lineage_capability_contract_repair", False,
|
||||||
|
),
|
||||||
|
FailureClass.CREDENTIAL_PROJECTION_ERROR: RepairDecision(
|
||||||
|
failure, RepairLevel.AUTO_REPAIR_TASK,
|
||||||
|
"restore_service_private_credential_projection", False,
|
||||||
|
),
|
||||||
|
FailureClass.DEPENDENCY_UNAVAILABLE: RepairDecision(
|
||||||
|
failure, RepairLevel.AUTO_LOCAL,
|
||||||
|
"retry_dependency_health_and_resume", False,
|
||||||
|
),
|
||||||
|
FailureClass.RESOURCE_UNAVAILABLE: RepairDecision(
|
||||||
|
failure, RepairLevel.AUTO_LOCAL,
|
||||||
|
"heal_runtime_resource_and_resume", False,
|
||||||
|
),
|
||||||
|
FailureClass.WORKER_CRASH: RepairDecision(
|
||||||
|
failure, RepairLevel.AUTO_LOCAL,
|
||||||
|
"restart_worker_and_resume_same_lineage", False,
|
||||||
|
),
|
||||||
|
FailureClass.LEASE_LOST: RepairDecision(
|
||||||
|
failure, RepairLevel.AUTO_LOCAL,
|
||||||
|
"recover_lease_and_resume_same_lineage", False,
|
||||||
|
),
|
||||||
|
FailureClass.POLICY_DENIED: RepairDecision(
|
||||||
|
failure, RepairLevel.OWNER_APPROVAL,
|
||||||
|
"request_bound_owner_authorization", False,
|
||||||
|
),
|
||||||
|
FailureClass.BUSINESS_PERMANENT: RepairDecision(
|
||||||
|
failure, RepairLevel.MANUAL,
|
||||||
|
"terminal_business_failure", True,
|
||||||
|
),
|
||||||
|
FailureClass.UNKNOWN: RepairDecision(
|
||||||
|
failure, RepairLevel.AUTO_REPAIR_TASK,
|
||||||
|
"create_diagnostic_repair_task", False,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
return table[failure]
|
||||||
|
|
||||||
|
|
||||||
|
def _probe(
|
||||||
|
name: str,
|
||||||
|
probes: Mapping[str, Callable[[], bool]],
|
||||||
|
) -> InvariantResult:
|
||||||
|
fn = probes.get(name)
|
||||||
|
if fn is None:
|
||||||
|
return InvariantResult(name, False, "PROBE_MISSING")
|
||||||
|
try:
|
||||||
|
passed = bool(fn())
|
||||||
|
except Exception as exc:
|
||||||
|
return InvariantResult(name, False, "PROBE_ERROR:" + type(exc).__name__)
|
||||||
|
return InvariantResult(name, passed, "PASS" if passed else "NOT_READY")
|
||||||
|
|
||||||
|
|
||||||
|
def runtime_admission(
|
||||||
|
*,
|
||||||
|
worker_capabilities: Iterable[object],
|
||||||
|
required_capabilities: Iterable[object],
|
||||||
|
requested_operations: Sequence[str] = (),
|
||||||
|
probes: Mapping[str, Callable[[], bool]] | None = None,
|
||||||
|
) -> AdmissionDecision:
|
||||||
|
"""Admission is evaluated before claim; failures do not consume attempts."""
|
||||||
|
|
||||||
|
worker_values = tuple(worker_capabilities)
|
||||||
|
required_values = tuple(required_capabilities)
|
||||||
|
probe_map = probes or {}
|
||||||
|
invariants: list[InvariantResult] = []
|
||||||
|
|
||||||
|
registry = validate_capability_contract_registry()
|
||||||
|
invariants.append(InvariantResult(
|
||||||
|
"capability_contract_registry",
|
||||||
|
bool(registry.get("valid")),
|
||||||
|
"PASS" if registry.get("valid") else "REGISTRY_INVALID",
|
||||||
|
))
|
||||||
|
|
||||||
|
task_known = all(registered_capability(item) for item in required_values)
|
||||||
|
worker_known = all(registered_capability(item, worker=True) for item in worker_values)
|
||||||
|
invariants.extend((
|
||||||
|
InvariantResult("task_capabilities_registered", task_known, "PASS" if task_known else "UNKNOWN_TASK_CAPABILITY"),
|
||||||
|
InvariantResult("worker_capabilities_registered", worker_known, "PASS" if worker_known else "UNKNOWN_WORKER_CAPABILITY"),
|
||||||
|
))
|
||||||
|
|
||||||
|
capability_match = (
|
||||||
|
task_known
|
||||||
|
and worker_known
|
||||||
|
and has_capabilities(worker_values, required_values)
|
||||||
|
)
|
||||||
|
invariants.append(InvariantResult(
|
||||||
|
"capability_match",
|
||||||
|
capability_match,
|
||||||
|
"PASS" if capability_match else "WORKER_NOT_ELIGIBLE",
|
||||||
|
))
|
||||||
|
|
||||||
|
normalized = normalize_required_capabilities(required_values)
|
||||||
|
contracts = [
|
||||||
|
capability_contract(item)
|
||||||
|
for item in normalized
|
||||||
|
if not str(item).startswith("exact:")
|
||||||
|
]
|
||||||
|
contracts = [item for item in contracts if item is not None]
|
||||||
|
|
||||||
|
forbidden: set[str] = set()
|
||||||
|
dependencies: set[str] = set()
|
||||||
|
health: set[str] = set()
|
||||||
|
for contract in contracts:
|
||||||
|
forbidden.update(contract.forbidden)
|
||||||
|
dependencies.update(contract.dependencies)
|
||||||
|
health.update(contract.health_requirements)
|
||||||
|
|
||||||
|
requested = {str(item).strip().lower() for item in requested_operations if str(item).strip()}
|
||||||
|
forbidden_hit = sorted(requested & {item.lower() for item in forbidden})
|
||||||
|
invariants.append(InvariantResult(
|
||||||
|
"forbidden_operations_absent",
|
||||||
|
not forbidden_hit,
|
||||||
|
"PASS" if not forbidden_hit else "FORBIDDEN:" + ",".join(forbidden_hit),
|
||||||
|
))
|
||||||
|
|
||||||
|
for name in sorted(dependencies | health):
|
||||||
|
invariants.append(_probe(name, probe_map))
|
||||||
|
|
||||||
|
failed = [item for item in invariants if not item.passed]
|
||||||
|
if not failed:
|
||||||
|
return AdmissionDecision(
|
||||||
|
True,
|
||||||
|
"EXECUTION_READY",
|
||||||
|
tuple(invariants),
|
||||||
|
)
|
||||||
|
|
||||||
|
details = " ".join(item.name + ":" + item.detail for item in failed)
|
||||||
|
failure = classify_failure(details, phase="admission")
|
||||||
|
if any(item.name == "forbidden_operations_absent" and not item.passed for item in failed):
|
||||||
|
failure = FailureClass.POLICY_DENIED
|
||||||
|
elif any("credential" in item.name for item in failed):
|
||||||
|
failure = FailureClass.CREDENTIAL_PROJECTION_ERROR
|
||||||
|
elif any(item.name in {"capability_contract_registry", "task_capabilities_registered", "worker_capabilities_registered", "capability_match"} for item in failed):
|
||||||
|
failure = FailureClass.CAPABILITY_CONTRACT_ERROR
|
||||||
|
elif any(item.name in dependencies or item.name in health for item in failed):
|
||||||
|
failure = FailureClass.DEPENDENCY_UNAVAILABLE
|
||||||
|
|
||||||
|
repair = repair_for_failure(failure)
|
||||||
|
return AdmissionDecision(
|
||||||
|
False,
|
||||||
|
"EXECUTION_READY_FAILED",
|
||||||
|
tuple(invariants),
|
||||||
|
failure,
|
||||||
|
repair.level,
|
||||||
|
repair.action,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def system_invariant_snapshot(
|
||||||
|
*,
|
||||||
|
worker_capabilities: Iterable[object],
|
||||||
|
required_capabilities: Iterable[object],
|
||||||
|
requested_operations: Sequence[str] = (),
|
||||||
|
probes: Mapping[str, Callable[[], bool]] | None = None,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
decision = runtime_admission(
|
||||||
|
worker_capabilities=worker_capabilities,
|
||||||
|
required_capabilities=required_capabilities,
|
||||||
|
requested_operations=requested_operations,
|
||||||
|
probes=probes,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"schema": "aios.system-invariants.v1",
|
||||||
|
"status": "HEALTHY" if decision.admitted else "DEGRADED",
|
||||||
|
"execution_ready": decision.admitted,
|
||||||
|
"admission": decision.as_dict(),
|
||||||
|
}
|
||||||
@@ -3,6 +3,8 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
|
from dataclasses import asdict, dataclass
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
_TASK_ALIASES: dict[str, frozenset[str]] = {
|
_TASK_ALIASES: dict[str, frozenset[str]] = {
|
||||||
@@ -75,6 +77,169 @@ _WORKER_ALIASES: dict[str, frozenset[str]] = {
|
|||||||
"release:candidate": frozenset({"release:candidate"}),
|
"release:candidate": frozenset({"release:candidate"}),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class CapabilityContract:
|
||||||
|
"""Versioned runtime contract attached to one canonical semantic capability."""
|
||||||
|
|
||||||
|
name: str
|
||||||
|
min_version: int = 1
|
||||||
|
max_version: int = 1
|
||||||
|
permissions: tuple[str, ...] = ()
|
||||||
|
forbidden: tuple[str, ...] = ()
|
||||||
|
dependencies: tuple[str, ...] = ()
|
||||||
|
health_requirements: tuple[str, ...] = ()
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, Any]:
|
||||||
|
return asdict(self)
|
||||||
|
|
||||||
|
|
||||||
|
# This registry is the semantic source of truth. Alias tables below are retained
|
||||||
|
# as compatibility adapters; startup/tests validate every alias target against
|
||||||
|
# this registry so a name cannot be accepted in one layer and be unknown in
|
||||||
|
# another.
|
||||||
|
_CAPABILITY_CONTRACTS: dict[str, CapabilityContract] = {
|
||||||
|
"repo.read": CapabilityContract("repo.read", permissions=("repository_read",)),
|
||||||
|
"test.execute": CapabilityContract("test.execute", permissions=("test_execution",)),
|
||||||
|
"governed.git.write.branch": CapabilityContract(
|
||||||
|
"governed.git.write.branch",
|
||||||
|
permissions=("governed_branch_write",),
|
||||||
|
dependencies=("writer_lease", "governed_gitops"),
|
||||||
|
),
|
||||||
|
"governed.git.write.file": CapabilityContract(
|
||||||
|
"governed.git.write.file",
|
||||||
|
permissions=("governed_file_write",),
|
||||||
|
dependencies=("writer_lease", "governed_gitops"),
|
||||||
|
),
|
||||||
|
"governed.git.write": CapabilityContract(
|
||||||
|
"governed.git.write",
|
||||||
|
permissions=("governed_git_write",),
|
||||||
|
dependencies=("writer_lease", "governed_gitops"),
|
||||||
|
),
|
||||||
|
"runtime.shell": CapabilityContract("runtime.shell", permissions=("runtime_shell",)),
|
||||||
|
"runtime.git": CapabilityContract("runtime.git", permissions=("runtime_git",)),
|
||||||
|
"runtime.python": CapabilityContract("runtime.python", permissions=("runtime_python",)),
|
||||||
|
"legacy.code.read": CapabilityContract("legacy.code.read"),
|
||||||
|
"legacy.code.review": CapabilityContract("legacy.code.review"),
|
||||||
|
"legacy.code.write": CapabilityContract("legacy.code.write"),
|
||||||
|
"legacy.code.write.patch": CapabilityContract("legacy.code.write.patch"),
|
||||||
|
"legacy.test.python": CapabilityContract("legacy.test.python"),
|
||||||
|
"legacy.test.shell.allowlisted": CapabilityContract("legacy.test.shell.allowlisted"),
|
||||||
|
"legacy.git.commit": CapabilityContract("legacy.git.commit"),
|
||||||
|
"legacy.git.diff.read": CapabilityContract("legacy.git.diff.read"),
|
||||||
|
"legacy.git.force_push": CapabilityContract(
|
||||||
|
"legacy.git.force_push",
|
||||||
|
forbidden=("protected_ref_bypass",),
|
||||||
|
),
|
||||||
|
"legacy.deploy.production": CapabilityContract(
|
||||||
|
"legacy.deploy.production",
|
||||||
|
permissions=("production_deploy",),
|
||||||
|
dependencies=("owner_approval", "deployment_controller"),
|
||||||
|
health_requirements=("approval_envelope_verified", "runtime_acceptance_available"),
|
||||||
|
),
|
||||||
|
"legacy.secret.transform": CapabilityContract("legacy.secret.transform"),
|
||||||
|
"legacy.secure_file_transformer": CapabilityContract("legacy.secure_file_transformer"),
|
||||||
|
"legacy.runtime.shadow": CapabilityContract("legacy.runtime.shadow"),
|
||||||
|
"legacy.review.security": CapabilityContract("legacy.review.security"),
|
||||||
|
"legacy.review.api-contract": CapabilityContract("legacy.review.api-contract"),
|
||||||
|
"legacy.gpu.train": CapabilityContract("legacy.gpu.train"),
|
||||||
|
"agent.runtime.hermes": CapabilityContract(
|
||||||
|
"agent.runtime.hermes",
|
||||||
|
dependencies=("hermes_runtime",),
|
||||||
|
health_requirements=("worker_liveness",),
|
||||||
|
),
|
||||||
|
"agent.runtime.openclaw": CapabilityContract(
|
||||||
|
"agent.runtime.openclaw",
|
||||||
|
dependencies=("openclaw_runtime",),
|
||||||
|
health_requirements=("worker_liveness",),
|
||||||
|
),
|
||||||
|
"release:candidate": CapabilityContract(
|
||||||
|
"release:candidate",
|
||||||
|
min_version=1,
|
||||||
|
max_version=1,
|
||||||
|
permissions=("prepare_release",),
|
||||||
|
forbidden=(
|
||||||
|
"production_deploy",
|
||||||
|
"submit_owner_approval",
|
||||||
|
"tag_publish",
|
||||||
|
"baseline_registry_finalize",
|
||||||
|
"runtime_switch",
|
||||||
|
),
|
||||||
|
dependencies=(
|
||||||
|
"credential_projection",
|
||||||
|
"trusted_builder",
|
||||||
|
),
|
||||||
|
health_requirements=(
|
||||||
|
"authenticated_probe",
|
||||||
|
"release_api_health",
|
||||||
|
"worker_liveness",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def capability_contract(name: object) -> CapabilityContract | None:
|
||||||
|
"""Return the contract for one canonical or registered capability name."""
|
||||||
|
|
||||||
|
text = _clean(name)
|
||||||
|
direct = _CAPABILITY_CONTRACTS.get(text)
|
||||||
|
if direct is not None:
|
||||||
|
return direct
|
||||||
|
mapped = _TASK_ALIASES.get(text) or _WORKER_ALIASES.get(text)
|
||||||
|
if mapped is None or len(mapped) != 1:
|
||||||
|
return None
|
||||||
|
return _CAPABILITY_CONTRACTS.get(next(iter(mapped)))
|
||||||
|
|
||||||
|
|
||||||
|
def registered_capability(name: object, *, worker: bool = False) -> bool:
|
||||||
|
"""Fail-closed validation used by registration/task admission."""
|
||||||
|
|
||||||
|
text = _clean(name)
|
||||||
|
if "*" in text:
|
||||||
|
return False
|
||||||
|
aliases = _WORKER_ALIASES if worker else _TASK_ALIASES
|
||||||
|
mapped = aliases.get(text)
|
||||||
|
if mapped is None:
|
||||||
|
return text in _CAPABILITY_CONTRACTS
|
||||||
|
return all(item in _CAPABILITY_CONTRACTS for item in mapped)
|
||||||
|
|
||||||
|
|
||||||
|
def validate_capability_contract_registry() -> dict[str, object]:
|
||||||
|
"""Prove alias/contract closure and contract safety invariants."""
|
||||||
|
|
||||||
|
alias_targets: set[str] = set()
|
||||||
|
for mapping in (_TASK_ALIASES, _WORKER_ALIASES):
|
||||||
|
for values in mapping.values():
|
||||||
|
alias_targets.update(values)
|
||||||
|
missing = sorted(alias_targets - set(_CAPABILITY_CONTRACTS))
|
||||||
|
invalid_versions = sorted(
|
||||||
|
name
|
||||||
|
for name, contract in _CAPABILITY_CONTRACTS.items()
|
||||||
|
if contract.min_version < 1 or contract.max_version < contract.min_version
|
||||||
|
)
|
||||||
|
permission_forbidden_overlap = sorted(
|
||||||
|
name
|
||||||
|
for name, contract in _CAPABILITY_CONTRACTS.items()
|
||||||
|
if set(contract.permissions) & set(contract.forbidden)
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"schema": "aios.capability-contract-registry.v1",
|
||||||
|
"contract_count": len(_CAPABILITY_CONTRACTS),
|
||||||
|
"missing_alias_targets": missing,
|
||||||
|
"invalid_versions": invalid_versions,
|
||||||
|
"permission_forbidden_overlap": permission_forbidden_overlap,
|
||||||
|
"valid": not missing and not invalid_versions and not permission_forbidden_overlap,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def capability_contract_snapshot() -> dict[str, dict[str, Any]]:
|
||||||
|
return {
|
||||||
|
name: contract.as_dict()
|
||||||
|
for name, contract in sorted(_CAPABILITY_CONTRACTS.items())
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
_UNKNOWN_PREFIX = "exact:"
|
_UNKNOWN_PREFIX = "exact:"
|
||||||
|
|
||||||
|
|
||||||
@@ -100,7 +265,11 @@ def normalize_required_capabilities(values: Iterable[object]) -> frozenset[str]:
|
|||||||
if "*" in text:
|
if "*" in text:
|
||||||
result.add(_UNKNOWN_PREFIX + text)
|
result.add(_UNKNOWN_PREFIX + text)
|
||||||
else:
|
else:
|
||||||
result.update(_TASK_ALIASES.get(text, frozenset({_UNKNOWN_PREFIX + text})))
|
mapped = _TASK_ALIASES.get(text)
|
||||||
|
if mapped is None or any(item not in _CAPABILITY_CONTRACTS for item in mapped):
|
||||||
|
result.add(_UNKNOWN_PREFIX + text)
|
||||||
|
else:
|
||||||
|
result.update(mapped)
|
||||||
return frozenset(result)
|
return frozenset(result)
|
||||||
|
|
||||||
|
|
||||||
@@ -113,7 +282,7 @@ def normalize_worker_capabilities(values: Iterable[object]) -> frozenset[str]:
|
|||||||
if "*" in text:
|
if "*" in text:
|
||||||
continue
|
continue
|
||||||
mapped = _WORKER_ALIASES.get(text)
|
mapped = _WORKER_ALIASES.get(text)
|
||||||
if mapped is not None:
|
if mapped is not None and all(item in _CAPABILITY_CONTRACTS for item in mapped):
|
||||||
result.update(mapped)
|
result.update(mapped)
|
||||||
if {
|
if {
|
||||||
"governed.git.write.branch",
|
"governed.git.write.branch",
|
||||||
|
|||||||
@@ -23,6 +23,11 @@ import time
|
|||||||
import urllib.error
|
import urllib.error
|
||||||
import urllib.request
|
import urllib.request
|
||||||
from typing import Any, Mapping
|
from typing import Any, Mapping
|
||||||
|
from aios_autonomous_control_plane import (
|
||||||
|
classify_failure,
|
||||||
|
repair_for_failure,
|
||||||
|
runtime_admission,
|
||||||
|
)
|
||||||
|
|
||||||
AGENT_ID = os.environ.get(
|
AGENT_ID = os.environ.get(
|
||||||
"AIOS_RELEASE_WORKER_AGENT_ID",
|
"AIOS_RELEASE_WORKER_AGENT_ID",
|
||||||
@@ -431,6 +436,53 @@ class Heartbeat:
|
|||||||
self.thread.join(timeout=2)
|
self.thread.join(timeout=2)
|
||||||
|
|
||||||
|
|
||||||
|
def release_admission(
|
||||||
|
task: Mapping[str, Any],
|
||||||
|
*,
|
||||||
|
release_token: str,
|
||||||
|
):
|
||||||
|
required = task.get("required_capabilities")
|
||||||
|
if not isinstance(required, list):
|
||||||
|
required = []
|
||||||
|
|
||||||
|
def release_api_health() -> bool:
|
||||||
|
status, value = http_json(
|
||||||
|
RELEASE_API,
|
||||||
|
"GET",
|
||||||
|
"/health",
|
||||||
|
"",
|
||||||
|
timeout=10,
|
||||||
|
)
|
||||||
|
return status == 200 and str(value.get("status") or "").lower() in {"ok", "healthy"}
|
||||||
|
|
||||||
|
def authenticated_probe() -> bool:
|
||||||
|
status, _ = http_json(
|
||||||
|
RELEASE_API,
|
||||||
|
"POST",
|
||||||
|
"/createReleaseCandidate",
|
||||||
|
release_token,
|
||||||
|
{},
|
||||||
|
timeout=10,
|
||||||
|
)
|
||||||
|
# Authentication + credential projection succeeded; the empty release
|
||||||
|
# request is intentionally rejected by request validation.
|
||||||
|
return status == 400
|
||||||
|
|
||||||
|
probes = {
|
||||||
|
"credential_projection": authenticated_probe,
|
||||||
|
"trusted_builder": release_api_health,
|
||||||
|
"authenticated_probe": authenticated_probe,
|
||||||
|
"release_api_health": release_api_health,
|
||||||
|
"worker_liveness": lambda: True,
|
||||||
|
}
|
||||||
|
return runtime_admission(
|
||||||
|
worker_capabilities=EXPECTED_CAPABILITIES,
|
||||||
|
required_capabilities=required,
|
||||||
|
requested_operations=("prepare_release",),
|
||||||
|
probes=probes,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def result_payload(
|
def result_payload(
|
||||||
task_id: str,
|
task_id: str,
|
||||||
release: ReleaseRequest,
|
release: ReleaseRequest,
|
||||||
@@ -478,6 +530,28 @@ def execute_task(
|
|||||||
) -> bool:
|
) -> bool:
|
||||||
task_id = str(task.get("task_id") or "")
|
task_id = str(task.get("task_id") or "")
|
||||||
release = ReleaseRequest.from_task(task)
|
release = ReleaseRequest.from_task(task)
|
||||||
|
admission = release_admission(
|
||||||
|
task,
|
||||||
|
release_token=release_token,
|
||||||
|
)
|
||||||
|
if not admission.admitted:
|
||||||
|
audit(
|
||||||
|
"TASK_ADMISSION_BLOCKED",
|
||||||
|
task_id=task_id,
|
||||||
|
admission=admission.as_dict(),
|
||||||
|
)
|
||||||
|
write_liveness(
|
||||||
|
"BLOCKED",
|
||||||
|
task_id=task_id,
|
||||||
|
stage="execution_ready",
|
||||||
|
error=admission.failure_class.value if admission.failure_class else "ADMISSION_FAILED",
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
audit(
|
||||||
|
"TASK_ADMISSION_PASSED",
|
||||||
|
task_id=task_id,
|
||||||
|
admission=admission.as_dict(),
|
||||||
|
)
|
||||||
status, claimed = http_json(
|
status, claimed = http_json(
|
||||||
TASK_API,
|
TASK_API,
|
||||||
"POST",
|
"POST",
|
||||||
@@ -610,11 +684,15 @@ def main() -> int:
|
|||||||
release_token=release_token,
|
release_token=release_token,
|
||||||
) or handled
|
) or handled
|
||||||
except ReleaseWorkerError as exc:
|
except ReleaseWorkerError as exc:
|
||||||
|
failure = classify_failure(exc.code, phase=exc.phase)
|
||||||
|
repair = repair_for_failure(failure)
|
||||||
audit(
|
audit(
|
||||||
"TASK_EXECUTION_FAILED",
|
"TASK_EXECUTION_FAILED",
|
||||||
task_id=raw.get("task_id"),
|
task_id=raw.get("task_id"),
|
||||||
phase=exc.phase,
|
phase=exc.phase,
|
||||||
error=exc.code,
|
error=exc.code,
|
||||||
|
failure_class=failure.value,
|
||||||
|
repair=repair.as_dict(),
|
||||||
)
|
)
|
||||||
time.sleep(0 if handled else POLL_SECONDS)
|
time.sleep(0 if handled else POLL_SECONDS)
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
|
|||||||
@@ -0,0 +1,210 @@
|
|||||||
|
# AIOS Autonomous Control Plane — Admission + Invariant + Repair
|
||||||
|
|
||||||
|
## Why this exists
|
||||||
|
|
||||||
|
Recent Agent registration, Task claim, release-worker, credential and retry failures shared one
|
||||||
|
failure mode:
|
||||||
|
|
||||||
|
> The control plane accepted an intent before the runtime had proved it could execute the intent.
|
||||||
|
|
||||||
|
This design does not add another fleet of daemons. It adds one shared library contract that Task API,
|
||||||
|
workers and governed host recovery consume.
|
||||||
|
|
||||||
|
## Three layers
|
||||||
|
|
||||||
|
```text
|
||||||
|
Intent
|
||||||
|
↓
|
||||||
|
Runtime Admission
|
||||||
|
↓
|
||||||
|
System Invariants
|
||||||
|
↓
|
||||||
|
Repair Policy
|
||||||
|
↓
|
||||||
|
Execution
|
||||||
|
```
|
||||||
|
|
||||||
|
### Runtime Admission
|
||||||
|
|
||||||
|
Admission occurs before Task claim.
|
||||||
|
|
||||||
|
A capability such as `release:candidate` is not considered executable merely because its name
|
||||||
|
matches. The capability contract also binds:
|
||||||
|
|
||||||
|
- supported contract version;
|
||||||
|
- allowed permission;
|
||||||
|
- forbidden operations;
|
||||||
|
- runtime dependencies;
|
||||||
|
- health requirements.
|
||||||
|
|
||||||
|
For `release:candidate`:
|
||||||
|
|
||||||
|
```text
|
||||||
|
permission:
|
||||||
|
prepare_release
|
||||||
|
|
||||||
|
forbidden:
|
||||||
|
production_deploy
|
||||||
|
submit_owner_approval
|
||||||
|
tag_publish
|
||||||
|
baseline_registry_finalize
|
||||||
|
runtime_switch
|
||||||
|
|
||||||
|
dependencies:
|
||||||
|
credential_projection
|
||||||
|
trusted_builder
|
||||||
|
|
||||||
|
health:
|
||||||
|
authenticated_probe
|
||||||
|
release_api_health
|
||||||
|
worker_liveness
|
||||||
|
```
|
||||||
|
|
||||||
|
Only a fully passing decision reaches:
|
||||||
|
|
||||||
|
```text
|
||||||
|
EXECUTION_READY
|
||||||
|
```
|
||||||
|
|
||||||
|
A failed decision remains:
|
||||||
|
|
||||||
|
```text
|
||||||
|
EXECUTION_READY_FAILED
|
||||||
|
```
|
||||||
|
|
||||||
|
and the worker does not call `/claim`.
|
||||||
|
|
||||||
|
Therefore an infrastructure defect discovered by admission does not consume a business attempt.
|
||||||
|
|
||||||
|
### System Invariants
|
||||||
|
|
||||||
|
The same admission checks are exposed as a machine-readable invariant snapshot.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
```text
|
||||||
|
capability_contract_registry
|
||||||
|
task_capabilities_registered
|
||||||
|
worker_capabilities_registered
|
||||||
|
capability_match
|
||||||
|
forbidden_operations_absent
|
||||||
|
credential_projection
|
||||||
|
trusted_builder
|
||||||
|
authenticated_probe
|
||||||
|
release_api_health
|
||||||
|
worker_liveness
|
||||||
|
```
|
||||||
|
|
||||||
|
A runtime can expose:
|
||||||
|
|
||||||
|
```text
|
||||||
|
HEALTHY
|
||||||
|
DEGRADED
|
||||||
|
```
|
||||||
|
|
||||||
|
without relying on a human reconstructing state from logs.
|
||||||
|
|
||||||
|
### Repair Policy
|
||||||
|
|
||||||
|
Failure classification maps to bounded recovery:
|
||||||
|
|
||||||
|
```text
|
||||||
|
DEPENDENCY_UNAVAILABLE → level 0 / auto local
|
||||||
|
RESOURCE_UNAVAILABLE → level 0 / auto local
|
||||||
|
WORKER_CRASH → level 0 / auto local
|
||||||
|
LEASE_LOST → level 0 / auto local
|
||||||
|
|
||||||
|
CAPABILITY_CONTRACT_ERROR → level 1 / same-lineage repair task
|
||||||
|
CREDENTIAL_PROJECTION_ERROR → level 1 / same-lineage repair task
|
||||||
|
UNKNOWN → level 1 / diagnostic repair task
|
||||||
|
|
||||||
|
POLICY_DENIED → level 2 / Owner authorization
|
||||||
|
BUSINESS_PERMANENT → level 3 / terminal/manual decision
|
||||||
|
```
|
||||||
|
|
||||||
|
Only business-permanent failure consumes the business retry budget in this first contract.
|
||||||
|
|
||||||
|
The current Task API does not yet have a first-class `EXECUTION_READY_FAILED` durable state. Until it
|
||||||
|
does, the Release Worker enforces the most important invariant by refusing to claim a Task whose
|
||||||
|
admission fails.
|
||||||
|
|
||||||
|
## Capability Contract Registry
|
||||||
|
|
||||||
|
`aios_capability_contract.py` remains backward-compatible with existing aliases but now validates
|
||||||
|
every alias target against a versioned semantic contract registry.
|
||||||
|
|
||||||
|
Unknown capabilities and wildcard capabilities remain fail-closed.
|
||||||
|
|
||||||
|
Agent registration and Task creation should next be changed to call `registered_capability()` so a
|
||||||
|
capability cannot be accepted at registration and rejected only later by the matcher.
|
||||||
|
|
||||||
|
## Credential rule
|
||||||
|
|
||||||
|
Admission probes credential usability, not merely credential existence.
|
||||||
|
|
||||||
|
For release candidate creation the authenticated probe intentionally sends an invalid empty
|
||||||
|
`/createReleaseCandidate` request.
|
||||||
|
|
||||||
|
Expected:
|
||||||
|
|
||||||
|
```text
|
||||||
|
HTTP 400
|
||||||
|
```
|
||||||
|
|
||||||
|
This proves:
|
||||||
|
|
||||||
|
```text
|
||||||
|
credential readable
|
||||||
|
→ authentication succeeded
|
||||||
|
→ scope accepted
|
||||||
|
→ request validation rejected empty business input
|
||||||
|
```
|
||||||
|
|
||||||
|
A permission failure/401/403/5xx is `EXECUTION_READY_FAILED`.
|
||||||
|
|
||||||
|
## Lineage
|
||||||
|
|
||||||
|
Repair actions are descriptive, not permission grants. Any repair Task must retain the existing
|
||||||
|
root/parent/attempt/authority chain and use governed same-lineage recovery.
|
||||||
|
|
||||||
|
No repair action may silently create a replacement business Task.
|
||||||
|
|
||||||
|
## Authority separation
|
||||||
|
|
||||||
|
Capability answers:
|
||||||
|
|
||||||
|
```text
|
||||||
|
can this runtime perform the operation?
|
||||||
|
```
|
||||||
|
|
||||||
|
Authority answers:
|
||||||
|
|
||||||
|
```text
|
||||||
|
is this operation currently authorized?
|
||||||
|
```
|
||||||
|
|
||||||
|
`release:candidate` never implies Owner approval, deployment, tag publication, baseline finalization
|
||||||
|
or runtime switch.
|
||||||
|
|
||||||
|
## Deployment sequence
|
||||||
|
|
||||||
|
Phase 1 (this candidate):
|
||||||
|
|
||||||
|
- Capability Contract Registry;
|
||||||
|
- shared Autonomous Control Plane library;
|
||||||
|
- Release Worker pre-claim admission;
|
||||||
|
- failure classification + repair evidence.
|
||||||
|
|
||||||
|
Phase 2:
|
||||||
|
|
||||||
|
- Task API uses the same library before returning Tasks from poll and again during claim CAS;
|
||||||
|
- Task schema gains durable `ADMITTED/EXECUTION_READY/EXECUTION_READY_FAILED` evidence without
|
||||||
|
counting pre-side-effect infrastructure failures against business attempts.
|
||||||
|
|
||||||
|
Phase 3:
|
||||||
|
|
||||||
|
- Agent registration validates against the same Capability Contract Registry;
|
||||||
|
- external connector writes only durable Intent/Command Inbox records and leaves execution to the
|
||||||
|
local control plane.
|
||||||
|
|
||||||
|
This sequencing avoids a second controller stack while converging every decision onto one contract.
|
||||||
@@ -0,0 +1,157 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
APP=ROOT/"active-runtime/app"
|
||||||
|
sys.path.insert(0,str(APP))
|
||||||
|
|
||||||
|
import aios_autonomous_control_plane as acp
|
||||||
|
import aios_capability_contract as contract
|
||||||
|
import aios_release_online_agent_worker as worker
|
||||||
|
|
||||||
|
|
||||||
|
def _ready_probes():
|
||||||
|
return {
|
||||||
|
"credential_projection": lambda: True,
|
||||||
|
"trusted_builder": lambda: True,
|
||||||
|
"authenticated_probe": lambda: True,
|
||||||
|
"release_api_health": lambda: True,
|
||||||
|
"worker_liveness": lambda: True,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _task():
|
||||||
|
return {
|
||||||
|
"task_id":"r42-autonomous-release-admission-test",
|
||||||
|
"required_capabilities":["release:candidate"],
|
||||||
|
"constraints":{
|
||||||
|
"release_candidate":{
|
||||||
|
"candidate_id":"r42-release-candidate-v2r2c9-test",
|
||||||
|
"source_commit":"1"*40,
|
||||||
|
"version":"V2R2C9",
|
||||||
|
"projection_id":"projection-"+"a"*32,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_capability_contract_registry_is_closed_and_release_is_versioned():
|
||||||
|
status=contract.validate_capability_contract_registry()
|
||||||
|
assert status["valid"] is True
|
||||||
|
release=contract.capability_contract("release:candidate")
|
||||||
|
assert release is not None
|
||||||
|
assert release.min_version==1
|
||||||
|
assert release.max_version==1
|
||||||
|
assert release.permissions==("prepare_release",)
|
||||||
|
assert "credential_projection" in release.dependencies
|
||||||
|
assert "trusted_builder" in release.dependencies
|
||||||
|
assert "authenticated_probe" in release.health_requirements
|
||||||
|
assert "production_deploy" in release.forbidden
|
||||||
|
assert "runtime_switch" in release.forbidden
|
||||||
|
|
||||||
|
|
||||||
|
def test_unknown_agent_capability_cannot_register_or_match():
|
||||||
|
assert contract.registered_capability("release:future",worker=True) is False
|
||||||
|
assert contract.registered_capability("release:*",worker=True) is False
|
||||||
|
assert contract.has_capabilities(["release:future"],["release:future"]) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_admission_requires_all_release_execution_readiness():
|
||||||
|
decision=acp.runtime_admission(
|
||||||
|
worker_capabilities=["release:candidate"],
|
||||||
|
required_capabilities=["release:candidate"],
|
||||||
|
requested_operations=["prepare_release"],
|
||||||
|
probes=_ready_probes(),
|
||||||
|
)
|
||||||
|
assert decision.admitted is True
|
||||||
|
assert decision.state=="EXECUTION_READY"
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_admission_blocks_credential_failure_before_execution():
|
||||||
|
probes=_ready_probes()
|
||||||
|
probes["credential_projection"]=lambda:False
|
||||||
|
probes["authenticated_probe"]=lambda:False
|
||||||
|
decision=acp.runtime_admission(
|
||||||
|
worker_capabilities=["release:candidate"],
|
||||||
|
required_capabilities=["release:candidate"],
|
||||||
|
requested_operations=["prepare_release"],
|
||||||
|
probes=probes,
|
||||||
|
)
|
||||||
|
assert decision.admitted is False
|
||||||
|
assert decision.state=="EXECUTION_READY_FAILED"
|
||||||
|
assert decision.failure_class==acp.FailureClass.CREDENTIAL_PROJECTION_ERROR
|
||||||
|
assert decision.repair_level==acp.RepairLevel.AUTO_REPAIR_TASK
|
||||||
|
|
||||||
|
|
||||||
|
def test_forbidden_release_operation_requires_owner_boundary():
|
||||||
|
decision=acp.runtime_admission(
|
||||||
|
worker_capabilities=["release:candidate"],
|
||||||
|
required_capabilities=["release:candidate"],
|
||||||
|
requested_operations=["production_deploy"],
|
||||||
|
probes=_ready_probes(),
|
||||||
|
)
|
||||||
|
assert decision.admitted is False
|
||||||
|
assert decision.failure_class==acp.FailureClass.POLICY_DENIED
|
||||||
|
assert decision.repair_level==acp.RepairLevel.OWNER_APPROVAL
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"error,phase,expected,level,consumes",
|
||||||
|
[
|
||||||
|
("Permission denied: gitops.tokens","release",acp.FailureClass.CREDENTIAL_PROJECTION_ERROR,acp.RepairLevel.AUTO_REPAIR_TASK,False),
|
||||||
|
("HTTP_TRANSPORT_UNAVAILABLE","network",acp.FailureClass.DEPENDENCY_UNAVAILABLE,acp.RepairLevel.AUTO_LOCAL,False),
|
||||||
|
("TASK_CAPABILITY_SCOPE_INVALID","validation",acp.FailureClass.CAPABILITY_CONTRACT_ERROR,acp.RepairLevel.AUTO_REPAIR_TASK,False),
|
||||||
|
("lease lost","heartbeat",acp.FailureClass.LEASE_LOST,acp.RepairLevel.AUTO_LOCAL,False),
|
||||||
|
("business permanent invalid release","release",acp.FailureClass.BUSINESS_PERMANENT,acp.RepairLevel.MANUAL,True),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_failure_classification_drives_repair_policy(error,phase,expected,level,consumes):
|
||||||
|
failure=acp.classify_failure(error,phase=phase)
|
||||||
|
assert failure==expected
|
||||||
|
repair=acp.repair_for_failure(failure)
|
||||||
|
assert repair.level==level
|
||||||
|
assert repair.consumes_business_attempt is consumes
|
||||||
|
|
||||||
|
|
||||||
|
def test_release_worker_does_not_claim_when_admission_is_not_ready(monkeypatch):
|
||||||
|
calls=[]
|
||||||
|
def fake_http(base,method,route,token,body=None,timeout=30):
|
||||||
|
calls.append((base,method,route,body))
|
||||||
|
if route=="/health":
|
||||||
|
return 200,{"status":"ok"}
|
||||||
|
if route=="/createReleaseCandidate" and body=={}:
|
||||||
|
return 500,{"error":"Permission denied: gitops.tokens"}
|
||||||
|
if route.endswith("/claim"):
|
||||||
|
raise AssertionError("claim must not happen before EXECUTION_READY")
|
||||||
|
raise AssertionError(route)
|
||||||
|
|
||||||
|
events=[]
|
||||||
|
monkeypatch.setattr(worker,"http_json",fake_http)
|
||||||
|
monkeypatch.setattr(worker,"audit",lambda event,**fields:events.append((event,fields)))
|
||||||
|
monkeypatch.setattr(worker,"write_liveness",lambda *args,**kwargs:None)
|
||||||
|
|
||||||
|
assert worker.execute_task(
|
||||||
|
_task(),
|
||||||
|
agent_token="a"*32,
|
||||||
|
release_token="r"*32,
|
||||||
|
) is False
|
||||||
|
assert not any(route.endswith("/claim") for _,_,route,_ in calls)
|
||||||
|
blocked=[item for item in events if item[0]=="TASK_ADMISSION_BLOCKED"]
|
||||||
|
assert len(blocked)==1
|
||||||
|
assert blocked[0][1]["admission"]["state"]=="EXECUTION_READY_FAILED"
|
||||||
|
|
||||||
|
|
||||||
|
def test_system_invariant_snapshot_is_degraded_when_dependency_is_missing():
|
||||||
|
probes=_ready_probes()
|
||||||
|
probes["trusted_builder"]=lambda:False
|
||||||
|
snapshot=acp.system_invariant_snapshot(
|
||||||
|
worker_capabilities=["release:candidate"],
|
||||||
|
required_capabilities=["release:candidate"],
|
||||||
|
probes=probes,
|
||||||
|
)
|
||||||
|
assert snapshot["status"]=="DEGRADED"
|
||||||
|
assert snapshot["execution_ready"] is False
|
||||||
@@ -120,6 +120,10 @@ def test_worker_calls_only_fixed_release_candidate_route(monkeypatch):
|
|||||||
|
|
||||||
def fake_http(base,method,route,token,body=None,timeout=30):
|
def fake_http(base,method,route,token,body=None,timeout=30):
|
||||||
calls.append((base,method,route,token,body))
|
calls.append((base,method,route,token,body))
|
||||||
|
if route=="/health":
|
||||||
|
return 200,{"status":"ok"}
|
||||||
|
if route=="/createReleaseCandidate" and body=={}:
|
||||||
|
return 400,{"error":"request_invalid"}
|
||||||
if route.endswith("/claim"):
|
if route.endswith("/claim"):
|
||||||
return 200,{"lease_token":"l"*32,"attempt":1}
|
return 200,{"lease_token":"l"*32,"attempt":1}
|
||||||
if route.endswith("/heartbeat"):
|
if route.endswith("/heartbeat"):
|
||||||
@@ -135,6 +139,7 @@ def test_worker_calls_only_fixed_release_candidate_route(monkeypatch):
|
|||||||
|
|
||||||
monkeypatch.setattr(worker,"http_json",fake_http)
|
monkeypatch.setattr(worker,"http_json",fake_http)
|
||||||
monkeypatch.setattr(worker,"audit",lambda *args,**kwargs:None)
|
monkeypatch.setattr(worker,"audit",lambda *args,**kwargs:None)
|
||||||
|
monkeypatch.setattr(worker,"write_liveness",lambda *args,**kwargs:None)
|
||||||
monkeypatch.setattr(worker.Heartbeat,"first",lambda self:None)
|
monkeypatch.setattr(worker.Heartbeat,"first",lambda self:None)
|
||||||
monkeypatch.setattr(worker.Heartbeat,"close",lambda self:None)
|
monkeypatch.setattr(worker.Heartbeat,"close",lambda self:None)
|
||||||
monkeypatch.setattr(worker.Heartbeat,"ensure",lambda self:None)
|
monkeypatch.setattr(worker.Heartbeat,"ensure",lambda self:None)
|
||||||
@@ -146,7 +151,12 @@ def test_worker_calls_only_fixed_release_candidate_route(monkeypatch):
|
|||||||
release_token="r"*32,
|
release_token="r"*32,
|
||||||
) is True
|
) is True
|
||||||
|
|
||||||
release_calls=[item for item in calls if item[0]==worker.RELEASE_API]
|
release_calls=[
|
||||||
|
item for item in calls
|
||||||
|
if item[0]==worker.RELEASE_API
|
||||||
|
and item[2]=="/createReleaseCandidate"
|
||||||
|
and item[4]!={}
|
||||||
|
]
|
||||||
assert len(release_calls)==1
|
assert len(release_calls)==1
|
||||||
assert release_calls[0][1]=="POST"
|
assert release_calls[0][1]=="POST"
|
||||||
assert release_calls[0][2]=="/createReleaseCandidate"
|
assert release_calls[0][2]=="/createReleaseCandidate"
|
||||||
|
|||||||
Reference in New Issue
Block a user