"""Durable Job side-effect executor for the AIOS campaign control plane.""" from __future__ import annotations import argparse import base64 from dataclasses import dataclass, field, replace from datetime import datetime, timedelta, timezone import json from pathlib import Path from typing import Any, Callable, Mapping from aios_authority_git_executor import AuthorityGitExecutor, AuthorityGitError from aios_campaign_recovery import RecoveryEvidenceError, load_recovered_authorization from aios_gitea_machine_broker import CampaignCapabilityReceipt, GiteaMachineBroker, MachineBrokerError from aios_gitops_job_controller import GitOpsJobController, JobError, JobState, JobStore, ServiceUnavailable TASK_ID = "AIOS_AUTONOMOUS_RELEASE_CAMPAIGN_V1" CAMPAIGN_ID = "AIOS-AUTONOMOUS-RELEASE-CAMPAIGN-20260807" REPOSITORY = "aios/aios" BRANCH = "agent/autonomous-release-campaign-v1-20260807" EXPECTED_BRANCH_OID = "bba675fab3b7e0c5fba7066720252b3c91b9f20d" AUTHORIZATION_DIGEST = "6b0bf550eb702126e967eeff2d561fa808444b924d8a71f5399accad508ed3b2" LINEAGE_ANCHOR = "69b45e81dcd2c3707c367003b08c55fd361faae8" class JobExecutionError(RuntimeError): pass @dataclass(slots=True) class DurableJobExecutor: controller: GitOpsJobController authority: AuthorityGitExecutor broker: GiteaMachineBroker recovery_path: Path clock: Callable[[], datetime] = field(default=lambda: datetime.now(timezone.utc)) lease_seconds: int = 60 _persistence_halted: bool = field(default=False, init=False, repr=False) def __post_init__(self) -> None: try: self.controller.assert_effect_admission() except JobError as exc: if str(exc) != "EFFECT_ADMISSION_LATCHED": raise self._persistence_halted = True def _request(self, job_id: str) -> tuple[dict[str, Any], dict[str, Any]]: row = self.controller.store.read(job_id) try: request = json.loads(row["request_json"]) except (KeyError, TypeError, json.JSONDecodeError) as exc: raise JobExecutionError("JOB_REQUEST_INVALID") from exc binding = dict(request.get("binding") or {}) # Payload location is derived from and protected by payload_digest, but # is intentionally outside the idempotency binding. Execution still # needs it after a durable round trip. for field_name in ("payload_ref", "payload_size"): if field_name in request: binding[field_name] = request[field_name] return row, binding def _validate_campaign(self, request: Mapping[str, Any]) -> str: operation = str(request.get("operation", "")) expected_branch_oid = request.get("expected_branch_oid") required = { "task_id": TASK_ID, "campaign_id": CAMPAIGN_ID, "repository": REPOSITORY, "branch": BRANCH, "authorization_recovery_digest": AUTHORIZATION_DIGEST, } for key, expected in required.items(): if request.get(key) != expected: raise JobExecutionError(f"JOB_{key.upper()}_MISMATCH") new_oid = request.get("new_branch_oid") if not isinstance(new_oid, str) or len(new_oid) != 40 or any(ch not in "0123456789abcdef" for ch in new_oid): raise JobExecutionError("NEW_BRANCH_OID_INVALID") if not isinstance(expected_branch_oid, str) or len(expected_branch_oid) != 40 or any(ch not in "0123456789abcdef" for ch in expected_branch_oid): raise JobExecutionError("EXPECTED_BRANCH_OID_INVALID") if operation == "ADVANCE_EXISTING_CAMPAIGN_BRANCH" and expected_branch_oid == new_oid: raise JobExecutionError("EXPECTED_BRANCH_OID_INVALID") if operation != "ADVANCE_EXISTING_CAMPAIGN_BRANCH" and expected_branch_oid != new_oid: raise JobExecutionError("POST_ADVANCE_HEAD_BINDING_INVALID") try: recovery = load_recovered_authorization(self.recovery_path) except RecoveryEvidenceError as exc: raise JobExecutionError(str(exc)) from exc if recovery["digest"] != AUTHORIZATION_DIGEST: raise JobExecutionError("RECOVERY_DIGEST_MISMATCH") for key, expected in required.items(): if key == "campaign_id": actual = recovery.get(key) elif key == "authorization_recovery_digest": continue else: actual = recovery.get({"expected_branch_oid": "expected_branch_oid"}.get(key, key)) if actual is not None and actual != expected: raise JobExecutionError("RECOVERY_BINDING_MISMATCH") if recovery.get("expected_branch_oid") != EXPECTED_BRANCH_OID: raise JobExecutionError("RECOVERY_BRANCH_OID_MISMATCH") authority_head = self.authority.read_ref(f"refs/heads/{BRANCH}") if authority_head not in {expected_branch_oid, new_oid}: raise JobExecutionError("AUTHORITY_BRANCH_EXPECTATION_MISMATCH") if not self.authority.ancestry_check(expected_branch_oid, new_oid): raise JobExecutionError("NEW_OID_NOT_DESCENDANT") if not self.authority.ancestry_check(EXPECTED_BRANCH_OID, new_oid): raise JobExecutionError("NEW_OID_OUTSIDE_AUTHORIZED_LINEAGE") if operation == "RECONCILE_AUTHORITY_MAIN": old_main = request.get("expected_main_oid") if ( not isinstance(old_main, str) or len(old_main) != 40 or any(ch not in "0123456789abcdef" for ch in old_main) ): raise JobExecutionError("MAIN_RECONCILIATION_BINDING_INVALID") if not self.authority.ancestry_check(old_main, new_oid): raise JobExecutionError("MAIN_OID_NOT_DESCENDANT") identity = self.broker.verify_identity() if identity.get("repository") != REPOSITORY or identity.get("login") == "aiman": raise JobExecutionError("MACHINE_IDENTITY_SCOPE_INVALID") return new_oid def dry_run(self, job_id: str) -> dict[str, Any]: row, request = self._request(job_id) operation = str(request.get("operation") or "") if operation not in { "ADVANCE_EXISTING_CAMPAIGN_BRANCH", "COMMIT_FILE", "SYNC_TASK_BRANCH_TO_GITEA", "CREATE_PR", "PUBLISH_CHECK", "PROTECTED_MERGE", "RECONCILE_AUTHORITY_MAIN", }: raise JobExecutionError("OPERATION_NOT_ALLOWED") self._validate_campaign(request) if operation == "COMMIT_FILE": self._commit_file_spec(row, request) observation = self._observe_operation(request) result = { "mode": "DRY_RUN", "operation": operation, "decision": "SAFE_TO_ADVANCE", "effect_performed": False, "readback": observation, } if operation == "ADVANCE_EXISTING_CAMPAIGN_BRANCH": result.update({ "authority_action": "CAS_REQUIRED", "gitea_action": "ALREADY_IN_SYNC" if observation["verification"] == "VERIFIED" else "FAST_FORWARD_REQUIRED", "previous_branch_oid": request["expected_branch_oid"], "final_branch_oid": request["new_branch_oid"], "force_push": False, }) return result def execute(self, job_id: str, *, dry_run: bool = False) -> dict[str, Any]: if dry_run: return self.dry_run(job_id) if self._persistence_halted: raise JobExecutionError("PERSISTENCE_FAILURE_LATCHED") try: return self._execute_unprotected(job_id) except ServiceUnavailable: self._persistence_halted = True raise except Exception as exc: try: row = self.controller.store.read(job_id) state = JobState(row["state"]) if state in { JobState.EXECUTING, JobState.LOCAL_VERIFIED, JobState.REMOTE_SYNCING, JobState.REMOTE_VERIFIED, }: self.controller.advance( job_id, JobState.RECONCILIATION_REQUIRED, side_effect_state="UNKNOWN", receipt={"error_code": type(exc).__name__, "verification": "UNKNOWN"}, ) elif state not in { JobState.COMPLETED, JobState.MANUAL_OWNER_REVIEW_REQUIRED, JobState.RECOVERABLE_FAILURE, JobState.RECONCILIATION_REQUIRED, }: self.controller.advance( job_id, JobState.RECOVERABLE_FAILURE, side_effect_state="NONE", receipt={"error_code": type(exc).__name__, "effect_performed": False}, ) except ServiceUnavailable: self._persistence_halted = True raise def _execute_unprotected(self, job_id: str) -> dict[str, Any]: row, request = self._request(job_id) state = JobState(row["state"]) if state == JobState.COMPLETED: return self.controller.store._receipt(row, duplicate=True).as_dict() if state in { JobState.RECOVERABLE_FAILURE, JobState.RECONCILIATION_REQUIRED, JobState.RECONCILING, JobState.EXECUTING, JobState.LOCAL_VERIFIED, JobState.REMOTE_SYNCING, JobState.REMOTE_VERIFIED, }: return self.reconcile(job_id) operation = str(request.get("operation") or "") if operation not in { "ADVANCE_EXISTING_CAMPAIGN_BRANCH", "COMMIT_FILE", "SYNC_TASK_BRANCH_TO_GITEA", "CREATE_PR", "PUBLISH_CHECK", "PROTECTED_MERGE", "RECONCILE_AUTHORITY_MAIN", }: raise JobExecutionError("OPERATION_NOT_ALLOWED") try: self.controller.assert_effect_admission(exclude_job_id=job_id) except JobError as exc: if str(exc) == "EFFECT_ADMISSION_LATCHED": self._persistence_halted = True raise JobExecutionError("PERSISTENCE_FAILURE_LATCHED") from exc raise self._validate_campaign(request) if operation == "COMMIT_FILE": self._commit_file_spec(row, request) row = self._prepare_intent(job_id, row, request) generation = int(request.get("generation", "1")) self.controller.assert_execution_lease(job_id, generation=generation) before = self._observe_operation(request) if before["verification"] == "VERIFIED": return self._complete(job_id, request, {"reconciled_before_effect": True}, before) if before["verification"] != "ABSENT": return self._record_unknown(job_id, request, before) effect = self._perform_effect(job_id, row, request, generation) after = self._observe_operation(request) if after["verification"] != "VERIFIED": return self._record_unknown(job_id, request, after, effect=effect) return self._complete(job_id, request, effect, after) def _prepare_intent( self, job_id: str, row: Mapping[str, Any], request: Mapping[str, Any] ) -> dict[str, Any]: state = JobState(row["state"]) if state == JobState.REQUESTED: self.controller.advance( job_id, JobState.AUTHORIZED, side_effect_state="AUTHORIZED", receipt={"authorization_digest": AUTHORIZATION_DIGEST, "task_id": TASK_ID}, expected_revision=int(row["revision"]), ) row = self.controller.store.read(job_id) state = JobState(row["state"]) if state == JobState.AUTHORIZED: deadline = self.clock().astimezone(timezone.utc) + timedelta(seconds=self.lease_seconds) self.controller.lease_job( job_id, lease_id=f"lease-{job_id}", lineage_anchor_oid=LINEAGE_ANCHOR, current_head_oid=str(request["expected_branch_oid"]), generation=int(request.get("generation", "1")), lease_expires_at=deadline.isoformat(), expected_revision=int(row["revision"]), ) row = self.controller.store.read(job_id) state = JobState(row["state"]) if state == JobState.LEASED: self.controller.advance( job_id, JobState.WORKTREE_READY, side_effect_state="WORKTREE_READY", expected_revision=int(row["revision"]), ) row = self.controller.store.read(job_id) state = JobState(row["state"]) if state == JobState.WORKTREE_READY: self.controller.advance( job_id, JobState.EXECUTING, side_effect_state="INTENT_COMMITTED", receipt={ "effect_id": job_id, "operation": request["operation"], "input_digest": row["payload_digest"], "intent_committed": True, }, expected_revision=int(row["revision"]), ) row = self.controller.store.read(job_id) if JobState(row["state"]) != JobState.EXECUTING: raise JobExecutionError("JOB_EXECUTION_STATE_INVALID") return dict(row) def _perform_effect( self, job_id: str, row: Mapping[str, Any], request: Mapping[str, Any], generation: int, ) -> dict[str, Any]: operation = str(request["operation"]) new_oid = str(request["new_branch_oid"]) self.controller.assert_execution_lease(job_id, generation=generation) if operation == "ADVANCE_EXISTING_CAMPAIGN_BRANCH": expected = str(request["expected_branch_oid"]) current = self.authority.read_ref(f"refs/heads/{BRANCH}") authority_oid = new_oid if current == new_oid else self.authority.advance_task_branch_CAS( f"refs/heads/{BRANCH}", expected, new_oid ) self.controller.assert_execution_lease(job_id, generation=generation) remote = self.broker.readback() if remote != new_oid: if hasattr(self.broker, "receipt"): self.broker.receipt = replace(self.broker.receipt, expected_oid=expected) self.broker.push_branch(new_oid) return {"authority_oid": authority_oid, "network_oid": new_oid, "force_push": False} if operation == "SYNC_TASK_BRANCH_TO_GITEA": return {"final_network_oid": self.broker.push_branch(new_oid), "force_push": False} if operation == "CREATE_PR": return dict(self.broker.create_pr( "AIOS autonomous release control plane", "Durable campaign continuation; exact head and recovery evidence are bound in the Job.", )) if operation == "PUBLISH_CHECK": return dict(self.broker.status( str(request.get("check_state", "success")), str(request.get("check_description", "Durable campaign control-plane check passed")), str(request.get("check_context", "aios/durable-release")), )) if operation == "PROTECTED_MERGE": number = int(request.get("pr_number", "0")) self.broker.transport.expected_merge_head = request["expected_branch_oid"] if request.get("expected_main_oid") is not None: self.broker.transport.expected_merge_base = request["expected_main_oid"] return dict(self.broker.merge(number)) if operation == "RECONCILE_AUTHORITY_MAIN": old = request.get("expected_main_oid") if not isinstance(old, str): raise JobExecutionError("MAIN_RECONCILIATION_BINDING_INVALID") if self.broker.transport.read_branch("main") != new_oid: raise JobExecutionError("GITEA_MAIN_EXPECTATION_MISMATCH") return {"authority_main": self.authority.ff_CAS_main_after_verified_protected_merge(old, new_oid)} if operation == "COMMIT_FILE": path, content = self._commit_file_spec(row, request) return { "new_oid": self.authority.commit_file_in_isolated_worktree( base_oid=row["expected_branch_oid"], files={path: content}, message="chore(gitops): durable commit-file", worktree_name=job_id, ) } raise JobExecutionError("OPERATION_NOT_ALLOWED") def _observe_operation(self, request: Mapping[str, Any]) -> dict[str, Any]: operation = str(request["operation"]) new_oid = str(request["new_branch_oid"]) if operation == "ADVANCE_EXISTING_CAMPAIGN_BRANCH": authority = self.authority.read_ref(f"refs/heads/{BRANCH}") remote = self.broker.readback() if authority == new_oid and remote == new_oid: return {"verification": "VERIFIED", "authority_oid": authority, "remote_oid": remote} expected = request["expected_branch_oid"] if authority == expected and remote in {None, expected}: return {"verification": "ABSENT", "authority_oid": authority, "remote_oid": remote} return {"verification": "UNKNOWN", "blocker": "PARTIAL_OR_CONFLICTING_BRANCH_EFFECT", "authority_oid": authority, "remote_oid": remote} if operation == "SYNC_TASK_BRANCH_TO_GITEA": remote = self.broker.readback() if remote == new_oid: return {"verification": "VERIFIED", "remote_oid": remote} if remote in {None, EXPECTED_BRANCH_OID}: return {"verification": "ABSENT", "remote_oid": remote} return {"verification": "UNKNOWN", "blocker": "REMOTE_BRANCH_CONFLICT", "remote_oid": remote} if operation == "RECONCILE_AUTHORITY_MAIN": current = self.authority.read_ref("refs/heads/main") if current == new_oid: return {"verification": "VERIFIED", "authority_main": current} if current == request.get("expected_main_oid"): return {"verification": "ABSENT", "authority_main": current} return {"verification": "UNKNOWN", "blocker": "AUTHORITY_MAIN_CONFLICT", "authority_main": current} if operation == "CREATE_PR" and hasattr(self.broker, "read_pr"): value = self.broker.read_pr(BRANCH, "main", new_oid) return {"verification": "VERIFIED" if value else "ABSENT", "pr": value} if operation == "PUBLISH_CHECK" and hasattr(self.broker, "read_status"): value = self.broker.read_status( new_oid, str(request.get("check_context", "aios/durable-release")) ) if value is None: return {"verification": "ABSENT"} expected_state = str(request.get("check_state", "success")) return {"verification": "VERIFIED" if value.get("status") == expected_state else "UNKNOWN", "status": dict(value), **({} if value.get("status") == expected_state else {"blocker": "STATUS_READBACK_MISMATCH"})} if operation == "PROTECTED_MERGE" and hasattr(self.broker, "read_merge"): value = self.broker.read_merge(int(request.get("pr_number", "0"))) if value is None: return {"verification": "ABSENT"} if value.get("merged") is True and value.get("head_oid") == request["expected_branch_oid"]: return {"verification": "VERIFIED", "merge": dict(value)} if value.get("merged") is False: return {"verification": "ABSENT", "merge": dict(value)} return {"verification": "UNKNOWN", "blocker": "MERGE_READBACK_MISMATCH", "merge": dict(value)} return {"verification": "UNKNOWN", "blocker": "EXACT_READBACK_UNSUPPORTED"} def _complete( self, job_id: str, request: Mapping[str, Any], effect: Mapping[str, Any], observation: Mapping[str, Any] ) -> dict[str, Any]: row = self.controller.store.read(job_id) receipt = { "effect_id": job_id, "operation": request["operation"], "effect": dict(effect), "verification": "VERIFIED", "readback": dict(observation), } completed = self.controller.advance( job_id, JobState.COMPLETED, side_effect_state="VERIFIED", receipt=receipt, expected_revision=int(row["revision"]), ) return completed.as_dict() def _record_unknown( self, job_id: str, request: Mapping[str, Any], observation: Mapping[str, Any], *, effect: Mapping[str, Any] | None = None, ) -> dict[str, Any]: row = self.controller.store.read(job_id) receipt = { "effect_id": job_id, "operation": request["operation"], "effect": dict(effect) if effect is not None else None, "verification": "UNKNOWN", "blocker": observation.get("blocker", "READBACK_INCONCLUSIVE"), "readback": dict(observation), } pending = self.controller.advance( job_id, JobState.RECONCILIATION_REQUIRED, side_effect_state="UNKNOWN", receipt=receipt, expected_revision=int(row["revision"]), ) return pending.as_dict() def reconcile(self, job_id: str) -> dict[str, Any]: def observer(_current: Mapping[str, Any]) -> Mapping[str, Any]: _row, request = self._request(job_id) self._validate_campaign(request) observation = self._observe_operation(request) if observation["verification"] == "VERIFIED": return { "state": JobState.COMPLETED.value, "side_effect_state": "VERIFIED", "receipt": { "effect_id": job_id, "operation": request["operation"], "effect": None, "verification": "VERIFIED", "readback": observation, "reconciled": True, }, } if observation["verification"] == "ABSENT": return { "state": JobState.REQUESTED.value, "side_effect_state": "RECONCILED_ABSENT", "receipt": { "effect_id": job_id, "operation": request["operation"], "verification": "ABSENT", "readback": observation, "reconciled": True, }, } return { "state": JobState.RECONCILIATION_REQUIRED.value, "side_effect_state": "UNKNOWN", "receipt": { "effect_id": job_id, "operation": request["operation"], "verification": "UNKNOWN", "blocker": observation.get("blocker", "READBACK_INCONCLUSIVE"), "readback": observation, "reconciled": True, }, } result = self.controller.reconcile(job_id, observer).as_dict() if result["state"] in {JobState.COMPLETED.value, JobState.REQUESTED.value}: self._persistence_halted = False return result def _commit_file_spec( self, row: Mapping[str, Any], request: Mapping[str, Any] ) -> tuple[str, bytes]: payload_ref = request.get("payload_ref") if not payload_ref: raise JobExecutionError("PAYLOAD_REQUIRED") payload = self.controller.store.payloads.read_verified(payload_ref, row["payload_digest"], int(request["payload_size"])) try: spec = json.loads(payload.decode()) path = str(spec["path"]) content = base64.b64decode(spec["content_base64"], validate=True) except Exception as exc: raise JobExecutionError("COMMIT_FILE_PAYLOAD_INVALID") from exc return path, content def build_from_environment(job_root: Path) -> DurableJobExecutor: from aios_gitea_machine_transport import GiteaMachineTransport from aios_gitea_machine_broker import MachineCredentialProjection account = __import__("os").environ.get("AIOS_GITEA_MACHINE_ACCOUNT", "aios-autonomy-agent") api = __import__("os").environ.get("AIOS_GITEA_API", "https://g.cnnetmkt.top/git/api/v1") remote = __import__("os").environ.get("AIOS_GITEA_REMOTE", "https://g.cnnetmkt.top/git/aios/aios.git") authority_dir = Path(__import__("os").environ.get("AIOS_GITOPS_AUTHORITY_GIT_DIR", "/data/aios-gitops/repo.git")) worktree_root = Path(__import__("os").environ.get("AIOS_GITOPS_WORKTREE_ROOT", "/var/lib/aios-gitops/worktrees")) recovery = Path(__import__("os").environ.get("AIOS_RELEASE_RECOVERY_RECEIPT", "/var/lib/aios-gitops/release-campaigns/AIOS-AUTONOMOUS-RELEASE-CAMPAIGN-20260807/authorization-recovery.json")) transport = GiteaMachineTransport(api, REPOSITORY, remote, authority_dir) credential = MachineCredentialProjection.from_mapping({"scheme": "systemd-credential", "name": "gitea-machine-token", "account": account}) receipt = CampaignCapabilityReceipt(CAMPAIGN_ID, TASK_ID, REPOSITORY, BRANCH, EXPECTED_BRANCH_OID, frozenset({"exact_task_branch_push", "pr_create_update", "status_publish", "protected_merge", "remote_readback"}), frozenset({"direct_main_push", "force_push", "tag_update", "tag_delete", "baseline_write", "production_deploy"})) return DurableJobExecutor(GitOpsJobController(JobStore(job_root)), AuthorityGitExecutor(authority_dir, worktree_root), GiteaMachineBroker(transport, credential, receipt), recovery) def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser() parser.add_argument("--job-root", required=True, type=Path) parser.add_argument("--job-id", required=True) parser.add_argument("--dry-run", action="store_true") args = parser.parse_args(argv) result = build_from_environment(args.job_root).execute(args.job_id, dry_run=args.dry_run) print(json.dumps(result, sort_keys=True)) return 0 if __name__ == "__main__": raise SystemExit(main())