Compare commits
33
Commits
@@ -0,0 +1,315 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""HTTP API for AIOS external-agent online task distribution.
|
||||||
|
|
||||||
|
The service is intended to bind to loopback and be published through the
|
||||||
|
existing TLS reverse proxy. Control operations use a server-managed control
|
||||||
|
Bearer token. Agents receive individual Bearer tokens at registration; only
|
||||||
|
token hashes are persisted.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import hmac
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
from typing import Any, Mapping
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
from aios_agent_task_protocol import TaskProtocolError, TaskStore
|
||||||
|
|
||||||
|
|
||||||
|
MAX_BODY = 1024 * 1024
|
||||||
|
TASK_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{2,179}$")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class APIConfig:
|
||||||
|
db_path: Path
|
||||||
|
control_token_path: Path
|
||||||
|
host: str = "127.0.0.1"
|
||||||
|
port: int = 18940
|
||||||
|
token_pepper: str = ""
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_env(cls) -> "APIConfig":
|
||||||
|
return cls(
|
||||||
|
db_path=Path(os.getenv(
|
||||||
|
"AIOS_AGENT_TASK_DB",
|
||||||
|
"/var/lib/aios-agent-task/state.sqlite3",
|
||||||
|
)),
|
||||||
|
control_token_path=Path(os.getenv(
|
||||||
|
"AIOS_AGENT_TASK_CONTROL_TOKEN_FILE",
|
||||||
|
"/run/credentials/aios-agent-task-api.service/control_token",
|
||||||
|
)),
|
||||||
|
host=os.getenv("AIOS_AGENT_TASK_HOST", "127.0.0.1"),
|
||||||
|
port=int(os.getenv("AIOS_AGENT_TASK_PORT", "18940")),
|
||||||
|
token_pepper=os.getenv("AIOS_AGENT_TASK_TOKEN_PEPPER", ""),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class AgentTaskApplication:
|
||||||
|
def __init__(self, config: APIConfig) -> None:
|
||||||
|
self.config = config
|
||||||
|
self.store = TaskStore(
|
||||||
|
config.db_path,
|
||||||
|
token_pepper=config.token_pepper,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _control_token(self) -> str:
|
||||||
|
path = self.config.control_token_path
|
||||||
|
if path.is_symlink() or not path.is_file():
|
||||||
|
raise TaskProtocolError("CONTROL_CREDENTIAL_UNAVAILABLE", 503)
|
||||||
|
value = path.read_text(encoding="utf-8").strip()
|
||||||
|
if len(value) < 32:
|
||||||
|
raise TaskProtocolError("CONTROL_CREDENTIAL_INVALID", 503)
|
||||||
|
return value
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _bearer(headers: Mapping[str, str]) -> str:
|
||||||
|
value = str(headers.get("Authorization") or "")
|
||||||
|
if not value.startswith("Bearer "):
|
||||||
|
return ""
|
||||||
|
return value[7:].strip()
|
||||||
|
|
||||||
|
def authenticate_control(self, headers: Mapping[str, str]) -> None:
|
||||||
|
supplied = self._bearer(headers)
|
||||||
|
expected = self._control_token()
|
||||||
|
if not supplied or not hmac.compare_digest(supplied, expected):
|
||||||
|
raise TaskProtocolError("UNAUTHORIZED", 401)
|
||||||
|
|
||||||
|
def authenticate_agent(self, headers: Mapping[str, str]) -> dict[str, Any]:
|
||||||
|
return self.store.authenticate_agent(self._bearer(headers))
|
||||||
|
|
||||||
|
def principal(self, headers: Mapping[str, str]) -> tuple[str, dict[str, Any] | None]:
|
||||||
|
supplied = self._bearer(headers)
|
||||||
|
if not supplied:
|
||||||
|
raise TaskProtocolError("UNAUTHORIZED", 401)
|
||||||
|
try:
|
||||||
|
expected = self._control_token()
|
||||||
|
except TaskProtocolError:
|
||||||
|
expected = ""
|
||||||
|
if expected and hmac.compare_digest(supplied, expected):
|
||||||
|
return "control", None
|
||||||
|
return "agent", self.store.authenticate_agent(supplied)
|
||||||
|
|
||||||
|
def health(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"status": "ok",
|
||||||
|
"schema": "aios.agent-task.v1",
|
||||||
|
"task_states": [
|
||||||
|
"QUEUED", "CLAIMED", "RUNNING", "VERIFYING",
|
||||||
|
"SUCCEEDED", "FAILED", "EXPIRED", "CANCELLED",
|
||||||
|
],
|
||||||
|
"lease_recovery": True,
|
||||||
|
"atomic_claim": True,
|
||||||
|
"idempotent_result_submission": True,
|
||||||
|
"server_side_verification_required": True,
|
||||||
|
"plaintext_credentials_persisted": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
def register_agent(self, headers: Mapping[str, str], body: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
|
self.authenticate_control(headers)
|
||||||
|
return self.store.register_agent(
|
||||||
|
agent_id=str(body.get("agent_id") or ""),
|
||||||
|
agent_type=str(body.get("agent_type") or ""),
|
||||||
|
agent_version=str(body.get("agent_version") or ""),
|
||||||
|
capabilities=body.get("capabilities") or [],
|
||||||
|
metadata=body.get("metadata") if isinstance(body.get("metadata"), Mapping) else {},
|
||||||
|
)
|
||||||
|
|
||||||
|
def create_task(self, headers: Mapping[str, str], body: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
|
self.authenticate_control(headers)
|
||||||
|
return self.store.create_task(
|
||||||
|
task_id=str(body.get("task_id") or "") or None,
|
||||||
|
goal=str(body.get("goal") or ""),
|
||||||
|
instructions=str(body.get("instructions") or ""),
|
||||||
|
success_criteria=body.get("success_criteria"),
|
||||||
|
required_capabilities=body.get("required_capabilities") or [],
|
||||||
|
constraints=body.get("constraints") if isinstance(body.get("constraints"), Mapping) else {},
|
||||||
|
metadata=body.get("metadata") if isinstance(body.get("metadata"), Mapping) else {},
|
||||||
|
deadline=str(body.get("deadline") or "") or None,
|
||||||
|
priority=body.get("priority", 0),
|
||||||
|
max_attempts=body.get("max_attempts", 3),
|
||||||
|
idempotency_key=str(body.get("idempotency_key") or ""),
|
||||||
|
)
|
||||||
|
|
||||||
|
def poll(self, headers: Mapping[str, str], body: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
|
agent = self.authenticate_agent(headers)
|
||||||
|
tasks = self.store.poll_tasks(
|
||||||
|
agent_id=agent["agent_id"],
|
||||||
|
limit=int(body.get("limit", 20)),
|
||||||
|
)
|
||||||
|
return {"agent_id": agent["agent_id"], "tasks": tasks}
|
||||||
|
|
||||||
|
def claim(self, task_id: str, headers: Mapping[str, str], body: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
|
agent = self.authenticate_agent(headers)
|
||||||
|
return self.store.claim_task(
|
||||||
|
task_id=task_id,
|
||||||
|
agent_id=agent["agent_id"],
|
||||||
|
lease_seconds=int(body.get("lease_seconds", 300)),
|
||||||
|
)
|
||||||
|
|
||||||
|
def heartbeat(self, task_id: str, headers: Mapping[str, str], body: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
|
agent = self.authenticate_agent(headers)
|
||||||
|
return self.store.heartbeat(
|
||||||
|
task_id=task_id,
|
||||||
|
agent_id=agent["agent_id"],
|
||||||
|
lease_token=str(body.get("lease_token") or ""),
|
||||||
|
progress=body.get("progress") if isinstance(body.get("progress"), Mapping) else {},
|
||||||
|
lease_seconds=int(body.get("lease_seconds", 300)),
|
||||||
|
)
|
||||||
|
|
||||||
|
def result(self, task_id: str, headers: Mapping[str, str], body: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
|
agent = self.authenticate_agent(headers)
|
||||||
|
result = body.get("result")
|
||||||
|
if not isinstance(result, Mapping):
|
||||||
|
raise TaskProtocolError("RESULT_INVALID", 400)
|
||||||
|
return self.store.submit_result(
|
||||||
|
task_id=task_id,
|
||||||
|
agent_id=agent["agent_id"],
|
||||||
|
lease_token=str(body.get("lease_token") or ""),
|
||||||
|
idempotency_key=str(body.get("idempotency_key") or ""),
|
||||||
|
result=result,
|
||||||
|
)
|
||||||
|
|
||||||
|
def verify(self, task_id: str, headers: Mapping[str, str], body: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
|
self.authenticate_control(headers)
|
||||||
|
return self.store.verify_task(
|
||||||
|
task_id=task_id,
|
||||||
|
decision=str(body.get("decision") or ""),
|
||||||
|
verification=body.get("verification") if isinstance(body.get("verification"), Mapping) else {},
|
||||||
|
)
|
||||||
|
|
||||||
|
def cancel(self, task_id: str, headers: Mapping[str, str], body: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
|
self.authenticate_control(headers)
|
||||||
|
return self.store.cancel_task(task_id=task_id, reason=str(body.get("reason") or ""))
|
||||||
|
|
||||||
|
def get_task(self, task_id: str, headers: Mapping[str, str]) -> dict[str, Any]:
|
||||||
|
kind, agent = self.principal(headers)
|
||||||
|
task = self.store.get_task(task_id)
|
||||||
|
if kind == "agent" and task.get("claimed_by") != agent["agent_id"]:
|
||||||
|
raise TaskProtocolError("FORBIDDEN", 403)
|
||||||
|
return task
|
||||||
|
|
||||||
|
def events(self, task_id: str, headers: Mapping[str, str]) -> dict[str, Any]:
|
||||||
|
self.authenticate_control(headers)
|
||||||
|
return {"task_id": task_id, "events": self.store.get_events(task_id)}
|
||||||
|
|
||||||
|
|
||||||
|
class Handler(BaseHTTPRequestHandler):
|
||||||
|
app: AgentTaskApplication | None = None
|
||||||
|
server_version = "AIOSAgentTaskAPI/1"
|
||||||
|
|
||||||
|
def log_message(self, _format: str, *_args: Any) -> None:
|
||||||
|
return
|
||||||
|
|
||||||
|
def _headers(self) -> dict[str, str]:
|
||||||
|
return {key: value for key, value in self.headers.items()}
|
||||||
|
|
||||||
|
def _json(self, value: Any, status: int = 200) -> None:
|
||||||
|
payload = json.dumps(value, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||||
|
self.send_response(status)
|
||||||
|
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||||||
|
self.send_header("Cache-Control", "no-store")
|
||||||
|
self.send_header("Content-Length", str(len(payload)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(payload)
|
||||||
|
|
||||||
|
def _body(self) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
size = int(self.headers.get("Content-Length", "0"))
|
||||||
|
except ValueError as exc:
|
||||||
|
raise TaskProtocolError("CONTENT_LENGTH_INVALID", 400) from exc
|
||||||
|
if size < 0 or size > MAX_BODY:
|
||||||
|
raise TaskProtocolError("REQUEST_TOO_LARGE", 413)
|
||||||
|
raw = self.rfile.read(size) if size else b"{}"
|
||||||
|
try:
|
||||||
|
value = json.loads(raw.decode("utf-8"))
|
||||||
|
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||||
|
raise TaskProtocolError("JSON_INVALID", 400) from exc
|
||||||
|
if not isinstance(value, dict):
|
||||||
|
raise TaskProtocolError("JSON_OBJECT_REQUIRED", 400)
|
||||||
|
return value
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _task_route(path: str) -> tuple[str, str | None]:
|
||||||
|
match = re.fullmatch(
|
||||||
|
r"/v1/tasks/([A-Za-z0-9][A-Za-z0-9._:-]{2,179})(?:/(claim|heartbeat|result|verify|cancel|events))?",
|
||||||
|
path,
|
||||||
|
)
|
||||||
|
if match:
|
||||||
|
return match.group(1), match.group(2)
|
||||||
|
return "", None
|
||||||
|
|
||||||
|
def _run(self, callback) -> None:
|
||||||
|
try:
|
||||||
|
callback()
|
||||||
|
except TaskProtocolError as exc:
|
||||||
|
self._json({"error": exc.code}, exc.http_status)
|
||||||
|
except Exception as exc:
|
||||||
|
self._json({"error": "INTERNAL_ERROR", "detail": type(exc).__name__}, 500)
|
||||||
|
|
||||||
|
def do_GET(self) -> None:
|
||||||
|
def action() -> None:
|
||||||
|
if self.app is None:
|
||||||
|
raise TaskProtocolError("SERVICE_UNAVAILABLE", 503)
|
||||||
|
path = urlparse(self.path).path.rstrip("/") or "/"
|
||||||
|
if path == "/health":
|
||||||
|
self._json(self.app.health())
|
||||||
|
return
|
||||||
|
task_id, suffix = self._task_route(path)
|
||||||
|
if task_id and suffix is None:
|
||||||
|
self._json(self.app.get_task(task_id, self._headers()))
|
||||||
|
return
|
||||||
|
if task_id and suffix == "events":
|
||||||
|
self._json(self.app.events(task_id, self._headers()))
|
||||||
|
return
|
||||||
|
self._json({"error": "NOT_FOUND"}, 404)
|
||||||
|
self._run(action)
|
||||||
|
|
||||||
|
def do_POST(self) -> None:
|
||||||
|
def action() -> None:
|
||||||
|
if self.app is None:
|
||||||
|
raise TaskProtocolError("SERVICE_UNAVAILABLE", 503)
|
||||||
|
path = urlparse(self.path).path.rstrip("/") or "/"
|
||||||
|
body = self._body()
|
||||||
|
if path == "/v1/agents/register":
|
||||||
|
self._json(self.app.register_agent(self._headers(), body), 201)
|
||||||
|
return
|
||||||
|
if path == "/v1/tasks":
|
||||||
|
self._json(self.app.create_task(self._headers(), body), 201)
|
||||||
|
return
|
||||||
|
if path == "/v1/tasks/poll":
|
||||||
|
self._json(self.app.poll(self._headers(), body))
|
||||||
|
return
|
||||||
|
task_id, suffix = self._task_route(path)
|
||||||
|
if not task_id:
|
||||||
|
self._json({"error": "NOT_FOUND"}, 404)
|
||||||
|
return
|
||||||
|
if suffix == "claim":
|
||||||
|
self._json(self.app.claim(task_id, self._headers(), body))
|
||||||
|
elif suffix == "heartbeat":
|
||||||
|
self._json(self.app.heartbeat(task_id, self._headers(), body))
|
||||||
|
elif suffix == "result":
|
||||||
|
self._json(self.app.result(task_id, self._headers(), body))
|
||||||
|
elif suffix == "verify":
|
||||||
|
self._json(self.app.verify(task_id, self._headers(), body))
|
||||||
|
elif suffix == "cancel":
|
||||||
|
self._json(self.app.cancel(task_id, self._headers(), body))
|
||||||
|
else:
|
||||||
|
self._json({"error": "NOT_FOUND"}, 404)
|
||||||
|
self._run(action)
|
||||||
|
|
||||||
|
|
||||||
|
def serve(config: APIConfig | None = None) -> None:
|
||||||
|
config = config or APIConfig.from_env()
|
||||||
|
application = AgentTaskApplication(config)
|
||||||
|
Handler.app = application
|
||||||
|
ThreadingHTTPServer((config.host, config.port), Handler).serve_forever()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
serve()
|
||||||
@@ -0,0 +1,722 @@
|
|||||||
|
"""Owner-controlled CAS-bound execution-envelope rebind for Online Agent Tasks.
|
||||||
|
|
||||||
|
This service exists for a narrow recovery case: a Task was projected/claimed with a read-only
|
||||||
|
source_capture envelope before its already-registered deterministic plan became authoritative.
|
||||||
|
|
||||||
|
It never creates Tasks/Jobs/branches and never accepts arbitrary metadata patches. It only replaces
|
||||||
|
metadata.execution + the derived execution_projection for the same governed binding, after an
|
||||||
|
official NEEDS_REWORK transition has returned the Task to QUEUED with no active lease.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import sqlite3
|
||||||
|
from typing import Any, Callable, Mapping, Sequence
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
from aios_online_agent_execution import (
|
||||||
|
GovernedExecutionBinding,
|
||||||
|
validate_execution_envelope,
|
||||||
|
)
|
||||||
|
|
||||||
|
TASK_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{2,179}$")
|
||||||
|
KEY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{7,159}$")
|
||||||
|
SHA_RE = re.compile(r"^[0-9a-f]{64}$")
|
||||||
|
MAX_BODY = 64 * 1024
|
||||||
|
AUTHORIZATION_ID_ENV = "AIOS_AGENT_TASK_EXECUTION_REBIND_AUTHORIZATION_ID"
|
||||||
|
|
||||||
|
|
||||||
|
class ExecutionRebindError(RuntimeError):
|
||||||
|
def __init__(self, code: str, status: int = 409) -> None:
|
||||||
|
super().__init__(code)
|
||||||
|
self.code = code
|
||||||
|
self.status = status
|
||||||
|
|
||||||
|
|
||||||
|
def _canonical(value: object) -> bytes:
|
||||||
|
return json.dumps(
|
||||||
|
value, sort_keys=True, separators=(",", ":"), ensure_ascii=False
|
||||||
|
).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _sha(value: object) -> str:
|
||||||
|
return hashlib.sha256(_canonical(value)).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _now() -> str:
|
||||||
|
return datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def _require_string(
|
||||||
|
value: object,
|
||||||
|
code: str,
|
||||||
|
pattern: re.Pattern[str] | None = None,
|
||||||
|
) -> str:
|
||||||
|
if not isinstance(value, str) or not value.strip():
|
||||||
|
raise ExecutionRebindError(code, 400)
|
||||||
|
value = value.strip()
|
||||||
|
if pattern is not None and pattern.fullmatch(value) is None:
|
||||||
|
raise ExecutionRebindError(code, 400)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _required_authorization_id() -> str:
|
||||||
|
value = os.environ.get(AUTHORIZATION_ID_ENV, "").strip()
|
||||||
|
if not value:
|
||||||
|
raise RuntimeError("TASK_EXECUTION_REBIND_AUTHORIZATION_ID_MISSING")
|
||||||
|
if KEY_RE.fullmatch(value) is None:
|
||||||
|
raise RuntimeError("TASK_EXECUTION_REBIND_AUTHORIZATION_ID_INVALID")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ExecutionRebindRequest:
|
||||||
|
expected_status: str
|
||||||
|
expected_revision: int
|
||||||
|
expected_attempt: int
|
||||||
|
previous_verification_sha256: str
|
||||||
|
expected_old_execution_digest: str
|
||||||
|
expected_new_execution_digest: str
|
||||||
|
authorization_id: str
|
||||||
|
idempotency_key: str
|
||||||
|
reason: str
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_mapping(cls, value: Mapping[str, Any]) -> "ExecutionRebindRequest":
|
||||||
|
allowed = {
|
||||||
|
"expected_status",
|
||||||
|
"expected_revision",
|
||||||
|
"expected_attempt",
|
||||||
|
"previous_verification_sha256",
|
||||||
|
"expected_old_execution_digest",
|
||||||
|
"expected_new_execution_digest",
|
||||||
|
"authorization_id",
|
||||||
|
"idempotency_key",
|
||||||
|
"reason",
|
||||||
|
}
|
||||||
|
if set(value) != allowed:
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_FIELDS_INVALID", 400)
|
||||||
|
|
||||||
|
status = _require_string(value["expected_status"], "EXPECTED_STATUS_INVALID")
|
||||||
|
if status != "QUEUED":
|
||||||
|
raise ExecutionRebindError("EXPECTED_STATUS_INVALID", 400)
|
||||||
|
|
||||||
|
revision = value["expected_revision"]
|
||||||
|
attempt = value["expected_attempt"]
|
||||||
|
if (
|
||||||
|
not isinstance(revision, int)
|
||||||
|
or isinstance(revision, bool)
|
||||||
|
or revision < 0
|
||||||
|
or not isinstance(attempt, int)
|
||||||
|
or isinstance(attempt, bool)
|
||||||
|
or attempt < 1
|
||||||
|
):
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_INTEGER_INVALID", 400)
|
||||||
|
|
||||||
|
verification = _require_string(
|
||||||
|
value["previous_verification_sha256"], "VERIFICATION_SHA256_INVALID"
|
||||||
|
)
|
||||||
|
old_digest = _require_string(
|
||||||
|
value["expected_old_execution_digest"], "OLD_EXECUTION_DIGEST_INVALID"
|
||||||
|
)
|
||||||
|
new_digest = _require_string(
|
||||||
|
value["expected_new_execution_digest"], "NEW_EXECUTION_DIGEST_INVALID"
|
||||||
|
)
|
||||||
|
for digest, code in (
|
||||||
|
(verification, "VERIFICATION_SHA256_INVALID"),
|
||||||
|
(old_digest, "OLD_EXECUTION_DIGEST_INVALID"),
|
||||||
|
(new_digest, "NEW_EXECUTION_DIGEST_INVALID"),
|
||||||
|
):
|
||||||
|
if SHA_RE.fullmatch(digest) is None:
|
||||||
|
raise ExecutionRebindError(code, 400)
|
||||||
|
if old_digest == new_digest:
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_DIGEST_UNCHANGED", 400)
|
||||||
|
|
||||||
|
authorization = _require_string(
|
||||||
|
value["authorization_id"], "AUTHORIZATION_ID_INVALID", KEY_RE
|
||||||
|
)
|
||||||
|
key = _require_string(value["idempotency_key"], "IDEMPOTENCY_KEY_INVALID", KEY_RE)
|
||||||
|
reason = _require_string(value["reason"], "EXECUTION_REBIND_REASON_INVALID")
|
||||||
|
if len(reason) > 512:
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_REASON_INVALID", 400)
|
||||||
|
|
||||||
|
return cls(
|
||||||
|
status,
|
||||||
|
revision,
|
||||||
|
attempt,
|
||||||
|
verification,
|
||||||
|
old_digest,
|
||||||
|
new_digest,
|
||||||
|
authorization,
|
||||||
|
key,
|
||||||
|
reason,
|
||||||
|
)
|
||||||
|
|
||||||
|
def public(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"expected_status": self.expected_status,
|
||||||
|
"expected_revision": self.expected_revision,
|
||||||
|
"expected_attempt": self.expected_attempt,
|
||||||
|
"previous_verification_sha256": self.previous_verification_sha256,
|
||||||
|
"expected_old_execution_digest": self.expected_old_execution_digest,
|
||||||
|
"expected_new_execution_digest": self.expected_new_execution_digest,
|
||||||
|
"authorization_id": self.authorization_id,
|
||||||
|
"idempotency_key": self.idempotency_key,
|
||||||
|
"reason": self.reason,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class CanonicalBridgeExecutionResolver:
|
||||||
|
"""Resolve the already-registered plan from the canonical governed Bridge."""
|
||||||
|
|
||||||
|
def __init__(self, base_url: str, control_token: str) -> None:
|
||||||
|
self.base_url = base_url.rstrip("/")
|
||||||
|
self.control_token = control_token
|
||||||
|
|
||||||
|
def __call__(
|
||||||
|
self,
|
||||||
|
task_id: str,
|
||||||
|
required_capabilities: Sequence[str],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
body = json.dumps(
|
||||||
|
{
|
||||||
|
"task_id": task_id,
|
||||||
|
"required_capabilities": list(required_capabilities),
|
||||||
|
},
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
).encode("utf-8")
|
||||||
|
request = urllib.request.Request(
|
||||||
|
self.base_url + "/v2/execution-envelope",
|
||||||
|
method="POST",
|
||||||
|
data=body,
|
||||||
|
headers={
|
||||||
|
"Authorization": "Bearer " + self.control_token,
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"Accept": "application/json",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(request, timeout=15) as response:
|
||||||
|
raw = response.read()
|
||||||
|
status = response.status
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
raw = exc.read()
|
||||||
|
status = exc.code
|
||||||
|
try:
|
||||||
|
value = json.loads(raw.decode("utf-8"))
|
||||||
|
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||||
|
raise ExecutionRebindError("CANONICAL_EXECUTION_RESPONSE_INVALID", 502) from exc
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise ExecutionRebindError("CANONICAL_EXECUTION_RESPONSE_INVALID", 502)
|
||||||
|
if status != 200:
|
||||||
|
raise ExecutionRebindError(
|
||||||
|
str(value.get("error") or "CANONICAL_EXECUTION_RESOLUTION_FAILED"),
|
||||||
|
502 if status >= 500 else 409,
|
||||||
|
)
|
||||||
|
return dict(value)
|
||||||
|
|
||||||
|
|
||||||
|
class ExecutionRebindStore:
|
||||||
|
"""Official owner-controlled mutation with CAS, plan resolution and durable audit."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
db_path: Path,
|
||||||
|
resolver: Callable[[str, Sequence[str]], Mapping[str, Any]],
|
||||||
|
) -> None:
|
||||||
|
self.db_path = Path(db_path)
|
||||||
|
self.resolver = resolver
|
||||||
|
|
||||||
|
def _connect(self) -> sqlite3.Connection:
|
||||||
|
connection = sqlite3.connect(self.db_path, timeout=30.0)
|
||||||
|
connection.row_factory = sqlite3.Row
|
||||||
|
connection.execute("PRAGMA foreign_keys=ON")
|
||||||
|
connection.execute("PRAGMA busy_timeout=30000")
|
||||||
|
return connection
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _columns(
|
||||||
|
connection: sqlite3.Connection,
|
||||||
|
table: str,
|
||||||
|
) -> dict[str, sqlite3.Row]:
|
||||||
|
return {
|
||||||
|
str(row["name"]): row
|
||||||
|
for row in connection.execute(f'PRAGMA table_info("{table}")')
|
||||||
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _json_value(raw: object, code: str) -> dict[str, Any]:
|
||||||
|
if isinstance(raw, Mapping):
|
||||||
|
return dict(raw)
|
||||||
|
if not isinstance(raw, str):
|
||||||
|
raise ExecutionRebindError(code, 409)
|
||||||
|
try:
|
||||||
|
value = json.loads(raw)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise ExecutionRebindError(code, 409) from exc
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise ExecutionRebindError(code, 409)
|
||||||
|
return dict(value)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _verification_hash(raw: object) -> str:
|
||||||
|
if raw in (None, ""):
|
||||||
|
value: object = {}
|
||||||
|
elif isinstance(raw, str):
|
||||||
|
try:
|
||||||
|
value = json.loads(raw)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise ExecutionRebindError("TASK_VERIFICATION_INVALID", 409) from exc
|
||||||
|
else:
|
||||||
|
value = raw
|
||||||
|
return _sha(value)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _metadata_column(columns: Mapping[str, sqlite3.Row]) -> str:
|
||||||
|
for name in ("metadata_json", "metadata"):
|
||||||
|
if name in columns:
|
||||||
|
return name
|
||||||
|
raise ExecutionRebindError("TASK_METADATA_SCHEMA_UNSUPPORTED", 503)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _required_capabilities(
|
||||||
|
task: sqlite3.Row,
|
||||||
|
columns: Mapping[str, sqlite3.Row],
|
||||||
|
old_execution: Mapping[str, Any],
|
||||||
|
) -> list[str]:
|
||||||
|
for field in ("required_capabilities_json", "required_capabilities"):
|
||||||
|
if field not in columns:
|
||||||
|
continue
|
||||||
|
raw = task[field]
|
||||||
|
if raw in (None, ""):
|
||||||
|
continue
|
||||||
|
if isinstance(raw, str):
|
||||||
|
try:
|
||||||
|
value = json.loads(raw)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
value = [item for item in raw.split(",") if item.strip()]
|
||||||
|
else:
|
||||||
|
value = raw
|
||||||
|
if isinstance(value, list) and all(isinstance(item, str) for item in value):
|
||||||
|
return [item.strip() for item in value if item.strip()]
|
||||||
|
value = old_execution.get("capability_scope")
|
||||||
|
if isinstance(value, list) and all(isinstance(item, str) for item in value):
|
||||||
|
return [item for item in value if item]
|
||||||
|
raise ExecutionRebindError("TASK_CAPABILITIES_UNAVAILABLE", 503)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _event_table(
|
||||||
|
connection: sqlite3.Connection,
|
||||||
|
) -> tuple[str, dict[str, sqlite3.Row]]:
|
||||||
|
tables = {
|
||||||
|
row["name"]
|
||||||
|
for row in connection.execute(
|
||||||
|
"SELECT name FROM sqlite_master WHERE type='table'"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
for name in ("task_events", "events"):
|
||||||
|
if name not in tables:
|
||||||
|
continue
|
||||||
|
columns = ExecutionRebindStore._columns(connection, name)
|
||||||
|
if "task_id" in columns and ("event_type" in columns or "type" in columns):
|
||||||
|
return name, columns
|
||||||
|
raise ExecutionRebindError("TASK_EVENT_TABLE_UNSUPPORTED", 503)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _insert_event(
|
||||||
|
connection: sqlite3.Connection,
|
||||||
|
task_id: str,
|
||||||
|
payload: Mapping[str, Any],
|
||||||
|
) -> None:
|
||||||
|
table, columns = ExecutionRebindStore._event_table(connection)
|
||||||
|
values: dict[str, Any] = {"task_id": task_id}
|
||||||
|
if "event_type" in columns:
|
||||||
|
values["event_type"] = "TASK_EXECUTION_REBIND_AUTHORIZED"
|
||||||
|
else:
|
||||||
|
values["type"] = "TASK_EXECUTION_REBIND_AUTHORIZED"
|
||||||
|
payload_text = json.dumps(payload, sort_keys=True, separators=(",", ":"))
|
||||||
|
if "payload_json" in columns:
|
||||||
|
values["payload_json"] = payload_text
|
||||||
|
elif "payload" in columns:
|
||||||
|
values["payload"] = payload_text
|
||||||
|
else:
|
||||||
|
raise ExecutionRebindError("TASK_EVENT_PAYLOAD_UNSUPPORTED", 503)
|
||||||
|
if "created_at" in columns:
|
||||||
|
values["created_at"] = _now()
|
||||||
|
if "agent_id" in columns:
|
||||||
|
values["agent_id"] = "owner-control"
|
||||||
|
if "actor_id" in columns:
|
||||||
|
values["actor_id"] = "owner-control"
|
||||||
|
names = list(values)
|
||||||
|
quoted = ",".join('"' + name + '"' for name in names)
|
||||||
|
placeholders = ",".join("?" for _ in names)
|
||||||
|
connection.execute(
|
||||||
|
f'INSERT INTO "{table}" ({quoted}) VALUES ({placeholders})',
|
||||||
|
tuple(values[name] for name in names),
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _ensure_rebind_table(connection: sqlite3.Connection) -> None:
|
||||||
|
connection.execute(
|
||||||
|
"""
|
||||||
|
CREATE TABLE IF NOT EXISTS task_execution_rebinds (
|
||||||
|
task_id TEXT NOT NULL,
|
||||||
|
idempotency_key TEXT NOT NULL,
|
||||||
|
request_sha256 TEXT NOT NULL,
|
||||||
|
authorization_id TEXT NOT NULL,
|
||||||
|
previous_verification_sha256 TEXT NOT NULL,
|
||||||
|
old_execution_digest TEXT NOT NULL,
|
||||||
|
new_execution_digest TEXT NOT NULL,
|
||||||
|
expected_revision INTEGER NOT NULL,
|
||||||
|
applied_revision INTEGER NOT NULL,
|
||||||
|
attempt INTEGER NOT NULL,
|
||||||
|
reason TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL,
|
||||||
|
PRIMARY KEY(task_id,idempotency_key)
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
def _load_and_validate(
|
||||||
|
self,
|
||||||
|
connection: sqlite3.Connection,
|
||||||
|
task_id: str,
|
||||||
|
request: ExecutionRebindRequest,
|
||||||
|
) -> tuple[sqlite3.Row, dict[str, sqlite3.Row], str, dict[str, Any], dict[str, Any]]:
|
||||||
|
columns = self._columns(connection, "tasks")
|
||||||
|
required = {
|
||||||
|
"task_id",
|
||||||
|
"status",
|
||||||
|
"attempt",
|
||||||
|
"revision",
|
||||||
|
"claimed_by",
|
||||||
|
"lease_token_hash",
|
||||||
|
"lease_expires_at",
|
||||||
|
"verification_json",
|
||||||
|
}
|
||||||
|
if not required.issubset(columns):
|
||||||
|
raise ExecutionRebindError("TASK_SCHEMA_EXECUTION_REBIND_UNSUPPORTED", 503)
|
||||||
|
metadata_column = self._metadata_column(columns)
|
||||||
|
task = connection.execute(
|
||||||
|
"SELECT * FROM tasks WHERE task_id=?",
|
||||||
|
(task_id,),
|
||||||
|
).fetchone()
|
||||||
|
if task is None:
|
||||||
|
raise ExecutionRebindError("TASK_NOT_FOUND", 404)
|
||||||
|
|
||||||
|
if (
|
||||||
|
str(task["status"]) != request.expected_status
|
||||||
|
or int(task["revision"]) != request.expected_revision
|
||||||
|
or int(task["attempt"]) != request.expected_attempt
|
||||||
|
):
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_CAS_MISMATCH", 409)
|
||||||
|
if task["lease_token_hash"] not in (None, "") or task["lease_expires_at"] not in (None, ""):
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_ACTIVE_LEASE_DENY", 409)
|
||||||
|
|
||||||
|
verification_sha = self._verification_hash(task["verification_json"])
|
||||||
|
if verification_sha != request.previous_verification_sha256:
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_VERIFICATION_DRIFT", 409)
|
||||||
|
|
||||||
|
metadata = self._json_value(
|
||||||
|
task[metadata_column],
|
||||||
|
"TASK_METADATA_INVALID",
|
||||||
|
)
|
||||||
|
execution = metadata.get("execution")
|
||||||
|
if not isinstance(execution, Mapping):
|
||||||
|
raise ExecutionRebindError("TASK_EXECUTION_METADATA_MISSING", 409)
|
||||||
|
old_execution = validate_execution_envelope(dict(execution))
|
||||||
|
if old_execution["execution_digest"] != request.expected_old_execution_digest:
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_OLD_DIGEST_DRIFT", 409)
|
||||||
|
if old_execution.get("action") != "source_capture":
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_SOURCE_CAPTURE_REQUIRED", 409)
|
||||||
|
|
||||||
|
projection = metadata.get("execution_projection")
|
||||||
|
if isinstance(projection, Mapping):
|
||||||
|
projected_digest = str(projection.get("execution_digest") or "")
|
||||||
|
if projected_digest and projected_digest != request.expected_old_execution_digest:
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_PROJECTION_DRIFT", 409)
|
||||||
|
return task, columns, metadata_column, metadata, old_execution
|
||||||
|
|
||||||
|
def rebind(
|
||||||
|
self,
|
||||||
|
task_id: str,
|
||||||
|
request: ExecutionRebindRequest,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
task_id = _require_string(task_id, "TASK_ID_INVALID", TASK_RE)
|
||||||
|
required_authorization = _required_authorization_id()
|
||||||
|
if request.authorization_id != required_authorization:
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_AUTHORIZATION_ID_MISMATCH", 403)
|
||||||
|
|
||||||
|
request_sha = _sha(request.public())
|
||||||
|
|
||||||
|
# Idempotent replay must win before CAS validation because a successful
|
||||||
|
# rebind necessarily increments the Task revision and changes the persisted envelope.
|
||||||
|
with self._connect() as connection:
|
||||||
|
self._ensure_rebind_table(connection)
|
||||||
|
replay = connection.execute(
|
||||||
|
"SELECT * FROM task_execution_rebinds WHERE task_id=? AND idempotency_key=?",
|
||||||
|
(task_id, request.idempotency_key),
|
||||||
|
).fetchone()
|
||||||
|
if replay is not None:
|
||||||
|
if replay["request_sha256"] != request_sha:
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_IDEMPOTENCY_CONFLICT", 409)
|
||||||
|
connection.commit()
|
||||||
|
return {
|
||||||
|
"task_id": task_id,
|
||||||
|
"status": "QUEUED",
|
||||||
|
"attempt": int(replay["attempt"]),
|
||||||
|
"revision": int(replay["applied_revision"]),
|
||||||
|
"old_execution_digest": replay["old_execution_digest"],
|
||||||
|
"new_execution_digest": replay["new_execution_digest"],
|
||||||
|
"authorization_id": replay["authorization_id"],
|
||||||
|
"idempotent": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Read/CAS snapshot first. Canonical Bridge resolution happens outside the DB write lock.
|
||||||
|
with self._connect() as connection:
|
||||||
|
task, columns, _, _, old_execution = self._load_and_validate(
|
||||||
|
connection, task_id, request
|
||||||
|
)
|
||||||
|
capabilities = self._required_capabilities(task, columns, old_execution)
|
||||||
|
|
||||||
|
binding = GovernedExecutionBinding.from_mapping(old_execution)
|
||||||
|
resolved = dict(self.resolver(task_id, capabilities))
|
||||||
|
new_execution = validate_execution_envelope(resolved, binding=binding)
|
||||||
|
if new_execution.get("action") != "deterministic":
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_DETERMINISTIC_PLAN_REQUIRED", 409)
|
||||||
|
if new_execution["execution_digest"] != request.expected_new_execution_digest:
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_NEW_DIGEST_DRIFT", 409)
|
||||||
|
if new_execution["execution_digest"] == old_execution["execution_digest"]:
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_DIGEST_UNCHANGED", 409)
|
||||||
|
|
||||||
|
# Re-read under BEGIN IMMEDIATE and apply only if the exact snapshot is unchanged.
|
||||||
|
with self._connect() as connection:
|
||||||
|
connection.execute("BEGIN IMMEDIATE")
|
||||||
|
self._ensure_rebind_table(connection)
|
||||||
|
|
||||||
|
replay = connection.execute(
|
||||||
|
"SELECT * FROM task_execution_rebinds WHERE task_id=? AND idempotency_key=?",
|
||||||
|
(task_id, request.idempotency_key),
|
||||||
|
).fetchone()
|
||||||
|
if replay is not None:
|
||||||
|
if replay["request_sha256"] != request_sha:
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_IDEMPOTENCY_CONFLICT", 409)
|
||||||
|
connection.commit()
|
||||||
|
return {
|
||||||
|
"task_id": task_id,
|
||||||
|
"status": "QUEUED",
|
||||||
|
"attempt": int(replay["attempt"]),
|
||||||
|
"revision": int(replay["applied_revision"]),
|
||||||
|
"old_execution_digest": replay["old_execution_digest"],
|
||||||
|
"new_execution_digest": replay["new_execution_digest"],
|
||||||
|
"authorization_id": replay["authorization_id"],
|
||||||
|
"idempotent": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
task, columns, metadata_column, metadata, current_old = self._load_and_validate(
|
||||||
|
connection, task_id, request
|
||||||
|
)
|
||||||
|
if current_old["execution_digest"] != old_execution["execution_digest"]:
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_OLD_DIGEST_DRIFT", 409)
|
||||||
|
|
||||||
|
new_metadata = dict(metadata)
|
||||||
|
new_metadata["execution"] = new_execution
|
||||||
|
new_metadata["execution_projection"] = {
|
||||||
|
"schema": "aios.online-agent.task-projection.v1",
|
||||||
|
"task_id": binding.task_id,
|
||||||
|
"job_id": binding.job_id,
|
||||||
|
"execution_digest": new_execution["execution_digest"],
|
||||||
|
}
|
||||||
|
applied_revision = request.expected_revision + 1
|
||||||
|
previous_claimed_by = task["claimed_by"]
|
||||||
|
|
||||||
|
cursor = connection.execute(
|
||||||
|
f"""
|
||||||
|
UPDATE tasks
|
||||||
|
SET "{metadata_column}"=?, revision=?, claimed_by=NULL,
|
||||||
|
lease_token_hash=NULL, lease_expires_at=NULL
|
||||||
|
WHERE task_id=? AND status=? AND revision=? AND attempt=?
|
||||||
|
AND claimed_by IS ? AND lease_token_hash IS NULL AND lease_expires_at IS NULL
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
json.dumps(new_metadata, sort_keys=True, separators=(",", ":")),
|
||||||
|
applied_revision,
|
||||||
|
task_id,
|
||||||
|
request.expected_status,
|
||||||
|
request.expected_revision,
|
||||||
|
request.expected_attempt,
|
||||||
|
previous_claimed_by,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if cursor.rowcount != 1:
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_CAS_MISMATCH", 409)
|
||||||
|
|
||||||
|
event_payload = {
|
||||||
|
"schema": "aios.task-execution-rebind.event.v1",
|
||||||
|
"task_id": task_id,
|
||||||
|
"job_id": binding.job_id,
|
||||||
|
"authorization_id": request.authorization_id,
|
||||||
|
"idempotency_key": request.idempotency_key,
|
||||||
|
"reason": request.reason,
|
||||||
|
"previous_verification_sha256": request.previous_verification_sha256,
|
||||||
|
"old_execution_digest": old_execution["execution_digest"],
|
||||||
|
"new_execution_digest": new_execution["execution_digest"],
|
||||||
|
"old_action": old_execution.get("action"),
|
||||||
|
"new_action": new_execution.get("action"),
|
||||||
|
"previous_revision": request.expected_revision,
|
||||||
|
"applied_revision": applied_revision,
|
||||||
|
"attempt": request.expected_attempt,
|
||||||
|
"request_sha256": request_sha,
|
||||||
|
"previous_claimed_by": previous_claimed_by,
|
||||||
|
"stale_claim_normalized": previous_claimed_by not in (None, ""),
|
||||||
|
}
|
||||||
|
self._insert_event(connection, task_id, event_payload)
|
||||||
|
connection.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO task_execution_rebinds(
|
||||||
|
task_id,idempotency_key,request_sha256,authorization_id,
|
||||||
|
previous_verification_sha256,old_execution_digest,new_execution_digest,
|
||||||
|
expected_revision,applied_revision,attempt,reason,created_at
|
||||||
|
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
task_id,
|
||||||
|
request.idempotency_key,
|
||||||
|
request_sha,
|
||||||
|
request.authorization_id,
|
||||||
|
request.previous_verification_sha256,
|
||||||
|
old_execution["execution_digest"],
|
||||||
|
new_execution["execution_digest"],
|
||||||
|
request.expected_revision,
|
||||||
|
applied_revision,
|
||||||
|
request.expected_attempt,
|
||||||
|
request.reason,
|
||||||
|
_now(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
connection.commit()
|
||||||
|
return {
|
||||||
|
"task_id": task_id,
|
||||||
|
"status": "QUEUED",
|
||||||
|
"attempt": request.expected_attempt,
|
||||||
|
"revision": applied_revision,
|
||||||
|
"old_execution_digest": old_execution["execution_digest"],
|
||||||
|
"new_execution_digest": new_execution["execution_digest"],
|
||||||
|
"authorization_id": request.authorization_id,
|
||||||
|
"idempotent": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class ExecutionRebindHandler(BaseHTTPRequestHandler):
|
||||||
|
store: ExecutionRebindStore | None = None
|
||||||
|
control_token: str = ""
|
||||||
|
|
||||||
|
def log_message(self, *_args: Any) -> None:
|
||||||
|
return
|
||||||
|
|
||||||
|
def _json(self, value: Mapping[str, Any], status: int) -> None:
|
||||||
|
data = json.dumps(
|
||||||
|
dict(value), sort_keys=True, separators=(",", ":")
|
||||||
|
).encode("utf-8")
|
||||||
|
self.send_response(status)
|
||||||
|
self.send_header("Content-Type", "application/json")
|
||||||
|
self.send_header("Content-Length", str(len(data)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(data)
|
||||||
|
|
||||||
|
def _authorized(self) -> bool:
|
||||||
|
header = self.headers.get("Authorization", "")
|
||||||
|
return (
|
||||||
|
header.startswith("Bearer ")
|
||||||
|
and bool(self.control_token)
|
||||||
|
and hmac.compare_digest(header[7:], self.control_token)
|
||||||
|
)
|
||||||
|
|
||||||
|
def do_GET(self) -> None:
|
||||||
|
if self.path == "/health":
|
||||||
|
self._json(
|
||||||
|
{"status": "ok", "service": "aios-agent-task-execution-rebind"},
|
||||||
|
200,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self._json({"error": "NOT_FOUND"}, 404)
|
||||||
|
|
||||||
|
def do_POST(self) -> None:
|
||||||
|
match = re.fullmatch(r"/v1/tasks/([^/]+)/execution-rebind", self.path)
|
||||||
|
if match is None:
|
||||||
|
self._json({"error": "NOT_FOUND"}, 404)
|
||||||
|
return
|
||||||
|
if not self._authorized():
|
||||||
|
self._json({"error": "UNAUTHORIZED"}, 401)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
length = int(self.headers.get("Content-Length", "0"))
|
||||||
|
except ValueError:
|
||||||
|
self._json({"error": "CONTENT_LENGTH_INVALID"}, 400)
|
||||||
|
return
|
||||||
|
if length < 1 or length > MAX_BODY:
|
||||||
|
self._json({"error": "REQUEST_SIZE_INVALID"}, 413)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
value = json.loads(self.rfile.read(length).decode("utf-8"))
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise ValueError
|
||||||
|
request = ExecutionRebindRequest.from_mapping(value)
|
||||||
|
if self.store is None:
|
||||||
|
raise ExecutionRebindError("RUNTIME_UNAVAILABLE", 503)
|
||||||
|
result = self.store.rebind(match.group(1), request)
|
||||||
|
self._json(result, 200 if result["idempotent"] else 201)
|
||||||
|
except ExecutionRebindError as exc:
|
||||||
|
self._json({"error": exc.code}, exc.status)
|
||||||
|
except (UnicodeDecodeError, json.JSONDecodeError, ValueError):
|
||||||
|
self._json({"error": "REQUEST_INVALID"}, 400)
|
||||||
|
|
||||||
|
|
||||||
|
def load_control_token(path: Path) -> str:
|
||||||
|
if path.is_symlink() or not path.is_file():
|
||||||
|
raise RuntimeError("TASK_EXECUTION_REBIND_CONTROL_CREDENTIAL_UNAVAILABLE")
|
||||||
|
value = path.read_text(encoding="utf-8").strip()
|
||||||
|
if len(value) < 32:
|
||||||
|
raise RuntimeError("TASK_EXECUTION_REBIND_CONTROL_CREDENTIAL_INVALID")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def serve() -> None:
|
||||||
|
db = Path(os.environ.get("AIOS_AGENT_TASK_DB", "/var/lib/aios-agent-task/state.sqlite3"))
|
||||||
|
_required_authorization_id()
|
||||||
|
token_path = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_AGENT_TASK_CONTROL_TOKEN_FILE",
|
||||||
|
str(
|
||||||
|
Path(os.environ.get("CREDENTIALS_DIRECTORY", "/run/credentials"))
|
||||||
|
/ "agent_task_control_token"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
token = load_control_token(token_path)
|
||||||
|
bridge = os.environ.get(
|
||||||
|
"AIOS_GOVERNED_WRITE_API_URL",
|
||||||
|
"http://127.0.0.1:18892",
|
||||||
|
)
|
||||||
|
host = os.environ.get("AIOS_TASK_EXECUTION_REBIND_HOST", "127.0.0.1")
|
||||||
|
port = int(os.environ.get("AIOS_TASK_EXECUTION_REBIND_PORT", "18942"))
|
||||||
|
ExecutionRebindHandler.store = ExecutionRebindStore(
|
||||||
|
db,
|
||||||
|
CanonicalBridgeExecutionResolver(bridge, token),
|
||||||
|
)
|
||||||
|
ExecutionRebindHandler.control_token = token
|
||||||
|
ThreadingHTTPServer((host, port), ExecutionRebindHandler).serve_forever()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
serve()
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,870 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Durable online task protocol for external AIOS agents.
|
||||||
|
|
||||||
|
The store is deliberately independent from Git/Gitea mutation. It provides a
|
||||||
|
small durable state machine for publishing work, atomic claim/lease ownership,
|
||||||
|
heartbeat, idempotent result submission, explicit server-side verification,
|
||||||
|
lease recovery, and audit events.
|
||||||
|
|
||||||
|
No plaintext agent credential or task lease token is persisted.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import secrets
|
||||||
|
import sqlite3
|
||||||
|
from typing import Any, Callable, Iterable, Mapping
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from aios_capability_contract import has_capabilities
|
||||||
|
from aios_online_agent_execution import (
|
||||||
|
ExecutionProjectionError,
|
||||||
|
validate_execution_envelope,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
TERMINAL_STATES = frozenset({"SUCCEEDED", "FAILED", "EXPIRED", "CANCELLED"})
|
||||||
|
ACTIVE_LEASE_STATES = frozenset({"CLAIMED", "RUNNING"})
|
||||||
|
TASK_STATES = frozenset({
|
||||||
|
"QUEUED", "CLAIMED", "RUNNING", "VERIFYING",
|
||||||
|
"SUCCEEDED", "FAILED", "EXPIRED", "CANCELLED",
|
||||||
|
})
|
||||||
|
VERIFY_DECISIONS = frozenset({"SUCCEEDED", "NEEDS_REWORK", "FAILED"})
|
||||||
|
|
||||||
|
|
||||||
|
GOVERNED_WRITE_CAPABILITIES = frozenset({
|
||||||
|
"governed.write.branch",
|
||||||
|
"governed.write.file",
|
||||||
|
})
|
||||||
|
EXECUTION_PROJECTION_SCHEMA = "aios.online-agent.task-projection.v1"
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_write_execution_binding(
|
||||||
|
task_id: str,
|
||||||
|
required_capabilities: list[str],
|
||||||
|
metadata: Mapping[str, Any],
|
||||||
|
) -> None:
|
||||||
|
required = set(required_capabilities)
|
||||||
|
if not required.intersection(GOVERNED_WRITE_CAPABILITIES):
|
||||||
|
return
|
||||||
|
execution = metadata.get("execution")
|
||||||
|
projection = metadata.get("execution_projection")
|
||||||
|
if not isinstance(execution, Mapping) or not isinstance(projection, Mapping):
|
||||||
|
raise TaskProtocolError("GOVERNED_EXECUTION_BINDING_REQUIRED", 409)
|
||||||
|
try:
|
||||||
|
canonical = validate_execution_envelope(dict(execution))
|
||||||
|
except (ExecutionProjectionError, TypeError, ValueError, KeyError) as exc:
|
||||||
|
raise TaskProtocolError("EXECUTION_ENVELOPE_INVALID", 409) from exc
|
||||||
|
if canonical.get("task_id") != task_id:
|
||||||
|
raise TaskProtocolError("TASK_BINDING_CONFLICT", 409)
|
||||||
|
if set(canonical.get("capability_scope") or ()) != required:
|
||||||
|
raise TaskProtocolError("EXECUTION_CAPABILITY_SCOPE_CONFLICT", 409)
|
||||||
|
if (
|
||||||
|
projection.get("schema") != EXECUTION_PROJECTION_SCHEMA
|
||||||
|
or projection.get("task_id") != task_id
|
||||||
|
or projection.get("job_id") != canonical.get("job_id")
|
||||||
|
or projection.get("execution_digest") != canonical.get("execution_digest")
|
||||||
|
):
|
||||||
|
raise TaskProtocolError("EXECUTION_PROJECTION_INVALID", 409)
|
||||||
|
|
||||||
|
|
||||||
|
class TaskProtocolError(RuntimeError):
|
||||||
|
def __init__(self, code: str, http_status: int = 409) -> None:
|
||||||
|
super().__init__(code)
|
||||||
|
self.code = code
|
||||||
|
self.http_status = http_status
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class Principal:
|
||||||
|
kind: str
|
||||||
|
identifier: str
|
||||||
|
|
||||||
|
|
||||||
|
def utcnow() -> datetime:
|
||||||
|
return datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def iso(value: datetime) -> str:
|
||||||
|
return value.astimezone(timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def parse_time(value: str | None) -> datetime | None:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
parsed = datetime.fromisoformat(value)
|
||||||
|
if parsed.tzinfo is None:
|
||||||
|
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||||
|
return parsed.astimezone(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def canonical_json(value: Any) -> str:
|
||||||
|
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||||
|
|
||||||
|
|
||||||
|
def digest_json(value: Any) -> str:
|
||||||
|
return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _string_list(value: Iterable[object] | None, field: str, *, max_items: int = 128) -> list[str]:
|
||||||
|
if value is None:
|
||||||
|
return []
|
||||||
|
if isinstance(value, (str, bytes)) or not isinstance(value, Iterable):
|
||||||
|
raise TaskProtocolError(field.upper() + "_INVALID", 400)
|
||||||
|
result: list[str] = []
|
||||||
|
for item in value:
|
||||||
|
text = str(item).strip()
|
||||||
|
if not text or len(text) > 160:
|
||||||
|
raise TaskProtocolError(field.upper() + "_INVALID", 400)
|
||||||
|
result.append(text)
|
||||||
|
if len(result) > max_items:
|
||||||
|
raise TaskProtocolError(field.upper() + "_TOO_LARGE", 400)
|
||||||
|
return sorted(set(result))
|
||||||
|
|
||||||
|
|
||||||
|
def _json_object(value: object | None, field: str, *, max_bytes: int = 128 * 1024) -> dict[str, Any]:
|
||||||
|
if value is None:
|
||||||
|
return {}
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise TaskProtocolError(field.upper() + "_INVALID", 400)
|
||||||
|
result = dict(value)
|
||||||
|
if len(canonical_json(result).encode("utf-8")) > max_bytes:
|
||||||
|
raise TaskProtocolError(field.upper() + "_TOO_LARGE", 413)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _criteria(value: object | None) -> list[Any]:
|
||||||
|
if value is None:
|
||||||
|
return []
|
||||||
|
if isinstance(value, (str, bytes)) or not isinstance(value, list):
|
||||||
|
raise TaskProtocolError("SUCCESS_CRITERIA_INVALID", 400)
|
||||||
|
if len(value) > 128:
|
||||||
|
raise TaskProtocolError("SUCCESS_CRITERIA_TOO_LARGE", 413)
|
||||||
|
if len(canonical_json(value).encode("utf-8")) > 128 * 1024:
|
||||||
|
raise TaskProtocolError("SUCCESS_CRITERIA_TOO_LARGE", 413)
|
||||||
|
return list(value)
|
||||||
|
|
||||||
|
|
||||||
|
class TaskStore:
|
||||||
|
"""SQLite-backed durable task store.
|
||||||
|
|
||||||
|
SQLite BEGIN IMMEDIATE serializes claims so two agents cannot successfully
|
||||||
|
claim the same queued task. WAL mode allows concurrent readers.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
path: str | Path,
|
||||||
|
*,
|
||||||
|
now: Callable[[], datetime] = utcnow,
|
||||||
|
token_pepper: str = "",
|
||||||
|
) -> None:
|
||||||
|
self.path = Path(path)
|
||||||
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
self.now = now
|
||||||
|
self.token_pepper = token_pepper
|
||||||
|
self._initialize()
|
||||||
|
|
||||||
|
def _connect(self) -> sqlite3.Connection:
|
||||||
|
connection = sqlite3.connect(self.path, timeout=10, isolation_level=None)
|
||||||
|
connection.row_factory = sqlite3.Row
|
||||||
|
connection.execute("PRAGMA foreign_keys=ON")
|
||||||
|
connection.execute("PRAGMA journal_mode=WAL")
|
||||||
|
connection.execute("PRAGMA synchronous=FULL")
|
||||||
|
connection.execute("PRAGMA busy_timeout=10000")
|
||||||
|
return connection
|
||||||
|
|
||||||
|
def _initialize(self) -> None:
|
||||||
|
with self._connect() as connection:
|
||||||
|
connection.executescript(
|
||||||
|
"""
|
||||||
|
CREATE TABLE IF NOT EXISTS agents (
|
||||||
|
agent_id TEXT PRIMARY KEY,
|
||||||
|
agent_type TEXT NOT NULL,
|
||||||
|
agent_version TEXT NOT NULL,
|
||||||
|
capabilities_json TEXT NOT NULL,
|
||||||
|
metadata_json TEXT NOT NULL,
|
||||||
|
token_hash TEXT NOT NULL UNIQUE,
|
||||||
|
status TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL,
|
||||||
|
last_seen_at TEXT NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS tasks (
|
||||||
|
task_id TEXT PRIMARY KEY,
|
||||||
|
goal TEXT NOT NULL,
|
||||||
|
instructions TEXT NOT NULL,
|
||||||
|
success_criteria_json TEXT NOT NULL,
|
||||||
|
required_capabilities_json TEXT NOT NULL,
|
||||||
|
constraints_json TEXT NOT NULL,
|
||||||
|
metadata_json TEXT NOT NULL,
|
||||||
|
deadline TEXT,
|
||||||
|
priority INTEGER NOT NULL,
|
||||||
|
max_attempts INTEGER NOT NULL,
|
||||||
|
attempt INTEGER NOT NULL DEFAULT 0,
|
||||||
|
status TEXT NOT NULL,
|
||||||
|
claimed_by TEXT,
|
||||||
|
lease_token_hash TEXT,
|
||||||
|
lease_expires_at TEXT,
|
||||||
|
result_json TEXT,
|
||||||
|
result_digest TEXT,
|
||||||
|
result_idempotency_key TEXT,
|
||||||
|
verification_json TEXT,
|
||||||
|
revision INTEGER NOT NULL DEFAULT 1,
|
||||||
|
created_at TEXT NOT NULL,
|
||||||
|
updated_at TEXT NOT NULL,
|
||||||
|
FOREIGN KEY(claimed_by) REFERENCES agents(agent_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS idempotency (
|
||||||
|
scope TEXT NOT NULL,
|
||||||
|
key TEXT NOT NULL,
|
||||||
|
payload_digest TEXT NOT NULL,
|
||||||
|
response_json TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL,
|
||||||
|
PRIMARY KEY(scope, key)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS events (
|
||||||
|
event_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
task_id TEXT,
|
||||||
|
agent_id TEXT,
|
||||||
|
event_type TEXT NOT NULL,
|
||||||
|
payload_json TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tasks_queue
|
||||||
|
ON tasks(status, priority DESC, created_at);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tasks_lease
|
||||||
|
ON tasks(status, lease_expires_at);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_events_task
|
||||||
|
ON events(task_id, event_id);
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
def _token_hash(self, token: str) -> str:
|
||||||
|
return hashlib.sha256((self.token_pepper + "\0" + token).encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _row_task(row: sqlite3.Row, *, include_result: bool = True) -> dict[str, Any]:
|
||||||
|
value: dict[str, Any] = {
|
||||||
|
"task_id": row["task_id"],
|
||||||
|
"goal": row["goal"],
|
||||||
|
"instructions": row["instructions"],
|
||||||
|
"success_criteria": json.loads(row["success_criteria_json"]),
|
||||||
|
"required_capabilities": json.loads(row["required_capabilities_json"]),
|
||||||
|
"constraints": json.loads(row["constraints_json"]),
|
||||||
|
"metadata": json.loads(row["metadata_json"]),
|
||||||
|
"deadline": row["deadline"],
|
||||||
|
"priority": row["priority"],
|
||||||
|
"max_attempts": row["max_attempts"],
|
||||||
|
"attempt": row["attempt"],
|
||||||
|
"status": row["status"],
|
||||||
|
"claimed_by": row["claimed_by"],
|
||||||
|
"lease_expires_at": row["lease_expires_at"],
|
||||||
|
"revision": row["revision"],
|
||||||
|
"created_at": row["created_at"],
|
||||||
|
"updated_at": row["updated_at"],
|
||||||
|
}
|
||||||
|
if include_result:
|
||||||
|
value["result"] = json.loads(row["result_json"]) if row["result_json"] else None
|
||||||
|
value["verification"] = (
|
||||||
|
json.loads(row["verification_json"]) if row["verification_json"] else None
|
||||||
|
)
|
||||||
|
return value
|
||||||
|
|
||||||
|
def _event(
|
||||||
|
self,
|
||||||
|
connection: sqlite3.Connection,
|
||||||
|
event_type: str,
|
||||||
|
*,
|
||||||
|
task_id: str | None = None,
|
||||||
|
agent_id: str | None = None,
|
||||||
|
payload: Mapping[str, Any] | None = None,
|
||||||
|
) -> None:
|
||||||
|
connection.execute(
|
||||||
|
"INSERT INTO events(task_id,agent_id,event_type,payload_json,created_at)"
|
||||||
|
" VALUES(?,?,?,?,?)",
|
||||||
|
(task_id, agent_id, event_type, canonical_json(dict(payload or {})), iso(self.now())),
|
||||||
|
)
|
||||||
|
|
||||||
|
def register_agent(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
agent_id: str,
|
||||||
|
agent_type: str,
|
||||||
|
agent_version: str,
|
||||||
|
capabilities: Iterable[object],
|
||||||
|
metadata: Mapping[str, Any] | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
agent_id = agent_id.strip()
|
||||||
|
agent_type = agent_type.strip()
|
||||||
|
agent_version = agent_version.strip()
|
||||||
|
if not agent_id or len(agent_id) > 160 or not agent_type or not agent_version:
|
||||||
|
raise TaskProtocolError("AGENT_IDENTITY_INVALID", 400)
|
||||||
|
caps = _string_list(capabilities, "capabilities")
|
||||||
|
meta = _json_object(metadata, "metadata")
|
||||||
|
token = "agt_" + secrets.token_urlsafe(32)
|
||||||
|
now = iso(self.now())
|
||||||
|
try:
|
||||||
|
with self._connect() as connection:
|
||||||
|
connection.execute("BEGIN IMMEDIATE")
|
||||||
|
connection.execute(
|
||||||
|
"INSERT INTO agents(agent_id,agent_type,agent_version,capabilities_json,"
|
||||||
|
"metadata_json,token_hash,status,created_at,last_seen_at)"
|
||||||
|
" VALUES(?,?,?,?,?,?,?,?,?)",
|
||||||
|
(
|
||||||
|
agent_id,
|
||||||
|
agent_type,
|
||||||
|
agent_version,
|
||||||
|
canonical_json(caps),
|
||||||
|
canonical_json(meta),
|
||||||
|
self._token_hash(token),
|
||||||
|
"ACTIVE",
|
||||||
|
now,
|
||||||
|
now,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self._event(
|
||||||
|
connection,
|
||||||
|
"AGENT_REGISTERED",
|
||||||
|
agent_id=agent_id,
|
||||||
|
payload={"agent_type": agent_type, "agent_version": agent_version},
|
||||||
|
)
|
||||||
|
connection.commit()
|
||||||
|
except sqlite3.IntegrityError as exc:
|
||||||
|
raise TaskProtocolError("AGENT_ALREADY_EXISTS", 409) from exc
|
||||||
|
return {
|
||||||
|
"agent_id": agent_id,
|
||||||
|
"agent_type": agent_type,
|
||||||
|
"agent_version": agent_version,
|
||||||
|
"capabilities": caps,
|
||||||
|
"agent_token": token,
|
||||||
|
"credential_returned_once": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
def authenticate_agent(self, token: str) -> dict[str, Any]:
|
||||||
|
if not token:
|
||||||
|
raise TaskProtocolError("UNAUTHORIZED", 401)
|
||||||
|
token_hash = self._token_hash(token)
|
||||||
|
with self._connect() as connection:
|
||||||
|
row = connection.execute(
|
||||||
|
"SELECT * FROM agents WHERE token_hash=? AND status='ACTIVE'",
|
||||||
|
(token_hash,),
|
||||||
|
).fetchone()
|
||||||
|
if row is None:
|
||||||
|
raise TaskProtocolError("UNAUTHORIZED", 401)
|
||||||
|
connection.execute(
|
||||||
|
"UPDATE agents SET last_seen_at=? WHERE agent_id=?",
|
||||||
|
(iso(self.now()), row["agent_id"]),
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"agent_id": row["agent_id"],
|
||||||
|
"agent_type": row["agent_type"],
|
||||||
|
"agent_version": row["agent_version"],
|
||||||
|
"capabilities": json.loads(row["capabilities_json"]),
|
||||||
|
"metadata": json.loads(row["metadata_json"]),
|
||||||
|
}
|
||||||
|
|
||||||
|
def _idempotent_lookup(
|
||||||
|
self,
|
||||||
|
connection: sqlite3.Connection,
|
||||||
|
*,
|
||||||
|
scope: str,
|
||||||
|
key: str,
|
||||||
|
payload_digest: str,
|
||||||
|
) -> dict[str, Any] | None:
|
||||||
|
row = connection.execute(
|
||||||
|
"SELECT payload_digest,response_json FROM idempotency WHERE scope=? AND key=?",
|
||||||
|
(scope, key),
|
||||||
|
).fetchone()
|
||||||
|
if row is None:
|
||||||
|
return None
|
||||||
|
if not hmac.compare_digest(row["payload_digest"], payload_digest):
|
||||||
|
raise TaskProtocolError("IDEMPOTENCY_CONFLICT", 409)
|
||||||
|
return json.loads(row["response_json"])
|
||||||
|
|
||||||
|
def _idempotent_store(
|
||||||
|
self,
|
||||||
|
connection: sqlite3.Connection,
|
||||||
|
*,
|
||||||
|
scope: str,
|
||||||
|
key: str,
|
||||||
|
payload_digest: str,
|
||||||
|
response: Mapping[str, Any],
|
||||||
|
) -> None:
|
||||||
|
connection.execute(
|
||||||
|
"INSERT INTO idempotency(scope,key,payload_digest,response_json,created_at)"
|
||||||
|
" VALUES(?,?,?,?,?)",
|
||||||
|
(scope, key, payload_digest, canonical_json(dict(response)), iso(self.now())),
|
||||||
|
)
|
||||||
|
|
||||||
|
def create_task(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
task_id: str | None,
|
||||||
|
goal: str,
|
||||||
|
instructions: str,
|
||||||
|
success_criteria: object | None,
|
||||||
|
required_capabilities: Iterable[object] | None = None,
|
||||||
|
constraints: Mapping[str, Any] | None = None,
|
||||||
|
metadata: Mapping[str, Any] | None = None,
|
||||||
|
deadline: str | None = None,
|
||||||
|
priority: int = 0,
|
||||||
|
max_attempts: int = 3,
|
||||||
|
idempotency_key: str,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
task_id = (task_id or ("task-" + uuid.uuid4().hex)).strip()
|
||||||
|
goal = goal.strip()
|
||||||
|
instructions = instructions.strip()
|
||||||
|
idempotency_key = idempotency_key.strip()
|
||||||
|
if not task_id or len(task_id) > 180 or not goal or not instructions:
|
||||||
|
raise TaskProtocolError("TASK_BINDING_INVALID", 400)
|
||||||
|
if not idempotency_key or len(idempotency_key) > 160:
|
||||||
|
raise TaskProtocolError("IDEMPOTENCY_KEY_INVALID", 400)
|
||||||
|
if not isinstance(priority, int) or isinstance(priority, bool) or not -100 <= priority <= 100:
|
||||||
|
raise TaskProtocolError("PRIORITY_INVALID", 400)
|
||||||
|
if not isinstance(max_attempts, int) or isinstance(max_attempts, bool) or not 1 <= max_attempts <= 20:
|
||||||
|
raise TaskProtocolError("MAX_ATTEMPTS_INVALID", 400)
|
||||||
|
parsed_deadline = parse_time(deadline)
|
||||||
|
if parsed_deadline and parsed_deadline <= self.now():
|
||||||
|
raise TaskProtocolError("DEADLINE_INVALID", 400)
|
||||||
|
criteria = _criteria(success_criteria)
|
||||||
|
required = _string_list(required_capabilities, "required_capabilities")
|
||||||
|
constraints_value = _json_object(constraints, "constraints")
|
||||||
|
metadata_value = _json_object(metadata, "metadata")
|
||||||
|
_validate_write_execution_binding(task_id, required, metadata_value)
|
||||||
|
payload = {
|
||||||
|
"task_id": task_id,
|
||||||
|
"goal": goal,
|
||||||
|
"instructions": instructions,
|
||||||
|
"success_criteria": criteria,
|
||||||
|
"required_capabilities": required,
|
||||||
|
"constraints": constraints_value,
|
||||||
|
"metadata": metadata_value,
|
||||||
|
"deadline": iso(parsed_deadline) if parsed_deadline else None,
|
||||||
|
"priority": priority,
|
||||||
|
"max_attempts": max_attempts,
|
||||||
|
}
|
||||||
|
payload_digest = digest_json(payload)
|
||||||
|
now = iso(self.now())
|
||||||
|
with self._connect() as connection:
|
||||||
|
connection.execute("BEGIN IMMEDIATE")
|
||||||
|
replay = self._idempotent_lookup(
|
||||||
|
connection,
|
||||||
|
scope="create_task",
|
||||||
|
key=idempotency_key,
|
||||||
|
payload_digest=payload_digest,
|
||||||
|
)
|
||||||
|
if replay is not None:
|
||||||
|
connection.commit()
|
||||||
|
return {**replay, "idempotent": True}
|
||||||
|
try:
|
||||||
|
connection.execute(
|
||||||
|
"INSERT INTO tasks(task_id,goal,instructions,success_criteria_json,"
|
||||||
|
"required_capabilities_json,constraints_json,metadata_json,deadline,priority,"
|
||||||
|
"max_attempts,attempt,status,revision,created_at,updated_at)"
|
||||||
|
" VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||||
|
(
|
||||||
|
task_id,
|
||||||
|
goal,
|
||||||
|
instructions,
|
||||||
|
canonical_json(criteria),
|
||||||
|
canonical_json(required),
|
||||||
|
canonical_json(constraints_value),
|
||||||
|
canonical_json(metadata_value),
|
||||||
|
payload["deadline"],
|
||||||
|
priority,
|
||||||
|
max_attempts,
|
||||||
|
0,
|
||||||
|
"QUEUED",
|
||||||
|
1,
|
||||||
|
now,
|
||||||
|
now,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
except sqlite3.IntegrityError as exc:
|
||||||
|
raise TaskProtocolError("TASK_ID_CONFLICT", 409) from exc
|
||||||
|
row = connection.execute("SELECT * FROM tasks WHERE task_id=?", (task_id,)).fetchone()
|
||||||
|
result = self._row_task(row)
|
||||||
|
self._idempotent_store(
|
||||||
|
connection,
|
||||||
|
scope="create_task",
|
||||||
|
key=idempotency_key,
|
||||||
|
payload_digest=payload_digest,
|
||||||
|
response=result,
|
||||||
|
)
|
||||||
|
self._event(connection, "TASK_CREATED", task_id=task_id, payload={"priority": priority})
|
||||||
|
connection.commit()
|
||||||
|
return result
|
||||||
|
|
||||||
|
def _expire_locked(self, connection: sqlite3.Connection) -> None:
|
||||||
|
now_dt = self.now()
|
||||||
|
now = iso(now_dt)
|
||||||
|
rows = connection.execute(
|
||||||
|
"SELECT * FROM tasks WHERE status IN ('QUEUED','CLAIMED','RUNNING')"
|
||||||
|
).fetchall()
|
||||||
|
for row in rows:
|
||||||
|
deadline = parse_time(row["deadline"])
|
||||||
|
if deadline is not None and deadline <= now_dt:
|
||||||
|
connection.execute(
|
||||||
|
"UPDATE tasks SET status='EXPIRED',lease_token_hash=NULL,"
|
||||||
|
"lease_expires_at=NULL,revision=revision+1,updated_at=? WHERE task_id=?",
|
||||||
|
(now, row["task_id"]),
|
||||||
|
)
|
||||||
|
self._event(connection, "TASK_EXPIRED", task_id=row["task_id"])
|
||||||
|
continue
|
||||||
|
if row["status"] in ACTIVE_LEASE_STATES:
|
||||||
|
lease_expires = parse_time(row["lease_expires_at"])
|
||||||
|
if lease_expires is not None and lease_expires <= now_dt:
|
||||||
|
if row["attempt"] >= row["max_attempts"]:
|
||||||
|
next_state = "FAILED"
|
||||||
|
else:
|
||||||
|
next_state = "QUEUED"
|
||||||
|
if next_state == "FAILED":
|
||||||
|
connection.execute(
|
||||||
|
"UPDATE tasks SET status=?,lease_token_hash=NULL,"
|
||||||
|
"lease_expires_at=NULL,revision=revision+1,updated_at=? WHERE task_id=?",
|
||||||
|
(next_state, now, row["task_id"]),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
connection.execute(
|
||||||
|
"UPDATE tasks SET status=?,claimed_by=NULL,lease_token_hash=NULL,"
|
||||||
|
"lease_expires_at=NULL,revision=revision+1,updated_at=? WHERE task_id=?",
|
||||||
|
(next_state, now, row["task_id"]),
|
||||||
|
)
|
||||||
|
self._event(
|
||||||
|
connection,
|
||||||
|
"LEASE_EXPIRED",
|
||||||
|
task_id=row["task_id"],
|
||||||
|
agent_id=row["claimed_by"],
|
||||||
|
payload={"next_state": next_state, "attempt": row["attempt"]},
|
||||||
|
)
|
||||||
|
|
||||||
|
def expire_leases(self) -> int:
|
||||||
|
with self._connect() as connection:
|
||||||
|
connection.execute("BEGIN IMMEDIATE")
|
||||||
|
before = connection.total_changes
|
||||||
|
self._expire_locked(connection)
|
||||||
|
changed = connection.total_changes - before
|
||||||
|
connection.commit()
|
||||||
|
return changed
|
||||||
|
|
||||||
|
def poll_tasks(self, *, agent_id: str, limit: int = 20) -> list[dict[str, Any]]:
|
||||||
|
if not 1 <= int(limit) <= 100:
|
||||||
|
raise TaskProtocolError("LIMIT_INVALID", 400)
|
||||||
|
with self._connect() as connection:
|
||||||
|
connection.execute("BEGIN IMMEDIATE")
|
||||||
|
self._expire_locked(connection)
|
||||||
|
agent = connection.execute(
|
||||||
|
"SELECT capabilities_json FROM agents WHERE agent_id=? AND status='ACTIVE'",
|
||||||
|
(agent_id,),
|
||||||
|
).fetchone()
|
||||||
|
if agent is None:
|
||||||
|
raise TaskProtocolError("AGENT_NOT_FOUND", 404)
|
||||||
|
capabilities = json.loads(agent["capabilities_json"])
|
||||||
|
rows = connection.execute(
|
||||||
|
"SELECT * FROM tasks WHERE status='QUEUED' ORDER BY priority DESC,created_at ASC"
|
||||||
|
).fetchall()
|
||||||
|
result = []
|
||||||
|
for row in rows:
|
||||||
|
required = json.loads(row["required_capabilities_json"])
|
||||||
|
if has_capabilities(capabilities, required):
|
||||||
|
result.append(self._row_task(row, include_result=False))
|
||||||
|
if len(result) >= int(limit):
|
||||||
|
break
|
||||||
|
connection.commit()
|
||||||
|
return result
|
||||||
|
|
||||||
|
def _agent_capabilities(self, connection: sqlite3.Connection, agent_id: str) -> set[str]:
|
||||||
|
row = connection.execute(
|
||||||
|
"SELECT capabilities_json FROM agents WHERE agent_id=? AND status='ACTIVE'",
|
||||||
|
(agent_id,),
|
||||||
|
).fetchone()
|
||||||
|
if row is None:
|
||||||
|
raise TaskProtocolError("AGENT_NOT_FOUND", 404)
|
||||||
|
return set(json.loads(row["capabilities_json"]))
|
||||||
|
|
||||||
|
def claim_task(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
task_id: str,
|
||||||
|
agent_id: str,
|
||||||
|
lease_seconds: int = 300,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if not isinstance(lease_seconds, int) or isinstance(lease_seconds, bool) or not 30 <= lease_seconds <= 3600:
|
||||||
|
raise TaskProtocolError("LEASE_SECONDS_INVALID", 400)
|
||||||
|
lease_token = "lease_" + secrets.token_urlsafe(32)
|
||||||
|
with self._connect() as connection:
|
||||||
|
connection.execute("BEGIN IMMEDIATE")
|
||||||
|
self._expire_locked(connection)
|
||||||
|
row = connection.execute("SELECT * FROM tasks WHERE task_id=?", (task_id,)).fetchone()
|
||||||
|
if row is None:
|
||||||
|
raise TaskProtocolError("TASK_NOT_FOUND", 404)
|
||||||
|
if row["status"] != "QUEUED":
|
||||||
|
raise TaskProtocolError("TASK_NOT_CLAIMABLE", 409)
|
||||||
|
capabilities = self._agent_capabilities(connection, agent_id)
|
||||||
|
required = json.loads(row["required_capabilities_json"])
|
||||||
|
if not has_capabilities(capabilities, required):
|
||||||
|
raise TaskProtocolError("AGENT_CAPABILITY_MISMATCH", 403)
|
||||||
|
attempt = int(row["attempt"]) + 1
|
||||||
|
if attempt > int(row["max_attempts"]):
|
||||||
|
raise TaskProtocolError("TASK_ATTEMPTS_EXHAUSTED", 409)
|
||||||
|
expires = self.now() + timedelta(seconds=lease_seconds)
|
||||||
|
now = iso(self.now())
|
||||||
|
updated = connection.execute(
|
||||||
|
"UPDATE tasks SET status='CLAIMED',claimed_by=?,lease_token_hash=?,"
|
||||||
|
"lease_expires_at=?,attempt=?,revision=revision+1,updated_at=? "
|
||||||
|
"WHERE task_id=? AND status='QUEUED'",
|
||||||
|
(
|
||||||
|
agent_id,
|
||||||
|
self._token_hash(lease_token),
|
||||||
|
iso(expires),
|
||||||
|
attempt,
|
||||||
|
now,
|
||||||
|
task_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if updated.rowcount != 1:
|
||||||
|
raise TaskProtocolError("TASK_CLAIM_CONFLICT", 409)
|
||||||
|
self._event(
|
||||||
|
connection,
|
||||||
|
"TASK_CLAIMED",
|
||||||
|
task_id=task_id,
|
||||||
|
agent_id=agent_id,
|
||||||
|
payload={"attempt": attempt, "lease_expires_at": iso(expires)},
|
||||||
|
)
|
||||||
|
row = connection.execute("SELECT * FROM tasks WHERE task_id=?", (task_id,)).fetchone()
|
||||||
|
connection.commit()
|
||||||
|
return {
|
||||||
|
**self._row_task(row, include_result=False),
|
||||||
|
"lease_token": lease_token,
|
||||||
|
"lease_token_returned_once": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
def _validate_lease(
|
||||||
|
self,
|
||||||
|
connection: sqlite3.Connection,
|
||||||
|
*,
|
||||||
|
task_id: str,
|
||||||
|
agent_id: str,
|
||||||
|
lease_token: str,
|
||||||
|
allowed_states: frozenset[str] = ACTIVE_LEASE_STATES,
|
||||||
|
) -> sqlite3.Row:
|
||||||
|
self._expire_locked(connection)
|
||||||
|
row = connection.execute("SELECT * FROM tasks WHERE task_id=?", (task_id,)).fetchone()
|
||||||
|
if row is None:
|
||||||
|
raise TaskProtocolError("TASK_NOT_FOUND", 404)
|
||||||
|
if row["status"] not in allowed_states:
|
||||||
|
raise TaskProtocolError("TASK_LEASE_NOT_ACTIVE", 409)
|
||||||
|
if row["claimed_by"] != agent_id:
|
||||||
|
raise TaskProtocolError("TASK_LEASE_OWNER_MISMATCH", 403)
|
||||||
|
if not row["lease_token_hash"] or not hmac.compare_digest(
|
||||||
|
row["lease_token_hash"], self._token_hash(lease_token)
|
||||||
|
):
|
||||||
|
raise TaskProtocolError("TASK_LEASE_TOKEN_INVALID", 403)
|
||||||
|
expires = parse_time(row["lease_expires_at"])
|
||||||
|
if expires is None or expires <= self.now():
|
||||||
|
raise TaskProtocolError("TASK_LEASE_EXPIRED", 409)
|
||||||
|
return row
|
||||||
|
|
||||||
|
def heartbeat(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
task_id: str,
|
||||||
|
agent_id: str,
|
||||||
|
lease_token: str,
|
||||||
|
progress: Mapping[str, Any] | None = None,
|
||||||
|
lease_seconds: int = 300,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if not 30 <= int(lease_seconds) <= 3600:
|
||||||
|
raise TaskProtocolError("LEASE_SECONDS_INVALID", 400)
|
||||||
|
progress_value = _json_object(progress, "progress", max_bytes=64 * 1024)
|
||||||
|
with self._connect() as connection:
|
||||||
|
connection.execute("BEGIN IMMEDIATE")
|
||||||
|
self._validate_lease(
|
||||||
|
connection,
|
||||||
|
task_id=task_id,
|
||||||
|
agent_id=agent_id,
|
||||||
|
lease_token=lease_token,
|
||||||
|
)
|
||||||
|
expires = self.now() + timedelta(seconds=int(lease_seconds))
|
||||||
|
now = iso(self.now())
|
||||||
|
connection.execute(
|
||||||
|
"UPDATE tasks SET status='RUNNING',lease_expires_at=?,revision=revision+1,"
|
||||||
|
"updated_at=? WHERE task_id=?",
|
||||||
|
(iso(expires), now, task_id),
|
||||||
|
)
|
||||||
|
self._event(
|
||||||
|
connection,
|
||||||
|
"TASK_HEARTBEAT",
|
||||||
|
task_id=task_id,
|
||||||
|
agent_id=agent_id,
|
||||||
|
payload={"progress": progress_value, "lease_expires_at": iso(expires)},
|
||||||
|
)
|
||||||
|
row = connection.execute("SELECT * FROM tasks WHERE task_id=?", (task_id,)).fetchone()
|
||||||
|
connection.commit()
|
||||||
|
return self._row_task(row, include_result=False)
|
||||||
|
|
||||||
|
def submit_result(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
task_id: str,
|
||||||
|
agent_id: str,
|
||||||
|
lease_token: str,
|
||||||
|
idempotency_key: str,
|
||||||
|
result: Mapping[str, Any],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
idempotency_key = idempotency_key.strip()
|
||||||
|
if not idempotency_key or len(idempotency_key) > 160:
|
||||||
|
raise TaskProtocolError("IDEMPOTENCY_KEY_INVALID", 400)
|
||||||
|
result_value = _json_object(result, "result", max_bytes=1024 * 1024)
|
||||||
|
result_digest = digest_json(result_value)
|
||||||
|
with self._connect() as connection:
|
||||||
|
connection.execute("BEGIN IMMEDIATE")
|
||||||
|
row = connection.execute("SELECT * FROM tasks WHERE task_id=?", (task_id,)).fetchone()
|
||||||
|
if row is None:
|
||||||
|
raise TaskProtocolError("TASK_NOT_FOUND", 404)
|
||||||
|
if row["result_idempotency_key"] == idempotency_key:
|
||||||
|
if not row["result_digest"] or not hmac.compare_digest(row["result_digest"], result_digest):
|
||||||
|
raise TaskProtocolError("IDEMPOTENCY_CONFLICT", 409)
|
||||||
|
connection.commit()
|
||||||
|
return {**self._row_task(row), "idempotent": True}
|
||||||
|
self._validate_lease(
|
||||||
|
connection,
|
||||||
|
task_id=task_id,
|
||||||
|
agent_id=agent_id,
|
||||||
|
lease_token=lease_token,
|
||||||
|
)
|
||||||
|
now = iso(self.now())
|
||||||
|
connection.execute(
|
||||||
|
"UPDATE tasks SET status='VERIFYING',result_json=?,result_digest=?,"
|
||||||
|
"result_idempotency_key=?,lease_token_hash=NULL,lease_expires_at=NULL,"
|
||||||
|
"revision=revision+1,updated_at=? WHERE task_id=?",
|
||||||
|
(
|
||||||
|
canonical_json(result_value),
|
||||||
|
result_digest,
|
||||||
|
idempotency_key,
|
||||||
|
now,
|
||||||
|
task_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self._event(
|
||||||
|
connection,
|
||||||
|
"RESULT_SUBMITTED",
|
||||||
|
task_id=task_id,
|
||||||
|
agent_id=agent_id,
|
||||||
|
payload={"result_digest": result_digest, "idempotency_key": idempotency_key},
|
||||||
|
)
|
||||||
|
row = connection.execute("SELECT * FROM tasks WHERE task_id=?", (task_id,)).fetchone()
|
||||||
|
connection.commit()
|
||||||
|
return self._row_task(row)
|
||||||
|
|
||||||
|
def verify_task(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
task_id: str,
|
||||||
|
decision: str,
|
||||||
|
verification: Mapping[str, Any] | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
decision = decision.strip().upper()
|
||||||
|
if decision not in VERIFY_DECISIONS:
|
||||||
|
raise TaskProtocolError("VERIFICATION_DECISION_INVALID", 400)
|
||||||
|
verification_value = _json_object(verification, "verification")
|
||||||
|
verification_value = {"decision": decision, **verification_value}
|
||||||
|
with self._connect() as connection:
|
||||||
|
connection.execute("BEGIN IMMEDIATE")
|
||||||
|
row = connection.execute("SELECT * FROM tasks WHERE task_id=?", (task_id,)).fetchone()
|
||||||
|
if row is None:
|
||||||
|
raise TaskProtocolError("TASK_NOT_FOUND", 404)
|
||||||
|
if row["status"] != "VERIFYING":
|
||||||
|
raise TaskProtocolError("TASK_NOT_VERIFYING", 409)
|
||||||
|
if decision == "SUCCEEDED":
|
||||||
|
next_state = "SUCCEEDED"
|
||||||
|
elif decision == "FAILED":
|
||||||
|
next_state = "FAILED"
|
||||||
|
elif int(row["attempt"]) >= int(row["max_attempts"]):
|
||||||
|
next_state = "FAILED"
|
||||||
|
verification_value["reason"] = verification_value.get(
|
||||||
|
"reason", "MAX_ATTEMPTS_EXHAUSTED"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
next_state = "QUEUED"
|
||||||
|
now = iso(self.now())
|
||||||
|
if next_state in {"SUCCEEDED", "FAILED"}:
|
||||||
|
connection.execute(
|
||||||
|
"UPDATE tasks SET status=?,lease_token_hash=NULL,"
|
||||||
|
"lease_expires_at=NULL,verification_json=?,revision=revision+1,updated_at=? "
|
||||||
|
"WHERE task_id=?",
|
||||||
|
(next_state, canonical_json(verification_value), now, task_id),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
connection.execute(
|
||||||
|
"UPDATE tasks SET status=?,claimed_by=NULL,lease_token_hash=NULL,"
|
||||||
|
"lease_expires_at=NULL,verification_json=?,revision=revision+1,updated_at=? "
|
||||||
|
"WHERE task_id=?",
|
||||||
|
(next_state, canonical_json(verification_value), now, task_id),
|
||||||
|
)
|
||||||
|
self._event(
|
||||||
|
connection,
|
||||||
|
"TASK_VERIFIED",
|
||||||
|
task_id=task_id,
|
||||||
|
agent_id=row["claimed_by"],
|
||||||
|
payload={"decision": decision, "next_state": next_state},
|
||||||
|
)
|
||||||
|
updated = connection.execute("SELECT * FROM tasks WHERE task_id=?", (task_id,)).fetchone()
|
||||||
|
connection.commit()
|
||||||
|
return self._row_task(updated)
|
||||||
|
|
||||||
|
def cancel_task(self, *, task_id: str, reason: str = "") -> dict[str, Any]:
|
||||||
|
with self._connect() as connection:
|
||||||
|
connection.execute("BEGIN IMMEDIATE")
|
||||||
|
row = connection.execute("SELECT * FROM tasks WHERE task_id=?", (task_id,)).fetchone()
|
||||||
|
if row is None:
|
||||||
|
raise TaskProtocolError("TASK_NOT_FOUND", 404)
|
||||||
|
if row["status"] in TERMINAL_STATES:
|
||||||
|
connection.commit()
|
||||||
|
return self._row_task(row)
|
||||||
|
connection.execute(
|
||||||
|
"UPDATE tasks SET status='CANCELLED',lease_token_hash=NULL,"
|
||||||
|
"lease_expires_at=NULL,revision=revision+1,updated_at=? WHERE task_id=?",
|
||||||
|
(iso(self.now()), task_id),
|
||||||
|
)
|
||||||
|
self._event(connection, "TASK_CANCELLED", task_id=task_id, payload={"reason": reason})
|
||||||
|
updated = connection.execute("SELECT * FROM tasks WHERE task_id=?", (task_id,)).fetchone()
|
||||||
|
connection.commit()
|
||||||
|
return self._row_task(updated)
|
||||||
|
|
||||||
|
def get_task(self, task_id: str) -> dict[str, Any]:
|
||||||
|
self.expire_leases()
|
||||||
|
with self._connect() as connection:
|
||||||
|
row = connection.execute("SELECT * FROM tasks WHERE task_id=?", (task_id,)).fetchone()
|
||||||
|
if row is None:
|
||||||
|
raise TaskProtocolError("TASK_NOT_FOUND", 404)
|
||||||
|
return self._row_task(row)
|
||||||
|
|
||||||
|
def get_events(self, task_id: str, *, limit: int = 200) -> list[dict[str, Any]]:
|
||||||
|
with self._connect() as connection:
|
||||||
|
rows = connection.execute(
|
||||||
|
"SELECT * FROM events WHERE task_id=? ORDER BY event_id DESC LIMIT ?",
|
||||||
|
(task_id, max(1, min(int(limit), 1000))),
|
||||||
|
).fetchall()
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"event_id": row["event_id"],
|
||||||
|
"task_id": row["task_id"],
|
||||||
|
"agent_id": row["agent_id"],
|
||||||
|
"event_type": row["event_type"],
|
||||||
|
"payload": json.loads(row["payload_json"]),
|
||||||
|
"created_at": row["created_at"],
|
||||||
|
}
|
||||||
|
for row in reversed(rows)
|
||||||
|
]
|
||||||
@@ -0,0 +1,427 @@
|
|||||||
|
"""Owner-controlled, CAS-bound retry extension for exhausted Online Agent Tasks."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import sqlite3
|
||||||
|
from typing import Any, Mapping
|
||||||
|
|
||||||
|
TASK_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{2,179}$")
|
||||||
|
KEY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{7,159}$")
|
||||||
|
MAX_BODY = 64 * 1024
|
||||||
|
AUTHORIZE_EXTENSION_ID_ENV = "AIOS_AGENT_TASK_AUTHORIZATION_ID"
|
||||||
|
|
||||||
|
|
||||||
|
class RetryExtensionError(RuntimeError):
|
||||||
|
def __init__(self, code: str, status: int = 409) -> None:
|
||||||
|
super().__init__(code)
|
||||||
|
self.code = code
|
||||||
|
self.status = status
|
||||||
|
|
||||||
|
|
||||||
|
def _canonical(value: object) -> bytes:
|
||||||
|
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _sha(value: object) -> str:
|
||||||
|
return hashlib.sha256(_canonical(value)).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _required_authorization_id() -> str:
|
||||||
|
value = os.environ.get(AUTHORIZE_EXTENSION_ID_ENV, "").strip()
|
||||||
|
if not value:
|
||||||
|
raise RuntimeError("TASK_RETRY_AUTHORIZATION_ID_MISSING")
|
||||||
|
if KEY_RE.fullmatch(value) is None:
|
||||||
|
raise RuntimeError("TASK_RETRY_AUTHORIZATION_ID_INVALID")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _now() -> str:
|
||||||
|
return datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def _require_string(value: object, code: str, pattern: re.Pattern[str] | None = None) -> str:
|
||||||
|
if not isinstance(value, str) or not value.strip():
|
||||||
|
raise RetryExtensionError(code, 400)
|
||||||
|
value = value.strip()
|
||||||
|
if pattern is not None and pattern.fullmatch(value) is None:
|
||||||
|
raise RetryExtensionError(code, 400)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class RetryExtensionRequest:
|
||||||
|
expected_status: str
|
||||||
|
expected_revision: int
|
||||||
|
expected_attempt: int
|
||||||
|
expected_max_attempts: int
|
||||||
|
next_max_attempts: int
|
||||||
|
previous_verification_sha256: str
|
||||||
|
authorization_id: str
|
||||||
|
idempotency_key: str
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_mapping(cls, value: Mapping[str, Any]) -> "RetryExtensionRequest":
|
||||||
|
allowed = {
|
||||||
|
"expected_status", "expected_revision", "expected_attempt", "expected_max_attempts",
|
||||||
|
"next_max_attempts", "previous_verification_sha256", "authorization_id", "idempotency_key",
|
||||||
|
}
|
||||||
|
if set(value) != allowed:
|
||||||
|
raise RetryExtensionError("RETRY_EXTENSION_FIELDS_INVALID", 400)
|
||||||
|
status = _require_string(value["expected_status"], "EXPECTED_STATUS_INVALID")
|
||||||
|
if status != "FAILED":
|
||||||
|
raise RetryExtensionError("EXPECTED_STATUS_INVALID", 400)
|
||||||
|
ints: dict[str, int] = {}
|
||||||
|
for field in ("expected_revision", "expected_attempt", "expected_max_attempts", "next_max_attempts"):
|
||||||
|
item = value[field]
|
||||||
|
if not isinstance(item, int) or isinstance(item, bool) or item < 0:
|
||||||
|
raise RetryExtensionError("RETRY_EXTENSION_INTEGER_INVALID", 400)
|
||||||
|
ints[field] = item
|
||||||
|
if ints["expected_attempt"] != ints["expected_max_attempts"]:
|
||||||
|
raise RetryExtensionError("RETRY_EXTENSION_NOT_EXHAUSTED", 409)
|
||||||
|
if ints["next_max_attempts"] != ints["expected_max_attempts"] + 1:
|
||||||
|
raise RetryExtensionError("RETRY_EXTENSION_MUST_BE_BOUNDED_ONE_STEP", 400)
|
||||||
|
if ints["next_max_attempts"] > 50:
|
||||||
|
raise RetryExtensionError("RETRY_EXTENSION_LIMIT_EXCEEDED", 400)
|
||||||
|
verification = _require_string(value["previous_verification_sha256"], "VERIFICATION_SHA256_INVALID")
|
||||||
|
if re.fullmatch(r"[0-9a-f]{64}", verification) is None:
|
||||||
|
raise RetryExtensionError("VERIFICATION_SHA256_INVALID", 400)
|
||||||
|
authorization = _require_string(value["authorization_id"], "AUTHORIZATION_ID_INVALID", KEY_RE)
|
||||||
|
key = _require_string(value["idempotency_key"], "IDEMPOTENCY_KEY_INVALID", KEY_RE)
|
||||||
|
return cls(
|
||||||
|
status,
|
||||||
|
ints["expected_revision"],
|
||||||
|
ints["expected_attempt"],
|
||||||
|
ints["expected_max_attempts"],
|
||||||
|
ints["next_max_attempts"],
|
||||||
|
verification,
|
||||||
|
authorization,
|
||||||
|
key,
|
||||||
|
)
|
||||||
|
|
||||||
|
def public(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"expected_status": self.expected_status,
|
||||||
|
"expected_revision": self.expected_revision,
|
||||||
|
"expected_attempt": self.expected_attempt,
|
||||||
|
"expected_max_attempts": self.expected_max_attempts,
|
||||||
|
"next_max_attempts": self.next_max_attempts,
|
||||||
|
"previous_verification_sha256": self.previous_verification_sha256,
|
||||||
|
"authorization_id": self.authorization_id,
|
||||||
|
"idempotency_key": self.idempotency_key,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class RetryExtensionStore:
|
||||||
|
"""Owns the official mutation. Direct/manual DB edits remain forbidden."""
|
||||||
|
|
||||||
|
def __init__(self, db_path: Path) -> None:
|
||||||
|
self.db_path = Path(db_path)
|
||||||
|
|
||||||
|
def _connect(self) -> sqlite3.Connection:
|
||||||
|
connection = sqlite3.connect(self.db_path, timeout=30.0)
|
||||||
|
connection.row_factory = sqlite3.Row
|
||||||
|
connection.execute("PRAGMA foreign_keys=ON")
|
||||||
|
connection.execute("PRAGMA busy_timeout=30000")
|
||||||
|
return connection
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _columns(connection: sqlite3.Connection, table: str) -> dict[str, sqlite3.Row]:
|
||||||
|
return {str(row["name"]): row for row in connection.execute(f'PRAGMA table_info("{table}")')}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _event_table(connection: sqlite3.Connection) -> tuple[str, dict[str, sqlite3.Row]]:
|
||||||
|
tables = {
|
||||||
|
row["name"]
|
||||||
|
for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'")
|
||||||
|
}
|
||||||
|
for name in ("task_events", "events"):
|
||||||
|
if name not in tables:
|
||||||
|
continue
|
||||||
|
columns = RetryExtensionStore._columns(connection, name)
|
||||||
|
if "task_id" in columns and ("event_type" in columns or "type" in columns):
|
||||||
|
return name, columns
|
||||||
|
raise RetryExtensionError("TASK_EVENT_TABLE_UNSUPPORTED", 503)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _verification_hash(raw: object) -> str:
|
||||||
|
if raw in (None, ""):
|
||||||
|
value: object = {}
|
||||||
|
elif isinstance(raw, str):
|
||||||
|
try:
|
||||||
|
value = json.loads(raw)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise RetryExtensionError("TASK_VERIFICATION_INVALID", 409) from exc
|
||||||
|
else:
|
||||||
|
value = raw
|
||||||
|
return _sha(value)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _ensure_extension_table(connection: sqlite3.Connection) -> None:
|
||||||
|
connection.execute(
|
||||||
|
"""
|
||||||
|
CREATE TABLE IF NOT EXISTS task_retry_extensions (
|
||||||
|
task_id TEXT NOT NULL,
|
||||||
|
idempotency_key TEXT NOT NULL,
|
||||||
|
request_sha256 TEXT NOT NULL,
|
||||||
|
authorization_id TEXT NOT NULL,
|
||||||
|
previous_verification_sha256 TEXT NOT NULL,
|
||||||
|
previous_max_attempts INTEGER NOT NULL,
|
||||||
|
next_max_attempts INTEGER NOT NULL,
|
||||||
|
expected_revision INTEGER NOT NULL,
|
||||||
|
applied_revision INTEGER NOT NULL,
|
||||||
|
created_at TEXT NOT NULL,
|
||||||
|
PRIMARY KEY(task_id, idempotency_key)
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _insert_event(
|
||||||
|
connection: sqlite3.Connection,
|
||||||
|
task_id: str,
|
||||||
|
payload: Mapping[str, Any],
|
||||||
|
) -> None:
|
||||||
|
table, columns = RetryExtensionStore._event_table(connection)
|
||||||
|
values: dict[str, Any] = {"task_id": task_id}
|
||||||
|
if "event_type" in columns:
|
||||||
|
values["event_type"] = "TASK_RETRY_EXTENSION_AUTHORIZED"
|
||||||
|
else:
|
||||||
|
values["type"] = "TASK_RETRY_EXTENSION_AUTHORIZED"
|
||||||
|
payload_text = json.dumps(payload, sort_keys=True, separators=(",", ":"))
|
||||||
|
if "payload_json" in columns:
|
||||||
|
values["payload_json"] = payload_text
|
||||||
|
elif "payload" in columns:
|
||||||
|
values["payload"] = payload_text
|
||||||
|
else:
|
||||||
|
raise RetryExtensionError("TASK_EVENT_PAYLOAD_UNSUPPORTED", 503)
|
||||||
|
if "created_at" in columns:
|
||||||
|
values["created_at"] = _now()
|
||||||
|
if "agent_id" in columns:
|
||||||
|
values["agent_id"] = "owner-control"
|
||||||
|
if "actor_id" in columns:
|
||||||
|
values["actor_id"] = "owner-control"
|
||||||
|
names = list(values)
|
||||||
|
sql = f'INSERT INTO "{table}" ({",".join(f"""\"{name}\"""" for name in names)}) VALUES ({",".join("?" for _ in names)})'
|
||||||
|
connection.execute(sql, tuple(values[name] for name in names))
|
||||||
|
|
||||||
|
def extend(self, task_id: str, request: RetryExtensionRequest) -> dict[str, Any]:
|
||||||
|
task_id = _require_string(task_id, "TASK_ID_INVALID", TASK_RE)
|
||||||
|
required_authorization_id = _required_authorization_id()
|
||||||
|
if request.authorization_id != required_authorization_id:
|
||||||
|
raise RetryExtensionError("RETRY_EXTENSION_AUTHORIZATION_ID_MISMATCH", 403)
|
||||||
|
request_payload = request.public()
|
||||||
|
request_sha = _sha(request_payload)
|
||||||
|
with self._connect() as connection:
|
||||||
|
connection.execute("BEGIN IMMEDIATE")
|
||||||
|
self._ensure_extension_table(connection)
|
||||||
|
replay = connection.execute(
|
||||||
|
"SELECT * FROM task_retry_extensions WHERE task_id=? AND idempotency_key=?",
|
||||||
|
(task_id, request.idempotency_key),
|
||||||
|
).fetchone()
|
||||||
|
if replay is not None:
|
||||||
|
if replay["request_sha256"] != request_sha:
|
||||||
|
raise RetryExtensionError("RETRY_EXTENSION_IDEMPOTENCY_CONFLICT", 409)
|
||||||
|
connection.commit()
|
||||||
|
return {
|
||||||
|
"task_id": task_id,
|
||||||
|
"status": "QUEUED",
|
||||||
|
"attempt": request.expected_attempt,
|
||||||
|
"max_attempts": int(replay["next_max_attempts"]),
|
||||||
|
"revision": int(replay["applied_revision"]),
|
||||||
|
"idempotent": True,
|
||||||
|
"authorization_id": replay["authorization_id"],
|
||||||
|
"previous_verification_sha256": replay["previous_verification_sha256"],
|
||||||
|
}
|
||||||
|
|
||||||
|
columns = self._columns(connection, "tasks")
|
||||||
|
required = {
|
||||||
|
"task_id", "status", "attempt", "max_attempts", "revision", "claimed_by",
|
||||||
|
"lease_token_hash", "lease_expires_at", "verification_json",
|
||||||
|
}
|
||||||
|
if not required.issubset(columns):
|
||||||
|
raise RetryExtensionError("TASK_SCHEMA_RETRY_EXTENSION_UNSUPPORTED", 503)
|
||||||
|
task = connection.execute("SELECT * FROM tasks WHERE task_id=?", (task_id,)).fetchone()
|
||||||
|
if task is None:
|
||||||
|
raise RetryExtensionError("TASK_NOT_FOUND", 404)
|
||||||
|
actual_verification_sha = self._verification_hash(task["verification_json"])
|
||||||
|
if actual_verification_sha != request.previous_verification_sha256:
|
||||||
|
raise RetryExtensionError("RETRY_EXTENSION_VERIFICATION_DRIFT", 409)
|
||||||
|
if (
|
||||||
|
task["status"] != request.expected_status
|
||||||
|
or int(task["revision"]) != request.expected_revision
|
||||||
|
or int(task["attempt"]) != request.expected_attempt
|
||||||
|
or int(task["max_attempts"]) != request.expected_max_attempts
|
||||||
|
):
|
||||||
|
raise RetryExtensionError("RETRY_EXTENSION_CAS_MISMATCH", 409)
|
||||||
|
for lease_field in ("lease_token_hash", "lease_expires_at"):
|
||||||
|
if lease_field in columns and task[lease_field] not in (None, ""):
|
||||||
|
raise RetryExtensionError("RETRY_EXTENSION_ACTIVE_LEASE_DENY", 409)
|
||||||
|
|
||||||
|
previous_claimed_by = task["claimed_by"]
|
||||||
|
applied_revision = request.expected_revision + 1
|
||||||
|
cursor = connection.execute(
|
||||||
|
"""
|
||||||
|
UPDATE tasks
|
||||||
|
SET status='QUEUED', max_attempts=?, revision=?, claimed_by=NULL,
|
||||||
|
lease_token_hash=NULL, lease_expires_at=NULL
|
||||||
|
WHERE task_id=? AND status=? AND revision=? AND attempt=? AND max_attempts=?
|
||||||
|
AND claimed_by IS ?
|
||||||
|
AND lease_token_hash IS NULL AND lease_expires_at IS NULL
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
request.next_max_attempts,
|
||||||
|
applied_revision,
|
||||||
|
task_id,
|
||||||
|
request.expected_status,
|
||||||
|
request.expected_revision,
|
||||||
|
request.expected_attempt,
|
||||||
|
request.expected_max_attempts,
|
||||||
|
previous_claimed_by,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if cursor.rowcount != 1:
|
||||||
|
raise RetryExtensionError("RETRY_EXTENSION_CAS_MISMATCH", 409)
|
||||||
|
|
||||||
|
event_payload = {
|
||||||
|
"schema": "aios.task-retry-extension.event.v1",
|
||||||
|
"task_id": task_id,
|
||||||
|
"authorization_id": request.authorization_id,
|
||||||
|
"idempotency_key": request.idempotency_key,
|
||||||
|
"previous_verification_sha256": request.previous_verification_sha256,
|
||||||
|
"previous_max_attempts": request.expected_max_attempts,
|
||||||
|
"next_max_attempts": request.next_max_attempts,
|
||||||
|
"previous_revision": request.expected_revision,
|
||||||
|
"applied_revision": applied_revision,
|
||||||
|
"request_sha256": request_sha,
|
||||||
|
"previous_claimed_by": previous_claimed_by,
|
||||||
|
"stale_terminal_claim_normalized": previous_claimed_by not in (None, ""),
|
||||||
|
}
|
||||||
|
self._insert_event(connection, task_id, event_payload)
|
||||||
|
connection.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO task_retry_extensions(
|
||||||
|
task_id,idempotency_key,request_sha256,authorization_id,previous_verification_sha256,
|
||||||
|
previous_max_attempts,next_max_attempts,expected_revision,applied_revision,created_at
|
||||||
|
) VALUES (?,?,?,?,?,?,?,?,?,?)
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
task_id,
|
||||||
|
request.idempotency_key,
|
||||||
|
request_sha,
|
||||||
|
request.authorization_id,
|
||||||
|
request.previous_verification_sha256,
|
||||||
|
request.expected_max_attempts,
|
||||||
|
request.next_max_attempts,
|
||||||
|
request.expected_revision,
|
||||||
|
applied_revision,
|
||||||
|
_now(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
connection.commit()
|
||||||
|
return {
|
||||||
|
"task_id": task_id,
|
||||||
|
"status": "QUEUED",
|
||||||
|
"attempt": request.expected_attempt,
|
||||||
|
"max_attempts": request.next_max_attempts,
|
||||||
|
"revision": applied_revision,
|
||||||
|
"idempotent": False,
|
||||||
|
"authorization_id": request.authorization_id,
|
||||||
|
"previous_verification_sha256": request.previous_verification_sha256,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class RetryExtensionHandler(BaseHTTPRequestHandler):
|
||||||
|
store: RetryExtensionStore | None = None
|
||||||
|
control_token: str = ""
|
||||||
|
|
||||||
|
def log_message(self, *_args: Any) -> None:
|
||||||
|
return
|
||||||
|
|
||||||
|
def _json(self, value: Mapping[str, Any], status: int) -> None:
|
||||||
|
data = json.dumps(dict(value), sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
self.send_response(status)
|
||||||
|
self.send_header("Content-Type", "application/json")
|
||||||
|
self.send_header("Content-Length", str(len(data)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(data)
|
||||||
|
|
||||||
|
def _authorized(self) -> bool:
|
||||||
|
header = self.headers.get("Authorization", "")
|
||||||
|
return (
|
||||||
|
header.startswith("Bearer ")
|
||||||
|
and bool(self.control_token)
|
||||||
|
and hmac.compare_digest(header[7:], self.control_token)
|
||||||
|
)
|
||||||
|
|
||||||
|
def do_GET(self) -> None:
|
||||||
|
if self.path == "/health":
|
||||||
|
self._json({"status": "ok", "service": "aios-agent-task-retry-extension"}, 200)
|
||||||
|
else:
|
||||||
|
self._json({"error": "NOT_FOUND"}, 404)
|
||||||
|
|
||||||
|
def do_POST(self) -> None:
|
||||||
|
match = re.fullmatch(r"/v1/tasks/([^/]+)/retry-extension", self.path)
|
||||||
|
if match is None:
|
||||||
|
self._json({"error": "NOT_FOUND"}, 404)
|
||||||
|
return
|
||||||
|
if not self._authorized():
|
||||||
|
self._json({"error": "UNAUTHORIZED"}, 401)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
length = int(self.headers.get("Content-Length", "0"))
|
||||||
|
except ValueError:
|
||||||
|
self._json({"error": "CONTENT_LENGTH_INVALID"}, 400)
|
||||||
|
return
|
||||||
|
if length < 1 or length > MAX_BODY:
|
||||||
|
self._json({"error": "REQUEST_SIZE_INVALID"}, 413)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
value = json.loads(self.rfile.read(length).decode("utf-8"))
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise ValueError
|
||||||
|
request = RetryExtensionRequest.from_mapping(value)
|
||||||
|
if self.store is None:
|
||||||
|
raise RetryExtensionError("RUNTIME_UNAVAILABLE", 503)
|
||||||
|
result = self.store.extend(match.group(1), request)
|
||||||
|
self._json(result, 200 if result["idempotent"] else 201)
|
||||||
|
except RetryExtensionError as exc:
|
||||||
|
self._json({"error": exc.code}, exc.status)
|
||||||
|
except (UnicodeDecodeError, json.JSONDecodeError, ValueError):
|
||||||
|
self._json({"error": "REQUEST_INVALID"}, 400)
|
||||||
|
|
||||||
|
|
||||||
|
def load_control_token(path: Path) -> str:
|
||||||
|
if path.is_symlink() or not path.is_file():
|
||||||
|
raise RuntimeError("TASK_RETRY_CONTROL_CREDENTIAL_UNAVAILABLE")
|
||||||
|
value = path.read_text(encoding="utf-8").strip()
|
||||||
|
if len(value) < 32:
|
||||||
|
raise RuntimeError("TASK_RETRY_CONTROL_CREDENTIAL_INVALID")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def serve() -> None:
|
||||||
|
db = Path(os.environ.get("AIOS_AGENT_TASK_DB", "/var/lib/aios-agent-task/state.sqlite3"))
|
||||||
|
_required_authorization_id()
|
||||||
|
token_path = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_AGENT_TASK_CONTROL_TOKEN_FILE",
|
||||||
|
str(Path(os.environ.get("CREDENTIALS_DIRECTORY", "/run/credentials")) / "agent_task_control_token"),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
host = os.environ.get("AIOS_TASK_RETRY_HOST", "127.0.0.1")
|
||||||
|
port = int(os.environ.get("AIOS_TASK_RETRY_PORT", "18941"))
|
||||||
|
RetryExtensionHandler.store = RetryExtensionStore(db)
|
||||||
|
RetryExtensionHandler.control_token = load_control_token(token_path)
|
||||||
|
ThreadingHTTPServer((host, port), RetryExtensionHandler).serve_forever()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
serve()
|
||||||
@@ -0,0 +1,655 @@
|
|||||||
|
"""CAS-bound projection from an unprivileged planner to a host coordinator."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import hmac
|
||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import sqlite3
|
||||||
|
from typing import Any, Callable, Mapping, Sequence
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
from aios_agent_task_initial_execution_projection import (
|
||||||
|
InitialExecutionProjectionError,
|
||||||
|
InitialExecutionProjectionStore,
|
||||||
|
JOB_RE,
|
||||||
|
KEY_RE,
|
||||||
|
MAX_BODY,
|
||||||
|
SHA_RE,
|
||||||
|
TASK_RE,
|
||||||
|
_canonical,
|
||||||
|
_now,
|
||||||
|
_required_authorization_id,
|
||||||
|
_require_string,
|
||||||
|
_sha,
|
||||||
|
load_control_token,
|
||||||
|
)
|
||||||
|
from aios_online_agent_execution import (
|
||||||
|
GovernedExecutionBinding,
|
||||||
|
validate_execution_envelope,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
RUNTIME_CAPABILITY = "agent.runtime.openclaw"
|
||||||
|
COORDINATOR_CAPABILITY = "engineering.autonomous"
|
||||||
|
COORDINATOR_CAPABILITIES = [RUNTIME_CAPABILITY, COORDINATOR_CAPABILITY]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SupervisorCoordinatorProjectionRequest:
|
||||||
|
expected_status: str
|
||||||
|
expected_revision: int
|
||||||
|
expected_attempt: int
|
||||||
|
expected_metadata_sha256: str
|
||||||
|
expected_required_capabilities_sha256: str
|
||||||
|
previous_verification_sha256: str
|
||||||
|
expected_planner_result_sha256: str
|
||||||
|
expected_old_execution_digest: str
|
||||||
|
expected_new_execution_digest: str
|
||||||
|
expected_job_id: str
|
||||||
|
authorization_id: str
|
||||||
|
idempotency_key: str
|
||||||
|
reason: str
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_mapping(
|
||||||
|
cls, value: Mapping[str, Any]
|
||||||
|
) -> "SupervisorCoordinatorProjectionRequest":
|
||||||
|
allowed = {
|
||||||
|
"expected_status",
|
||||||
|
"expected_revision",
|
||||||
|
"expected_attempt",
|
||||||
|
"expected_metadata_sha256",
|
||||||
|
"expected_required_capabilities_sha256",
|
||||||
|
"previous_verification_sha256",
|
||||||
|
"expected_planner_result_sha256",
|
||||||
|
"expected_old_execution_digest",
|
||||||
|
"expected_new_execution_digest",
|
||||||
|
"expected_job_id",
|
||||||
|
"authorization_id",
|
||||||
|
"idempotency_key",
|
||||||
|
"reason",
|
||||||
|
}
|
||||||
|
if set(value) != allowed:
|
||||||
|
raise InitialExecutionProjectionError(
|
||||||
|
"SUPERVISOR_COORDINATOR_PROJECTION_FIELDS_INVALID", 400
|
||||||
|
)
|
||||||
|
status = _require_string(value["expected_status"], "EXPECTED_STATUS_INVALID")
|
||||||
|
revision = value["expected_revision"]
|
||||||
|
attempt = value["expected_attempt"]
|
||||||
|
if (
|
||||||
|
status != "QUEUED"
|
||||||
|
or not isinstance(revision, int)
|
||||||
|
or isinstance(revision, bool)
|
||||||
|
or revision < 2
|
||||||
|
or not isinstance(attempt, int)
|
||||||
|
or isinstance(attempt, bool)
|
||||||
|
or attempt < 1
|
||||||
|
):
|
||||||
|
raise InitialExecutionProjectionError(
|
||||||
|
"SUPERVISOR_COORDINATOR_PROJECTION_STATE_INVALID", 400
|
||||||
|
)
|
||||||
|
names = (
|
||||||
|
"expected_metadata_sha256",
|
||||||
|
"expected_required_capabilities_sha256",
|
||||||
|
"previous_verification_sha256",
|
||||||
|
"expected_planner_result_sha256",
|
||||||
|
"expected_old_execution_digest",
|
||||||
|
"expected_new_execution_digest",
|
||||||
|
)
|
||||||
|
digests = {
|
||||||
|
name: _require_string(value[name], name.upper() + "_INVALID", SHA_RE)
|
||||||
|
for name in names
|
||||||
|
}
|
||||||
|
job_id = _require_string(value["expected_job_id"], "JOB_ID_INVALID", JOB_RE)
|
||||||
|
authorization_id = _require_string(
|
||||||
|
value["authorization_id"], "AUTHORIZATION_ID_INVALID", KEY_RE
|
||||||
|
)
|
||||||
|
idempotency_key = _require_string(
|
||||||
|
value["idempotency_key"], "IDEMPOTENCY_KEY_INVALID", KEY_RE
|
||||||
|
)
|
||||||
|
reason = _require_string(value["reason"], "PROJECTION_REASON_INVALID")
|
||||||
|
if len(reason) > 512:
|
||||||
|
raise InitialExecutionProjectionError("PROJECTION_REASON_INVALID", 400)
|
||||||
|
return cls(
|
||||||
|
status,
|
||||||
|
revision,
|
||||||
|
attempt,
|
||||||
|
digests["expected_metadata_sha256"],
|
||||||
|
digests["expected_required_capabilities_sha256"],
|
||||||
|
digests["previous_verification_sha256"],
|
||||||
|
digests["expected_planner_result_sha256"],
|
||||||
|
digests["expected_old_execution_digest"],
|
||||||
|
digests["expected_new_execution_digest"],
|
||||||
|
job_id,
|
||||||
|
authorization_id,
|
||||||
|
idempotency_key,
|
||||||
|
reason,
|
||||||
|
)
|
||||||
|
|
||||||
|
def public(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"expected_status": self.expected_status,
|
||||||
|
"expected_revision": self.expected_revision,
|
||||||
|
"expected_attempt": self.expected_attempt,
|
||||||
|
"expected_metadata_sha256": self.expected_metadata_sha256,
|
||||||
|
"expected_required_capabilities_sha256": self.expected_required_capabilities_sha256,
|
||||||
|
"previous_verification_sha256": self.previous_verification_sha256,
|
||||||
|
"expected_planner_result_sha256": self.expected_planner_result_sha256,
|
||||||
|
"expected_old_execution_digest": self.expected_old_execution_digest,
|
||||||
|
"expected_new_execution_digest": self.expected_new_execution_digest,
|
||||||
|
"expected_job_id": self.expected_job_id,
|
||||||
|
"authorization_id": self.authorization_id,
|
||||||
|
"idempotency_key": self.idempotency_key,
|
||||||
|
"reason": self.reason,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class CanonicalCoordinatorExecutionResolver:
|
||||||
|
def __init__(self, base_url: str, control_token: str) -> None:
|
||||||
|
self.base_url = base_url.rstrip("/")
|
||||||
|
self.control_token = control_token
|
||||||
|
|
||||||
|
def __call__(
|
||||||
|
self,
|
||||||
|
task_id: str,
|
||||||
|
expected_job_id: str,
|
||||||
|
runtime_input: Mapping[str, Any],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
body = {
|
||||||
|
"task_id": task_id,
|
||||||
|
"job_id": expected_job_id,
|
||||||
|
"required_capabilities": COORDINATOR_CAPABILITIES,
|
||||||
|
"execution": {
|
||||||
|
"action": "agent_runtime",
|
||||||
|
"adapter": "openclaw",
|
||||||
|
"capability_scope": COORDINATOR_CAPABILITIES,
|
||||||
|
"runtime_input": dict(runtime_input),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
request = urllib.request.Request(
|
||||||
|
self.base_url + "/v2/execution-envelope",
|
||||||
|
method="POST",
|
||||||
|
data=_canonical(body),
|
||||||
|
headers={
|
||||||
|
"Authorization": "Bearer " + self.control_token,
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"Accept": "application/json",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(request, timeout=15) as response:
|
||||||
|
raw, status = response.read(), response.status
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
raw, status = exc.read(), exc.code
|
||||||
|
try:
|
||||||
|
value = json.loads(raw.decode("utf-8"))
|
||||||
|
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||||
|
raise InitialExecutionProjectionError(
|
||||||
|
"CANONICAL_COORDINATOR_EXECUTION_RESPONSE_INVALID", 502
|
||||||
|
) from exc
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise InitialExecutionProjectionError(
|
||||||
|
"CANONICAL_COORDINATOR_EXECUTION_RESPONSE_INVALID", 502
|
||||||
|
)
|
||||||
|
if status != 200:
|
||||||
|
raise InitialExecutionProjectionError(
|
||||||
|
str(value.get("error") or "CANONICAL_COORDINATOR_EXECUTION_FAILED"),
|
||||||
|
502 if status >= 500 else 409,
|
||||||
|
)
|
||||||
|
return dict(value)
|
||||||
|
|
||||||
|
|
||||||
|
class SupervisorCoordinatorProjectionStore(InitialExecutionProjectionStore):
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
db_path: Path,
|
||||||
|
resolver: Callable[[str, str, Mapping[str, Any]], Mapping[str, Any]],
|
||||||
|
) -> None:
|
||||||
|
super().__init__(db_path, resolver) # type: ignore[arg-type]
|
||||||
|
self.coordinator_resolver = resolver
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _ensure_table(connection: sqlite3.Connection) -> None:
|
||||||
|
connection.execute(
|
||||||
|
"""
|
||||||
|
CREATE TABLE IF NOT EXISTS task_supervisor_coordinator_projections (
|
||||||
|
task_id TEXT NOT NULL,
|
||||||
|
idempotency_key TEXT NOT NULL,
|
||||||
|
request_sha256 TEXT NOT NULL,
|
||||||
|
authorization_id TEXT NOT NULL,
|
||||||
|
expected_job_id TEXT NOT NULL,
|
||||||
|
planner_result_sha256 TEXT NOT NULL,
|
||||||
|
old_execution_digest TEXT NOT NULL,
|
||||||
|
new_execution_digest TEXT NOT NULL,
|
||||||
|
expected_revision INTEGER NOT NULL,
|
||||||
|
applied_revision INTEGER NOT NULL,
|
||||||
|
attempt INTEGER NOT NULL,
|
||||||
|
created_at TEXT NOT NULL,
|
||||||
|
PRIMARY KEY(task_id,idempotency_key)
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _planner_event_chain(
|
||||||
|
cls, connection: sqlite3.Connection, task_id: str
|
||||||
|
) -> None:
|
||||||
|
table, columns = cls._event_table(connection)
|
||||||
|
ordering = "event_id" if "event_id" in columns else "rowid"
|
||||||
|
rows = connection.execute(
|
||||||
|
f'SELECT event_type,payload_json FROM "{table}" '
|
||||||
|
f'WHERE task_id=? ORDER BY {ordering}',
|
||||||
|
(task_id,),
|
||||||
|
).fetchall()
|
||||||
|
events = [str(row[0]) for row in rows]
|
||||||
|
try:
|
||||||
|
projection = len(events) - 1 - events[::-1].index(
|
||||||
|
"TASK_INITIAL_EXECUTION_PROJECTION_AUTHORIZED"
|
||||||
|
)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise InitialExecutionProjectionError(
|
||||||
|
"SUPERVISOR_PLANNER_PROJECTION_EVENT_MISSING", 409
|
||||||
|
) from exc
|
||||||
|
tail = events[projection + 1 :]
|
||||||
|
if (
|
||||||
|
len(tail) < 4
|
||||||
|
or tail[0] != "TASK_CLAIMED"
|
||||||
|
or tail.count("TASK_CLAIMED") != 1
|
||||||
|
or tail.count("RESULT_SUBMITTED") != 1
|
||||||
|
or tail.count("TASK_VERIFIED") != 1
|
||||||
|
or tail[-1] != "TASK_VERIFIED"
|
||||||
|
):
|
||||||
|
raise InitialExecutionProjectionError(
|
||||||
|
"SUPERVISOR_PLANNER_EVENT_CHAIN_INVALID", 409
|
||||||
|
)
|
||||||
|
result = tail.index("RESULT_SUBMITTED")
|
||||||
|
if (
|
||||||
|
not any(event == "TASK_HEARTBEAT" for event in tail[1:result])
|
||||||
|
or any(
|
||||||
|
event not in {
|
||||||
|
"TASK_CLAIMED",
|
||||||
|
"TASK_HEARTBEAT",
|
||||||
|
"RESULT_SUBMITTED",
|
||||||
|
"TASK_VERIFIED",
|
||||||
|
}
|
||||||
|
for event in tail
|
||||||
|
)
|
||||||
|
):
|
||||||
|
raise InitialExecutionProjectionError(
|
||||||
|
"SUPERVISOR_PLANNER_EVENT_CHAIN_INVALID", 409
|
||||||
|
)
|
||||||
|
verification_event = json.loads(rows[-1][1])
|
||||||
|
if (
|
||||||
|
verification_event.get("decision") != "NEEDS_REWORK"
|
||||||
|
or verification_event.get("next_state") != "QUEUED"
|
||||||
|
):
|
||||||
|
raise InitialExecutionProjectionError(
|
||||||
|
"SUPERVISOR_PLANNER_NEEDS_REWORK_EVENT_REQUIRED", 409
|
||||||
|
)
|
||||||
|
|
||||||
|
def _load(
|
||||||
|
self,
|
||||||
|
connection: sqlite3.Connection,
|
||||||
|
task_id: str,
|
||||||
|
request: SupervisorCoordinatorProjectionRequest,
|
||||||
|
) -> tuple[sqlite3.Row, dict[str, Any], dict[str, Any]]:
|
||||||
|
task = connection.execute(
|
||||||
|
"SELECT * FROM tasks WHERE task_id=?", (task_id,)
|
||||||
|
).fetchone()
|
||||||
|
if task is None:
|
||||||
|
raise InitialExecutionProjectionError("TASK_NOT_FOUND", 404)
|
||||||
|
if (
|
||||||
|
task["status"] != request.expected_status
|
||||||
|
or int(task["revision"]) != request.expected_revision
|
||||||
|
or int(task["attempt"]) != request.expected_attempt
|
||||||
|
):
|
||||||
|
raise InitialExecutionProjectionError(
|
||||||
|
"SUPERVISOR_COORDINATOR_PROJECTION_CAS_MISMATCH", 409
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
task["claimed_by"] not in (None, "")
|
||||||
|
or task["lease_token_hash"] not in (None, "")
|
||||||
|
or task["lease_expires_at"] not in (None, "")
|
||||||
|
):
|
||||||
|
raise InitialExecutionProjectionError(
|
||||||
|
"SUPERVISOR_COORDINATOR_ACTIVE_LEASE_DENY", 409
|
||||||
|
)
|
||||||
|
metadata = self._json_value(task["metadata_json"], "TASK_METADATA_INVALID")
|
||||||
|
capabilities = self._json_value(
|
||||||
|
task["required_capabilities_json"], "TASK_CAPABILITIES_INVALID"
|
||||||
|
)
|
||||||
|
result = self._json_value(task["result_json"], "PLANNER_RESULT_INVALID")
|
||||||
|
verification = self._json_value(
|
||||||
|
task["verification_json"], "PLANNER_VERIFICATION_INVALID"
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
not isinstance(metadata, Mapping)
|
||||||
|
or not isinstance(capabilities, list)
|
||||||
|
or not isinstance(result, Mapping)
|
||||||
|
or not isinstance(verification, Mapping)
|
||||||
|
):
|
||||||
|
raise InitialExecutionProjectionError(
|
||||||
|
"SUPERVISOR_COORDINATOR_STATE_INVALID", 409
|
||||||
|
)
|
||||||
|
metadata, result, verification = dict(metadata), dict(result), dict(verification)
|
||||||
|
if _sha(metadata) != request.expected_metadata_sha256:
|
||||||
|
raise InitialExecutionProjectionError("TASK_METADATA_DRIFT", 409)
|
||||||
|
if _sha(capabilities) != request.expected_required_capabilities_sha256:
|
||||||
|
raise InitialExecutionProjectionError("TASK_CAPABILITIES_DRIFT", 409)
|
||||||
|
if _sha(result) != request.expected_planner_result_sha256:
|
||||||
|
raise InitialExecutionProjectionError("PLANNER_RESULT_DRIFT", 409)
|
||||||
|
if _sha(verification) != request.previous_verification_sha256:
|
||||||
|
raise InitialExecutionProjectionError("PLANNER_VERIFICATION_DRIFT", 409)
|
||||||
|
old = metadata.get("execution")
|
||||||
|
if not isinstance(old, Mapping):
|
||||||
|
raise InitialExecutionProjectionError("TASK_EXECUTION_METADATA_MISSING", 409)
|
||||||
|
old_execution = validate_execution_envelope(dict(old))
|
||||||
|
if (
|
||||||
|
old_execution.get("action") != "agent_runtime"
|
||||||
|
or old_execution.get("adapter") != "openclaw"
|
||||||
|
or old_execution.get("capability_scope") != [RUNTIME_CAPABILITY]
|
||||||
|
or old_execution.get("execution_digest")
|
||||||
|
!= request.expected_old_execution_digest
|
||||||
|
or capabilities != [RUNTIME_CAPABILITY]
|
||||||
|
):
|
||||||
|
raise InitialExecutionProjectionError(
|
||||||
|
"SUPERVISOR_PLANNER_EXECUTION_CONTRACT_INVALID", 409
|
||||||
|
)
|
||||||
|
expected_result = {
|
||||||
|
"task_id": task_id,
|
||||||
|
"job_id": request.expected_job_id,
|
||||||
|
"status": "SUCCEEDED",
|
||||||
|
"execution_mode": "agent_runtime",
|
||||||
|
"agent_runtime": "openclaw",
|
||||||
|
"execution_digest": request.expected_old_execution_digest,
|
||||||
|
"control_plane_access_granted_to_runtime": False,
|
||||||
|
"credential_material_exposed": False,
|
||||||
|
"secret_material_exposed": False,
|
||||||
|
"changed_files": [],
|
||||||
|
"writer_lease": "NONE",
|
||||||
|
"direct_main_write": False,
|
||||||
|
"production_write": False,
|
||||||
|
"force_push": False,
|
||||||
|
"protected_ref_bypass": False,
|
||||||
|
}
|
||||||
|
if any(result.get(name) != expected for name, expected in expected_result.items()):
|
||||||
|
raise InitialExecutionProjectionError(
|
||||||
|
"SUPERVISOR_PLANNER_RESULT_CONTRACT_INVALID", 409
|
||||||
|
)
|
||||||
|
if verification.get("decision") != "NEEDS_REWORK":
|
||||||
|
raise InitialExecutionProjectionError(
|
||||||
|
"SUPERVISOR_PLANNER_NEEDS_REWORK_REQUIRED", 409
|
||||||
|
)
|
||||||
|
self._planner_event_chain(connection, task_id)
|
||||||
|
return task, metadata, result
|
||||||
|
|
||||||
|
def project(
|
||||||
|
self,
|
||||||
|
task_id: str,
|
||||||
|
request: SupervisorCoordinatorProjectionRequest,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
task_id = _require_string(task_id, "TASK_ID_INVALID", TASK_RE)
|
||||||
|
if request.authorization_id != _required_authorization_id():
|
||||||
|
raise InitialExecutionProjectionError("AUTHORIZATION_ID_MISMATCH", 403)
|
||||||
|
request_sha = _sha(request.public())
|
||||||
|
with self._connect() as connection:
|
||||||
|
self._ensure_table(connection)
|
||||||
|
replay = connection.execute(
|
||||||
|
"SELECT * FROM task_supervisor_coordinator_projections "
|
||||||
|
"WHERE task_id=? AND idempotency_key=?",
|
||||||
|
(task_id, request.idempotency_key),
|
||||||
|
).fetchone()
|
||||||
|
if replay is not None:
|
||||||
|
if replay["request_sha256"] != request_sha:
|
||||||
|
raise InitialExecutionProjectionError("IDEMPOTENCY_CONFLICT", 409)
|
||||||
|
return {
|
||||||
|
"task_id": task_id,
|
||||||
|
"status": "QUEUED",
|
||||||
|
"attempt": replay["attempt"],
|
||||||
|
"revision": replay["applied_revision"],
|
||||||
|
"execution_digest": replay["new_execution_digest"],
|
||||||
|
"required_capabilities": COORDINATOR_CAPABILITIES,
|
||||||
|
"idempotent": True,
|
||||||
|
}
|
||||||
|
with self._connect() as connection:
|
||||||
|
task, _, result = self._load(connection, task_id, request)
|
||||||
|
runtime_input = self._runtime_input(task)
|
||||||
|
resolved = dict(
|
||||||
|
self.coordinator_resolver(task_id, request.expected_job_id, runtime_input)
|
||||||
|
)
|
||||||
|
binding = GovernedExecutionBinding.from_mapping(resolved)
|
||||||
|
if binding.task_id != task_id or binding.job_id != request.expected_job_id:
|
||||||
|
raise InitialExecutionProjectionError("GOVERNED_BINDING_CONFLICT", 409)
|
||||||
|
result_binding = {
|
||||||
|
name: result.get(name)
|
||||||
|
for name in (
|
||||||
|
"task_id",
|
||||||
|
"job_id",
|
||||||
|
"repository",
|
||||||
|
"base_oid",
|
||||||
|
"expected_branch_oid",
|
||||||
|
"target_branch",
|
||||||
|
"generation",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
expected_binding = {
|
||||||
|
"task_id": binding.task_id,
|
||||||
|
"job_id": binding.job_id,
|
||||||
|
"repository": binding.repository,
|
||||||
|
"base_oid": binding.base_oid,
|
||||||
|
"expected_branch_oid": binding.expected_branch_oid,
|
||||||
|
"target_branch": binding.target_branch,
|
||||||
|
"generation": binding.generation,
|
||||||
|
}
|
||||||
|
if result_binding != expected_binding:
|
||||||
|
raise InitialExecutionProjectionError(
|
||||||
|
"SUPERVISOR_PLANNER_BINDING_CONFLICT", 409
|
||||||
|
)
|
||||||
|
execution = validate_execution_envelope(resolved, binding=binding)
|
||||||
|
if (
|
||||||
|
execution.get("action") != "agent_runtime"
|
||||||
|
or execution.get("adapter") != "openclaw"
|
||||||
|
or execution.get("capability_scope") != COORDINATOR_CAPABILITIES
|
||||||
|
or execution.get("change_set") != []
|
||||||
|
or execution.get("verification") != []
|
||||||
|
or execution.get("governance", {}).get("writer_lease") is not False
|
||||||
|
or execution.get("execution_digest") != request.expected_new_execution_digest
|
||||||
|
):
|
||||||
|
raise InitialExecutionProjectionError(
|
||||||
|
"SUPERVISOR_COORDINATOR_ENVELOPE_INVALID", 409
|
||||||
|
)
|
||||||
|
with self._connect() as connection:
|
||||||
|
connection.execute("BEGIN IMMEDIATE")
|
||||||
|
self._ensure_table(connection)
|
||||||
|
task, metadata, _ = self._load(connection, task_id, request)
|
||||||
|
new_metadata = dict(metadata)
|
||||||
|
new_metadata["execution"] = execution
|
||||||
|
new_metadata["execution_projection"] = {
|
||||||
|
"schema": "aios.online-agent.task-projection.v1",
|
||||||
|
"task_id": task_id,
|
||||||
|
"job_id": request.expected_job_id,
|
||||||
|
"execution_digest": execution["execution_digest"],
|
||||||
|
}
|
||||||
|
applied_revision = request.expected_revision + 1
|
||||||
|
cursor = connection.execute(
|
||||||
|
"""
|
||||||
|
UPDATE tasks SET metadata_json=?,required_capabilities_json=?,
|
||||||
|
revision=?,updated_at=?
|
||||||
|
WHERE task_id=? AND status=? AND revision=? AND attempt=?
|
||||||
|
AND claimed_by IS NULL AND lease_token_hash IS NULL
|
||||||
|
AND lease_expires_at IS NULL AND metadata_json=?
|
||||||
|
AND required_capabilities_json=? AND result_json=?
|
||||||
|
AND verification_json=?
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
json.dumps(new_metadata, sort_keys=True, separators=(",", ":")),
|
||||||
|
json.dumps(COORDINATOR_CAPABILITIES, separators=(",", ":")),
|
||||||
|
applied_revision,
|
||||||
|
_now(),
|
||||||
|
task_id,
|
||||||
|
request.expected_status,
|
||||||
|
request.expected_revision,
|
||||||
|
request.expected_attempt,
|
||||||
|
task["metadata_json"],
|
||||||
|
task["required_capabilities_json"],
|
||||||
|
task["result_json"],
|
||||||
|
task["verification_json"],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if cursor.rowcount != 1:
|
||||||
|
raise InitialExecutionProjectionError(
|
||||||
|
"SUPERVISOR_COORDINATOR_PROJECTION_CAS_MISMATCH", 409
|
||||||
|
)
|
||||||
|
event = {
|
||||||
|
"schema": "aios.supervisor-coordinator-projection.event.v1",
|
||||||
|
"task_id": task_id,
|
||||||
|
"job_id": request.expected_job_id,
|
||||||
|
"authorization_id": request.authorization_id,
|
||||||
|
"idempotency_key": request.idempotency_key,
|
||||||
|
"reason": request.reason,
|
||||||
|
"old_execution_digest": request.expected_old_execution_digest,
|
||||||
|
"new_execution_digest": execution["execution_digest"],
|
||||||
|
"required_capabilities": COORDINATOR_CAPABILITIES,
|
||||||
|
"previous_revision": request.expected_revision,
|
||||||
|
"applied_revision": applied_revision,
|
||||||
|
"attempt": request.expected_attempt,
|
||||||
|
"request_sha256": request_sha,
|
||||||
|
"credential_forwarded_to_planner": False,
|
||||||
|
"writer_lease": False,
|
||||||
|
"main_write": False,
|
||||||
|
"production_write": False,
|
||||||
|
}
|
||||||
|
table, columns = self._event_table(connection)
|
||||||
|
values = {
|
||||||
|
"task_id": task_id,
|
||||||
|
"event_type": "TASK_SUPERVISOR_COORDINATOR_PROJECTION_AUTHORIZED",
|
||||||
|
"payload_json": json.dumps(event, sort_keys=True, separators=(",", ":")),
|
||||||
|
"created_at": _now(),
|
||||||
|
"agent_id": "owner-control",
|
||||||
|
}
|
||||||
|
names = [name for name in values if name in columns]
|
||||||
|
connection.execute(
|
||||||
|
f'INSERT INTO "{table}" ({",".join(chr(34)+name+chr(34) for name in names)}) '
|
||||||
|
f'VALUES ({",".join("?" for _ in names)})',
|
||||||
|
tuple(values[name] for name in names),
|
||||||
|
)
|
||||||
|
connection.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO task_supervisor_coordinator_projections(
|
||||||
|
task_id,idempotency_key,request_sha256,authorization_id,
|
||||||
|
expected_job_id,planner_result_sha256,old_execution_digest,
|
||||||
|
new_execution_digest,expected_revision,applied_revision,
|
||||||
|
attempt,created_at
|
||||||
|
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
task_id,
|
||||||
|
request.idempotency_key,
|
||||||
|
request_sha,
|
||||||
|
request.authorization_id,
|
||||||
|
request.expected_job_id,
|
||||||
|
request.expected_planner_result_sha256,
|
||||||
|
request.expected_old_execution_digest,
|
||||||
|
execution["execution_digest"],
|
||||||
|
request.expected_revision,
|
||||||
|
applied_revision,
|
||||||
|
request.expected_attempt,
|
||||||
|
_now(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
connection.commit()
|
||||||
|
return {
|
||||||
|
"task_id": task_id,
|
||||||
|
"status": "QUEUED",
|
||||||
|
"attempt": request.expected_attempt,
|
||||||
|
"revision": applied_revision,
|
||||||
|
"execution_digest": execution["execution_digest"],
|
||||||
|
"required_capabilities": COORDINATOR_CAPABILITIES,
|
||||||
|
"credential_forwarded_to_planner": False,
|
||||||
|
"idempotent": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class Handler(BaseHTTPRequestHandler):
|
||||||
|
store: SupervisorCoordinatorProjectionStore | None = None
|
||||||
|
control_token: str = ""
|
||||||
|
|
||||||
|
def log_message(self, *_args: Any) -> None:
|
||||||
|
return
|
||||||
|
|
||||||
|
def send_json(self, value: Mapping[str, Any], status: int) -> None:
|
||||||
|
raw = _canonical(dict(value))
|
||||||
|
self.send_response(status)
|
||||||
|
self.send_header("Content-Type", "application/json")
|
||||||
|
self.send_header("Content-Length", str(len(raw)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(raw)
|
||||||
|
|
||||||
|
def do_GET(self) -> None:
|
||||||
|
if self.path == "/health":
|
||||||
|
self.send_json(
|
||||||
|
{"status": "ok", "service": "aios-supervisor-coordinator-projection"},
|
||||||
|
200,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.send_json({"error": "NOT_FOUND"}, 404)
|
||||||
|
|
||||||
|
def do_POST(self) -> None:
|
||||||
|
match = re.fullmatch(
|
||||||
|
r"/v1/tasks/([^/]+)/supervisor-coordinator-projection", self.path
|
||||||
|
)
|
||||||
|
header = self.headers.get("Authorization", "")
|
||||||
|
if match is None:
|
||||||
|
self.send_json({"error": "NOT_FOUND"}, 404)
|
||||||
|
return
|
||||||
|
if (
|
||||||
|
not header.startswith("Bearer ")
|
||||||
|
or not hmac.compare_digest(header[7:], self.control_token)
|
||||||
|
):
|
||||||
|
self.send_json({"error": "UNAUTHORIZED"}, 401)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
length = int(self.headers.get("Content-Length", "0"))
|
||||||
|
if length < 1 or length > MAX_BODY:
|
||||||
|
raise InitialExecutionProjectionError("REQUEST_SIZE_INVALID", 413)
|
||||||
|
value = json.loads(self.rfile.read(length).decode("utf-8"))
|
||||||
|
if not isinstance(value, Mapping) or self.store is None:
|
||||||
|
raise InitialExecutionProjectionError("REQUEST_INVALID", 400)
|
||||||
|
request = SupervisorCoordinatorProjectionRequest.from_mapping(value)
|
||||||
|
result = self.store.project(match.group(1), request)
|
||||||
|
self.send_json(result, 200 if result["idempotent"] else 201)
|
||||||
|
except InitialExecutionProjectionError as exc:
|
||||||
|
self.send_json({"error": exc.code}, exc.status)
|
||||||
|
except (ValueError, UnicodeDecodeError, json.JSONDecodeError):
|
||||||
|
self.send_json({"error": "REQUEST_INVALID"}, 400)
|
||||||
|
|
||||||
|
|
||||||
|
def serve() -> None:
|
||||||
|
credential = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_AGENT_TASK_CONTROL_TOKEN_FILE",
|
||||||
|
str(
|
||||||
|
Path(os.environ.get("CREDENTIALS_DIRECTORY", "/run/credentials"))
|
||||||
|
/ "agent_task_control_token"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
token = load_control_token(credential)
|
||||||
|
bridge = os.environ.get("AIOS_GOVERNED_WRITE_API_URL", "http://127.0.0.1:18892")
|
||||||
|
db = Path(os.environ.get("AIOS_AGENT_TASK_DB", "/var/lib/aios-agent-task/state.sqlite3"))
|
||||||
|
Handler.control_token = token
|
||||||
|
Handler.store = SupervisorCoordinatorProjectionStore(
|
||||||
|
db, CanonicalCoordinatorExecutionResolver(bridge, token)
|
||||||
|
)
|
||||||
|
host = os.environ.get("AIOS_SUPERVISOR_COORDINATOR_PROJECTION_HOST", "127.0.0.1")
|
||||||
|
port = int(os.environ.get("AIOS_SUPERVISOR_COORDINATOR_PROJECTION_PORT", "18944"))
|
||||||
|
ThreadingHTTPServer((host, port), Handler).serve_forever()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
serve()
|
||||||
@@ -0,0 +1,470 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Canonical AIOS all-modified-files evidence packager.
|
||||||
|
|
||||||
|
The bundle contract is intentionally compatible with the historical
|
||||||
|
aios_package_all_modified_files_v1.sh output while keeping secret scanning in
|
||||||
|
version-controlled, tested code.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import zipfile
|
||||||
|
from typing import Iterable
|
||||||
|
|
||||||
|
OID_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||||
|
TASK_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{2,179}$")
|
||||||
|
|
||||||
|
# A PEM private-key finding requires the complete PEM header. Source code,
|
||||||
|
# tests, docs, or error text containing only "BEGIN PRIVATE KEY" are not
|
||||||
|
# credentials and must not be rejected.
|
||||||
|
PEM_PRIVATE_KEY_RE = re.compile(
|
||||||
|
rb"-----BEGIN (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
BEARER_RE = re.compile(
|
||||||
|
rb"authorization\s*:\s*bearer\s+[A-Za-z0-9._~+/=-]{20,}",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
KEY_VALUE_SECRET_RE = re.compile(
|
||||||
|
rb"(?:password|passwd|secret|api[_-]?key)\s*[:=]\s*[^\s,;]{12,}",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
|
||||||
|
DOCUMENT_SUFFIXES = {".md", ".rst", ".adoc", ".txt"}
|
||||||
|
DOCUMENT_PREFIXES = (
|
||||||
|
"README",
|
||||||
|
"CHANGELOG",
|
||||||
|
"RUNBOOK",
|
||||||
|
"ADR",
|
||||||
|
"ARCHITECTURE",
|
||||||
|
"DEPLOYMENT",
|
||||||
|
"TROUBLESHOOTING",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class PackagerError(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def sha256(data: bytes) -> str:
|
||||||
|
return hashlib.sha256(data).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def secret_finding(data: bytes) -> str | None:
|
||||||
|
if PEM_PRIVATE_KEY_RE.search(data):
|
||||||
|
return "PEM_PRIVATE_KEY"
|
||||||
|
if BEARER_RE.search(data):
|
||||||
|
return "AUTHORIZATION_BEARER"
|
||||||
|
if KEY_VALUE_SECRET_RE.search(data):
|
||||||
|
return "KEY_VALUE_SECRET"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def scan_secret(data: bytes, label: str) -> None:
|
||||||
|
finding = secret_finding(data)
|
||||||
|
if finding is not None:
|
||||||
|
raise PackagerError(
|
||||||
|
f"MODIFIED_FILES_SECRET_SCAN_MATCH:{finding}:{label}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def git_bytes(git_dir: Path, oid: str, path: str) -> bytes:
|
||||||
|
try:
|
||||||
|
return subprocess.check_output(
|
||||||
|
[
|
||||||
|
"/usr/bin/git",
|
||||||
|
f"--git-dir={git_dir}",
|
||||||
|
"show",
|
||||||
|
f"{oid}:{path}",
|
||||||
|
],
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
except subprocess.CalledProcessError as exc:
|
||||||
|
raise PackagerError(
|
||||||
|
"MODIFIED_FILES_GIT_SHOW_FAILED:" + path
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
|
||||||
|
def git_text(git_dir: Path, *args: str) -> str:
|
||||||
|
try:
|
||||||
|
return subprocess.check_output(
|
||||||
|
["/usr/bin/git", f"--git-dir={git_dir}", *args],
|
||||||
|
text=True,
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
except subprocess.CalledProcessError as exc:
|
||||||
|
raise PackagerError(
|
||||||
|
"MODIFIED_FILES_GIT_COMMAND_FAILED:" + " ".join(args)
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
|
||||||
|
def validate_lineage(git_dir: Path, base: str, candidate: str) -> None:
|
||||||
|
if OID_RE.fullmatch(base) is None or OID_RE.fullmatch(candidate) is None:
|
||||||
|
raise PackagerError("MODIFIED_FILES_OID_INVALID")
|
||||||
|
for oid in (base, candidate):
|
||||||
|
try:
|
||||||
|
subprocess.run(
|
||||||
|
[
|
||||||
|
"/usr/bin/git",
|
||||||
|
f"--git-dir={git_dir}",
|
||||||
|
"cat-file",
|
||||||
|
"-e",
|
||||||
|
f"{oid}^{{commit}}",
|
||||||
|
],
|
||||||
|
check=True,
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
except subprocess.CalledProcessError as exc:
|
||||||
|
raise PackagerError("MODIFIED_FILES_COMMIT_MISSING:" + oid) from exc
|
||||||
|
if subprocess.run(
|
||||||
|
[
|
||||||
|
"/usr/bin/git",
|
||||||
|
f"--git-dir={git_dir}",
|
||||||
|
"merge-base",
|
||||||
|
"--is-ancestor",
|
||||||
|
base,
|
||||||
|
candidate,
|
||||||
|
],
|
||||||
|
check=False,
|
||||||
|
timeout=30,
|
||||||
|
).returncode != 0:
|
||||||
|
raise PackagerError("MODIFIED_FILES_LINEAGE_INVALID")
|
||||||
|
|
||||||
|
|
||||||
|
def safe_destination(root: Path, relative: str) -> Path:
|
||||||
|
path = Path(relative)
|
||||||
|
if path.is_absolute() or ".." in path.parts:
|
||||||
|
raise PackagerError("MODIFIED_FILE_PATH_INVALID:" + relative)
|
||||||
|
destination = root / path
|
||||||
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
return destination
|
||||||
|
|
||||||
|
|
||||||
|
def is_document_path(path: str) -> bool:
|
||||||
|
value = Path(path)
|
||||||
|
if path.startswith("docs/"):
|
||||||
|
return True
|
||||||
|
if value.suffix.lower() in DOCUMENT_SUFFIXES:
|
||||||
|
return True
|
||||||
|
upper = value.name.upper()
|
||||||
|
return any(upper.startswith(prefix) for prefix in DOCUMENT_PREFIXES)
|
||||||
|
|
||||||
|
|
||||||
|
def diff_entries(
|
||||||
|
git_dir: Path,
|
||||||
|
base: str,
|
||||||
|
candidate: str,
|
||||||
|
before_root: Path,
|
||||||
|
after_root: Path,
|
||||||
|
) -> list[dict[str, object]]:
|
||||||
|
raw = git_text(
|
||||||
|
git_dir,
|
||||||
|
"diff",
|
||||||
|
"--name-status",
|
||||||
|
"-M",
|
||||||
|
"-C",
|
||||||
|
base,
|
||||||
|
candidate,
|
||||||
|
)
|
||||||
|
entries: list[dict[str, object]] = []
|
||||||
|
for line in raw.splitlines():
|
||||||
|
if not line.strip():
|
||||||
|
continue
|
||||||
|
parts = line.split("\t")
|
||||||
|
status = parts[0]
|
||||||
|
code = status[0]
|
||||||
|
old_path: str | None = None
|
||||||
|
new_path: str | None = None
|
||||||
|
if code in {"R", "C"}:
|
||||||
|
if len(parts) != 3:
|
||||||
|
raise PackagerError(
|
||||||
|
"MODIFIED_FILES_DIFF_PARSE_INVALID:" + line
|
||||||
|
)
|
||||||
|
old_path, new_path = parts[1], parts[2]
|
||||||
|
else:
|
||||||
|
if len(parts) != 2:
|
||||||
|
raise PackagerError(
|
||||||
|
"MODIFIED_FILES_DIFF_PARSE_INVALID:" + line
|
||||||
|
)
|
||||||
|
new_path = parts[1] if code != "D" else None
|
||||||
|
old_path = parts[1] if code != "A" else None
|
||||||
|
|
||||||
|
entry: dict[str, object] = {"status": status}
|
||||||
|
if old_path is not None:
|
||||||
|
entry["previous_path"] = old_path
|
||||||
|
before = git_bytes(git_dir, base, old_path)
|
||||||
|
scan_secret(before, "before:" + old_path)
|
||||||
|
safe_destination(before_root, old_path).write_bytes(before)
|
||||||
|
entry["before_sha256"] = sha256(before)
|
||||||
|
entry["before_bytes"] = len(before)
|
||||||
|
if new_path is not None:
|
||||||
|
entry["path"] = new_path
|
||||||
|
after = git_bytes(git_dir, candidate, new_path)
|
||||||
|
scan_secret(after, "after:" + new_path)
|
||||||
|
safe_destination(after_root, new_path).write_bytes(after)
|
||||||
|
entry["after_sha256"] = sha256(after)
|
||||||
|
entry["after_bytes"] = len(after)
|
||||||
|
entry["deleted"] = code == "D"
|
||||||
|
entry["added"] = code == "A"
|
||||||
|
entry["renamed"] = code == "R"
|
||||||
|
entry["copied"] = code == "C"
|
||||||
|
entries.append(entry)
|
||||||
|
return entries
|
||||||
|
|
||||||
|
|
||||||
|
def execution_entries(
|
||||||
|
extra_root: Path | None,
|
||||||
|
execution_root: Path,
|
||||||
|
) -> list[dict[str, object]]:
|
||||||
|
if extra_root is None or not extra_root.exists():
|
||||||
|
return []
|
||||||
|
if extra_root.is_symlink() or not extra_root.is_dir():
|
||||||
|
raise PackagerError("EXTRA_FILES_DIR_INVALID")
|
||||||
|
|
||||||
|
entries: list[dict[str, object]] = []
|
||||||
|
for path in sorted(extra_root.rglob("*")):
|
||||||
|
if path.is_symlink():
|
||||||
|
raise PackagerError(
|
||||||
|
"EXTRA_FILE_SYMLINK_REJECTED:" + str(path)
|
||||||
|
)
|
||||||
|
if not path.is_file():
|
||||||
|
continue
|
||||||
|
relative = path.relative_to(extra_root).as_posix()
|
||||||
|
data = path.read_bytes()
|
||||||
|
scan_secret(data, "execution:" + relative)
|
||||||
|
safe_destination(execution_root, relative).write_bytes(data)
|
||||||
|
entries.append(
|
||||||
|
{
|
||||||
|
"path": relative,
|
||||||
|
"sha256": sha256(data),
|
||||||
|
"bytes": len(data),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return entries
|
||||||
|
|
||||||
|
|
||||||
|
def write_checksums(bundle: Path) -> None:
|
||||||
|
checks: list[tuple[str, str]] = []
|
||||||
|
for path in sorted(bundle.rglob("*")):
|
||||||
|
if path.is_file() and path.name != "checksums.sha256":
|
||||||
|
checks.append(
|
||||||
|
(
|
||||||
|
sha256(path.read_bytes()),
|
||||||
|
path.relative_to(bundle).as_posix(),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
(bundle / "checksums.sha256").write_text(
|
||||||
|
"".join(f"{digest} {name}\n" for digest, name in checks),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def verify_checksums(bundle: Path) -> None:
|
||||||
|
checksum_path = bundle / "checksums.sha256"
|
||||||
|
for line in checksum_path.read_text(encoding="utf-8").splitlines():
|
||||||
|
digest, relative = line.split(" ", 1)
|
||||||
|
path = bundle / relative
|
||||||
|
if not path.is_file() or sha256(path.read_bytes()) != digest:
|
||||||
|
raise PackagerError(
|
||||||
|
"MODIFIED_FILES_CHECKSUM_CLOSURE_INVALID:" + relative
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def build_bundle(
|
||||||
|
*,
|
||||||
|
git_dir: Path,
|
||||||
|
base: str,
|
||||||
|
candidate: str,
|
||||||
|
task_id: str,
|
||||||
|
extra_dir: Path | None = None,
|
||||||
|
output_root: Path | None = None,
|
||||||
|
now: datetime | None = None,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
if TASK_RE.fullmatch(task_id) is None:
|
||||||
|
raise PackagerError("TASK_ID_INVALID")
|
||||||
|
validate_lineage(git_dir, base, candidate)
|
||||||
|
|
||||||
|
timestamp = (now or datetime.now(timezone.utc)).strftime(
|
||||||
|
"%Y%m%dT%H%M%SZ"
|
||||||
|
)
|
||||||
|
safe_task = re.sub(r"[^A-Za-z0-9._-]+", "-", task_id).strip("-")
|
||||||
|
if output_root is None:
|
||||||
|
output_root = Path(
|
||||||
|
tempfile.mkdtemp(
|
||||||
|
prefix=f"aios-modified-files-bundle-{safe_task}-{timestamp}-",
|
||||||
|
dir="/var/tmp",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
output_root.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
bundle = output_root / "bundle"
|
||||||
|
before_root = bundle / "repo" / "before"
|
||||||
|
after_root = bundle / "repo" / "after"
|
||||||
|
execution_root = bundle / "execution"
|
||||||
|
before_root.mkdir(parents=True)
|
||||||
|
after_root.mkdir(parents=True)
|
||||||
|
execution_root.mkdir(parents=True)
|
||||||
|
|
||||||
|
changes = diff_entries(
|
||||||
|
git_dir,
|
||||||
|
base,
|
||||||
|
candidate,
|
||||||
|
before_root,
|
||||||
|
after_root,
|
||||||
|
)
|
||||||
|
execution_files = execution_entries(extra_dir, execution_root)
|
||||||
|
|
||||||
|
document_changes = [
|
||||||
|
entry
|
||||||
|
for entry in changes
|
||||||
|
if is_document_path(
|
||||||
|
str(entry.get("path") or entry.get("previous_path") or "")
|
||||||
|
)
|
||||||
|
]
|
||||||
|
execution_documents = [
|
||||||
|
entry
|
||||||
|
for entry in execution_files
|
||||||
|
if is_document_path(str(entry["path"]))
|
||||||
|
]
|
||||||
|
|
||||||
|
documents_index = {
|
||||||
|
"schema": "aios.modified-documents.index.v1",
|
||||||
|
"task_id": task_id,
|
||||||
|
"base_oid": base,
|
||||||
|
"candidate_oid": candidate,
|
||||||
|
"repository_documents": document_changes,
|
||||||
|
"execution_documents": execution_documents,
|
||||||
|
"repository_document_count": len(document_changes),
|
||||||
|
"execution_document_count": len(execution_documents),
|
||||||
|
}
|
||||||
|
(bundle / "documents-index.json").write_text(
|
||||||
|
json.dumps(documents_index, indent=2, sort_keys=True) + "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
manifest = {
|
||||||
|
"schema": "aios.all-modified-files.bundle.v1",
|
||||||
|
"task_id": task_id,
|
||||||
|
"base_oid": base,
|
||||||
|
"candidate_oid": candidate,
|
||||||
|
"generated_at": (
|
||||||
|
now or datetime.now(timezone.utc)
|
||||||
|
).isoformat(),
|
||||||
|
"changed_file_count": len(changes),
|
||||||
|
"execution_file_count": len(execution_files),
|
||||||
|
"repository_document_count": len(document_changes),
|
||||||
|
"execution_document_count": len(execution_documents),
|
||||||
|
"changes": changes,
|
||||||
|
"execution_files": execution_files,
|
||||||
|
}
|
||||||
|
(bundle / "all-modified-files-manifest.json").write_text(
|
||||||
|
json.dumps(manifest, indent=2, sort_keys=True) + "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
write_checksums(bundle)
|
||||||
|
verify_checksums(bundle)
|
||||||
|
|
||||||
|
zip_name = (
|
||||||
|
f"AIOS_{task_id}_ALL_MODIFIED_FILES_{candidate[:12]}.zip"
|
||||||
|
.replace("/", "-")
|
||||||
|
)
|
||||||
|
zip_path = output_root / zip_name
|
||||||
|
with zipfile.ZipFile(
|
||||||
|
zip_path,
|
||||||
|
"w",
|
||||||
|
compression=zipfile.ZIP_DEFLATED,
|
||||||
|
) as archive:
|
||||||
|
for path in sorted(bundle.rglob("*")):
|
||||||
|
if path.is_symlink():
|
||||||
|
raise PackagerError(
|
||||||
|
"BUNDLE_SYMLINK_REJECTED:" + str(path)
|
||||||
|
)
|
||||||
|
if path.is_file():
|
||||||
|
archive.write(
|
||||||
|
path,
|
||||||
|
arcname=path.relative_to(bundle).as_posix(),
|
||||||
|
)
|
||||||
|
|
||||||
|
zip_sha = sha256(zip_path.read_bytes())
|
||||||
|
summary = {
|
||||||
|
"schema": "aios.all-modified-files.bundle-summary.v1",
|
||||||
|
"task_id": task_id,
|
||||||
|
"base_oid": base,
|
||||||
|
"candidate_oid": candidate,
|
||||||
|
"bundle_path": str(zip_path),
|
||||||
|
"bundle_sha256": zip_sha,
|
||||||
|
"changed_file_count": len(changes),
|
||||||
|
"execution_file_count": len(execution_files),
|
||||||
|
"repository_document_count": len(document_changes),
|
||||||
|
"execution_document_count": len(execution_documents),
|
||||||
|
}
|
||||||
|
(output_root / "ALL_MODIFIED_FILES_BUNDLE.json").write_text(
|
||||||
|
json.dumps(summary, indent=2, sort_keys=True) + "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
return summary
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("base")
|
||||||
|
parser.add_argument("candidate")
|
||||||
|
parser.add_argument("task_id")
|
||||||
|
parser.add_argument("extra_dir", nargs="?")
|
||||||
|
parser.add_argument(
|
||||||
|
"--authority-git-dir",
|
||||||
|
default=os.environ.get(
|
||||||
|
"AIOS_AUTHORITY_GIT_DIR",
|
||||||
|
"/data/aios-gitops/repo.git",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
parser.add_argument("--output-root")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
try:
|
||||||
|
summary = build_bundle(
|
||||||
|
git_dir=Path(args.authority_git_dir),
|
||||||
|
base=args.base,
|
||||||
|
candidate=args.candidate,
|
||||||
|
task_id=args.task_id,
|
||||||
|
extra_dir=Path(args.extra_dir) if args.extra_dir else None,
|
||||||
|
output_root=(
|
||||||
|
Path(args.output_root) if args.output_root else None
|
||||||
|
),
|
||||||
|
)
|
||||||
|
except PackagerError as exc:
|
||||||
|
print("FATAL:" + str(exc))
|
||||||
|
return 20
|
||||||
|
|
||||||
|
print("ALL_MODIFIED_FILES_BUNDLE=PASS")
|
||||||
|
print("ALL_MODIFIED_FILES_BUNDLE_PATH=" + str(summary["bundle_path"]))
|
||||||
|
print("ALL_MODIFIED_FILES_BUNDLE_SHA256=" + str(summary["bundle_sha256"]))
|
||||||
|
print("CHANGED_FILE_COUNT=" + str(summary["changed_file_count"]))
|
||||||
|
print("EXECUTION_FILE_COUNT=" + str(summary["execution_file_count"]))
|
||||||
|
print(
|
||||||
|
"REPOSITORY_DOCUMENT_COUNT="
|
||||||
|
+ str(summary["repository_document_count"])
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"EXECUTION_DOCUMENT_COUNT="
|
||||||
|
+ str(summary["execution_document_count"])
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"ALL_MODIFIED_FILES_EVIDENCE="
|
||||||
|
+ str(Path(str(summary["bundle_path"])).parent)
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,288 @@
|
|||||||
|
"""Unified autonomous control-plane primitives.
|
||||||
|
|
||||||
|
This is deliberately a library, not another daemon. Task API, workers and host
|
||||||
|
recovery code consume the same Admission -> Invariant -> Repair semantics.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import asdict, dataclass
|
||||||
|
from enum import Enum
|
||||||
|
import re
|
||||||
|
from typing import Callable, Iterable, Mapping, Sequence
|
||||||
|
|
||||||
|
from aios_capability_contract import (
|
||||||
|
capability_contract,
|
||||||
|
has_capabilities,
|
||||||
|
normalize_required_capabilities,
|
||||||
|
normalize_worker_capabilities,
|
||||||
|
registered_capability,
|
||||||
|
validate_capability_contract_registry,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FailureClass(str, Enum):
|
||||||
|
CAPABILITY_CONTRACT_ERROR = "CAPABILITY_CONTRACT_ERROR"
|
||||||
|
CREDENTIAL_PROJECTION_ERROR = "CREDENTIAL_PROJECTION_ERROR"
|
||||||
|
DEPENDENCY_UNAVAILABLE = "DEPENDENCY_UNAVAILABLE"
|
||||||
|
POLICY_DENIED = "POLICY_DENIED"
|
||||||
|
RESOURCE_UNAVAILABLE = "RESOURCE_UNAVAILABLE"
|
||||||
|
WORKER_CRASH = "WORKER_CRASH"
|
||||||
|
LEASE_LOST = "LEASE_LOST"
|
||||||
|
BUSINESS_PERMANENT = "BUSINESS_PERMANENT"
|
||||||
|
UNKNOWN = "UNKNOWN"
|
||||||
|
|
||||||
|
|
||||||
|
class RepairLevel(int, Enum):
|
||||||
|
AUTO_LOCAL = 0
|
||||||
|
AUTO_REPAIR_TASK = 1
|
||||||
|
OWNER_APPROVAL = 2
|
||||||
|
MANUAL = 3
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class InvariantResult:
|
||||||
|
name: str
|
||||||
|
passed: bool
|
||||||
|
detail: str = ""
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, object]:
|
||||||
|
return asdict(self)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class AdmissionDecision:
|
||||||
|
admitted: bool
|
||||||
|
state: str
|
||||||
|
invariants: tuple[InvariantResult, ...]
|
||||||
|
failure_class: FailureClass | None = None
|
||||||
|
repair_level: RepairLevel | None = None
|
||||||
|
repair_action: str | None = None
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"schema": "aios.runtime-admission.decision.v1",
|
||||||
|
"admitted": self.admitted,
|
||||||
|
"state": self.state,
|
||||||
|
"invariants": [item.as_dict() for item in self.invariants],
|
||||||
|
"failure_class": None if self.failure_class is None else self.failure_class.value,
|
||||||
|
"repair_level": None if self.repair_level is None else int(self.repair_level),
|
||||||
|
"repair_action": self.repair_action,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class RepairDecision:
|
||||||
|
failure_class: FailureClass
|
||||||
|
level: RepairLevel
|
||||||
|
action: str
|
||||||
|
consumes_business_attempt: bool
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"failure_class": self.failure_class.value,
|
||||||
|
"level": int(self.level),
|
||||||
|
"action": self.action,
|
||||||
|
"consumes_business_attempt": self.consumes_business_attempt,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def classify_failure(error: object, *, phase: str = "") -> FailureClass:
|
||||||
|
text = (str(error) + " " + str(phase)).lower()
|
||||||
|
if any(token in text for token in (
|
||||||
|
"credential", "permission denied", "token registry", "secret", "unauthorized"
|
||||||
|
)):
|
||||||
|
return FailureClass.CREDENTIAL_PROJECTION_ERROR
|
||||||
|
if any(token in text for token in (
|
||||||
|
"transport", "unavailable", "timeout", "connection", "bad gateway", "http_5"
|
||||||
|
)):
|
||||||
|
return FailureClass.DEPENDENCY_UNAVAILABLE
|
||||||
|
if any(token in text for token in (
|
||||||
|
"capability", "eligible", "matcher", "contract"
|
||||||
|
)):
|
||||||
|
return FailureClass.CAPABILITY_CONTRACT_ERROR
|
||||||
|
if any(token in text for token in (
|
||||||
|
"policy", "forbidden", "approval required", "not authorized"
|
||||||
|
)):
|
||||||
|
return FailureClass.POLICY_DENIED
|
||||||
|
if any(token in text for token in ("lease lost", "lease_expired", "heartbeat_failed")):
|
||||||
|
return FailureClass.LEASE_LOST
|
||||||
|
if any(token in text for token in ("out of memory", "disk full", "resource")):
|
||||||
|
return FailureClass.RESOURCE_UNAVAILABLE
|
||||||
|
if any(token in text for token in ("worker crash", "process exited", "signal")):
|
||||||
|
return FailureClass.WORKER_CRASH
|
||||||
|
if any(token in text for token in ("invalid release", "business permanent")):
|
||||||
|
return FailureClass.BUSINESS_PERMANENT
|
||||||
|
return FailureClass.UNKNOWN
|
||||||
|
|
||||||
|
|
||||||
|
def repair_for_failure(failure: FailureClass) -> RepairDecision:
|
||||||
|
table = {
|
||||||
|
FailureClass.CAPABILITY_CONTRACT_ERROR: RepairDecision(
|
||||||
|
failure, RepairLevel.AUTO_REPAIR_TASK,
|
||||||
|
"create_same_lineage_capability_contract_repair", False,
|
||||||
|
),
|
||||||
|
FailureClass.CREDENTIAL_PROJECTION_ERROR: RepairDecision(
|
||||||
|
failure, RepairLevel.AUTO_REPAIR_TASK,
|
||||||
|
"restore_service_private_credential_projection", False,
|
||||||
|
),
|
||||||
|
FailureClass.DEPENDENCY_UNAVAILABLE: RepairDecision(
|
||||||
|
failure, RepairLevel.AUTO_LOCAL,
|
||||||
|
"retry_dependency_health_and_resume", False,
|
||||||
|
),
|
||||||
|
FailureClass.RESOURCE_UNAVAILABLE: RepairDecision(
|
||||||
|
failure, RepairLevel.AUTO_LOCAL,
|
||||||
|
"heal_runtime_resource_and_resume", False,
|
||||||
|
),
|
||||||
|
FailureClass.WORKER_CRASH: RepairDecision(
|
||||||
|
failure, RepairLevel.AUTO_LOCAL,
|
||||||
|
"restart_worker_and_resume_same_lineage", False,
|
||||||
|
),
|
||||||
|
FailureClass.LEASE_LOST: RepairDecision(
|
||||||
|
failure, RepairLevel.AUTO_LOCAL,
|
||||||
|
"recover_lease_and_resume_same_lineage", False,
|
||||||
|
),
|
||||||
|
FailureClass.POLICY_DENIED: RepairDecision(
|
||||||
|
failure, RepairLevel.OWNER_APPROVAL,
|
||||||
|
"request_bound_owner_authorization", False,
|
||||||
|
),
|
||||||
|
FailureClass.BUSINESS_PERMANENT: RepairDecision(
|
||||||
|
failure, RepairLevel.MANUAL,
|
||||||
|
"terminal_business_failure", True,
|
||||||
|
),
|
||||||
|
FailureClass.UNKNOWN: RepairDecision(
|
||||||
|
failure, RepairLevel.AUTO_REPAIR_TASK,
|
||||||
|
"create_diagnostic_repair_task", False,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
return table[failure]
|
||||||
|
|
||||||
|
|
||||||
|
def _probe(
|
||||||
|
name: str,
|
||||||
|
probes: Mapping[str, Callable[[], bool]],
|
||||||
|
) -> InvariantResult:
|
||||||
|
fn = probes.get(name)
|
||||||
|
if fn is None:
|
||||||
|
return InvariantResult(name, False, "PROBE_MISSING")
|
||||||
|
try:
|
||||||
|
passed = bool(fn())
|
||||||
|
except Exception as exc:
|
||||||
|
return InvariantResult(name, False, "PROBE_ERROR:" + type(exc).__name__)
|
||||||
|
return InvariantResult(name, passed, "PASS" if passed else "NOT_READY")
|
||||||
|
|
||||||
|
|
||||||
|
def runtime_admission(
|
||||||
|
*,
|
||||||
|
worker_capabilities: Iterable[object],
|
||||||
|
required_capabilities: Iterable[object],
|
||||||
|
requested_operations: Sequence[str] = (),
|
||||||
|
probes: Mapping[str, Callable[[], bool]] | None = None,
|
||||||
|
) -> AdmissionDecision:
|
||||||
|
"""Admission is evaluated before claim; failures do not consume attempts."""
|
||||||
|
|
||||||
|
worker_values = tuple(worker_capabilities)
|
||||||
|
required_values = tuple(required_capabilities)
|
||||||
|
probe_map = probes or {}
|
||||||
|
invariants: list[InvariantResult] = []
|
||||||
|
|
||||||
|
registry = validate_capability_contract_registry()
|
||||||
|
invariants.append(InvariantResult(
|
||||||
|
"capability_contract_registry",
|
||||||
|
bool(registry.get("valid")),
|
||||||
|
"PASS" if registry.get("valid") else "REGISTRY_INVALID",
|
||||||
|
))
|
||||||
|
|
||||||
|
task_known = all(registered_capability(item) for item in required_values)
|
||||||
|
worker_known = all(registered_capability(item, worker=True) for item in worker_values)
|
||||||
|
invariants.extend((
|
||||||
|
InvariantResult("task_capabilities_registered", task_known, "PASS" if task_known else "UNKNOWN_TASK_CAPABILITY"),
|
||||||
|
InvariantResult("worker_capabilities_registered", worker_known, "PASS" if worker_known else "UNKNOWN_WORKER_CAPABILITY"),
|
||||||
|
))
|
||||||
|
|
||||||
|
capability_match = (
|
||||||
|
task_known
|
||||||
|
and worker_known
|
||||||
|
and has_capabilities(worker_values, required_values)
|
||||||
|
)
|
||||||
|
invariants.append(InvariantResult(
|
||||||
|
"capability_match",
|
||||||
|
capability_match,
|
||||||
|
"PASS" if capability_match else "WORKER_NOT_ELIGIBLE",
|
||||||
|
))
|
||||||
|
|
||||||
|
normalized = normalize_required_capabilities(required_values)
|
||||||
|
contracts = [
|
||||||
|
capability_contract(item)
|
||||||
|
for item in normalized
|
||||||
|
if not str(item).startswith("exact:")
|
||||||
|
]
|
||||||
|
contracts = [item for item in contracts if item is not None]
|
||||||
|
|
||||||
|
forbidden: set[str] = set()
|
||||||
|
dependencies: set[str] = set()
|
||||||
|
health: set[str] = set()
|
||||||
|
for contract in contracts:
|
||||||
|
forbidden.update(contract.forbidden)
|
||||||
|
dependencies.update(contract.dependencies)
|
||||||
|
health.update(contract.health_requirements)
|
||||||
|
|
||||||
|
requested = {str(item).strip().lower() for item in requested_operations if str(item).strip()}
|
||||||
|
forbidden_hit = sorted(requested & {item.lower() for item in forbidden})
|
||||||
|
invariants.append(InvariantResult(
|
||||||
|
"forbidden_operations_absent",
|
||||||
|
not forbidden_hit,
|
||||||
|
"PASS" if not forbidden_hit else "FORBIDDEN:" + ",".join(forbidden_hit),
|
||||||
|
))
|
||||||
|
|
||||||
|
for name in sorted(dependencies | health):
|
||||||
|
invariants.append(_probe(name, probe_map))
|
||||||
|
|
||||||
|
failed = [item for item in invariants if not item.passed]
|
||||||
|
if not failed:
|
||||||
|
return AdmissionDecision(
|
||||||
|
True,
|
||||||
|
"EXECUTION_READY",
|
||||||
|
tuple(invariants),
|
||||||
|
)
|
||||||
|
|
||||||
|
details = " ".join(item.name + ":" + item.detail for item in failed)
|
||||||
|
failure = classify_failure(details, phase="admission")
|
||||||
|
if any(item.name == "forbidden_operations_absent" and not item.passed for item in failed):
|
||||||
|
failure = FailureClass.POLICY_DENIED
|
||||||
|
elif any("credential" in item.name for item in failed):
|
||||||
|
failure = FailureClass.CREDENTIAL_PROJECTION_ERROR
|
||||||
|
elif any(item.name in {"capability_contract_registry", "task_capabilities_registered", "worker_capabilities_registered", "capability_match"} for item in failed):
|
||||||
|
failure = FailureClass.CAPABILITY_CONTRACT_ERROR
|
||||||
|
elif any(item.name in dependencies or item.name in health for item in failed):
|
||||||
|
failure = FailureClass.DEPENDENCY_UNAVAILABLE
|
||||||
|
|
||||||
|
repair = repair_for_failure(failure)
|
||||||
|
return AdmissionDecision(
|
||||||
|
False,
|
||||||
|
"EXECUTION_READY_FAILED",
|
||||||
|
tuple(invariants),
|
||||||
|
failure,
|
||||||
|
repair.level,
|
||||||
|
repair.action,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def system_invariant_snapshot(
|
||||||
|
*,
|
||||||
|
worker_capabilities: Iterable[object],
|
||||||
|
required_capabilities: Iterable[object],
|
||||||
|
requested_operations: Sequence[str] = (),
|
||||||
|
probes: Mapping[str, Callable[[], bool]] | None = None,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
decision = runtime_admission(
|
||||||
|
worker_capabilities=worker_capabilities,
|
||||||
|
required_capabilities=required_capabilities,
|
||||||
|
requested_operations=requested_operations,
|
||||||
|
probes=probes,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"schema": "aios.system-invariants.v1",
|
||||||
|
"status": "HEALTHY" if decision.admitted else "DEGRADED",
|
||||||
|
"execution_ready": decision.admitted,
|
||||||
|
"admission": decision.as_dict(),
|
||||||
|
}
|
||||||
@@ -0,0 +1,142 @@
|
|||||||
|
"""Private Unix-socket caller projection and rehydration client.
|
||||||
|
|
||||||
|
The broker response body is deliberately public metadata only. The short-lived
|
||||||
|
credential projection is delivered in a private response header, consumed
|
||||||
|
inside the model API process, and never returned by an HTTP API or persisted.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import hashlib
|
||||||
|
import http.client
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import socket
|
||||||
|
from typing import Any, Mapping
|
||||||
|
|
||||||
|
CALLER = "chatgpt-governed-implementer-singlehost"
|
||||||
|
SCOPE = "release:candidate"
|
||||||
|
GOVERNED_TASK_SCOPE = "governed-task:project"
|
||||||
|
PROJECTED_SCOPES = frozenset({GOVERNED_TASK_SCOPE, SCOPE})
|
||||||
|
CREDENTIAL_HEADER = "X-AIOS-Caller-Projection-Credential"
|
||||||
|
|
||||||
|
|
||||||
|
class CallerProjectionError(RuntimeError):
|
||||||
|
def __init__(self, code: str, status: int = 503) -> None:
|
||||||
|
super().__init__(code)
|
||||||
|
self.code = code
|
||||||
|
self.status = status
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ProjectionResult:
|
||||||
|
public: dict[str, Any]
|
||||||
|
credential_loaded: bool
|
||||||
|
|
||||||
|
|
||||||
|
class _UnixHTTPConnection(http.client.HTTPConnection):
|
||||||
|
def __init__(self, socket_path: Path, timeout: float) -> None:
|
||||||
|
super().__init__("localhost", timeout=timeout)
|
||||||
|
self.socket_path = socket_path
|
||||||
|
|
||||||
|
def connect(self) -> None:
|
||||||
|
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||||
|
sock.settimeout(self.timeout)
|
||||||
|
try:
|
||||||
|
sock.connect(str(self.socket_path))
|
||||||
|
except Exception:
|
||||||
|
sock.close()
|
||||||
|
raise
|
||||||
|
self.sock = sock
|
||||||
|
|
||||||
|
|
||||||
|
class CallerProjectionClient:
|
||||||
|
def __init__(self, socket_path: Path, *, timeout: float = 2.0) -> None:
|
||||||
|
self.socket_path = Path(socket_path)
|
||||||
|
self.timeout = timeout
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _idempotency_key(caller: str, scope: str) -> str:
|
||||||
|
digest = hashlib.sha256((caller + "\0" + scope).encode()).hexdigest()
|
||||||
|
return "caller-projection-" + digest[:32] + "-v1"
|
||||||
|
|
||||||
|
def rehydrate(
|
||||||
|
self,
|
||||||
|
caller: str,
|
||||||
|
scope: str,
|
||||||
|
request: Mapping[str, Any] | None = None,
|
||||||
|
) -> ProjectionResult:
|
||||||
|
if caller != CALLER:
|
||||||
|
raise CallerProjectionError("CALLER_MISMATCH", 403)
|
||||||
|
if scope not in PROJECTED_SCOPES:
|
||||||
|
raise CallerProjectionError("SCOPE_MISMATCH", 403)
|
||||||
|
body = {
|
||||||
|
"caller": caller,
|
||||||
|
"scope": scope,
|
||||||
|
"idempotency_key": self._idempotency_key(caller, scope),
|
||||||
|
"request": dict(request or {}),
|
||||||
|
}
|
||||||
|
connection = _UnixHTTPConnection(self.socket_path, self.timeout)
|
||||||
|
try:
|
||||||
|
connection.request(
|
||||||
|
"POST",
|
||||||
|
"/v1/project-caller",
|
||||||
|
body=json.dumps(body, sort_keys=True, separators=(",", ":")).encode(),
|
||||||
|
headers={
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"Accept": "application/json",
|
||||||
|
"X-AIOS-Account": caller,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
response = connection.getresponse()
|
||||||
|
status = response.status
|
||||||
|
raw = response.read()
|
||||||
|
credential = response.getheader(CREDENTIAL_HEADER, "")
|
||||||
|
except (OSError, TimeoutError, http.client.HTTPException) as exc:
|
||||||
|
raise CallerProjectionError("PROJECTION_BROKER_UNAVAILABLE", 503) from exc
|
||||||
|
finally:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
try:
|
||||||
|
value = json.loads(raw.decode("utf-8"))
|
||||||
|
except Exception as exc:
|
||||||
|
raise CallerProjectionError("PROJECTION_RESPONSE_INVALID", 503) from exc
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise CallerProjectionError("PROJECTION_RESPONSE_INVALID", 503)
|
||||||
|
if status != 200:
|
||||||
|
code = str(value.get("error") or "PROJECTION_DENIED")
|
||||||
|
raise CallerProjectionError(code, 403 if status < 500 else 503)
|
||||||
|
if any(
|
||||||
|
str(key).lower() in {
|
||||||
|
"credential",
|
||||||
|
"credential_material",
|
||||||
|
"token",
|
||||||
|
"bearer",
|
||||||
|
"secret",
|
||||||
|
"password",
|
||||||
|
}
|
||||||
|
for key in value
|
||||||
|
):
|
||||||
|
raise CallerProjectionError("PROJECTION_SECRET_FIELD_REJECTED", 503)
|
||||||
|
if value.get("caller") != CALLER or value.get("scope") != scope:
|
||||||
|
raise CallerProjectionError("PROJECTION_BINDING_INVALID", 503)
|
||||||
|
if value.get("projection_state") != "ACKED":
|
||||||
|
raise CallerProjectionError("PROJECTION_STATE_INVALID", 503)
|
||||||
|
if value.get("credential_state") != "REHYDRATED":
|
||||||
|
raise CallerProjectionError("CREDENTIAL_REHYDRATION_INVALID", 503)
|
||||||
|
if value.get("rehydration_state") not in {"GENERATED", "RECOVERED"}:
|
||||||
|
raise CallerProjectionError("REHYDRATION_STATE_INVALID", 503)
|
||||||
|
effective_scopes = value.get("effective_scopes")
|
||||||
|
if (
|
||||||
|
not isinstance(effective_scopes, list)
|
||||||
|
or len(effective_scopes) != len(set(effective_scopes))
|
||||||
|
or not all(isinstance(item, str) for item in effective_scopes)
|
||||||
|
or scope not in effective_scopes
|
||||||
|
or not set(effective_scopes).issubset(PROJECTED_SCOPES)
|
||||||
|
):
|
||||||
|
raise CallerProjectionError("PROJECTION_SCOPE_INVALID", 503)
|
||||||
|
if value.get("secret_exposed") is not False:
|
||||||
|
raise CallerProjectionError("PROJECTION_SECRET_EXPOSURE", 503)
|
||||||
|
if not isinstance(credential, str) or len(credential) < 32:
|
||||||
|
raise CallerProjectionError("CREDENTIAL_REHYDRATION_MISSING", 503)
|
||||||
|
return ProjectionResult(public=dict(value), credential_loaded=True)
|
||||||
@@ -0,0 +1,341 @@
|
|||||||
|
"""Canonical fail-closed capability semantics for task eligibility."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Iterable
|
||||||
|
from dataclasses import asdict, dataclass
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
_TASK_ALIASES: dict[str, frozenset[str]] = {
|
||||||
|
"repository_read": frozenset({"repo.read"}),
|
||||||
|
"git.read": frozenset({"repo.read"}),
|
||||||
|
"repo.read": frozenset({"repo.read"}),
|
||||||
|
"test_execution": frozenset({"test.execute"}),
|
||||||
|
"git.test": frozenset({"test.execute"}),
|
||||||
|
"test.execute": frozenset({"test.execute"}),
|
||||||
|
"governed_git_write": frozenset({"governed.git.write.branch", "governed.git.write.file"}),
|
||||||
|
"governed.git.write": frozenset({"governed.git.write.branch", "governed.git.write.file"}),
|
||||||
|
"governed.write.branch": frozenset({"governed.git.write.branch"}),
|
||||||
|
"governed.write.file": frozenset({"governed.git.write.file"}),
|
||||||
|
"shell": frozenset({"runtime.shell"}),
|
||||||
|
"git": frozenset({"runtime.git"}),
|
||||||
|
"python": frozenset({"runtime.python"}),
|
||||||
|
"code.read": frozenset({"legacy.code.read"}),
|
||||||
|
"code.review": frozenset({"legacy.code.review"}),
|
||||||
|
"code.write": frozenset({"legacy.code.write"}),
|
||||||
|
"code.write.patch": frozenset({"legacy.code.write.patch"}),
|
||||||
|
"test.python": frozenset({"legacy.test.python"}),
|
||||||
|
"test.shell.allowlisted": frozenset({"legacy.test.shell.allowlisted"}),
|
||||||
|
"git.commit": frozenset({"legacy.git.commit"}),
|
||||||
|
"git.diff.read": frozenset({"legacy.git.diff.read"}),
|
||||||
|
"git.force_push": frozenset({"legacy.git.force_push"}),
|
||||||
|
"deploy.production": frozenset({"legacy.deploy.production"}),
|
||||||
|
"secret.transform": frozenset({"legacy.secret.transform"}),
|
||||||
|
"secure_file_transformer": frozenset({"legacy.secure_file_transformer"}),
|
||||||
|
"runtime.shadow": frozenset({"legacy.runtime.shadow"}),
|
||||||
|
"review.security": frozenset({"legacy.review.security"}),
|
||||||
|
"review.api-contract": frozenset({"legacy.review.api-contract"}),
|
||||||
|
"gpu.train": frozenset({"legacy.gpu.train"}),
|
||||||
|
"agent.runtime.hermes": frozenset({"agent.runtime.hermes"}),
|
||||||
|
"agent.runtime.openclaw": frozenset({"agent.runtime.openclaw"}),
|
||||||
|
"engineering.autonomous": frozenset({"engineering.autonomous"}),
|
||||||
|
"release:candidate": frozenset({"release:candidate"}),
|
||||||
|
}
|
||||||
|
|
||||||
|
_WORKER_ALIASES: dict[str, frozenset[str]] = {
|
||||||
|
"git.read": frozenset({"repo.read"}),
|
||||||
|
"repo.read": frozenset({"repo.read"}),
|
||||||
|
"repository_read": frozenset({"repo.read"}),
|
||||||
|
"git.test": frozenset({"test.execute"}),
|
||||||
|
"test.execute": frozenset({"test.execute"}),
|
||||||
|
"test_execution": frozenset({"test.execute"}),
|
||||||
|
"governed.write.branch": frozenset({"governed.git.write.branch"}),
|
||||||
|
"governed.git.write.branch": frozenset({"governed.git.write.branch"}),
|
||||||
|
"governed.write.file": frozenset({"governed.git.write.file"}),
|
||||||
|
"governed.git.write.file": frozenset({"governed.git.write.file"}),
|
||||||
|
"shell": frozenset({"runtime.shell"}),
|
||||||
|
"git": frozenset({"runtime.git"}),
|
||||||
|
"python": frozenset({"runtime.python"}),
|
||||||
|
"code.read": frozenset({"legacy.code.read"}),
|
||||||
|
"code.review": frozenset({"legacy.code.review"}),
|
||||||
|
"code.write": frozenset({"legacy.code.write"}),
|
||||||
|
"code.write.patch": frozenset({"legacy.code.write.patch"}),
|
||||||
|
"test.python": frozenset({"legacy.test.python"}),
|
||||||
|
"test.shell.allowlisted": frozenset({"legacy.test.shell.allowlisted"}),
|
||||||
|
"git.commit": frozenset({"legacy.git.commit"}),
|
||||||
|
"git.diff.read": frozenset({"legacy.git.diff.read"}),
|
||||||
|
"git.force_push": frozenset({"legacy.git.force_push"}),
|
||||||
|
"deploy.production": frozenset({"legacy.deploy.production"}),
|
||||||
|
"secret.transform": frozenset({"legacy.secret.transform"}),
|
||||||
|
"secure_file_transformer": frozenset({"legacy.secure_file_transformer"}),
|
||||||
|
"runtime.shadow": frozenset({"legacy.runtime.shadow"}),
|
||||||
|
"review.security": frozenset({"legacy.review.security"}),
|
||||||
|
"review.api-contract": frozenset({"legacy.review.api-contract"}),
|
||||||
|
"gpu.train": frozenset({"legacy.gpu.train"}),
|
||||||
|
"agent.runtime.hermes": frozenset({"agent.runtime.hermes"}),
|
||||||
|
"agent.runtime.openclaw": frozenset({"agent.runtime.openclaw"}),
|
||||||
|
"engineering.autonomous": frozenset({"engineering.autonomous"}),
|
||||||
|
"release:candidate": frozenset({"release:candidate"}),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class CapabilityContract:
|
||||||
|
"""Versioned runtime contract attached to one canonical semantic capability."""
|
||||||
|
|
||||||
|
name: str
|
||||||
|
min_version: int = 1
|
||||||
|
max_version: int = 1
|
||||||
|
permissions: tuple[str, ...] = ()
|
||||||
|
forbidden: tuple[str, ...] = ()
|
||||||
|
dependencies: tuple[str, ...] = ()
|
||||||
|
health_requirements: tuple[str, ...] = ()
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, Any]:
|
||||||
|
return asdict(self)
|
||||||
|
|
||||||
|
|
||||||
|
# This registry is the semantic source of truth. Alias tables below are retained
|
||||||
|
# as compatibility adapters; startup/tests validate every alias target against
|
||||||
|
# this registry so a name cannot be accepted in one layer and be unknown in
|
||||||
|
# another.
|
||||||
|
_CAPABILITY_CONTRACTS: dict[str, CapabilityContract] = {
|
||||||
|
"repo.read": CapabilityContract("repo.read", permissions=("repository_read",)),
|
||||||
|
"test.execute": CapabilityContract("test.execute", permissions=("test_execution",)),
|
||||||
|
"governed.git.write.branch": CapabilityContract(
|
||||||
|
"governed.git.write.branch",
|
||||||
|
permissions=("governed_branch_write",),
|
||||||
|
dependencies=("writer_lease", "governed_gitops"),
|
||||||
|
),
|
||||||
|
"governed.git.write.file": CapabilityContract(
|
||||||
|
"governed.git.write.file",
|
||||||
|
permissions=("governed_file_write",),
|
||||||
|
dependencies=("writer_lease", "governed_gitops"),
|
||||||
|
),
|
||||||
|
"governed.git.write": CapabilityContract(
|
||||||
|
"governed.git.write",
|
||||||
|
permissions=("governed_git_write",),
|
||||||
|
dependencies=("writer_lease", "governed_gitops"),
|
||||||
|
),
|
||||||
|
"runtime.shell": CapabilityContract("runtime.shell", permissions=("runtime_shell",)),
|
||||||
|
"runtime.git": CapabilityContract("runtime.git", permissions=("runtime_git",)),
|
||||||
|
"runtime.python": CapabilityContract("runtime.python", permissions=("runtime_python",)),
|
||||||
|
"legacy.code.read": CapabilityContract("legacy.code.read"),
|
||||||
|
"legacy.code.review": CapabilityContract("legacy.code.review"),
|
||||||
|
"legacy.code.write": CapabilityContract("legacy.code.write"),
|
||||||
|
"legacy.code.write.patch": CapabilityContract("legacy.code.write.patch"),
|
||||||
|
"legacy.test.python": CapabilityContract("legacy.test.python"),
|
||||||
|
"legacy.test.shell.allowlisted": CapabilityContract("legacy.test.shell.allowlisted"),
|
||||||
|
"legacy.git.commit": CapabilityContract("legacy.git.commit"),
|
||||||
|
"legacy.git.diff.read": CapabilityContract("legacy.git.diff.read"),
|
||||||
|
"legacy.git.force_push": CapabilityContract(
|
||||||
|
"legacy.git.force_push",
|
||||||
|
forbidden=("protected_ref_bypass",),
|
||||||
|
),
|
||||||
|
"legacy.deploy.production": CapabilityContract(
|
||||||
|
"legacy.deploy.production",
|
||||||
|
permissions=("production_deploy",),
|
||||||
|
dependencies=("owner_approval", "deployment_controller"),
|
||||||
|
health_requirements=("approval_envelope_verified", "runtime_acceptance_available"),
|
||||||
|
),
|
||||||
|
"legacy.secret.transform": CapabilityContract("legacy.secret.transform"),
|
||||||
|
"legacy.secure_file_transformer": CapabilityContract("legacy.secure_file_transformer"),
|
||||||
|
"legacy.runtime.shadow": CapabilityContract("legacy.runtime.shadow"),
|
||||||
|
"legacy.review.security": CapabilityContract("legacy.review.security"),
|
||||||
|
"legacy.review.api-contract": CapabilityContract("legacy.review.api-contract"),
|
||||||
|
"legacy.gpu.train": CapabilityContract("legacy.gpu.train"),
|
||||||
|
"agent.runtime.hermes": CapabilityContract(
|
||||||
|
"agent.runtime.hermes",
|
||||||
|
dependencies=("hermes_runtime",),
|
||||||
|
health_requirements=("worker_liveness",),
|
||||||
|
),
|
||||||
|
"agent.runtime.openclaw": CapabilityContract(
|
||||||
|
"agent.runtime.openclaw",
|
||||||
|
dependencies=("openclaw_runtime",),
|
||||||
|
health_requirements=("worker_liveness",),
|
||||||
|
),
|
||||||
|
"engineering.autonomous": CapabilityContract(
|
||||||
|
"engineering.autonomous",
|
||||||
|
dependencies=("agent_task_control_plane",),
|
||||||
|
health_requirements=("worker_liveness",),
|
||||||
|
),
|
||||||
|
"release:candidate": CapabilityContract(
|
||||||
|
"release:candidate",
|
||||||
|
min_version=1,
|
||||||
|
max_version=1,
|
||||||
|
permissions=("prepare_release",),
|
||||||
|
forbidden=(
|
||||||
|
"production_deploy",
|
||||||
|
"submit_owner_approval",
|
||||||
|
"tag_publish",
|
||||||
|
"baseline_registry_finalize",
|
||||||
|
"runtime_switch",
|
||||||
|
),
|
||||||
|
dependencies=(
|
||||||
|
"credential_projection",
|
||||||
|
"trusted_builder",
|
||||||
|
),
|
||||||
|
health_requirements=(
|
||||||
|
"authenticated_probe",
|
||||||
|
"release_api_health",
|
||||||
|
"worker_liveness",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def capability_contract(name: object) -> CapabilityContract | None:
|
||||||
|
"""Return the contract for one canonical or registered capability name."""
|
||||||
|
|
||||||
|
text = _clean(name)
|
||||||
|
direct = _CAPABILITY_CONTRACTS.get(text)
|
||||||
|
if direct is not None:
|
||||||
|
return direct
|
||||||
|
mapped = _TASK_ALIASES.get(text) or _WORKER_ALIASES.get(text)
|
||||||
|
if mapped is None or len(mapped) != 1:
|
||||||
|
return None
|
||||||
|
return _CAPABILITY_CONTRACTS.get(next(iter(mapped)))
|
||||||
|
|
||||||
|
|
||||||
|
def registered_capability(name: object, *, worker: bool = False) -> bool:
|
||||||
|
"""Fail-closed validation used by registration/task admission."""
|
||||||
|
|
||||||
|
text = _clean(name)
|
||||||
|
if "*" in text:
|
||||||
|
return False
|
||||||
|
aliases = _WORKER_ALIASES if worker else _TASK_ALIASES
|
||||||
|
mapped = aliases.get(text)
|
||||||
|
if mapped is None:
|
||||||
|
return text in _CAPABILITY_CONTRACTS
|
||||||
|
return all(item in _CAPABILITY_CONTRACTS for item in mapped)
|
||||||
|
|
||||||
|
|
||||||
|
def validate_capability_contract_registry() -> dict[str, object]:
|
||||||
|
"""Prove alias/contract closure and contract safety invariants."""
|
||||||
|
|
||||||
|
alias_targets: set[str] = set()
|
||||||
|
for mapping in (_TASK_ALIASES, _WORKER_ALIASES):
|
||||||
|
for values in mapping.values():
|
||||||
|
alias_targets.update(values)
|
||||||
|
missing = sorted(alias_targets - set(_CAPABILITY_CONTRACTS))
|
||||||
|
invalid_versions = sorted(
|
||||||
|
name
|
||||||
|
for name, contract in _CAPABILITY_CONTRACTS.items()
|
||||||
|
if contract.min_version < 1 or contract.max_version < contract.min_version
|
||||||
|
)
|
||||||
|
permission_forbidden_overlap = sorted(
|
||||||
|
name
|
||||||
|
for name, contract in _CAPABILITY_CONTRACTS.items()
|
||||||
|
if set(contract.permissions) & set(contract.forbidden)
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"schema": "aios.capability-contract-registry.v1",
|
||||||
|
"contract_count": len(_CAPABILITY_CONTRACTS),
|
||||||
|
"missing_alias_targets": missing,
|
||||||
|
"invalid_versions": invalid_versions,
|
||||||
|
"permission_forbidden_overlap": permission_forbidden_overlap,
|
||||||
|
"valid": not missing and not invalid_versions and not permission_forbidden_overlap,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def capability_contract_snapshot() -> dict[str, dict[str, Any]]:
|
||||||
|
return {
|
||||||
|
name: contract.as_dict()
|
||||||
|
for name, contract in sorted(_CAPABILITY_CONTRACTS.items())
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
_UNKNOWN_PREFIX = "exact:"
|
||||||
|
|
||||||
|
|
||||||
|
def _clean(value: object) -> str:
|
||||||
|
if not isinstance(value, str):
|
||||||
|
raise ValueError("capability must be a string")
|
||||||
|
text = value.strip().lower()
|
||||||
|
if not text:
|
||||||
|
raise ValueError("capability must not be empty")
|
||||||
|
return text
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_required_capabilities(values: Iterable[object]) -> frozenset[str]:
|
||||||
|
"""Map task-facing names to one semantic vocabulary.
|
||||||
|
|
||||||
|
Unknown names remain explicit exact requirements. They are never treated
|
||||||
|
as wildcards or granted by a failed normalization.
|
||||||
|
"""
|
||||||
|
|
||||||
|
result: set[str] = set()
|
||||||
|
for value in values:
|
||||||
|
text = _clean(value)
|
||||||
|
if "*" in text:
|
||||||
|
result.add(_UNKNOWN_PREFIX + text)
|
||||||
|
else:
|
||||||
|
mapped = _TASK_ALIASES.get(text)
|
||||||
|
if mapped is None or any(item not in _CAPABILITY_CONTRACTS for item in mapped):
|
||||||
|
result.add(_UNKNOWN_PREFIX + text)
|
||||||
|
else:
|
||||||
|
result.update(mapped)
|
||||||
|
return frozenset(result)
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_worker_capabilities(values: Iterable[object]) -> frozenset[str]:
|
||||||
|
"""Map registered worker names to effective semantic capabilities."""
|
||||||
|
|
||||||
|
result: set[str] = set()
|
||||||
|
for value in values:
|
||||||
|
text = _clean(value)
|
||||||
|
if "*" in text:
|
||||||
|
continue
|
||||||
|
mapped = _WORKER_ALIASES.get(text)
|
||||||
|
if mapped is not None and all(item in _CAPABILITY_CONTRACTS for item in mapped):
|
||||||
|
result.update(mapped)
|
||||||
|
if {
|
||||||
|
"governed.git.write.branch",
|
||||||
|
"governed.git.write.file",
|
||||||
|
}.issubset(result):
|
||||||
|
result.add("governed.git.write")
|
||||||
|
return frozenset(result)
|
||||||
|
|
||||||
|
|
||||||
|
def has_capabilities(
|
||||||
|
worker_capabilities: Iterable[object],
|
||||||
|
required_capabilities: Iterable[object],
|
||||||
|
) -> bool:
|
||||||
|
"""Return the single eligibility decision used by poll and claim."""
|
||||||
|
|
||||||
|
required = normalize_required_capabilities(required_capabilities)
|
||||||
|
effective = normalize_worker_capabilities(worker_capabilities)
|
||||||
|
if any(value.startswith(_UNKNOWN_PREFIX) for value in required):
|
||||||
|
return False
|
||||||
|
return required.issubset(effective)
|
||||||
|
|
||||||
|
|
||||||
|
def required_capabilities_subset(
|
||||||
|
granted_requirements: Iterable[object],
|
||||||
|
required_requirements: Iterable[object],
|
||||||
|
) -> bool:
|
||||||
|
"""Check semantic capability containment for delegation constraints."""
|
||||||
|
granted = normalize_required_capabilities(granted_requirements)
|
||||||
|
required = normalize_required_capabilities(required_requirements)
|
||||||
|
if any(value.startswith(_UNKNOWN_PREFIX) for value in granted | required):
|
||||||
|
return False
|
||||||
|
return required.issubset(granted)
|
||||||
|
|
||||||
|
|
||||||
|
def normalized_capability_matrix(
|
||||||
|
worker_capabilities: Iterable[object],
|
||||||
|
required_capabilities: Iterable[object],
|
||||||
|
) -> dict[str, object]:
|
||||||
|
"""Expose non-secret decision evidence for tests and diagnostics."""
|
||||||
|
|
||||||
|
worker_values = tuple(worker_capabilities)
|
||||||
|
required_values = tuple(required_capabilities)
|
||||||
|
worker = normalize_worker_capabilities(worker_values)
|
||||||
|
required = normalize_required_capabilities(required_values)
|
||||||
|
return {
|
||||||
|
"worker_effective": sorted(worker),
|
||||||
|
"task_required": sorted(required),
|
||||||
|
"eligible": has_capabilities(worker_values, required_values),
|
||||||
|
}
|
||||||
@@ -1,9 +1,10 @@
|
|||||||
"""Loopback-only governed Bridge v2 HTTP gateway with durable handlers."""
|
"""Loopback-only governed Bridge v2 HTTP gateway with durable handlers."""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from dataclasses import dataclass, replace
|
from dataclasses import asdict, dataclass, replace
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
import base64
|
import base64
|
||||||
|
import hmac
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
@@ -12,6 +13,8 @@ import re
|
|||||||
import sqlite3
|
import sqlite3
|
||||||
import threading
|
import threading
|
||||||
from typing import Any, Mapping
|
from typing import Any, Mapping
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
from aios_authority_git_executor import AuthorityGitExecutor
|
from aios_authority_git_executor import AuthorityGitExecutor
|
||||||
from aios_chatgpt_bridge_v2 import BridgeV2Adapter, BridgeV2Error, OPERATION_REQUIRED_FIELDS
|
from aios_chatgpt_bridge_v2 import BridgeV2Adapter, BridgeV2Error, OPERATION_REQUIRED_FIELDS
|
||||||
@@ -19,7 +22,9 @@ from aios_continuation_executor import AuthorityGitAdapter, ContinuationCommit,
|
|||||||
from aios_continuation_lease import WriterLeaseRequestV2
|
from aios_continuation_lease import WriterLeaseRequestV2
|
||||||
from aios_control_plane_snapshot import build_control_plane_snapshot
|
from aios_control_plane_snapshot import build_control_plane_snapshot
|
||||||
from aios_gitops_job_controller import GitOpsJobController, JobError, JobState, JobStore
|
from aios_gitops_job_controller import GitOpsJobController, JobError, JobState, JobStore
|
||||||
from aios_governed_task_bootstrap import BootstrapError, GovernedTaskBootstrap
|
from aios_governed_task_bootstrap import BootstrapError, GovernedTaskBinding, GovernedTaskBootstrap, _legacy_safe, _task_key
|
||||||
|
from aios_online_agent_execution import ExecutionProjectionError, OnlineTaskProjectionClient
|
||||||
|
from aios_online_agent_plan_store import ExecutionPlanError, ExecutionPlanStore
|
||||||
from governed_production_release import build_executor_config
|
from governed_production_release import build_executor_config
|
||||||
|
|
||||||
MAX_BODY = 2 * 1024 * 1024
|
MAX_BODY = 2 * 1024 * 1024
|
||||||
@@ -27,6 +32,8 @@ ROUTES = {
|
|||||||
"/v2/runtime": "getRuntimeStatus",
|
"/v2/runtime": "getRuntimeStatus",
|
||||||
"/v2/capabilities": "getGovernedWriteCapabilities",
|
"/v2/capabilities": "getGovernedWriteCapabilities",
|
||||||
"/v2/governed-task": "createGovernedTask",
|
"/v2/governed-task": "createGovernedTask",
|
||||||
|
"/v2/execution-envelope": "projectOnlineAgentExecution",
|
||||||
|
"/v2/execution-plan": "registerOnlineAgentExecutionPlan",
|
||||||
"/v2/writer-lease": "acquireWriterLease",
|
"/v2/writer-lease": "acquireWriterLease",
|
||||||
"/v2/writer-lease/bootstrap": "bootstrapWriterLease",
|
"/v2/writer-lease/bootstrap": "bootstrapWriterLease",
|
||||||
"/v2/writer-lease/renew": "renewWriterLease",
|
"/v2/writer-lease/renew": "renewWriterLease",
|
||||||
@@ -62,6 +69,9 @@ class RuntimeConfig:
|
|||||||
registry_root: Path = Path("/var/lib/aios-gitops/runtime/registry-projection")
|
registry_root: Path = Path("/var/lib/aios-gitops/runtime/registry-projection")
|
||||||
baseline_path: Path = Path("/var/lib/aios-gitops-runtime-anchor/baseline/current.json")
|
baseline_path: Path = Path("/var/lib/aios-gitops-runtime-anchor/baseline/current.json")
|
||||||
health_url: str = "http://127.0.0.1:18927/health"
|
health_url: str = "http://127.0.0.1:18927/health"
|
||||||
|
agent_task_api_url: str = "http://127.0.0.1:18940"
|
||||||
|
agent_task_control_token_path: Path = Path("/run/credentials/aios-chatgpt-bridge-v2-staging.service/agent_task_control_token")
|
||||||
|
online_task_projection_enabled: bool = False
|
||||||
|
|
||||||
|
|
||||||
def _safe(value: str) -> str:
|
def _safe(value: str) -> str:
|
||||||
@@ -74,6 +84,16 @@ class GovernedRuntime:
|
|||||||
config.state_root.mkdir(parents=True, exist_ok=True)
|
config.state_root.mkdir(parents=True, exist_ok=True)
|
||||||
self.authority = AuthorityGitExecutor(config.authoritative_git_dir, config.state_root / "worktrees")
|
self.authority = AuthorityGitExecutor(config.authoritative_git_dir, config.state_root / "worktrees")
|
||||||
self.bootstrap = GovernedTaskBootstrap(self.authority, config.state_root, remote_name=config.remote_name, token_path=config.token_path, ssh_key_path=config.ssh_key_path, api_base=config.api_base)
|
self.bootstrap = GovernedTaskBootstrap(self.authority, config.state_root, remote_name=config.remote_name, token_path=config.token_path, ssh_key_path=config.ssh_key_path, api_base=config.api_base)
|
||||||
|
self.online_task_projector = (
|
||||||
|
OnlineTaskProjectionClient(config.agent_task_api_url, config.agent_task_control_token_path)
|
||||||
|
if config.online_task_projection_enabled
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
self.projection_outbox_root = config.state_root / "online-task-projection-outbox"
|
||||||
|
self.projection_outbox_root.mkdir(parents=True, exist_ok=True)
|
||||||
|
self.execution_plan_store = ExecutionPlanStore(config.state_root / "online-agent-execution-plans")
|
||||||
|
self._projection_locks_guard = threading.Lock()
|
||||||
|
self._projection_locks: dict[str, threading.RLock] = {}
|
||||||
self.runtimes: dict[tuple[str, int], tuple[Any, GitOpsJobController, ContinuationExecutor]] = {}
|
self.runtimes: dict[tuple[str, int], tuple[Any, GitOpsJobController, ContinuationExecutor]] = {}
|
||||||
self.controllers: dict[str, GitOpsJobController] = {}
|
self.controllers: dict[str, GitOpsJobController] = {}
|
||||||
self._recovery_lock = threading.RLock()
|
self._recovery_lock = threading.RLock()
|
||||||
@@ -93,6 +113,54 @@ class GovernedRuntime:
|
|||||||
askpass.write_text('#!/bin/sh\ncase "$1" in *Username*) printf "%s\\n" "$GITEA_MACHINE_ACTOR" ;; *) cat "$CREDENTIAL_FILE" ;; esac\n', encoding="utf-8")
|
askpass.write_text('#!/bin/sh\ncase "$1" in *Username*) printf "%s\\n" "$GITEA_MACHINE_ACTOR" ;; *) cat "$CREDENTIAL_FILE" ;; esac\n', encoding="utf-8")
|
||||||
askpass.chmod(0o700)
|
askpass.chmod(0o700)
|
||||||
os.environ.setdefault("GIT_ASKPASS", str(askpass)); os.environ.setdefault("GIT_ASKPASS_REQUIRE", "force"); os.environ.setdefault("GIT_TERMINAL_PROMPT", "0"); os.environ.setdefault("GITEA_MACHINE_ACTOR", "aios-autonomy-agent")
|
os.environ.setdefault("GIT_ASKPASS", str(askpass)); os.environ.setdefault("GIT_ASKPASS_REQUIRE", "force"); os.environ.setdefault("GIT_TERMINAL_PROMPT", "0"); os.environ.setdefault("GITEA_MACHINE_ACTOR", "aios-autonomy-agent")
|
||||||
|
if self.online_task_projector is not None:
|
||||||
|
threading.Thread(target=self._projection_reconciler, name="online-task-projection-reconciler", daemon=True).start()
|
||||||
|
|
||||||
|
def _projection_lock(self, task_id: str) -> threading.RLock:
|
||||||
|
guard = getattr(self, "_projection_locks_guard", None)
|
||||||
|
if guard is None:
|
||||||
|
guard = threading.Lock()
|
||||||
|
self._projection_locks_guard = guard
|
||||||
|
self._projection_locks = {}
|
||||||
|
with guard:
|
||||||
|
return self._projection_locks.setdefault(task_id, threading.RLock())
|
||||||
|
|
||||||
|
def _projection_outbox_path(self, task_id: str) -> Path:
|
||||||
|
return self.projection_outbox_root / (_task_key(task_id) + ".json")
|
||||||
|
|
||||||
|
def _write_projection_outbox(self, task_id: str, projection: Mapping[str, Any], binding: Any) -> Path:
|
||||||
|
path = self._projection_outbox_path(task_id)
|
||||||
|
temporary = path.with_name(path.name + f".{os.getpid()}.{threading.get_ident()}.tmp")
|
||||||
|
temporary.write_text(json.dumps({"task": dict(projection), "binding": asdict(binding)}, sort_keys=True, separators=(",", ":")), encoding="utf-8")
|
||||||
|
temporary.replace(path)
|
||||||
|
return path
|
||||||
|
|
||||||
|
def _deliver_projection_outbox(self, path: Path) -> dict[str, Any]:
|
||||||
|
value = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
if not isinstance(value, Mapping) or not isinstance(value.get("task"), Mapping) or not isinstance(value.get("binding"), Mapping):
|
||||||
|
raise RuntimeError("ONLINE_TASK_PROJECTION_OUTBOX_INVALID")
|
||||||
|
binding = self.bootstrap.execution_binding(GovernedTaskBinding(**dict(value["binding"])))
|
||||||
|
if self.online_task_projector is None:
|
||||||
|
raise RuntimeError("ONLINE_TASK_PROJECTION_CONTROL_UNAVAILABLE")
|
||||||
|
created = self.online_task_projector.create(dict(value["task"]), binding=binding)
|
||||||
|
path.unlink(missing_ok=True)
|
||||||
|
return created
|
||||||
|
|
||||||
|
def _projection_reconciler(self) -> None:
|
||||||
|
while True:
|
||||||
|
for path in sorted(self.projection_outbox_root.glob("*.json")):
|
||||||
|
try:
|
||||||
|
value = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
binding = value.get("binding") if isinstance(value, Mapping) else None
|
||||||
|
task_id = str(binding.get("task_id") or "") if isinstance(binding, Mapping) else ""
|
||||||
|
if not task_id:
|
||||||
|
continue
|
||||||
|
with self._projection_lock(task_id):
|
||||||
|
if path.exists():
|
||||||
|
self._deliver_projection_outbox(path)
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
threading.Event().wait(5)
|
||||||
|
|
||||||
def runtime_status(self) -> dict[str, Any]:
|
def runtime_status(self) -> dict[str, Any]:
|
||||||
return {
|
return {
|
||||||
@@ -113,7 +181,10 @@ class GovernedRuntime:
|
|||||||
}
|
}
|
||||||
|
|
||||||
def _job_root(self, task_id: str) -> Path:
|
def _job_root(self, task_id: str) -> Path:
|
||||||
return self.config.state_root / "jobs" / _safe(task_id)
|
parent = self.config.state_root / "jobs"
|
||||||
|
keyed = parent / _task_key(task_id)
|
||||||
|
legacy = parent / _legacy_safe(task_id)
|
||||||
|
return keyed if keyed.exists() or not legacy.exists() else legacy
|
||||||
|
|
||||||
def _build_runtime(
|
def _build_runtime(
|
||||||
self,
|
self,
|
||||||
@@ -443,7 +514,85 @@ class GovernedRuntime:
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
def create_task(self, request: Mapping[str, Any]) -> dict[str, Any]:
|
def create_task(self, request: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
result = self.bootstrap.create(request); self.bindings[result["task_id"]] = result; return result
|
online_request = request.get("online_task")
|
||||||
|
if online_request is not None and not isinstance(online_request, Mapping):
|
||||||
|
raise ValueError("ONLINE_TASK_REQUEST_INVALID")
|
||||||
|
execution_request = request.get("execution")
|
||||||
|
if execution_request is not None and not isinstance(execution_request, Mapping):
|
||||||
|
raise ValueError("EXECUTION_REQUEST_INVALID")
|
||||||
|
online_task_projector = getattr(self, "online_task_projector", None)
|
||||||
|
projection_required = (
|
||||||
|
online_request is not None
|
||||||
|
or execution_request is not None
|
||||||
|
or request.get("require_online_task_projection") is True
|
||||||
|
)
|
||||||
|
if online_task_projector is None:
|
||||||
|
if projection_required:
|
||||||
|
raise RuntimeError("ONLINE_TASK_PROJECTION_CONTROL_UNAVAILABLE")
|
||||||
|
return self.bootstrap.create(request)
|
||||||
|
online = dict(online_request or {})
|
||||||
|
online.setdefault("goal", str(request.get("description") or "Governed engineering task"))
|
||||||
|
online.setdefault("instructions", "Execute only the exact governed binding and return server-verifiable evidence.")
|
||||||
|
online.setdefault("success_criteria", [{"id": "governed", "description": "Complete the exact governed task lifecycle."}])
|
||||||
|
online.setdefault("required_capabilities", ["git.read", "git.test", "governed.write.branch", "governed.write.file"])
|
||||||
|
online.setdefault("priority", int(request.get("priority") or 0))
|
||||||
|
online.setdefault("max_attempts", int(request.get("max_attempts") or 10))
|
||||||
|
normalized = dict(request)
|
||||||
|
normalized["online_task"] = online
|
||||||
|
normalized["execution"] = dict(execution_request or {})
|
||||||
|
normalized["require_online_task_projection"] = True
|
||||||
|
result = self.bootstrap.create(normalized)
|
||||||
|
self.bindings[result["task_id"]] = result
|
||||||
|
projection = self.bootstrap.online_task_projection(
|
||||||
|
result["task_id"],
|
||||||
|
online,
|
||||||
|
normalized["execution"],
|
||||||
|
expected_job_id=str(result.get("job_id") or "") or None,
|
||||||
|
)
|
||||||
|
durable_binding = self.bootstrap.get_binding(result["task_id"])
|
||||||
|
with self._projection_lock(result["task_id"]):
|
||||||
|
path = self._write_projection_outbox(result["task_id"], projection, durable_binding)
|
||||||
|
created = self._deliver_projection_outbox(path)
|
||||||
|
return {
|
||||||
|
**result,
|
||||||
|
"online_task_projection": {
|
||||||
|
"status": "PROJECTED",
|
||||||
|
"task_id": created["task_id"],
|
||||||
|
"job_id": result["job_id"],
|
||||||
|
"task_status": created["status"],
|
||||||
|
"execution_digest": projection["metadata"]["execution"]["execution_digest"],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def project_online_agent_execution(self, request: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
|
capabilities = request.get("required_capabilities") or []
|
||||||
|
if not isinstance(capabilities, list) or not all(isinstance(item, str) for item in capabilities):
|
||||||
|
raise ValueError("CAPABILITY_SCOPE_INVALID")
|
||||||
|
execution = request.get("execution")
|
||||||
|
if execution is not None and not isinstance(execution, Mapping):
|
||||||
|
raise ValueError("EXECUTION_REQUEST_INVALID")
|
||||||
|
task_id = str(request["task_id"])
|
||||||
|
durable = self.bootstrap.get_binding(task_id)
|
||||||
|
binding = self.bootstrap.execution_binding(durable)
|
||||||
|
expected_job_id = str(request.get("job_id") or "") or None
|
||||||
|
if expected_job_id and expected_job_id != binding.job_id:
|
||||||
|
raise BootstrapError("JOB_BINDING_CONFLICT")
|
||||||
|
if execution is None:
|
||||||
|
planned = self.execution_plan_store.resolve(binding)
|
||||||
|
if planned is not None:
|
||||||
|
execution = planned["execution"]
|
||||||
|
return self.bootstrap.execution_envelope(
|
||||||
|
task_id,
|
||||||
|
execution,
|
||||||
|
required_capabilities=tuple(capabilities),
|
||||||
|
expected_job_id=expected_job_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
def register_online_agent_execution_plan(self, request: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
|
task_id = str(request["task_id"])
|
||||||
|
durable = self.bootstrap.get_binding(task_id)
|
||||||
|
binding = self.bootstrap.execution_binding(durable)
|
||||||
|
return self.execution_plan_store.register(binding, request)
|
||||||
|
|
||||||
def acquire_lease(self, request: Mapping[str, Any]) -> dict[str, Any]:
|
def acquire_lease(self, request: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
binding, controller, _ = self._runtime(request)
|
binding, controller, _ = self._runtime(request)
|
||||||
@@ -1340,7 +1489,19 @@ class GovernedRuntime:
|
|||||||
controller = self._controller_for_job(job_id)
|
controller = self._controller_for_job(job_id)
|
||||||
row = controller.store.read(job_id)
|
row = controller.store.read(job_id)
|
||||||
if action == "get":
|
if action == "get":
|
||||||
return controller.store._receipt(row, duplicate=True).as_dict()
|
response = controller.store._receipt(row, duplicate=True).as_dict()
|
||||||
|
try:
|
||||||
|
with controller.store._connect() as connection:
|
||||||
|
connection.row_factory = sqlite3.Row
|
||||||
|
leases = connection.execute(
|
||||||
|
"SELECT lease_id,job_id,branch,lineage_anchor_oid,current_head_oid,generation,"
|
||||||
|
"lease_expires_at,state,revision FROM branch_leases WHERE job_id=? ORDER BY revision DESC",
|
||||||
|
(job_id,),
|
||||||
|
).fetchall()
|
||||||
|
except sqlite3.Error as exc:
|
||||||
|
raise RuntimeError("JOB_DB_UNAVAILABLE") from exc
|
||||||
|
response["branch_leases"] = [dict(item) for item in leases]
|
||||||
|
return response
|
||||||
if action == "resume":
|
if action == "resume":
|
||||||
return controller.resume(job_id).as_dict()
|
return controller.resume(job_id).as_dict()
|
||||||
return self._executor_from_row(row, controller).reconcile(job_id)
|
return self._executor_from_row(row, controller).reconcile(job_id)
|
||||||
@@ -1405,6 +1566,8 @@ class GovernedRuntime:
|
|||||||
def handlers(self) -> dict[str, Any]:
|
def handlers(self) -> dict[str, Any]:
|
||||||
return {
|
return {
|
||||||
"createGovernedTask": self.create_task,
|
"createGovernedTask": self.create_task,
|
||||||
|
"projectOnlineAgentExecution": self.project_online_agent_execution,
|
||||||
|
"registerOnlineAgentExecutionPlan": self.register_online_agent_execution_plan,
|
||||||
"acquireWriterLease": self.acquire_lease,
|
"acquireWriterLease": self.acquire_lease,
|
||||||
"bootstrapWriterLease": self.bootstrap_writer_lease,
|
"bootstrapWriterLease": self.bootstrap_writer_lease,
|
||||||
"renewWriterLease": self.renew_writer_lease,
|
"renewWriterLease": self.renew_writer_lease,
|
||||||
@@ -1434,6 +1597,8 @@ class _HTTPError(RuntimeError): pass
|
|||||||
|
|
||||||
class BridgeHandler(BaseHTTPRequestHandler):
|
class BridgeHandler(BaseHTTPRequestHandler):
|
||||||
adapter: BridgeV2Adapter | None = None
|
adapter: BridgeV2Adapter | None = None
|
||||||
|
control_token: str | None = None
|
||||||
|
agent_task_api_url: str = "http://127.0.0.1:18940"
|
||||||
def log_message(self, *_args: Any) -> None: return
|
def log_message(self, *_args: Any) -> None: return
|
||||||
def _json(self, value: Any, status: int = 200) -> None:
|
def _json(self, value: Any, status: int = 200) -> None:
|
||||||
raw = json.dumps(value, sort_keys=True, separators=(",", ":")).encode(); self.send_response(status); self.send_header("Content-Type", "application/json"); self.send_header("Content-Length", str(len(raw))); self.end_headers(); self.wfile.write(raw)
|
raw = json.dumps(value, sort_keys=True, separators=(",", ":")).encode(); self.send_response(status); self.send_header("Content-Type", "application/json"); self.send_header("Content-Length", str(len(raw))); self.end_headers(); self.wfile.write(raw)
|
||||||
@@ -1451,7 +1616,50 @@ class BridgeHandler(BaseHTTPRequestHandler):
|
|||||||
allowed = set(OPERATION_REQUIRED_FIELDS[operation]) | {"schema", "job_id", "lease_seconds", "create_target_branch", "worktree_mode", "cas_ref_update_required", "expected_branch_oid", "generation", "branch_mode"}
|
allowed = set(OPERATION_REQUIRED_FIELDS[operation]) | {"schema", "job_id", "lease_seconds", "create_target_branch", "worktree_mode", "cas_ref_update_required", "expected_branch_oid", "generation", "branch_mode"}
|
||||||
if operation == "commitFilesGoverned": allowed.add("files")
|
if operation == "commitFilesGoverned": allowed.add("files")
|
||||||
allowed.update({"owner_approval", "execute"})
|
allowed.update({"owner_approval", "execute"})
|
||||||
|
if operation == "projectOnlineAgentExecution":
|
||||||
|
allowed.update({"execution", "required_capabilities"})
|
||||||
|
if operation == "registerOnlineAgentExecutionPlan":
|
||||||
|
allowed.update({"execution", "source_head_oid", "expected_previous_plan_digest"})
|
||||||
|
if operation == "createGovernedTask":
|
||||||
|
allowed.update({"online_task", "execution", "require_online_task_projection", "priority", "max_attempts", "parent_task_id", "parent_job_id"})
|
||||||
if set(body) - allowed: raise _HTTPError("UNKNOWN_FIELDS")
|
if set(body) - allowed: raise _HTTPError("UNKNOWN_FIELDS")
|
||||||
|
def _authorized(self, operation: str, body: Mapping[str, Any]) -> bool:
|
||||||
|
header = self.headers.get("Authorization", "")
|
||||||
|
if not header.startswith("Bearer "):
|
||||||
|
return False
|
||||||
|
supplied = header[7:]
|
||||||
|
if self.control_token and hmac.compare_digest(supplied, self.control_token):
|
||||||
|
return True
|
||||||
|
poll = urllib.request.Request(
|
||||||
|
self.agent_task_api_url.rstrip("/") + "/v1/tasks/poll",
|
||||||
|
method="POST",
|
||||||
|
data=b'{"limit":1}',
|
||||||
|
headers={"Authorization": header, "Content-Type": "application/json", "Accept": "application/json"},
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(poll, timeout=5) as response:
|
||||||
|
principal = json.loads(response.read().decode("utf-8"))
|
||||||
|
except (OSError, urllib.error.URLError, urllib.error.HTTPError, ValueError):
|
||||||
|
return False
|
||||||
|
agent_id = str(principal.get("agent_id") or "") if isinstance(principal, Mapping) else ""
|
||||||
|
if not agent_id:
|
||||||
|
return False
|
||||||
|
claim_bound = {"acquireWriterLease", "bootstrapWriterLease", "renewWriterLease", "commitFileGoverned", "commitFilesGoverned", "projectOnlineAgentExecution", "getGitOpsJob"}
|
||||||
|
if operation not in claim_bound:
|
||||||
|
return False
|
||||||
|
task_id = str(body.get("task_id") or "")
|
||||||
|
if not task_id:
|
||||||
|
return False
|
||||||
|
task_request = urllib.request.Request(
|
||||||
|
self.agent_task_api_url.rstrip("/") + "/v1/tasks/" + task_id,
|
||||||
|
headers={"Authorization": header, "Accept": "application/json"},
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(task_request, timeout=5) as response:
|
||||||
|
task = json.loads(response.read().decode("utf-8"))
|
||||||
|
except (OSError, urllib.error.URLError, urllib.error.HTTPError, ValueError):
|
||||||
|
return False
|
||||||
|
return isinstance(task, Mapping) and task.get("task_id") == task_id and task.get("claimed_by") == agent_id and task.get("status") in {"CLAIMED", "RUNNING"}
|
||||||
def do_GET(self) -> None:
|
def do_GET(self) -> None:
|
||||||
if self.adapter is None: self._json({"error": "RUNTIME_UNAVAILABLE"}, 503); return
|
if self.adapter is None: self._json({"error": "RUNTIME_UNAVAILABLE"}, 503); return
|
||||||
if self.path == "/v2/runtime": self._json(self.adapter.getRuntimeStatus()); return
|
if self.path == "/v2/runtime": self._json(self.adapter.getRuntimeStatus()); return
|
||||||
@@ -1462,14 +1670,23 @@ class BridgeHandler(BaseHTTPRequestHandler):
|
|||||||
operation = ROUTES.get(self.path)
|
operation = ROUTES.get(self.path)
|
||||||
if operation is None or operation in ("getRuntimeStatus", "getGovernedWriteCapabilities") or self.adapter is None: self._json({"error": "NOT_FOUND"}, 404); return
|
if operation is None or operation in ("getRuntimeStatus", "getGovernedWriteCapabilities") or self.adapter is None: self._json({"error": "NOT_FOUND"}, 404); return
|
||||||
try:
|
try:
|
||||||
body = self._body(); self._allowed(operation, body); self._json(getattr(self.adapter, operation)(body))
|
body = self._body()
|
||||||
|
except _HTTPError as exc: self._json({"error": str(exc)}, 413 if str(exc) == "REQUEST_TOO_LARGE" else 400); return
|
||||||
|
if not self._authorized(operation, body): self._json({"error": "BRIDGE_CALLER_UNAUTHORIZED"}, 401); return
|
||||||
|
try:
|
||||||
|
self._allowed(operation, body); self._json(getattr(self.adapter, operation)(body))
|
||||||
except _HTTPError as exc: self._json({"error": str(exc)}, 413 if str(exc) == "REQUEST_TOO_LARGE" else 400)
|
except _HTTPError as exc: self._json({"error": str(exc)}, 413 if str(exc) == "REQUEST_TOO_LARGE" else 400)
|
||||||
except (BootstrapError, ValueError, BridgeV2Error) as exc: self._json({"error": str(exc)}, 409 if "CONFLICT" in str(exc) else 400)
|
except (BootstrapError, ExecutionProjectionError, ExecutionPlanError, ValueError, BridgeV2Error) as exc: self._json({"error": str(exc)}, 409 if any(token in str(exc) for token in ("CONFLICT", "DRIFT", "CAS")) else 400)
|
||||||
except Exception as exc: self._json({"error": str(exc) or type(exc).__name__}, 409 if any(token in str(exc) for token in ("CONFLICT", "EXPECTATION", "MONOTONIC")) else 503)
|
except Exception as exc: self._json({"error": str(exc) or type(exc).__name__}, 409 if any(token in str(exc) for token in ("CONFLICT", "EXPECTATION", "MONOTONIC")) else 503)
|
||||||
|
|
||||||
|
|
||||||
def serve(adapter: BridgeV2Adapter, host: str = "127.0.0.1", port: int = 18892) -> None:
|
def serve(adapter: BridgeV2Adapter, host: str = "127.0.0.1", port: int = 18892, *, control_token_path: Path, agent_task_api_url: str) -> None:
|
||||||
BridgeHandler.adapter = adapter; ThreadingHTTPServer((host, port), BridgeHandler).serve_forever()
|
if not control_token_path.is_file() or control_token_path.is_symlink():
|
||||||
|
raise RuntimeError("BRIDGE_CONTROL_CREDENTIAL_UNAVAILABLE")
|
||||||
|
control_token = control_token_path.read_text(encoding="utf-8").strip()
|
||||||
|
if len(control_token) < 32:
|
||||||
|
raise RuntimeError("BRIDGE_CONTROL_CREDENTIAL_INVALID")
|
||||||
|
BridgeHandler.adapter = adapter; BridgeHandler.control_token = control_token; BridgeHandler.agent_task_api_url = agent_task_api_url; ThreadingHTTPServer((host, port), BridgeHandler).serve_forever()
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
@@ -1488,11 +1705,21 @@ def main() -> None:
|
|||||||
remote_name=os.environ.get("AIOS_GITEA_REMOTE", "gitea-http"),
|
remote_name=os.environ.get("AIOS_GITEA_REMOTE", "gitea-http"),
|
||||||
api_base=os.environ.get("AIOS_GITEA_API_BASE", "https://g.cnnetmkt.top/git/api/v1"),
|
api_base=os.environ.get("AIOS_GITEA_API_BASE", "https://g.cnnetmkt.top/git/api/v1"),
|
||||||
target_version=os.environ.get("AIOS_CONTROL_PLANE_TARGET_VERSION", "V2R2C1"),
|
target_version=os.environ.get("AIOS_CONTROL_PLANE_TARGET_VERSION", "V2R2C1"),
|
||||||
|
agent_task_api_url=os.environ.get("AIOS_AGENT_TASK_API_URL", "http://127.0.0.1:18940"),
|
||||||
|
agent_task_control_token_path=Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_AGENT_TASK_CONTROL_TOKEN_FILE",
|
||||||
|
str(Path(os.environ.get("CREDENTIALS_DIRECTORY", "/run/credentials/aios-chatgpt-bridge-v2-staging.service")) / "agent_task_control_token"),
|
||||||
|
)
|
||||||
|
),
|
||||||
|
online_task_projection_enabled=os.environ.get("AIOS_ONLINE_TASK_PROJECTION_ENABLED", "1") == "1",
|
||||||
)
|
)
|
||||||
runtime = GovernedRuntime(config)
|
runtime = GovernedRuntime(config)
|
||||||
serve(
|
serve(
|
||||||
BridgeV2Adapter(runtime_status=runtime.runtime_status(), handlers=runtime.handlers()),
|
BridgeV2Adapter(runtime_status=runtime.runtime_status(), handlers=runtime.handlers()),
|
||||||
port=int(os.environ.get("AIOS_BRIDGE_V2_PORT", "18892")),
|
port=int(os.environ.get("AIOS_BRIDGE_V2_PORT", "18892")),
|
||||||
|
control_token_path=config.agent_task_control_token_path,
|
||||||
|
agent_task_api_url=config.agent_task_api_url,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -157,7 +157,7 @@ class DetachedGitWorkspace:
|
|||||||
def materialize_files(self, files: Mapping[str, bytes]) -> None:
|
def materialize_files(self, files: Mapping[str, bytes]) -> None:
|
||||||
dirty = self._dirty_paths()
|
dirty = self._dirty_paths()
|
||||||
expected_paths = set(files)
|
expected_paths = set(files)
|
||||||
if dirty and dirty != expected_paths:
|
if dirty and not dirty.issubset(expected_paths):
|
||||||
raise ContinuationExecutionError("WORKTREE_DIRTY_SCOPE_MISMATCH")
|
raise ContinuationExecutionError("WORKTREE_DIRTY_SCOPE_MISMATCH")
|
||||||
for relative, content in files.items():
|
for relative, content in files.items():
|
||||||
target = (self.path / relative).resolve()
|
target = (self.path / relative).resolve()
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -2,13 +2,20 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from dataclasses import dataclass, asdict
|
from dataclasses import dataclass, asdict
|
||||||
|
from contextlib import contextmanager
|
||||||
|
import fcntl
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import re
|
import re
|
||||||
from typing import Any, Mapping
|
from typing import Any, Mapping
|
||||||
|
|
||||||
from aios_gitops_job_controller import GitOpsJobController, JobStore
|
from aios_gitops_job_controller import GitOpsJobController, JobStore, sha256_json
|
||||||
|
from aios_online_agent_execution import (
|
||||||
|
GovernedExecutionBinding,
|
||||||
|
project_execution_envelope,
|
||||||
|
project_online_task,
|
||||||
|
)
|
||||||
|
|
||||||
OID_RE = re.compile(r"^[0-9a-f]{40}$")
|
OID_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||||
BRANCH_RE = re.compile(r"^agent/[A-Za-z0-9][A-Za-z0-9._/-]{2,179}$")
|
BRANCH_RE = re.compile(r"^agent/[A-Za-z0-9][A-Za-z0-9._/-]{2,179}$")
|
||||||
@@ -36,35 +43,161 @@ class GovernedTaskBinding:
|
|||||||
return {"task_id": self.task_id, "repository": self.repository, "base_oid": self.base_oid, "target_branch": self.target_branch, "expected_branch_oid": self.expected_branch_oid, "generation": self.generation, "branch_mode": self.branch_mode, "create_target_branch": self.branch_mode == "CREATE_NEW", "worktree_mode": "detached_at_base_oid" if self.branch_mode == "CREATE_NEW" else "detached_at_expected_head", "cas_ref_update_required": True, "idempotent": idempotent, "state": "BOUND", "job_id": self.job_id}
|
return {"task_id": self.task_id, "repository": self.repository, "base_oid": self.base_oid, "target_branch": self.target_branch, "expected_branch_oid": self.expected_branch_oid, "generation": self.generation, "branch_mode": self.branch_mode, "create_target_branch": self.branch_mode == "CREATE_NEW", "worktree_mode": "detached_at_base_oid" if self.branch_mode == "CREATE_NEW" else "detached_at_expected_head", "cas_ref_update_required": True, "idempotent": idempotent, "state": "BOUND", "job_id": self.job_id}
|
||||||
|
|
||||||
|
|
||||||
def _safe(value: str) -> str:
|
def _legacy_safe(value: str) -> str:
|
||||||
return re.sub(r"[^A-Za-z0-9_.-]+", "-", value).strip("-")[:120] or "task"
|
return re.sub(r"[^A-Za-z0-9_.-]+", "-", value).strip("-")[:120] or "task"
|
||||||
|
|
||||||
|
|
||||||
|
def _task_key(value: str) -> str:
|
||||||
|
readable = _legacy_safe(value)[:72]
|
||||||
|
return f"{readable}-{hashlib.sha256(value.encode('utf-8')).hexdigest()[:24]}"
|
||||||
|
|
||||||
|
|
||||||
class GovernedTaskBootstrap:
|
class GovernedTaskBootstrap:
|
||||||
def __init__(self, authority: Any, root: Path, *, remote_name: str, token_path: Path, ssh_key_path: Path, api_base: str = "https://g.cnnetmkt.top/git/api/v1") -> None:
|
def __init__(self, authority: Any, root: Path, *, remote_name: str, token_path: Path, ssh_key_path: Path, api_base: str = "https://g.cnnetmkt.top/git/api/v1") -> None:
|
||||||
self.authority = authority; self.root = Path(root); self.remote_name = remote_name; self.token_path = Path(token_path); self.ssh_key_path = Path(ssh_key_path); self.api_base = api_base; self.task_meta_root = self.root / "task-bindings"; self.job_root = self.root / "jobs"; self.task_meta_root.mkdir(parents=True, exist_ok=True); self.job_root.mkdir(parents=True, exist_ok=True)
|
self.authority = authority; self.root = Path(root); self.remote_name = remote_name; self.token_path = Path(token_path); self.ssh_key_path = Path(ssh_key_path); self.api_base = api_base; self.task_meta_root = self.root / "task-bindings"; self.job_root = self.root / "jobs"; self.lock_root = self.root / "task-locks"; self.task_meta_root.mkdir(parents=True, exist_ok=True); self.job_root.mkdir(parents=True, exist_ok=True); self.lock_root.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
def _store(self, task_id: str) -> tuple[JobStore, GitOpsJobController]:
|
def _store(self, task_id: str) -> tuple[JobStore, GitOpsJobController]:
|
||||||
root = self.job_root / _safe(task_id); store = JobStore(root, payload_root=root / "payloads"); return store, GitOpsJobController(store)
|
legacy = self.job_root / _legacy_safe(task_id)
|
||||||
|
keyed = self.job_root / _task_key(task_id)
|
||||||
|
root = legacy if legacy.is_dir() and not keyed.exists() else keyed
|
||||||
|
store = JobStore(root, payload_root=root / "payloads"); return store, GitOpsJobController(store)
|
||||||
|
|
||||||
def _metadata(self, task_id: str) -> Path:
|
def _metadata(self, task_id: str) -> Path:
|
||||||
return self.task_meta_root / (_safe(task_id) + ".json")
|
return self.task_meta_root / (_task_key(task_id) + ".json")
|
||||||
|
|
||||||
|
def _legacy_metadata(self, task_id: str) -> Path:
|
||||||
|
return self.task_meta_root / (_legacy_safe(task_id) + ".json")
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def _task_lock(self, task_id: str):
|
||||||
|
path = self.lock_root / (_task_key(task_id) + ".lock")
|
||||||
|
with path.open("a+b") as handle:
|
||||||
|
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
|
||||||
|
try:
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def _branch_lock(self, repository: str, target_branch: str):
|
||||||
|
key = hashlib.sha256((repository + "\0" + target_branch).encode("utf-8")).hexdigest()
|
||||||
|
path = self.lock_root / ("branch-" + key + ".lock")
|
||||||
|
with path.open("a+b") as handle:
|
||||||
|
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
|
||||||
|
try:
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
||||||
|
|
||||||
def _read_existing(self, task_id: str) -> GovernedTaskBinding | None:
|
def _read_existing(self, task_id: str) -> GovernedTaskBinding | None:
|
||||||
path = self._metadata(task_id)
|
path = self._metadata(task_id)
|
||||||
|
if not path.exists():
|
||||||
|
path = self._legacy_metadata(task_id)
|
||||||
if not path.is_file() or path.is_symlink(): return None
|
if not path.is_file() or path.is_symlink(): return None
|
||||||
return GovernedTaskBinding(**json.loads(path.read_text(encoding="utf-8")))
|
binding = GovernedTaskBinding(**json.loads(path.read_text(encoding="utf-8")))
|
||||||
|
if binding.task_id != task_id:
|
||||||
|
raise BootstrapError("TASK_BINDING_KEY_COLLISION")
|
||||||
|
return binding
|
||||||
|
|
||||||
|
def get_binding(self, task_id: str) -> GovernedTaskBinding:
|
||||||
|
"""Return the exact durable binding; never synthesize missing lineage."""
|
||||||
|
existing = self._read_existing(str(task_id))
|
||||||
|
if existing is None or not existing.job_id:
|
||||||
|
raise BootstrapError("TASK_EXECUTION_PROJECTION_MISSING")
|
||||||
|
return existing
|
||||||
|
|
||||||
|
def execution_binding(self, binding: GovernedTaskBinding) -> GovernedExecutionBinding:
|
||||||
|
expected = binding.expected_branch_oid
|
||||||
|
mode = binding.branch_mode
|
||||||
|
if mode == "CREATE_NEW":
|
||||||
|
current = self.authority.read_ref("refs/heads/" + binding.target_branch)
|
||||||
|
if current is not None:
|
||||||
|
store, _ = self._store(binding.task_id)
|
||||||
|
with store._connect() as connection:
|
||||||
|
rows = connection.execute(
|
||||||
|
"SELECT current_head_oid,generation,state,lineage_anchor_oid "
|
||||||
|
"FROM branch_leases WHERE job_id=? AND branch=? AND state='ACTIVE' "
|
||||||
|
"ORDER BY revision DESC",
|
||||||
|
(binding.job_id, binding.target_branch),
|
||||||
|
).fetchall()
|
||||||
|
if len(rows) != 1:
|
||||||
|
raise BootstrapError("SAME_JOB_CONTINUATION_LEASE_INVALID")
|
||||||
|
row = rows[0]
|
||||||
|
if (
|
||||||
|
str(row["current_head_oid"] or "") != current
|
||||||
|
or int(row["generation"] or 0) != binding.generation
|
||||||
|
or str(row["lineage_anchor_oid"] or "") != binding.base_oid
|
||||||
|
):
|
||||||
|
raise BootstrapError("SAME_JOB_CONTINUATION_BINDING_INVALID")
|
||||||
|
expected = current
|
||||||
|
mode = "CONTINUE_EXISTING"
|
||||||
|
return GovernedExecutionBinding.from_mapping(
|
||||||
|
{
|
||||||
|
"task_id": binding.task_id,
|
||||||
|
"job_id": binding.job_id,
|
||||||
|
"repository": binding.repository,
|
||||||
|
"base_oid": binding.base_oid,
|
||||||
|
"expected_branch_oid": expected,
|
||||||
|
"target_branch": binding.target_branch,
|
||||||
|
"branch_mode": mode,
|
||||||
|
"generation": binding.generation,
|
||||||
|
"idempotency_key": binding.idempotency_key,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
def execution_envelope(
|
||||||
|
self,
|
||||||
|
task_id: str,
|
||||||
|
execution_request: Mapping[str, Any] | None = None,
|
||||||
|
*,
|
||||||
|
required_capabilities: tuple[str, ...] = (),
|
||||||
|
expected_job_id: str | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
binding = self.get_binding(task_id)
|
||||||
|
if expected_job_id and binding.job_id != expected_job_id:
|
||||||
|
raise BootstrapError("JOB_BINDING_CONFLICT")
|
||||||
|
return project_execution_envelope(
|
||||||
|
self.execution_binding(binding),
|
||||||
|
execution_request,
|
||||||
|
required_capabilities=required_capabilities,
|
||||||
|
)
|
||||||
|
|
||||||
|
def online_task_projection(
|
||||||
|
self,
|
||||||
|
task_id: str,
|
||||||
|
task_request: Mapping[str, Any],
|
||||||
|
execution_request: Mapping[str, Any] | None = None,
|
||||||
|
*,
|
||||||
|
expected_job_id: str | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
binding = self.get_binding(task_id)
|
||||||
|
if expected_job_id and binding.job_id != expected_job_id:
|
||||||
|
raise BootstrapError("JOB_BINDING_CONFLICT")
|
||||||
|
return project_online_task(self.execution_binding(binding), task_request, execution_request)
|
||||||
|
|
||||||
def _write(self, binding: GovernedTaskBinding) -> None:
|
def _write(self, binding: GovernedTaskBinding) -> None:
|
||||||
path = self._metadata(binding.task_id); temporary = path.with_suffix(".tmp"); temporary.write_text(json.dumps(asdict(binding), sort_keys=True), encoding="utf-8"); temporary.replace(path)
|
path = self._metadata(binding.task_id); temporary = path.with_suffix(".tmp"); temporary.write_text(json.dumps(asdict(binding), sort_keys=True), encoding="utf-8"); temporary.replace(path)
|
||||||
|
|
||||||
def create(self, request: Mapping[str, Any]) -> dict[str, Any]:
|
def create(self, request: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
|
task_id = str(request.get("task_id") or "")
|
||||||
|
if not task_id:
|
||||||
|
raise BootstrapError("TASK_BINDING_INVALID")
|
||||||
|
repository = str(request.get("repository") or "")
|
||||||
|
target_branch = str(request.get("target_branch") or "")
|
||||||
|
with self._task_lock(task_id):
|
||||||
|
with self._branch_lock(repository, target_branch):
|
||||||
|
return self._create_locked(request)
|
||||||
|
|
||||||
|
def _create_locked(self, request: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
values = {key: str(request[key]) for key in ("task_id", "repository", "base_oid", "target_branch", "risk", "description", "idempotency_key")}
|
values = {key: str(request[key]) for key in ("task_id", "repository", "base_oid", "target_branch", "risk", "description", "idempotency_key")}
|
||||||
if values["repository"] != "aios/aios" or not OID_RE.fullmatch(values["base_oid"]): raise BootstrapError("TASK_BINDING_INVALID")
|
if values["repository"] != "aios/aios" or not OID_RE.fullmatch(values["base_oid"]): raise BootstrapError("TASK_BINDING_INVALID")
|
||||||
if not BRANCH_RE.fullmatch(values["target_branch"]): raise BootstrapError("TARGET_BRANCH_INVALID")
|
if not BRANCH_RE.fullmatch(values["target_branch"]): raise BootstrapError("TARGET_BRANCH_INVALID")
|
||||||
existing = self._read_existing(values["task_id"])
|
existing = self._read_existing(values["task_id"])
|
||||||
if existing is not None:
|
if existing is not None:
|
||||||
if any(getattr(existing, key) != values[key] for key in ("repository", "base_oid", "target_branch", "risk", "description")): raise BootstrapError("TASK_BINDING_CONFLICT")
|
if any(getattr(existing, key) != values[key] for key in ("repository", "base_oid", "target_branch", "risk", "description", "idempotency_key")): raise BootstrapError("TASK_BINDING_CONFLICT")
|
||||||
|
optional = {"expected_branch_oid": existing.expected_branch_oid, "branch_mode": existing.branch_mode, "generation": existing.generation}
|
||||||
|
for key, durable in optional.items():
|
||||||
|
if key in request and str(request[key]) != str(durable): raise BootstrapError("TASK_BINDING_CONFLICT")
|
||||||
return existing.response(idempotent=True)
|
return existing.response(idempotent=True)
|
||||||
if not self.authority.commit_exists(values["base_oid"]): raise BootstrapError("BASE_COMMIT_NOT_FOUND")
|
if not self.authority.commit_exists(values["base_oid"]): raise BootstrapError("BASE_COMMIT_NOT_FOUND")
|
||||||
current = self.authority.read_ref("refs/heads/" + values["target_branch"])
|
current = self.authority.read_ref("refs/heads/" + values["target_branch"])
|
||||||
@@ -74,8 +207,61 @@ class GovernedTaskBootstrap:
|
|||||||
if expected != values["base_oid"]: raise BootstrapError("EXPECTED_BASE_CONFLICT")
|
if expected != values["base_oid"]: raise BootstrapError("EXPECTED_BASE_CONFLICT")
|
||||||
else:
|
else:
|
||||||
if expected != current: raise BootstrapError("EXISTING_BRANCH_OID_CONFLICT")
|
if expected != current: raise BootstrapError("EXISTING_BRANCH_OID_CONFLICT")
|
||||||
mode, generation = "CONTINUE_EXISTING", int(request.get("generation", 2))
|
parent_task_id = str(request.get("parent_task_id") or "")
|
||||||
if generation < 1: raise BootstrapError("GENERATION_INVALID")
|
parent_job_id = str(request.get("parent_job_id") or "")
|
||||||
|
if not parent_task_id or not parent_job_id:
|
||||||
|
raise BootstrapError("EXISTING_BRANCH_OWNERSHIP_REQUIRED")
|
||||||
|
parent = self._read_existing(parent_task_id)
|
||||||
|
if parent is None or parent.job_id != parent_job_id or parent.repository != values["repository"] or parent.target_branch != values["target_branch"]:
|
||||||
|
raise BootstrapError("EXISTING_BRANCH_OWNERSHIP_CONFLICT")
|
||||||
|
store, _ = self._store(parent_task_id)
|
||||||
|
with store._connect() as connection:
|
||||||
|
rows = connection.execute(
|
||||||
|
"SELECT current_head_oid,generation,state FROM branch_leases WHERE job_id=? AND branch=? AND state='ACTIVE' ORDER BY revision DESC",
|
||||||
|
(parent_job_id, values["target_branch"]),
|
||||||
|
).fetchall()
|
||||||
|
if len(rows) != 1 or str(rows[0]["current_head_oid"] or "") != current:
|
||||||
|
raise BootstrapError("EXISTING_BRANCH_OWNERSHIP_CONFLICT")
|
||||||
|
mode, generation = "CONTINUE_EXISTING", int(rows[0]["generation"]) + 1
|
||||||
|
if "generation" in request and int(request["generation"]) != generation:
|
||||||
|
raise BootstrapError("GENERATION_BINDING_CONFLICT")
|
||||||
|
online_request = request.get("online_task")
|
||||||
|
execution_request = request.get("execution")
|
||||||
|
projection_required = (
|
||||||
|
online_request is not None
|
||||||
|
or execution_request is not None
|
||||||
|
or request.get("require_online_task_projection") is True
|
||||||
|
)
|
||||||
|
if projection_required:
|
||||||
|
if not isinstance(online_request, Mapping):
|
||||||
|
raise BootstrapError("ONLINE_TASK_REQUEST_INVALID")
|
||||||
|
if execution_request is not None and not isinstance(execution_request, Mapping):
|
||||||
|
raise BootstrapError("EXECUTION_REQUEST_INVALID")
|
||||||
|
provisional_job_id = "job-" + sha256_json(
|
||||||
|
{"client_idempotency_key": values["idempotency_key"]}
|
||||||
|
)[:32]
|
||||||
|
try:
|
||||||
|
provisional_binding = GovernedExecutionBinding.from_mapping(
|
||||||
|
{
|
||||||
|
"task_id": values["task_id"],
|
||||||
|
"job_id": provisional_job_id,
|
||||||
|
"repository": values["repository"],
|
||||||
|
"base_oid": values["base_oid"],
|
||||||
|
"expected_branch_oid": expected,
|
||||||
|
"target_branch": values["target_branch"],
|
||||||
|
"branch_mode": mode,
|
||||||
|
"generation": generation,
|
||||||
|
"idempotency_key": values["idempotency_key"],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
project_online_task(
|
||||||
|
provisional_binding,
|
||||||
|
dict(online_request),
|
||||||
|
dict(execution_request or {}),
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
raise BootstrapError("EXECUTION_BINDING_INVALID") from exc
|
||||||
|
|
||||||
payload = json.dumps(values, sort_keys=True, separators=(",", ":")).encode()
|
payload = json.dumps(values, sort_keys=True, separators=(",", ":")).encode()
|
||||||
_, controller = self._store(values["task_id"])
|
_, controller = self._store(values["task_id"])
|
||||||
receipt = controller.submit({"idempotency_key": values["idempotency_key"], "campaign_id": values["task_id"], "task_id": values["task_id"], "repository": values["repository"], "operation": "GOVERNED_TASK_CREATE", "branch": values["target_branch"], "expected_branch_oid": expected, "payload_digest": hashlib.sha256(payload).hexdigest(), "lineage_anchor_oid": values["base_oid"]}, payload=payload)
|
receipt = controller.submit({"idempotency_key": values["idempotency_key"], "campaign_id": values["task_id"], "task_id": values["task_id"], "repository": values["repository"], "operation": "GOVERNED_TASK_CREATE", "branch": values["target_branch"], "expected_branch_oid": expected, "payload_digest": hashlib.sha256(payload).hexdigest(), "lineage_anchor_oid": values["base_oid"]}, payload=payload)
|
||||||
|
|||||||
@@ -0,0 +1,456 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Disabled-by-default TLS edge for the read-only GPT Online Task ingress.
|
||||||
|
|
||||||
|
The internal AIOS ingress remains loopback-only. This service is a separate edge
|
||||||
|
that may bind non-loopback only when the owner explicitly approves it. It
|
||||||
|
forwards only the tiny read-only ingress API and never proxies the canonical
|
||||||
|
Bridge or Task API.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections import defaultdict, deque
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import secrets
|
||||||
|
import ssl
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from typing import Any, Mapping
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
|
UPSTREAM = "http://127.0.0.1:18945"
|
||||||
|
MAX_BODY = 64 * 1024
|
||||||
|
MAX_RESPONSE = 256 * 1024
|
||||||
|
MAX_CONCURRENT_REQUESTS = 32
|
||||||
|
REQUEST_TIMEOUT_SECONDS = 20
|
||||||
|
RATE_WINDOW_SECONDS = 60
|
||||||
|
RATE_LIMIT_REQUESTS = 60
|
||||||
|
RATE_BURST_REQUESTS = 15
|
||||||
|
RATE_MAX_CLIENTS = 4096
|
||||||
|
_AUDIT_SALT = secrets.token_bytes(32)
|
||||||
|
PUBLIC_APPROVAL_ENV = "AIOS_GPT_EDGE_PUBLIC_BIND_APPROVED"
|
||||||
|
PUBLIC_DOMAIN_ENV = "AIOS_GPT_EDGE_PUBLIC_DOMAIN"
|
||||||
|
TASK_PATH_RE = re.compile(r"^/v1/read-only-tasks(?:/[A-Za-z0-9][A-Za-z0-9._:-]{7,127})?$")
|
||||||
|
|
||||||
|
|
||||||
|
def _client_pseudonym(value: str) -> str:
|
||||||
|
return hashlib.sha256(_AUDIT_SALT + value.encode("utf-8", "replace")).hexdigest()[:16]
|
||||||
|
|
||||||
|
|
||||||
|
def audit(event: str, **fields: object) -> None:
|
||||||
|
safe = {
|
||||||
|
"event": event,
|
||||||
|
"ts_unix_ms": int(time.time() * 1000),
|
||||||
|
}
|
||||||
|
for key, value in fields.items():
|
||||||
|
if key in {"authorization", "body", "token", "cookie", "client_ip"}:
|
||||||
|
continue
|
||||||
|
safe[key] = value
|
||||||
|
print(
|
||||||
|
json.dumps(safe, sort_keys=True, separators=(",", ":")),
|
||||||
|
file=sys.stderr,
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class EdgeError(RuntimeError):
|
||||||
|
def __init__(self, code: str, status: int = 400) -> None:
|
||||||
|
super().__init__(code)
|
||||||
|
self.code = code
|
||||||
|
self.status = status
|
||||||
|
|
||||||
|
|
||||||
|
def canonical(value: object) -> bytes:
|
||||||
|
return json.dumps(
|
||||||
|
value,
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def public_bind_allowed(host: str, approval: str) -> str:
|
||||||
|
value = host.strip()
|
||||||
|
if value in {"127.0.0.1", "::1", "localhost"}:
|
||||||
|
return value
|
||||||
|
if approval != "1":
|
||||||
|
raise RuntimeError("GPT_EDGE_PUBLIC_BIND_OWNER_APPROVAL_REQUIRED")
|
||||||
|
if value not in {"0.0.0.0", "::"}:
|
||||||
|
raise RuntimeError("GPT_EDGE_BIND_ADDRESS_INVALID")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def validated_public_domain(value: str) -> str:
|
||||||
|
domain = value.strip().lower()
|
||||||
|
if not domain:
|
||||||
|
return ""
|
||||||
|
if len(domain) > 253 or re.fullmatch(
|
||||||
|
r"(?=.{1,253}\Z)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?",
|
||||||
|
domain,
|
||||||
|
) is None:
|
||||||
|
raise RuntimeError("GPT_EDGE_PUBLIC_DOMAIN_INVALID")
|
||||||
|
return domain
|
||||||
|
|
||||||
|
|
||||||
|
def validated_edge_identity(host: str, approval: str, domain: str) -> tuple[str, str]:
|
||||||
|
bind_host = public_bind_allowed(host, approval)
|
||||||
|
public_domain = validated_public_domain(domain)
|
||||||
|
if bind_host not in {"127.0.0.1", "::1", "localhost"} and not public_domain:
|
||||||
|
raise RuntimeError("GPT_EDGE_PUBLIC_DOMAIN_REQUIRED")
|
||||||
|
return bind_host, public_domain
|
||||||
|
|
||||||
|
|
||||||
|
def validate_tls_file(path: Path, *, key: bool) -> Path:
|
||||||
|
if path.is_symlink() or not path.is_file():
|
||||||
|
raise RuntimeError("GPT_EDGE_TLS_FILE_INVALID")
|
||||||
|
size = path.stat().st_size
|
||||||
|
if size < 16 or size > 256 * 1024:
|
||||||
|
raise RuntimeError("GPT_EDGE_TLS_FILE_INVALID")
|
||||||
|
if key and (path.stat().st_mode & 0o077):
|
||||||
|
raise RuntimeError("GPT_EDGE_TLS_KEY_MODE_INVALID")
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def build_tls_context(cert_path: Path, key_path: Path) -> ssl.SSLContext:
|
||||||
|
cert = validate_tls_file(cert_path, key=False)
|
||||||
|
key = validate_tls_file(key_path, key=True)
|
||||||
|
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||||
|
context.minimum_version = ssl.TLSVersion.TLSv1_2
|
||||||
|
context.options |= ssl.OP_NO_COMPRESSION
|
||||||
|
context.load_cert_chain(certfile=str(cert), keyfile=str(key))
|
||||||
|
return context
|
||||||
|
|
||||||
|
|
||||||
|
def allowed_route(method: str, raw_path: str) -> str:
|
||||||
|
parsed = urlsplit(raw_path)
|
||||||
|
if parsed.query or parsed.fragment:
|
||||||
|
raise EdgeError("QUERY_OR_FRAGMENT_DENY", 400)
|
||||||
|
path = parsed.path
|
||||||
|
if method == "GET" and path in {"/health", "/openapi.json"}:
|
||||||
|
return path
|
||||||
|
if method == "POST" and path == "/v1/read-only-tasks":
|
||||||
|
return path
|
||||||
|
if method == "GET" and TASK_PATH_RE.fullmatch(path):
|
||||||
|
return path
|
||||||
|
raise EdgeError("ROUTE_DENY", 404)
|
||||||
|
|
||||||
|
|
||||||
|
def route_label(method: str, path: str) -> str:
|
||||||
|
if method == "GET" and path == "/health":
|
||||||
|
return "health"
|
||||||
|
if method == "GET" and path == "/openapi.json":
|
||||||
|
return "openapi"
|
||||||
|
if method == "POST" and path == "/v1/read-only-tasks":
|
||||||
|
return "task_create"
|
||||||
|
if method == "GET" and path.startswith("/v1/read-only-tasks/"):
|
||||||
|
return "task_status"
|
||||||
|
return "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
def forwarded_headers(headers: Mapping[str, str], *, body: bool) -> dict[str, str]:
|
||||||
|
result = {"Accept": "application/json"}
|
||||||
|
authorization = headers.get("Authorization")
|
||||||
|
if not isinstance(authorization, str) or not authorization.startswith("Bearer "):
|
||||||
|
raise EdgeError("AUTHORIZATION_REQUIRED", 401)
|
||||||
|
token = authorization[7:]
|
||||||
|
if len(token) < 32 or len(token) > 4096 or any(ch.isspace() for ch in token):
|
||||||
|
raise EdgeError("AUTHORIZATION_INVALID", 401)
|
||||||
|
result["Authorization"] = authorization
|
||||||
|
if body:
|
||||||
|
content_type = headers.get("Content-Type", "")
|
||||||
|
if content_type.split(";", 1)[0].strip().lower() != "application/json":
|
||||||
|
raise EdgeError("CONTENT_TYPE_INVALID", 415)
|
||||||
|
result["Content-Type"] = "application/json"
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
class RateLimiter:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
window_seconds: int = RATE_WINDOW_SECONDS,
|
||||||
|
limit: int = RATE_LIMIT_REQUESTS,
|
||||||
|
burst: int = RATE_BURST_REQUESTS,
|
||||||
|
max_clients: int = RATE_MAX_CLIENTS,
|
||||||
|
now_fn=time.monotonic,
|
||||||
|
) -> None:
|
||||||
|
self.window_seconds = window_seconds
|
||||||
|
self.limit = limit
|
||||||
|
self.burst = burst
|
||||||
|
self.max_clients = max_clients
|
||||||
|
self.now_fn = now_fn
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
self._events: dict[str, deque[float]] = defaultdict(deque)
|
||||||
|
|
||||||
|
def _prune_stale(self, cutoff: float) -> None:
|
||||||
|
stale = [
|
||||||
|
key
|
||||||
|
for key, events in self._events.items()
|
||||||
|
if not events or events[-1] <= cutoff
|
||||||
|
]
|
||||||
|
for key in stale:
|
||||||
|
self._events.pop(key, None)
|
||||||
|
|
||||||
|
def allowed(self, key: str) -> bool:
|
||||||
|
now = self.now_fn()
|
||||||
|
cutoff = now - self.window_seconds
|
||||||
|
with self._lock:
|
||||||
|
if key not in self._events and len(self._events) >= self.max_clients:
|
||||||
|
self._prune_stale(cutoff)
|
||||||
|
if len(self._events) >= self.max_clients:
|
||||||
|
return False
|
||||||
|
events = self._events[key]
|
||||||
|
while events and events[0] <= cutoff:
|
||||||
|
events.popleft()
|
||||||
|
if len(events) >= self.limit + self.burst:
|
||||||
|
return False
|
||||||
|
events.append(now)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Proxy:
|
||||||
|
upstream: str = UPSTREAM
|
||||||
|
timeout: int = REQUEST_TIMEOUT_SECONDS
|
||||||
|
|
||||||
|
def request(
|
||||||
|
self,
|
||||||
|
method: str,
|
||||||
|
path: str,
|
||||||
|
headers: Mapping[str, str],
|
||||||
|
body: bytes | None,
|
||||||
|
) -> tuple[int, bytes, str]:
|
||||||
|
if self.upstream != UPSTREAM:
|
||||||
|
raise EdgeError("UPSTREAM_DRIFT_DENY", 503)
|
||||||
|
req = urllib.request.Request(
|
||||||
|
self.upstream + path,
|
||||||
|
method=method,
|
||||||
|
data=body,
|
||||||
|
headers=dict(headers),
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=self.timeout) as response:
|
||||||
|
raw = response.read(MAX_RESPONSE + 1)
|
||||||
|
status = response.status
|
||||||
|
content_type = response.headers.get("Content-Type", "application/json")
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
raw = exc.read(MAX_RESPONSE + 1)
|
||||||
|
status = exc.code
|
||||||
|
content_type = exc.headers.get("Content-Type", "application/json")
|
||||||
|
except Exception as exc:
|
||||||
|
raise EdgeError("UPSTREAM_UNAVAILABLE", 502) from exc
|
||||||
|
if len(raw) > MAX_RESPONSE:
|
||||||
|
raise EdgeError("UPSTREAM_RESPONSE_TOO_LARGE", 502)
|
||||||
|
if content_type.split(";", 1)[0].strip().lower() != "application/json":
|
||||||
|
raise EdgeError("UPSTREAM_CONTENT_TYPE_INVALID", 502)
|
||||||
|
return status, raw, "application/json"
|
||||||
|
|
||||||
|
|
||||||
|
class EdgeServer(ThreadingHTTPServer):
|
||||||
|
daemon_threads = True
|
||||||
|
request_queue_size = 64
|
||||||
|
allow_reuse_address = True
|
||||||
|
|
||||||
|
|
||||||
|
class Handler(BaseHTTPRequestHandler):
|
||||||
|
proxy = Proxy()
|
||||||
|
limiter = RateLimiter()
|
||||||
|
concurrency = threading.BoundedSemaphore(MAX_CONCURRENT_REQUESTS)
|
||||||
|
expected_domain = ""
|
||||||
|
|
||||||
|
server_version = "AIOSGPTIngressEdge/1.0"
|
||||||
|
sys_version = ""
|
||||||
|
|
||||||
|
def log_message(self, *_args: Any) -> None:
|
||||||
|
return
|
||||||
|
|
||||||
|
def respond_json(self, value: Mapping[str, Any], status: int) -> None:
|
||||||
|
raw = canonical(value)
|
||||||
|
self.send_response(status)
|
||||||
|
self.send_header("Content-Type", "application/json")
|
||||||
|
self.send_header("Cache-Control", "no-store")
|
||||||
|
self.send_header("X-Content-Type-Options", "nosniff")
|
||||||
|
self.send_header("Referrer-Policy", "no-referrer")
|
||||||
|
self.send_header("Content-Security-Policy", "default-src 'none'")
|
||||||
|
self.send_header("Strict-Transport-Security", "max-age=31536000")
|
||||||
|
self.send_header("Content-Length", str(len(raw)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(raw)
|
||||||
|
|
||||||
|
def send_proxy_response(self, status: int, raw: bytes, content_type: str) -> None:
|
||||||
|
self.send_response(status)
|
||||||
|
self.send_header("Content-Type", content_type)
|
||||||
|
self.send_header("Cache-Control", "no-store")
|
||||||
|
self.send_header("X-Content-Type-Options", "nosniff")
|
||||||
|
self.send_header("Referrer-Policy", "no-referrer")
|
||||||
|
self.send_header("Content-Security-Policy", "default-src 'none'")
|
||||||
|
self.send_header("Strict-Transport-Security", "max-age=31536000")
|
||||||
|
self.send_header("Content-Length", str(len(raw)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(raw)
|
||||||
|
|
||||||
|
def client_key(self) -> str:
|
||||||
|
return str(self.client_address[0])
|
||||||
|
|
||||||
|
def validate_host(self) -> None:
|
||||||
|
if not self.expected_domain:
|
||||||
|
return
|
||||||
|
host = self.headers.get("Host", "").split(":", 1)[0].lower()
|
||||||
|
if not hmac.compare_digest(host, self.expected_domain):
|
||||||
|
raise EdgeError("HOST_HEADER_DENY", 421)
|
||||||
|
|
||||||
|
def body(self) -> bytes:
|
||||||
|
try:
|
||||||
|
length = int(self.headers.get("Content-Length", "-1"))
|
||||||
|
except ValueError as exc:
|
||||||
|
raise EdgeError("CONTENT_LENGTH_INVALID") from exc
|
||||||
|
if length < 1 or length > MAX_BODY:
|
||||||
|
raise EdgeError("REQUEST_SIZE_INVALID", 413)
|
||||||
|
raw = self.rfile.read(length)
|
||||||
|
if len(raw) != length:
|
||||||
|
raise EdgeError("REQUEST_BODY_TRUNCATED", 400)
|
||||||
|
# Validate JSON at the edge so malformed bodies do not consume upstream work.
|
||||||
|
try:
|
||||||
|
value = json.loads(raw.decode("utf-8"))
|
||||||
|
except Exception as exc:
|
||||||
|
raise EdgeError("REQUEST_JSON_INVALID", 400) from exc
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise EdgeError("REQUEST_JSON_OBJECT_REQUIRED", 400)
|
||||||
|
return canonical(value)
|
||||||
|
|
||||||
|
def handle_proxy(self, method: str) -> None:
|
||||||
|
client_id = _client_pseudonym(self.client_key())
|
||||||
|
started = time.monotonic()
|
||||||
|
acquired = self.concurrency.acquire(blocking=False)
|
||||||
|
if not acquired:
|
||||||
|
audit("EDGE_BUSY", client_id=client_id, method=method)
|
||||||
|
self.respond_json({"error": "EDGE_BUSY"}, 503)
|
||||||
|
return
|
||||||
|
label = "unknown"
|
||||||
|
try:
|
||||||
|
self.validate_host()
|
||||||
|
path = allowed_route(method, self.path)
|
||||||
|
label = route_label(method, path)
|
||||||
|
if path == "/health" and method == "GET":
|
||||||
|
self.respond_json(
|
||||||
|
{
|
||||||
|
"status": "ok",
|
||||||
|
"service": "aios-gpt-https-edge",
|
||||||
|
"upstream": "loopback-readonly-ingress",
|
||||||
|
},
|
||||||
|
200,
|
||||||
|
)
|
||||||
|
audit(
|
||||||
|
"EDGE_REQUEST",
|
||||||
|
client_id=client_id,
|
||||||
|
method=method,
|
||||||
|
route=label,
|
||||||
|
status=200,
|
||||||
|
duration_ms=int((time.monotonic() - started) * 1000),
|
||||||
|
)
|
||||||
|
return
|
||||||
|
if not self.limiter.allowed(self.client_key()):
|
||||||
|
audit(
|
||||||
|
"EDGE_RATE_LIMIT",
|
||||||
|
client_id=client_id,
|
||||||
|
method=method,
|
||||||
|
route=label,
|
||||||
|
)
|
||||||
|
self.respond_json({"error": "RATE_LIMITED"}, 429)
|
||||||
|
return
|
||||||
|
has_body = method == "POST"
|
||||||
|
headers = forwarded_headers(self.headers, body=has_body)
|
||||||
|
body = self.body() if has_body else None
|
||||||
|
status, raw, content_type = self.proxy.request(
|
||||||
|
method,
|
||||||
|
path,
|
||||||
|
headers,
|
||||||
|
body,
|
||||||
|
)
|
||||||
|
self.send_proxy_response(status, raw, content_type)
|
||||||
|
audit(
|
||||||
|
"EDGE_REQUEST",
|
||||||
|
client_id=client_id,
|
||||||
|
method=method,
|
||||||
|
route=label,
|
||||||
|
status=status,
|
||||||
|
duration_ms=int((time.monotonic() - started) * 1000),
|
||||||
|
)
|
||||||
|
except EdgeError as exc:
|
||||||
|
audit(
|
||||||
|
"EDGE_REJECT",
|
||||||
|
client_id=client_id,
|
||||||
|
method=method,
|
||||||
|
route=label,
|
||||||
|
status=exc.status,
|
||||||
|
error=exc.code,
|
||||||
|
)
|
||||||
|
self.respond_json({"error": exc.code}, exc.status)
|
||||||
|
finally:
|
||||||
|
self.concurrency.release()
|
||||||
|
|
||||||
|
def do_GET(self) -> None:
|
||||||
|
self.handle_proxy("GET")
|
||||||
|
|
||||||
|
def do_POST(self) -> None:
|
||||||
|
self.handle_proxy("POST")
|
||||||
|
|
||||||
|
def do_PUT(self) -> None:
|
||||||
|
self.respond_json({"error": "METHOD_DENY"}, 405)
|
||||||
|
|
||||||
|
def do_PATCH(self) -> None:
|
||||||
|
self.respond_json({"error": "METHOD_DENY"}, 405)
|
||||||
|
|
||||||
|
def do_DELETE(self) -> None:
|
||||||
|
self.respond_json({"error": "METHOD_DENY"}, 405)
|
||||||
|
|
||||||
|
def do_CONNECT(self) -> None:
|
||||||
|
self.respond_json({"error": "METHOD_DENY"}, 405)
|
||||||
|
|
||||||
|
def do_OPTIONS(self) -> None:
|
||||||
|
self.respond_json({"error": "METHOD_DENY"}, 405)
|
||||||
|
|
||||||
|
|
||||||
|
def serve() -> None:
|
||||||
|
host, public_domain = validated_edge_identity(
|
||||||
|
os.environ.get("AIOS_GPT_EDGE_HOST", "127.0.0.1"),
|
||||||
|
os.environ.get(PUBLIC_APPROVAL_ENV, "0"),
|
||||||
|
os.environ.get(PUBLIC_DOMAIN_ENV, ""),
|
||||||
|
)
|
||||||
|
port = int(os.environ.get("AIOS_GPT_EDGE_PORT", "18443"))
|
||||||
|
if not 1024 <= port <= 65535:
|
||||||
|
raise RuntimeError("GPT_EDGE_PORT_INVALID")
|
||||||
|
Handler.expected_domain = public_domain
|
||||||
|
|
||||||
|
credentials = Path(os.environ.get("CREDENTIALS_DIRECTORY", "/run/credentials"))
|
||||||
|
cert_path = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_GPT_EDGE_TLS_CERT_FILE",
|
||||||
|
str(credentials / "tls_cert"),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
key_path = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_GPT_EDGE_TLS_KEY_FILE",
|
||||||
|
str(credentials / "tls_key"),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
context = build_tls_context(cert_path, key_path)
|
||||||
|
|
||||||
|
server = EdgeServer((host, port), Handler)
|
||||||
|
server.socket = context.wrap_socket(server.socket, server_side=True)
|
||||||
|
server.serve_forever()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
serve()
|
||||||
@@ -0,0 +1,561 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Owner-approved activation/rollback controller for the AIOS GPT HTTPS edge.
|
||||||
|
|
||||||
|
This controller never creates approval, TLS, or ingress credentials. It consumes
|
||||||
|
owner-managed inputs, applies one exact public-bind overlay, verifies the live
|
||||||
|
edge, and provides an idempotent rollback path.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import socket
|
||||||
|
import ssl
|
||||||
|
import stat
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
import urllib.request
|
||||||
|
from typing import Any, Mapping
|
||||||
|
|
||||||
|
APPROVAL_SCHEMA = "aios.gpt-https-edge.public-approval.v1"
|
||||||
|
STATE_SCHEMA = "aios.gpt-https-edge.activation-state.v1"
|
||||||
|
|
||||||
|
OID_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||||
|
SHA_RE = re.compile(r"^[0-9a-f]{64}$")
|
||||||
|
APPROVAL_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{7,127}$")
|
||||||
|
DOMAIN_RE = re.compile(
|
||||||
|
r"(?=.{1,253}\Z)"
|
||||||
|
r"(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+"
|
||||||
|
r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?"
|
||||||
|
)
|
||||||
|
|
||||||
|
EDGE_UNIT = "aios-gpt-https-edge.service"
|
||||||
|
INGRESS_UNIT = "aios-gpt-online-task-ingress.service"
|
||||||
|
PUBLIC_HOST = "0.0.0.0"
|
||||||
|
PUBLIC_PORT = 18443
|
||||||
|
INGRESS_HEALTH = "http://127.0.0.1:18945/health"
|
||||||
|
DROPIN_NAME = "zzzzzzzzzz-owner-approved-public.conf"
|
||||||
|
|
||||||
|
|
||||||
|
class ActivationError(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def canonical(value: object) -> bytes:
|
||||||
|
return json.dumps(
|
||||||
|
value,
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def sha256_bytes(data: bytes) -> str:
|
||||||
|
return hashlib.sha256(data).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def sha256_file(path: Path) -> str:
|
||||||
|
return sha256_bytes(path.read_bytes())
|
||||||
|
|
||||||
|
|
||||||
|
def parse_time(value: object, *, field: str) -> datetime:
|
||||||
|
if not isinstance(value, str) or not value.strip():
|
||||||
|
raise ActivationError("APPROVAL_" + field.upper() + "_INVALID")
|
||||||
|
text = value.strip()
|
||||||
|
if text.endswith("Z"):
|
||||||
|
text = text[:-1] + "+00:00"
|
||||||
|
try:
|
||||||
|
parsed = datetime.fromisoformat(text)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise ActivationError("APPROVAL_" + field.upper() + "_INVALID") from exc
|
||||||
|
if parsed.tzinfo is None:
|
||||||
|
raise ActivationError("APPROVAL_" + field.upper() + "_INVALID")
|
||||||
|
return parsed.astimezone(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def validate_domain(value: object) -> str:
|
||||||
|
if not isinstance(value, str):
|
||||||
|
raise ActivationError("APPROVAL_DOMAIN_INVALID")
|
||||||
|
domain = value.strip().lower()
|
||||||
|
if DOMAIN_RE.fullmatch(domain) is None:
|
||||||
|
raise ActivationError("APPROVAL_DOMAIN_INVALID")
|
||||||
|
return domain
|
||||||
|
|
||||||
|
|
||||||
|
def validate_root_file(
|
||||||
|
path: Path,
|
||||||
|
*,
|
||||||
|
exact_mode: int | None = None,
|
||||||
|
max_size: int = 256 * 1024,
|
||||||
|
) -> Path:
|
||||||
|
if path.is_symlink() or not path.is_file():
|
||||||
|
raise ActivationError("OWNER_FILE_INVALID")
|
||||||
|
info = path.stat()
|
||||||
|
if info.st_uid != 0 or info.st_gid != 0:
|
||||||
|
raise ActivationError("OWNER_FILE_OWNERSHIP_INVALID")
|
||||||
|
mode = stat.S_IMODE(info.st_mode)
|
||||||
|
if exact_mode is not None and mode != exact_mode:
|
||||||
|
raise ActivationError("OWNER_FILE_MODE_INVALID")
|
||||||
|
if info.st_size < 16 or info.st_size > max_size:
|
||||||
|
raise ActivationError("OWNER_FILE_SIZE_INVALID")
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def certificate_dns_names(cert_path: Path) -> tuple[str, ...]:
|
||||||
|
try:
|
||||||
|
decoded = ssl._ssl._test_decode_cert(str(cert_path))
|
||||||
|
except Exception as exc:
|
||||||
|
raise ActivationError("TLS_CERT_DECODE_INVALID") from exc
|
||||||
|
names = [
|
||||||
|
value
|
||||||
|
for kind, value in decoded.get("subjectAltName", ())
|
||||||
|
if kind == "DNS"
|
||||||
|
]
|
||||||
|
if not names:
|
||||||
|
for rdn in decoded.get("subject", ()):
|
||||||
|
for key, value in rdn:
|
||||||
|
if key == "commonName":
|
||||||
|
names.append(value)
|
||||||
|
return tuple(names)
|
||||||
|
|
||||||
|
|
||||||
|
def cert_matches_domain(cert_path: Path, domain: str) -> bool:
|
||||||
|
names = certificate_dns_names(cert_path)
|
||||||
|
try:
|
||||||
|
return bool(names) and any(ssl._dnsname_match(name, domain) for name in names)
|
||||||
|
except ssl.CertificateError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def validate_tls_pair(
|
||||||
|
cert_path: Path,
|
||||||
|
key_path: Path,
|
||||||
|
domain: str,
|
||||||
|
*,
|
||||||
|
now_fn=time.time,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
validate_root_file(cert_path)
|
||||||
|
validate_root_file(key_path, exact_mode=0o600)
|
||||||
|
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||||
|
context.minimum_version = ssl.TLSVersion.TLSv1_2
|
||||||
|
context.options |= ssl.OP_NO_COMPRESSION
|
||||||
|
try:
|
||||||
|
context.load_cert_chain(str(cert_path), str(key_path))
|
||||||
|
except Exception as exc:
|
||||||
|
raise ActivationError("TLS_CERT_KEY_PAIR_INVALID") from exc
|
||||||
|
if not cert_matches_domain(cert_path, domain):
|
||||||
|
raise ActivationError("TLS_CERT_DOMAIN_MISMATCH")
|
||||||
|
decoded = ssl._ssl._test_decode_cert(str(cert_path))
|
||||||
|
not_after = decoded.get("notAfter")
|
||||||
|
if not not_after:
|
||||||
|
raise ActivationError("TLS_CERT_EXPIRY_MISSING")
|
||||||
|
remaining = ssl.cert_time_to_seconds(not_after) - now_fn()
|
||||||
|
if remaining < 7 * 86400:
|
||||||
|
raise ActivationError("TLS_CERT_EXPIRY_TOO_CLOSE")
|
||||||
|
return {
|
||||||
|
"cert_sha256": sha256_file(cert_path),
|
||||||
|
"remaining_days": int(remaining // 86400),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Approval:
|
||||||
|
approval_id: str
|
||||||
|
candidate_oid: str
|
||||||
|
domain: str
|
||||||
|
bind_host: str
|
||||||
|
port: int
|
||||||
|
tls_cert_sha256: str
|
||||||
|
owner: str
|
||||||
|
issued_at: datetime
|
||||||
|
not_before: datetime
|
||||||
|
expires_at: datetime
|
||||||
|
raw_sha256: str
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def load(
|
||||||
|
cls,
|
||||||
|
path: Path,
|
||||||
|
*,
|
||||||
|
candidate_oid: str,
|
||||||
|
cert_path: Path,
|
||||||
|
now: datetime | None = None,
|
||||||
|
) -> "Approval":
|
||||||
|
validate_root_file(path, exact_mode=0o600, max_size=64 * 1024)
|
||||||
|
try:
|
||||||
|
value = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
except Exception as exc:
|
||||||
|
raise ActivationError("APPROVAL_JSON_INVALID") from exc
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise ActivationError("APPROVAL_JSON_INVALID")
|
||||||
|
allowed = {
|
||||||
|
"schema",
|
||||||
|
"approval_id",
|
||||||
|
"candidate_oid",
|
||||||
|
"domain",
|
||||||
|
"public_bind",
|
||||||
|
"bind_host",
|
||||||
|
"port",
|
||||||
|
"tls_cert_sha256",
|
||||||
|
"issued_at",
|
||||||
|
"not_before",
|
||||||
|
"expires_at",
|
||||||
|
"owner",
|
||||||
|
}
|
||||||
|
if set(value) != allowed or value.get("schema") != APPROVAL_SCHEMA:
|
||||||
|
raise ActivationError("APPROVAL_FIELDS_INVALID")
|
||||||
|
|
||||||
|
approval_id = value.get("approval_id")
|
||||||
|
if not isinstance(approval_id, str) or APPROVAL_ID_RE.fullmatch(approval_id) is None:
|
||||||
|
raise ActivationError("APPROVAL_ID_INVALID")
|
||||||
|
if not OID_RE.fullmatch(str(value.get("candidate_oid") or "")):
|
||||||
|
raise ActivationError("APPROVAL_CANDIDATE_INVALID")
|
||||||
|
if value["candidate_oid"] != candidate_oid:
|
||||||
|
raise ActivationError("APPROVAL_CANDIDATE_MISMATCH")
|
||||||
|
if value.get("public_bind") is not True:
|
||||||
|
raise ActivationError("APPROVAL_PUBLIC_BIND_REQUIRED")
|
||||||
|
if value.get("bind_host") != PUBLIC_HOST or value.get("port") != PUBLIC_PORT:
|
||||||
|
raise ActivationError("APPROVAL_BINDING_INVALID")
|
||||||
|
|
||||||
|
cert_sha = str(value.get("tls_cert_sha256") or "")
|
||||||
|
if SHA_RE.fullmatch(cert_sha) is None:
|
||||||
|
raise ActivationError("APPROVAL_CERT_SHA_INVALID")
|
||||||
|
if not cert_path.is_file() or sha256_file(cert_path) != cert_sha:
|
||||||
|
raise ActivationError("APPROVAL_CERT_SHA_MISMATCH")
|
||||||
|
|
||||||
|
domain = validate_domain(value.get("domain"))
|
||||||
|
owner = value.get("owner")
|
||||||
|
if not isinstance(owner, str) or not owner.strip() or len(owner) > 255:
|
||||||
|
raise ActivationError("APPROVAL_OWNER_INVALID")
|
||||||
|
|
||||||
|
issued_at = parse_time(value.get("issued_at"), field="issued_at")
|
||||||
|
not_before = parse_time(value.get("not_before"), field="not_before")
|
||||||
|
expires_at = parse_time(value.get("expires_at"), field="expires_at")
|
||||||
|
current = now or datetime.now(timezone.utc)
|
||||||
|
if not (issued_at <= not_before <= expires_at):
|
||||||
|
raise ActivationError("APPROVAL_TIME_ORDER_INVALID")
|
||||||
|
if current < not_before:
|
||||||
|
raise ActivationError("APPROVAL_NOT_YET_VALID")
|
||||||
|
if current >= expires_at:
|
||||||
|
raise ActivationError("APPROVAL_EXPIRED")
|
||||||
|
if expires_at - issued_at > __import__("datetime").timedelta(days=7):
|
||||||
|
raise ActivationError("APPROVAL_VALIDITY_TOO_LONG")
|
||||||
|
|
||||||
|
return cls(
|
||||||
|
approval_id=approval_id,
|
||||||
|
candidate_oid=candidate_oid,
|
||||||
|
domain=domain,
|
||||||
|
bind_host=PUBLIC_HOST,
|
||||||
|
port=PUBLIC_PORT,
|
||||||
|
tls_cert_sha256=cert_sha,
|
||||||
|
owner=owner.strip(),
|
||||||
|
issued_at=issued_at,
|
||||||
|
not_before=not_before,
|
||||||
|
expires_at=expires_at,
|
||||||
|
raw_sha256=sha256_file(path),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def validate_release(release_root: Path) -> dict[str, Path]:
|
||||||
|
if release_root.is_symlink() or not release_root.is_dir():
|
||||||
|
raise ActivationError("IMMUTABLE_RELEASE_INVALID")
|
||||||
|
edge = release_root / "active-runtime/app/aios_gpt_https_edge.py"
|
||||||
|
controller = release_root / "active-runtime/app/aios_gpt_https_edge_activation.py"
|
||||||
|
for path in (edge, controller):
|
||||||
|
if path.is_symlink() or not path.is_file():
|
||||||
|
raise ActivationError("IMMUTABLE_RELEASE_INVALID")
|
||||||
|
return {"edge": edge, "controller": controller}
|
||||||
|
|
||||||
|
|
||||||
|
def render_dropin(approval: Approval, release_root: Path) -> str:
|
||||||
|
app = release_root / "active-runtime/app/aios_gpt_https_edge.py"
|
||||||
|
return (
|
||||||
|
"[Service]\n"
|
||||||
|
f"Environment=AIOS_GPT_EDGE_HOST={approval.bind_host}\n"
|
||||||
|
f"Environment=AIOS_GPT_EDGE_PORT={approval.port}\n"
|
||||||
|
"Environment=AIOS_GPT_EDGE_PUBLIC_BIND_APPROVED=1\n"
|
||||||
|
f"Environment=AIOS_GPT_EDGE_PUBLIC_DOMAIN={approval.domain}\n"
|
||||||
|
"ExecStart=\n"
|
||||||
|
f"ExecStart=/usr/bin/python3 -B {app}\n"
|
||||||
|
f"ReadOnlyPaths={release_root}\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def atomic_write(path: Path, data: str, mode: int = 0o600) -> None:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
fd, name = tempfile.mkstemp(prefix="." + path.name + ".", dir=path.parent)
|
||||||
|
try:
|
||||||
|
os.fchmod(fd, mode)
|
||||||
|
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||||
|
handle.write(data)
|
||||||
|
handle.flush()
|
||||||
|
os.fsync(handle.fileno())
|
||||||
|
os.replace(name, path)
|
||||||
|
finally:
|
||||||
|
if os.path.exists(name):
|
||||||
|
os.unlink(name)
|
||||||
|
|
||||||
|
|
||||||
|
class Systemd:
|
||||||
|
def run(self, *args: str, check: bool = True) -> subprocess.CompletedProcess[str]:
|
||||||
|
return subprocess.run(
|
||||||
|
["/usr/bin/systemctl", *args],
|
||||||
|
check=check,
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
|
||||||
|
def active(self, unit: str) -> bool:
|
||||||
|
return self.run("is-active", "--quiet", unit, check=False).returncode == 0
|
||||||
|
|
||||||
|
def enabled(self, unit: str) -> bool:
|
||||||
|
return self.run("is-enabled", "--quiet", unit, check=False).returncode == 0
|
||||||
|
|
||||||
|
def show(self, unit: str, prop: str) -> str:
|
||||||
|
return self.run("show", unit, f"--property={prop}", "--value").stdout.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def require_ingress_ready(systemd: Systemd) -> None:
|
||||||
|
if not systemd.active(INGRESS_UNIT):
|
||||||
|
raise ActivationError("INGRESS_SERVICE_NOT_ACTIVE")
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(INGRESS_HEALTH, timeout=5) as response:
|
||||||
|
if response.status != 200:
|
||||||
|
raise ActivationError("INGRESS_HEALTH_FAILED")
|
||||||
|
value = json.loads(response.read().decode())
|
||||||
|
except ActivationError:
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
raise ActivationError("INGRESS_HEALTH_FAILED") from exc
|
||||||
|
if value.get("status") != "ok":
|
||||||
|
raise ActivationError("INGRESS_HEALTH_FAILED")
|
||||||
|
|
||||||
|
|
||||||
|
def tls_probe(domain: str, cert_path: Path, *, port: int = PUBLIC_PORT) -> None:
|
||||||
|
context = ssl.create_default_context(cafile=str(cert_path))
|
||||||
|
context.check_hostname = True
|
||||||
|
context.verify_mode = ssl.CERT_REQUIRED
|
||||||
|
try:
|
||||||
|
with socket.create_connection(("127.0.0.1", port), timeout=5) as sock:
|
||||||
|
with context.wrap_socket(sock, server_hostname=domain) as tls_sock:
|
||||||
|
if tls_sock.version() not in {"TLSv1.2", "TLSv1.3"}:
|
||||||
|
raise ActivationError("EDGE_TLS_VERSION_INVALID")
|
||||||
|
except ActivationError:
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
raise ActivationError("EDGE_TLS_PROBE_FAILED") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def receipt_for(
|
||||||
|
approval: Approval,
|
||||||
|
release_root: Path,
|
||||||
|
*,
|
||||||
|
state: str,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"schema": STATE_SCHEMA,
|
||||||
|
"state": state,
|
||||||
|
"approval_id": approval.approval_id,
|
||||||
|
"approval_sha256": approval.raw_sha256,
|
||||||
|
"candidate_oid": approval.candidate_oid,
|
||||||
|
"domain": approval.domain,
|
||||||
|
"bind_host": approval.bind_host,
|
||||||
|
"port": approval.port,
|
||||||
|
"tls_cert_sha256": approval.tls_cert_sha256,
|
||||||
|
"release_root": str(release_root),
|
||||||
|
"updated_at": datetime.now(timezone.utc).isoformat(),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def activate(
|
||||||
|
*,
|
||||||
|
approval: Approval,
|
||||||
|
release_root: Path,
|
||||||
|
cert_path: Path,
|
||||||
|
key_path: Path,
|
||||||
|
dropin_path: Path,
|
||||||
|
state_path: Path,
|
||||||
|
systemd: Systemd,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if os.geteuid() != 0:
|
||||||
|
raise ActivationError("ROOT_REQUIRED")
|
||||||
|
release = validate_release(release_root)
|
||||||
|
app = release["edge"]
|
||||||
|
require_ingress_ready(systemd)
|
||||||
|
validate_tls_pair(cert_path, key_path, approval.domain)
|
||||||
|
|
||||||
|
if state_path.is_file():
|
||||||
|
try:
|
||||||
|
existing = json.loads(state_path.read_text(encoding="utf-8"))
|
||||||
|
except Exception:
|
||||||
|
existing = {}
|
||||||
|
if (
|
||||||
|
existing.get("schema") == STATE_SCHEMA
|
||||||
|
and existing.get("state") == "ACTIVE"
|
||||||
|
and existing.get("approval_sha256") == approval.raw_sha256
|
||||||
|
and existing.get("candidate_oid") == approval.candidate_oid
|
||||||
|
and systemd.active(EDGE_UNIT)
|
||||||
|
):
|
||||||
|
return existing
|
||||||
|
|
||||||
|
dropin = render_dropin(approval, release_root)
|
||||||
|
modified = False
|
||||||
|
try:
|
||||||
|
atomic_write(dropin_path, dropin, mode=0o644)
|
||||||
|
modified = True
|
||||||
|
systemd.run("daemon-reload")
|
||||||
|
systemd.run("enable", EDGE_UNIT)
|
||||||
|
systemd.run("restart", EDGE_UNIT)
|
||||||
|
if not systemd.active(EDGE_UNIT):
|
||||||
|
raise ActivationError("EDGE_SERVICE_NOT_ACTIVE")
|
||||||
|
exec_start = systemd.show(EDGE_UNIT, "ExecStart")
|
||||||
|
environment = systemd.show(EDGE_UNIT, "Environment")
|
||||||
|
if str(app) not in exec_start:
|
||||||
|
raise ActivationError("EDGE_EFFECTIVE_EXECSTART_INVALID")
|
||||||
|
for expected in (
|
||||||
|
f"AIOS_GPT_EDGE_HOST={approval.bind_host}",
|
||||||
|
f"AIOS_GPT_EDGE_PORT={approval.port}",
|
||||||
|
"AIOS_GPT_EDGE_PUBLIC_BIND_APPROVED=1",
|
||||||
|
f"AIOS_GPT_EDGE_PUBLIC_DOMAIN={approval.domain}",
|
||||||
|
):
|
||||||
|
if expected not in environment:
|
||||||
|
raise ActivationError("EDGE_EFFECTIVE_ENVIRONMENT_INVALID")
|
||||||
|
tls_probe(approval.domain, cert_path, port=approval.port)
|
||||||
|
receipt = receipt_for(approval, release_root, state="ACTIVE")
|
||||||
|
atomic_write(state_path, json.dumps(receipt, indent=2, sort_keys=True) + "\n")
|
||||||
|
return receipt
|
||||||
|
except Exception:
|
||||||
|
if modified:
|
||||||
|
rollback(
|
||||||
|
dropin_path=dropin_path,
|
||||||
|
state_path=state_path,
|
||||||
|
systemd=systemd,
|
||||||
|
reason="activation_failed",
|
||||||
|
)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def rollback(
|
||||||
|
*,
|
||||||
|
dropin_path: Path,
|
||||||
|
state_path: Path,
|
||||||
|
systemd: Systemd,
|
||||||
|
reason: str,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if os.geteuid() != 0:
|
||||||
|
raise ActivationError("ROOT_REQUIRED")
|
||||||
|
systemd.run("stop", EDGE_UNIT, check=False)
|
||||||
|
systemd.run("disable", EDGE_UNIT, check=False)
|
||||||
|
dropin_path.unlink(missing_ok=True)
|
||||||
|
systemd.run("daemon-reload")
|
||||||
|
if systemd.active(EDGE_UNIT):
|
||||||
|
raise ActivationError("EDGE_ROLLBACK_SERVICE_STILL_ACTIVE")
|
||||||
|
previous: dict[str, Any] = {}
|
||||||
|
if state_path.is_file():
|
||||||
|
try:
|
||||||
|
previous = json.loads(state_path.read_text(encoding="utf-8"))
|
||||||
|
except Exception:
|
||||||
|
previous = {}
|
||||||
|
receipt = {
|
||||||
|
"schema": STATE_SCHEMA,
|
||||||
|
"state": "ROLLED_BACK",
|
||||||
|
"reason": reason[:255],
|
||||||
|
"candidate_oid": previous.get("candidate_oid"),
|
||||||
|
"approval_id": previous.get("approval_id"),
|
||||||
|
"domain": previous.get("domain"),
|
||||||
|
"updated_at": datetime.now(timezone.utc).isoformat(),
|
||||||
|
}
|
||||||
|
atomic_write(state_path, json.dumps(receipt, indent=2, sort_keys=True) + "\n")
|
||||||
|
return receipt
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("command", choices=("check", "activate", "rollback"))
|
||||||
|
parser.add_argument("--candidate")
|
||||||
|
parser.add_argument("--approval", default="/etc/aios-gitops/gpt-https-edge.approved")
|
||||||
|
parser.add_argument("--tls-cert", default="/etc/aios-gitops/gpt-https-edge-tls.crt")
|
||||||
|
parser.add_argument("--tls-key", default="/etc/aios-gitops/gpt-https-edge-tls.key")
|
||||||
|
parser.add_argument("--release-root")
|
||||||
|
parser.add_argument(
|
||||||
|
"--dropin",
|
||||||
|
default=f"/etc/systemd/system/{EDGE_UNIT}.d/{DROPIN_NAME}",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--state",
|
||||||
|
default="/var/lib/aios-gpt-https-edge-activation/state.json",
|
||||||
|
)
|
||||||
|
parser.add_argument("--reason", default="owner_requested")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
systemd = Systemd()
|
||||||
|
if args.command == "rollback":
|
||||||
|
value = rollback(
|
||||||
|
dropin_path=Path(args.dropin),
|
||||||
|
state_path=Path(args.state),
|
||||||
|
systemd=systemd,
|
||||||
|
reason=args.reason,
|
||||||
|
)
|
||||||
|
print(json.dumps(value, indent=2, sort_keys=True))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
candidate = str(args.candidate or "")
|
||||||
|
if OID_RE.fullmatch(candidate) is None:
|
||||||
|
raise SystemExit("CANDIDATE_REQUIRED")
|
||||||
|
release_root = (
|
||||||
|
Path(args.release_root)
|
||||||
|
if args.release_root
|
||||||
|
else Path(f"/opt/aios/releases/{candidate}-gpt-https-edge-public")
|
||||||
|
)
|
||||||
|
cert_path = Path(args.tls_cert)
|
||||||
|
key_path = Path(args.tls_key)
|
||||||
|
approval = Approval.load(
|
||||||
|
Path(args.approval),
|
||||||
|
candidate_oid=candidate,
|
||||||
|
cert_path=cert_path,
|
||||||
|
)
|
||||||
|
tls = validate_tls_pair(cert_path, key_path, approval.domain)
|
||||||
|
release = validate_release(release_root)
|
||||||
|
require_ingress_ready(systemd)
|
||||||
|
|
||||||
|
if args.command == "check":
|
||||||
|
value = {
|
||||||
|
"status": "READY",
|
||||||
|
"candidate_oid": candidate,
|
||||||
|
"approval_id": approval.approval_id,
|
||||||
|
"approval_sha256": approval.raw_sha256,
|
||||||
|
"domain": approval.domain,
|
||||||
|
"bind_host": approval.bind_host,
|
||||||
|
"port": approval.port,
|
||||||
|
"edge_sha256": sha256_file(release["edge"]),
|
||||||
|
"controller_sha256": sha256_file(release["controller"]),
|
||||||
|
**tls,
|
||||||
|
"public_network_mutation": False,
|
||||||
|
}
|
||||||
|
print(json.dumps(value, indent=2, sort_keys=True))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
value = activate(
|
||||||
|
approval=approval,
|
||||||
|
release_root=release_root,
|
||||||
|
cert_path=cert_path,
|
||||||
|
key_path=key_path,
|
||||||
|
dropin_path=Path(args.dropin),
|
||||||
|
state_path=Path(args.state),
|
||||||
|
systemd=systemd,
|
||||||
|
)
|
||||||
|
print(json.dumps(value, indent=2, sort_keys=True))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,816 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Credential-isolated host-side ingress for externally scheduled read-only Online Tasks.
|
||||||
|
|
||||||
|
This service is intentionally not a public AIOS control plane. It accepts a tiny,
|
||||||
|
authenticated external intent and translates it to the canonical loopback Bridge.
|
||||||
|
Callers cannot select repository coordinates, capabilities, execution action,
|
||||||
|
worker, branch policy, credentials, verification decisions, or write permissions.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import sqlite3
|
||||||
|
import threading
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from typing import Any, Mapping
|
||||||
|
|
||||||
|
SCHEMA = "aios.gpt-online-task-ingress.request.v1"
|
||||||
|
RESPONSE_SCHEMA = "aios.gpt-online-task-ingress.response.v1"
|
||||||
|
OPENAPI_VERSION = "3.1.0"
|
||||||
|
|
||||||
|
REQUEST_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{7,127}$")
|
||||||
|
TASK_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{2,179}$")
|
||||||
|
JOB_ID_RE = re.compile(r"^job-[0-9a-f]{32}$")
|
||||||
|
OID_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||||
|
MAX_BODY = 64 * 1024
|
||||||
|
MAX_GOAL = 12_000
|
||||||
|
MAX_INSTRUCTIONS = 24_000
|
||||||
|
MAX_CRITERIA = 16
|
||||||
|
MAX_CRITERION_TEXT = 1_000
|
||||||
|
|
||||||
|
READONLY_CAPABILITIES = ("git.read", "git.test")
|
||||||
|
REPOSITORY = "aios/aios"
|
||||||
|
|
||||||
|
|
||||||
|
class IngressError(RuntimeError):
|
||||||
|
def __init__(self, code: str, status: int = 400) -> None:
|
||||||
|
super().__init__(code)
|
||||||
|
self.code = code
|
||||||
|
self.status = status
|
||||||
|
|
||||||
|
|
||||||
|
def canonical(value: object) -> bytes:
|
||||||
|
return json.dumps(
|
||||||
|
value,
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def digest(value: object) -> str:
|
||||||
|
return hashlib.sha256(canonical(value)).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def now() -> str:
|
||||||
|
return datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def load_secret(path: Path, *, code: str) -> str:
|
||||||
|
if path.is_symlink() or not path.is_file():
|
||||||
|
raise RuntimeError(code + "_UNAVAILABLE")
|
||||||
|
value = path.read_text(encoding="utf-8").strip()
|
||||||
|
if len(value) < 32 or "\n" in value or "\r" in value:
|
||||||
|
raise RuntimeError(code + "_INVALID")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def normalized_request(value: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
|
allowed = {
|
||||||
|
"schema",
|
||||||
|
"request_id",
|
||||||
|
"goal",
|
||||||
|
"instructions",
|
||||||
|
"success_criteria",
|
||||||
|
"constraints",
|
||||||
|
}
|
||||||
|
if set(value) - allowed:
|
||||||
|
raise IngressError("UNKNOWN_FIELDS")
|
||||||
|
if value.get("schema") != SCHEMA:
|
||||||
|
raise IngressError("SCHEMA_INVALID")
|
||||||
|
|
||||||
|
request_id = value.get("request_id")
|
||||||
|
if not isinstance(request_id, str) or REQUEST_ID_RE.fullmatch(request_id) is None:
|
||||||
|
raise IngressError("REQUEST_ID_INVALID")
|
||||||
|
|
||||||
|
goal = value.get("goal")
|
||||||
|
if not isinstance(goal, str) or not goal.strip() or len(goal) > MAX_GOAL:
|
||||||
|
raise IngressError("GOAL_INVALID")
|
||||||
|
instructions = value.get("instructions", "")
|
||||||
|
if not isinstance(instructions, str) or len(instructions) > MAX_INSTRUCTIONS:
|
||||||
|
raise IngressError("INSTRUCTIONS_INVALID")
|
||||||
|
|
||||||
|
criteria = value.get("success_criteria") or []
|
||||||
|
if not isinstance(criteria, list) or len(criteria) > MAX_CRITERIA:
|
||||||
|
raise IngressError("SUCCESS_CRITERIA_INVALID")
|
||||||
|
normalized_criteria = []
|
||||||
|
for item in criteria:
|
||||||
|
if not isinstance(item, Mapping) or set(item) != {"id", "description"}:
|
||||||
|
raise IngressError("SUCCESS_CRITERIA_INVALID")
|
||||||
|
criterion_id = item.get("id")
|
||||||
|
description = item.get("description")
|
||||||
|
if (
|
||||||
|
not isinstance(criterion_id, str)
|
||||||
|
or not criterion_id.strip()
|
||||||
|
or len(criterion_id) > 120
|
||||||
|
or not isinstance(description, str)
|
||||||
|
or not description.strip()
|
||||||
|
or len(description) > MAX_CRITERION_TEXT
|
||||||
|
):
|
||||||
|
raise IngressError("SUCCESS_CRITERIA_INVALID")
|
||||||
|
normalized_criteria.append(
|
||||||
|
{"id": criterion_id.strip(), "description": description.strip()}
|
||||||
|
)
|
||||||
|
|
||||||
|
constraints = value.get("constraints") or {}
|
||||||
|
if not isinstance(constraints, Mapping):
|
||||||
|
raise IngressError("CONSTRAINTS_INVALID")
|
||||||
|
allowed_constraints = {
|
||||||
|
"max_files",
|
||||||
|
"focus_paths",
|
||||||
|
"notes",
|
||||||
|
}
|
||||||
|
if set(constraints) - allowed_constraints:
|
||||||
|
raise IngressError("CONSTRAINTS_FIELDS_INVALID")
|
||||||
|
normalized_constraints: dict[str, Any] = {}
|
||||||
|
if "max_files" in constraints:
|
||||||
|
max_files = constraints["max_files"]
|
||||||
|
if (
|
||||||
|
not isinstance(max_files, int)
|
||||||
|
or isinstance(max_files, bool)
|
||||||
|
or not 1 <= max_files <= 500
|
||||||
|
):
|
||||||
|
raise IngressError("MAX_FILES_INVALID")
|
||||||
|
normalized_constraints["max_files"] = max_files
|
||||||
|
if "focus_paths" in constraints:
|
||||||
|
paths = constraints["focus_paths"]
|
||||||
|
if (
|
||||||
|
not isinstance(paths, list)
|
||||||
|
or len(paths) > 32
|
||||||
|
or not all(
|
||||||
|
isinstance(path, str)
|
||||||
|
and path
|
||||||
|
and len(path) <= 512
|
||||||
|
and not path.startswith("/")
|
||||||
|
and ".." not in Path(path).parts
|
||||||
|
for path in paths
|
||||||
|
)
|
||||||
|
):
|
||||||
|
raise IngressError("FOCUS_PATHS_INVALID")
|
||||||
|
normalized_constraints["focus_paths"] = paths
|
||||||
|
if "notes" in constraints:
|
||||||
|
notes = constraints["notes"]
|
||||||
|
if not isinstance(notes, str) or len(notes) > 4_000:
|
||||||
|
raise IngressError("CONSTRAINT_NOTES_INVALID")
|
||||||
|
normalized_constraints["notes"] = notes
|
||||||
|
|
||||||
|
return {
|
||||||
|
"schema": SCHEMA,
|
||||||
|
"request_id": request_id,
|
||||||
|
"goal": goal.strip(),
|
||||||
|
"instructions": instructions,
|
||||||
|
"success_criteria": normalized_criteria,
|
||||||
|
"constraints": normalized_constraints,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def validated_bind_host(value: str) -> str:
|
||||||
|
host = value.strip()
|
||||||
|
if host not in {"127.0.0.1", "::1", "localhost"}:
|
||||||
|
raise RuntimeError("GPT_INGRESS_NON_LOOPBACK_BIND_DENY")
|
||||||
|
return host
|
||||||
|
|
||||||
|
|
||||||
|
def task_id_for(request_id: str) -> str:
|
||||||
|
suffix = hashlib.sha256(request_id.encode("utf-8")).hexdigest()[:24]
|
||||||
|
return "gpt-readonly-" + suffix
|
||||||
|
|
||||||
|
|
||||||
|
def branch_for(task_id: str) -> str:
|
||||||
|
return "agent/external-gpt-readonly/" + task_id[-24:]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Upstream:
|
||||||
|
bridge_base: str
|
||||||
|
task_base: str
|
||||||
|
control_token: str
|
||||||
|
timeout: int = 20
|
||||||
|
|
||||||
|
def _request(
|
||||||
|
self,
|
||||||
|
base: str,
|
||||||
|
method: str,
|
||||||
|
path: str,
|
||||||
|
body: Mapping[str, Any] | None = None,
|
||||||
|
) -> tuple[int, dict[str, Any]]:
|
||||||
|
raw_body = None if body is None else canonical(body)
|
||||||
|
headers = {
|
||||||
|
"Authorization": "Bearer " + self.control_token,
|
||||||
|
"Accept": "application/json",
|
||||||
|
}
|
||||||
|
if raw_body is not None:
|
||||||
|
headers["Content-Type"] = "application/json"
|
||||||
|
req = urllib.request.Request(
|
||||||
|
base.rstrip("/") + path,
|
||||||
|
method=method,
|
||||||
|
data=raw_body,
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=self.timeout) as response:
|
||||||
|
raw = response.read()
|
||||||
|
status = response.status
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
raw = exc.read()
|
||||||
|
status = exc.code
|
||||||
|
except Exception as exc:
|
||||||
|
raise IngressError("UPSTREAM_TRANSPORT_FAILED", 503) from exc
|
||||||
|
try:
|
||||||
|
value = json.loads(raw.decode("utf-8"))
|
||||||
|
except Exception as exc:
|
||||||
|
raise IngressError("UPSTREAM_RESPONSE_INVALID", 502) from exc
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise IngressError("UPSTREAM_RESPONSE_INVALID", 502)
|
||||||
|
return status, dict(value)
|
||||||
|
|
||||||
|
def control_snapshot(self) -> dict[str, Any]:
|
||||||
|
status, value = self._request(
|
||||||
|
self.bridge_base,
|
||||||
|
"GET",
|
||||||
|
"/v2/control-plane-snapshot",
|
||||||
|
)
|
||||||
|
if status != 200:
|
||||||
|
raise IngressError("CONTROL_SNAPSHOT_UNAVAILABLE", 503)
|
||||||
|
return value
|
||||||
|
|
||||||
|
def create_governed_task(
|
||||||
|
self,
|
||||||
|
body: Mapping[str, Any],
|
||||||
|
) -> tuple[int, dict[str, Any]]:
|
||||||
|
return self._request(
|
||||||
|
self.bridge_base,
|
||||||
|
"POST",
|
||||||
|
"/v2/governed-task",
|
||||||
|
body,
|
||||||
|
)
|
||||||
|
|
||||||
|
def execution_envelope(
|
||||||
|
self,
|
||||||
|
body: Mapping[str, Any],
|
||||||
|
) -> tuple[int, dict[str, Any]]:
|
||||||
|
return self._request(
|
||||||
|
self.bridge_base,
|
||||||
|
"POST",
|
||||||
|
"/v2/execution-envelope",
|
||||||
|
body,
|
||||||
|
)
|
||||||
|
|
||||||
|
def get_task(self, task_id: str) -> tuple[int, dict[str, Any]]:
|
||||||
|
return self._request(
|
||||||
|
self.task_base,
|
||||||
|
"GET",
|
||||||
|
"/v1/tasks/" + task_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class Store:
|
||||||
|
def __init__(self, path: Path) -> None:
|
||||||
|
self.path = Path(path)
|
||||||
|
self._lock = threading.RLock()
|
||||||
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with self._connect() as con:
|
||||||
|
con.execute(
|
||||||
|
"""
|
||||||
|
CREATE TABLE IF NOT EXISTS external_task_requests(
|
||||||
|
request_id TEXT PRIMARY KEY,
|
||||||
|
request_sha256 TEXT NOT NULL,
|
||||||
|
task_id TEXT NOT NULL UNIQUE,
|
||||||
|
job_id TEXT,
|
||||||
|
base_oid TEXT,
|
||||||
|
state TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL,
|
||||||
|
updated_at TEXT NOT NULL,
|
||||||
|
response_json TEXT
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
con.commit()
|
||||||
|
|
||||||
|
def _connect(self) -> sqlite3.Connection:
|
||||||
|
con = sqlite3.connect(self.path, timeout=30)
|
||||||
|
con.row_factory = sqlite3.Row
|
||||||
|
con.execute("PRAGMA busy_timeout=30000")
|
||||||
|
return con
|
||||||
|
|
||||||
|
def get(self, request_id: str) -> sqlite3.Row | None:
|
||||||
|
with self._connect() as con:
|
||||||
|
return con.execute(
|
||||||
|
"SELECT * FROM external_task_requests WHERE request_id=?",
|
||||||
|
(request_id,),
|
||||||
|
).fetchone()
|
||||||
|
|
||||||
|
def reserve(self, request: Mapping[str, Any], task_id: str) -> sqlite3.Row:
|
||||||
|
request_sha = digest(request)
|
||||||
|
timestamp = now()
|
||||||
|
with self._lock, self._connect() as con:
|
||||||
|
con.execute("BEGIN IMMEDIATE")
|
||||||
|
row = con.execute(
|
||||||
|
"SELECT * FROM external_task_requests WHERE request_id=?",
|
||||||
|
(request["request_id"],),
|
||||||
|
).fetchone()
|
||||||
|
if row is not None:
|
||||||
|
if row["request_sha256"] != request_sha:
|
||||||
|
raise IngressError("IDEMPOTENCY_CONFLICT", 409)
|
||||||
|
con.commit()
|
||||||
|
return row
|
||||||
|
con.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO external_task_requests(
|
||||||
|
request_id,request_sha256,task_id,state,created_at,updated_at
|
||||||
|
) VALUES(?,?,?,?,?,?)
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
request["request_id"],
|
||||||
|
request_sha,
|
||||||
|
task_id,
|
||||||
|
"PENDING",
|
||||||
|
timestamp,
|
||||||
|
timestamp,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
con.commit()
|
||||||
|
row = self.get(request["request_id"])
|
||||||
|
assert row is not None
|
||||||
|
return row
|
||||||
|
|
||||||
|
def bound(
|
||||||
|
self,
|
||||||
|
request_id: str,
|
||||||
|
*,
|
||||||
|
job_id: str,
|
||||||
|
base_oid: str,
|
||||||
|
response: Mapping[str, Any],
|
||||||
|
) -> None:
|
||||||
|
with self._lock, self._connect() as con:
|
||||||
|
con.execute(
|
||||||
|
"""
|
||||||
|
UPDATE external_task_requests
|
||||||
|
SET job_id=?,base_oid=?,state='BOUND',response_json=?,updated_at=?
|
||||||
|
WHERE request_id=?
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
job_id,
|
||||||
|
base_oid,
|
||||||
|
json.dumps(response, sort_keys=True, separators=(",", ":")),
|
||||||
|
now(),
|
||||||
|
request_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
con.commit()
|
||||||
|
|
||||||
|
|
||||||
|
class IngressService:
|
||||||
|
def __init__(self, upstream: Upstream, store: Store) -> None:
|
||||||
|
self.upstream = upstream
|
||||||
|
self.store = store
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _main_oid(snapshot: Mapping[str, Any]) -> str:
|
||||||
|
candidates = [
|
||||||
|
snapshot.get("authoritative_head_oid"),
|
||||||
|
snapshot.get("head"),
|
||||||
|
(snapshot.get("aggregate") or {}).get("authoritative_head_oid")
|
||||||
|
if isinstance(snapshot.get("aggregate"), Mapping)
|
||||||
|
else None,
|
||||||
|
]
|
||||||
|
for candidate in candidates:
|
||||||
|
if isinstance(candidate, str) and OID_RE.fullmatch(candidate):
|
||||||
|
return candidate
|
||||||
|
raise IngressError("AUTHORITATIVE_HEAD_UNAVAILABLE", 503)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _public_response(
|
||||||
|
request_id: str,
|
||||||
|
task_id: str,
|
||||||
|
job_id: str,
|
||||||
|
state: str,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"schema": RESPONSE_SCHEMA,
|
||||||
|
"request_id": request_id,
|
||||||
|
"task_id": task_id,
|
||||||
|
"job_id": job_id,
|
||||||
|
"state": state,
|
||||||
|
"execution_mode": "source_capture",
|
||||||
|
"capabilities": list(READONLY_CAPABILITIES),
|
||||||
|
"write_access": False,
|
||||||
|
"official_verification_required": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
def _recover_existing(
|
||||||
|
self,
|
||||||
|
request: Mapping[str, Any],
|
||||||
|
row: sqlite3.Row,
|
||||||
|
) -> dict[str, Any] | None:
|
||||||
|
task_id = str(row["task_id"])
|
||||||
|
status, task = self.upstream.get_task(task_id)
|
||||||
|
if status == 404:
|
||||||
|
return None
|
||||||
|
if status != 200:
|
||||||
|
raise IngressError("TASK_RECOVERY_FAILED", 503)
|
||||||
|
metadata = task.get("metadata")
|
||||||
|
execution = (
|
||||||
|
metadata.get("execution")
|
||||||
|
if isinstance(metadata, Mapping)
|
||||||
|
and isinstance(metadata.get("execution"), Mapping)
|
||||||
|
else {}
|
||||||
|
)
|
||||||
|
job_id = str(execution.get("job_id") or row["job_id"] or "")
|
||||||
|
base_oid = str(execution.get("base_oid") or row["base_oid"] or "")
|
||||||
|
if (
|
||||||
|
task.get("task_id") != task_id
|
||||||
|
or not JOB_ID_RE.fullmatch(job_id)
|
||||||
|
or not OID_RE.fullmatch(base_oid)
|
||||||
|
or execution.get("action") != "source_capture"
|
||||||
|
or sorted(execution.get("capability_scope") or [])
|
||||||
|
!= sorted(READONLY_CAPABILITIES)
|
||||||
|
or (execution.get("governance") or {}).get("writer_lease") is not False
|
||||||
|
):
|
||||||
|
raise IngressError("TASK_RECOVERY_BINDING_INVALID", 409)
|
||||||
|
response = self._public_response(
|
||||||
|
request["request_id"],
|
||||||
|
task_id,
|
||||||
|
job_id,
|
||||||
|
str(task.get("status") or "BOUND"),
|
||||||
|
)
|
||||||
|
self.store.bound(
|
||||||
|
request["request_id"],
|
||||||
|
job_id=job_id,
|
||||||
|
base_oid=base_oid,
|
||||||
|
response=response,
|
||||||
|
)
|
||||||
|
return response
|
||||||
|
|
||||||
|
def create(self, raw: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
|
request = normalized_request(raw)
|
||||||
|
task_id = task_id_for(request["request_id"])
|
||||||
|
row = self.store.reserve(request, task_id)
|
||||||
|
if row["state"] == "BOUND" and row["response_json"]:
|
||||||
|
return json.loads(row["response_json"])
|
||||||
|
|
||||||
|
recovered = self._recover_existing(request, row)
|
||||||
|
if recovered is not None:
|
||||||
|
return recovered
|
||||||
|
|
||||||
|
snapshot = self.upstream.control_snapshot()
|
||||||
|
base_oid = self._main_oid(snapshot)
|
||||||
|
target_branch = branch_for(task_id)
|
||||||
|
binding_key = "gpt-readonly-binding-" + digest(
|
||||||
|
{"request_id": request["request_id"], "base_oid": base_oid}
|
||||||
|
)[:24]
|
||||||
|
constraints = {
|
||||||
|
**request["constraints"],
|
||||||
|
"repository_mutation": "forbidden",
|
||||||
|
"control_plane_side_effects": "forbidden",
|
||||||
|
"credential_access": "forbidden",
|
||||||
|
}
|
||||||
|
body = {
|
||||||
|
"task_id": task_id,
|
||||||
|
"repository": REPOSITORY,
|
||||||
|
"base_oid": base_oid,
|
||||||
|
"expected_branch_oid": base_oid,
|
||||||
|
"target_branch": target_branch,
|
||||||
|
"branch_mode": "CREATE_NEW",
|
||||||
|
"generation": 1,
|
||||||
|
"risk": "low",
|
||||||
|
"description": "Externally requested read-only governed Online Task.",
|
||||||
|
"idempotency_key": binding_key,
|
||||||
|
"priority": 0,
|
||||||
|
"max_attempts": 3,
|
||||||
|
"require_online_task_projection": True,
|
||||||
|
"online_task": {
|
||||||
|
"goal": request["goal"],
|
||||||
|
"instructions": request["instructions"],
|
||||||
|
"success_criteria": request["success_criteria"],
|
||||||
|
"constraints": constraints,
|
||||||
|
"required_capabilities": list(READONLY_CAPABILITIES),
|
||||||
|
"priority": 0,
|
||||||
|
"max_attempts": 3,
|
||||||
|
},
|
||||||
|
"execution": {
|
||||||
|
"action": "source_capture",
|
||||||
|
"adapter": "source_capture",
|
||||||
|
"capability_scope": list(READONLY_CAPABILITIES),
|
||||||
|
"source_signals": [
|
||||||
|
"external read-only source analysis request",
|
||||||
|
"no repository mutation no credential access no control-plane side effects",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
status, value = self.upstream.create_governed_task(body)
|
||||||
|
if status not in {200, 201, 202}:
|
||||||
|
if status == 409:
|
||||||
|
recovered = self._recover_existing(request, row)
|
||||||
|
if recovered is not None:
|
||||||
|
return recovered
|
||||||
|
raise IngressError(
|
||||||
|
"GOVERNED_TASK_CREATE_FAILED",
|
||||||
|
409 if status == 409 else 503,
|
||||||
|
)
|
||||||
|
job_id = str(value.get("job_id") or "")
|
||||||
|
if value.get("task_id") != task_id or JOB_ID_RE.fullmatch(job_id) is None:
|
||||||
|
raise IngressError("GOVERNED_TASK_RESPONSE_INVALID", 502)
|
||||||
|
|
||||||
|
envelope_status, envelope = self.upstream.execution_envelope(
|
||||||
|
{
|
||||||
|
"task_id": task_id,
|
||||||
|
"job_id": job_id,
|
||||||
|
"required_capabilities": list(READONLY_CAPABILITIES),
|
||||||
|
"execution": body["execution"],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if envelope_status != 200:
|
||||||
|
raise IngressError("EXECUTION_PROJECTION_FAILED", 503)
|
||||||
|
if (
|
||||||
|
envelope.get("task_id") != task_id
|
||||||
|
or envelope.get("job_id") != job_id
|
||||||
|
or envelope.get("base_oid") != base_oid
|
||||||
|
or envelope.get("action") != "source_capture"
|
||||||
|
or sorted(envelope.get("capability_scope") or [])
|
||||||
|
!= sorted(READONLY_CAPABILITIES)
|
||||||
|
or (envelope.get("governance") or {}).get("writer_lease") is not False
|
||||||
|
):
|
||||||
|
raise IngressError("EXECUTION_PROJECTION_INVALID", 502)
|
||||||
|
|
||||||
|
response = self._public_response(
|
||||||
|
request["request_id"],
|
||||||
|
task_id,
|
||||||
|
job_id,
|
||||||
|
"BOUND",
|
||||||
|
)
|
||||||
|
self.store.bound(
|
||||||
|
request["request_id"],
|
||||||
|
job_id=job_id,
|
||||||
|
base_oid=base_oid,
|
||||||
|
response=response,
|
||||||
|
)
|
||||||
|
return response
|
||||||
|
|
||||||
|
def status(self, request_id: str) -> dict[str, Any]:
|
||||||
|
if REQUEST_ID_RE.fullmatch(request_id) is None:
|
||||||
|
raise IngressError("REQUEST_ID_INVALID")
|
||||||
|
row = self.store.get(request_id)
|
||||||
|
if row is None:
|
||||||
|
raise IngressError("REQUEST_NOT_FOUND", 404)
|
||||||
|
task_id = str(row["task_id"])
|
||||||
|
status, task = self.upstream.get_task(task_id)
|
||||||
|
if status == 200 and not row["job_id"]:
|
||||||
|
metadata = task.get("metadata")
|
||||||
|
execution = (
|
||||||
|
metadata.get("execution")
|
||||||
|
if isinstance(metadata, Mapping)
|
||||||
|
and isinstance(metadata.get("execution"), Mapping)
|
||||||
|
else {}
|
||||||
|
)
|
||||||
|
job_id = str(execution.get("job_id") or "")
|
||||||
|
base_oid = str(execution.get("base_oid") or "")
|
||||||
|
if (
|
||||||
|
JOB_ID_RE.fullmatch(job_id)
|
||||||
|
and OID_RE.fullmatch(base_oid)
|
||||||
|
and execution.get("action") == "source_capture"
|
||||||
|
and sorted(execution.get("capability_scope") or [])
|
||||||
|
== sorted(READONLY_CAPABILITIES)
|
||||||
|
and (execution.get("governance") or {}).get("writer_lease") is False
|
||||||
|
):
|
||||||
|
recovered = self._public_response(
|
||||||
|
request_id,
|
||||||
|
task_id,
|
||||||
|
job_id,
|
||||||
|
str(task.get("status") or "BOUND"),
|
||||||
|
)
|
||||||
|
self.store.bound(
|
||||||
|
request_id,
|
||||||
|
job_id=job_id,
|
||||||
|
base_oid=base_oid,
|
||||||
|
response=recovered,
|
||||||
|
)
|
||||||
|
row = self.store.get(request_id) or row
|
||||||
|
if status == 404:
|
||||||
|
return {
|
||||||
|
"schema": RESPONSE_SCHEMA,
|
||||||
|
"request_id": request_id,
|
||||||
|
"task_id": task_id,
|
||||||
|
"state": "PENDING",
|
||||||
|
"official_verification_required": True,
|
||||||
|
}
|
||||||
|
if status != 200:
|
||||||
|
raise IngressError("TASK_STATUS_UNAVAILABLE", 503)
|
||||||
|
|
||||||
|
result = task.get("result") if isinstance(task.get("result"), Mapping) else {}
|
||||||
|
verification = (
|
||||||
|
task.get("verification")
|
||||||
|
if isinstance(task.get("verification"), Mapping)
|
||||||
|
else {}
|
||||||
|
)
|
||||||
|
safe = {
|
||||||
|
"schema": RESPONSE_SCHEMA,
|
||||||
|
"request_id": request_id,
|
||||||
|
"task_id": task_id,
|
||||||
|
"job_id": str(row["job_id"] or ""),
|
||||||
|
"state": str(task.get("status") or "UNKNOWN"),
|
||||||
|
"attempt": int(task.get("attempt") or 0),
|
||||||
|
"claimed": bool(task.get("claimed_by")),
|
||||||
|
"result_status": result.get("status"),
|
||||||
|
"result_classification": result.get("classification"),
|
||||||
|
"official_verification": verification.get("decision"),
|
||||||
|
"official_classification": verification.get("classification"),
|
||||||
|
"write_access": False,
|
||||||
|
}
|
||||||
|
if result.get("execution_mode") == "source_capture":
|
||||||
|
snapshot = result.get("source_snapshot")
|
||||||
|
if isinstance(snapshot, Mapping):
|
||||||
|
safe["source_snapshot_sha256"] = snapshot.get("payload_sha256")
|
||||||
|
safe["source_snapshot_file_count"] = snapshot.get("file_count")
|
||||||
|
return safe
|
||||||
|
|
||||||
|
|
||||||
|
def openapi_document() -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"openapi": OPENAPI_VERSION,
|
||||||
|
"info": {
|
||||||
|
"title": "AIOS Read-Only Online Task Ingress",
|
||||||
|
"version": "1.0.0",
|
||||||
|
},
|
||||||
|
"paths": {
|
||||||
|
"/v1/read-only-tasks": {
|
||||||
|
"post": {
|
||||||
|
"operationId": "createReadOnlyOnlineTask",
|
||||||
|
"security": [{"bearerAuth": []}],
|
||||||
|
"requestBody": {"required": True},
|
||||||
|
"responses": {"200": {"description": "Task accepted"}},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"/v1/read-only-tasks/{request_id}": {
|
||||||
|
"get": {
|
||||||
|
"operationId": "getReadOnlyOnlineTaskStatus",
|
||||||
|
"security": [{"bearerAuth": []}],
|
||||||
|
"responses": {"200": {"description": "Sanitized task status"}},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"components": {
|
||||||
|
"securitySchemes": {
|
||||||
|
"bearerAuth": {
|
||||||
|
"type": "http",
|
||||||
|
"scheme": "bearer",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class Handler(BaseHTTPRequestHandler):
|
||||||
|
service: IngressService | None = None
|
||||||
|
ingress_token: str = ""
|
||||||
|
|
||||||
|
def log_message(self, *_args: Any) -> None:
|
||||||
|
return
|
||||||
|
|
||||||
|
def respond(self, value: Mapping[str, Any], status: int) -> None:
|
||||||
|
raw = canonical(value)
|
||||||
|
self.send_response(status)
|
||||||
|
self.send_header("Content-Type", "application/json")
|
||||||
|
self.send_header("Cache-Control", "no-store")
|
||||||
|
self.send_header("Content-Length", str(len(raw)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(raw)
|
||||||
|
|
||||||
|
def authorized(self) -> bool:
|
||||||
|
header = self.headers.get("Authorization", "")
|
||||||
|
return (
|
||||||
|
header.startswith("Bearer ")
|
||||||
|
and bool(self.ingress_token)
|
||||||
|
and hmac.compare_digest(header[7:], self.ingress_token)
|
||||||
|
)
|
||||||
|
|
||||||
|
def body(self) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
length = int(self.headers.get("Content-Length", "-1"))
|
||||||
|
except ValueError as exc:
|
||||||
|
raise IngressError("CONTENT_LENGTH_INVALID") from exc
|
||||||
|
if length < 1 or length > MAX_BODY:
|
||||||
|
raise IngressError("REQUEST_SIZE_INVALID", 413)
|
||||||
|
try:
|
||||||
|
value = json.loads(self.rfile.read(length).decode("utf-8"))
|
||||||
|
except Exception as exc:
|
||||||
|
raise IngressError("REQUEST_JSON_INVALID") from exc
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise IngressError("REQUEST_OBJECT_REQUIRED")
|
||||||
|
return dict(value)
|
||||||
|
|
||||||
|
def do_GET(self) -> None:
|
||||||
|
if self.path == "/health":
|
||||||
|
self.respond(
|
||||||
|
{
|
||||||
|
"status": "ok",
|
||||||
|
"service": "aios-gpt-online-task-ingress",
|
||||||
|
"mode": "read-only",
|
||||||
|
},
|
||||||
|
200,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
if self.path == "/openapi.json":
|
||||||
|
self.respond(openapi_document(), 200)
|
||||||
|
return
|
||||||
|
if not self.authorized():
|
||||||
|
self.respond({"error": "UNAUTHORIZED"}, 401)
|
||||||
|
return
|
||||||
|
match = re.fullmatch(r"/v1/read-only-tasks/([^/?#]+)", self.path)
|
||||||
|
if match is None:
|
||||||
|
self.respond({"error": "NOT_FOUND"}, 404)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
if self.service is None:
|
||||||
|
raise IngressError("RUNTIME_UNAVAILABLE", 503)
|
||||||
|
self.respond(self.service.status(match.group(1)), 200)
|
||||||
|
except IngressError as exc:
|
||||||
|
self.respond({"error": exc.code}, exc.status)
|
||||||
|
|
||||||
|
def do_POST(self) -> None:
|
||||||
|
if self.path != "/v1/read-only-tasks":
|
||||||
|
self.respond({"error": "NOT_FOUND"}, 404)
|
||||||
|
return
|
||||||
|
if not self.authorized():
|
||||||
|
self.respond({"error": "UNAUTHORIZED"}, 401)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
if self.service is None:
|
||||||
|
raise IngressError("RUNTIME_UNAVAILABLE", 503)
|
||||||
|
value = self.service.create(self.body())
|
||||||
|
self.respond(value, 200)
|
||||||
|
except IngressError as exc:
|
||||||
|
self.respond({"error": exc.code}, exc.status)
|
||||||
|
|
||||||
|
|
||||||
|
def serve() -> None:
|
||||||
|
ingress_token_path = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_GPT_INGRESS_TOKEN_FILE",
|
||||||
|
str(
|
||||||
|
Path(os.environ.get("CREDENTIALS_DIRECTORY", "/run/credentials"))
|
||||||
|
/ "gpt_ingress_token"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
control_token_path = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_AGENT_TASK_CONTROL_TOKEN_FILE",
|
||||||
|
str(
|
||||||
|
Path(os.environ.get("CREDENTIALS_DIRECTORY", "/run/credentials"))
|
||||||
|
/ "agent_task_control_token"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
ingress_token = load_secret(
|
||||||
|
ingress_token_path,
|
||||||
|
code="GPT_INGRESS_CREDENTIAL",
|
||||||
|
)
|
||||||
|
control_token = load_secret(
|
||||||
|
control_token_path,
|
||||||
|
code="CONTROL_PLANE_CREDENTIAL",
|
||||||
|
)
|
||||||
|
|
||||||
|
state_root = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_GPT_INGRESS_STATE_ROOT",
|
||||||
|
"/var/lib/aios-gpt-online-task-ingress",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
host = validated_bind_host(
|
||||||
|
os.environ.get("AIOS_GPT_INGRESS_HOST", "127.0.0.1")
|
||||||
|
)
|
||||||
|
port = int(os.environ.get("AIOS_GPT_INGRESS_PORT", "18945"))
|
||||||
|
if not 1024 <= port <= 65535:
|
||||||
|
raise RuntimeError("GPT_INGRESS_PORT_INVALID")
|
||||||
|
|
||||||
|
Handler.service = IngressService(
|
||||||
|
Upstream(
|
||||||
|
bridge_base=os.environ.get(
|
||||||
|
"AIOS_GOVERNED_WRITE_API_URL",
|
||||||
|
"http://127.0.0.1:18892",
|
||||||
|
),
|
||||||
|
task_base=os.environ.get(
|
||||||
|
"AIOS_AGENT_TASK_API_URL",
|
||||||
|
"http://127.0.0.1:18940",
|
||||||
|
),
|
||||||
|
control_token=control_token,
|
||||||
|
),
|
||||||
|
Store(state_root / "state.sqlite3"),
|
||||||
|
)
|
||||||
|
Handler.ingress_token = ingress_token
|
||||||
|
ThreadingHTTPServer((host, port), Handler).serve_forever()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
serve()
|
||||||
@@ -0,0 +1,306 @@
|
|||||||
|
"""Uncredentialed deterministic executor for governed Online Agent changes.
|
||||||
|
|
||||||
|
This process never receives Agent/Owner/Bridge credentials and never writes Git refs.
|
||||||
|
It operates only on an exchange bundle and returns hash-bound changed files + test evidence.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import errno
|
||||||
|
import hashlib
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import socket
|
||||||
|
import subprocess
|
||||||
|
import tarfile
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
from typing import Any, Mapping, Sequence
|
||||||
|
|
||||||
|
ALLOWED_PROGRAMS = {"/usr/bin/python3", "/usr/bin/git", "/bin/true", "/usr/bin/true"}
|
||||||
|
MAX_ARCHIVE_BYTES = 64 * 1024 * 1024
|
||||||
|
MAX_FILE_BYTES = 4 * 1024 * 1024
|
||||||
|
SECRET_ENV_TOKENS = ("TOKEN", "SECRET", "PASSWORD", "CREDENTIAL", "AUTHORIZATION", "PRIVATE_KEY")
|
||||||
|
REQUEST_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{15,254}$")
|
||||||
|
SENSITIVE_VALUE_RE = re.compile(
|
||||||
|
r"(?i)(authorization\s*[:=]\s*bearer\s+\S+|authorization\s+bearer\s+\S+|"
|
||||||
|
r"(?:password|passwd|secret|token|api[_-]?key)\s*[:=]\s*\S+)"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class IsolatedExecutorError(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def sha256(data: bytes) -> str:
|
||||||
|
return hashlib.sha256(data).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def canonical(value: object) -> bytes:
|
||||||
|
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def safe_relative(value: object) -> str:
|
||||||
|
if not isinstance(value, str) or not value or value.startswith("/"):
|
||||||
|
raise IsolatedExecutorError("PATH_INVALID")
|
||||||
|
path = Path(value)
|
||||||
|
if ".." in path.parts or "." in path.parts or any(part in {"", ".git"} for part in path.parts):
|
||||||
|
raise IsolatedExecutorError("PATH_INVALID")
|
||||||
|
return path.as_posix()
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_under(root: Path, relative: str) -> Path:
|
||||||
|
root = root.resolve()
|
||||||
|
path = (root / safe_relative(relative)).resolve(strict=False)
|
||||||
|
if path == root or root not in path.parents:
|
||||||
|
raise IsolatedExecutorError("PATH_ESCAPE")
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def _credential_exposure(directory: Path) -> str | None:
|
||||||
|
"""Return an exposure code; EACCES is the intended credential isolation proof."""
|
||||||
|
try:
|
||||||
|
if not directory.exists():
|
||||||
|
return None
|
||||||
|
if directory.is_symlink():
|
||||||
|
return "CREDENTIAL_MOUNT_SYMLINK"
|
||||||
|
if os.access(directory, os.R_OK):
|
||||||
|
return "CREDENTIAL_MOUNT_ACCESSIBLE"
|
||||||
|
if directory.is_dir():
|
||||||
|
for child in directory.iterdir():
|
||||||
|
if child.is_file() and os.access(child, os.R_OK):
|
||||||
|
return "CREDENTIAL_FILE_ACCESSIBLE"
|
||||||
|
except PermissionError:
|
||||||
|
return None
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def assert_uncredentialed() -> None:
|
||||||
|
credentials = Path("/run/credentials")
|
||||||
|
candidates = {credentials}
|
||||||
|
credentials_directory = Path(os.environ.get("CREDENTIALS_DIRECTORY", "/run/credentials"))
|
||||||
|
candidates.add(credentials_directory)
|
||||||
|
leaked = sorted(
|
||||||
|
key for key, value in os.environ.items()
|
||||||
|
if value
|
||||||
|
and (any(token in key.upper() for token in SECRET_ENV_TOKENS) or SENSITIVE_VALUE_RE.search(str(value)))
|
||||||
|
)
|
||||||
|
if leaked:
|
||||||
|
raise IsolatedExecutorError("CREDENTIAL_ENV_EXPOSED:" + ",".join(leaked))
|
||||||
|
for directory in candidates:
|
||||||
|
exposure = _credential_exposure(directory)
|
||||||
|
if exposure is not None:
|
||||||
|
raise IsolatedExecutorError(exposure)
|
||||||
|
# In the intended PrivateNetwork namespace host loopback services are unreachable.
|
||||||
|
try:
|
||||||
|
probe = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||||
|
except OSError as exc:
|
||||||
|
if exc.errno in {errno.EACCES, errno.EPERM, errno.EAFNOSUPPORT}:
|
||||||
|
return
|
||||||
|
raise IsolatedExecutorError("NETWORK_ISOLATION_PROBE_FAILED") from exc
|
||||||
|
probe.settimeout(0.15)
|
||||||
|
try:
|
||||||
|
if probe.connect_ex(("127.0.0.1", 18892)) == 0:
|
||||||
|
raise IsolatedExecutorError("GOVERNED_BRIDGE_REACHABLE")
|
||||||
|
finally:
|
||||||
|
probe.close()
|
||||||
|
|
||||||
|
|
||||||
|
def extract_source(archive: Path, root: Path) -> None:
|
||||||
|
if archive.is_symlink() or not archive.is_file() or archive.stat().st_size > MAX_ARCHIVE_BYTES:
|
||||||
|
raise IsolatedExecutorError("SOURCE_ARCHIVE_INVALID")
|
||||||
|
with tarfile.open(archive, "r:gz") as tf:
|
||||||
|
for member in tf.getmembers():
|
||||||
|
path = Path(member.name)
|
||||||
|
if path.is_absolute() or ".." in path.parts or ".git" in path.parts:
|
||||||
|
raise IsolatedExecutorError("SOURCE_ARCHIVE_PATH_INVALID")
|
||||||
|
if member.issym() or member.islnk():
|
||||||
|
raise IsolatedExecutorError("SOURCE_ARCHIVE_LINK_DENY")
|
||||||
|
tf.extractall(root, filter="data")
|
||||||
|
|
||||||
|
|
||||||
|
def run_command(item: Mapping[str, Any], root: Path) -> dict[str, Any]:
|
||||||
|
argv = item.get("argv")
|
||||||
|
if not isinstance(argv, Sequence) or isinstance(argv, (str, bytes)) or not argv:
|
||||||
|
raise IsolatedExecutorError("COMMAND_INVALID")
|
||||||
|
argv = [str(part) for part in argv]
|
||||||
|
if argv[0] not in ALLOWED_PROGRAMS:
|
||||||
|
raise IsolatedExecutorError("COMMAND_PROGRAM_DENY")
|
||||||
|
if argv[0] == "/usr/bin/python3" and "-m" in argv:
|
||||||
|
index = argv.index("-m")
|
||||||
|
if index + 1 >= len(argv) or argv[index + 1] not in {"pytest", "py_compile"}:
|
||||||
|
raise IsolatedExecutorError("PYTHON_MODULE_DENY")
|
||||||
|
if argv[0] == "/usr/bin/git" and any(
|
||||||
|
token in {"push", "update-ref", "commit", "merge", "rebase", "reset", "clean", "branch"}
|
||||||
|
for token in argv[1:]
|
||||||
|
):
|
||||||
|
raise IsolatedExecutorError("GIT_MUTATION_DENY")
|
||||||
|
timeout = int(item.get("timeout_seconds") or 300)
|
||||||
|
if not 1 <= timeout <= 1800:
|
||||||
|
raise IsolatedExecutorError("COMMAND_TIMEOUT_INVALID")
|
||||||
|
cwd = root if item.get("cwd", ".") == "." else resolve_under(root, str(item.get("cwd")))
|
||||||
|
started = time.monotonic()
|
||||||
|
completed = subprocess.run(
|
||||||
|
argv,
|
||||||
|
cwd=cwd,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=False,
|
||||||
|
timeout=timeout,
|
||||||
|
env={"PATH": "/usr/bin:/bin", "LC_ALL": "C.UTF-8"},
|
||||||
|
)
|
||||||
|
result = {
|
||||||
|
"label": str(item.get("label") or "command"),
|
||||||
|
"exit_code": completed.returncode,
|
||||||
|
"stdout_sha256": sha256(completed.stdout.encode("utf-8", "replace")),
|
||||||
|
"stderr_sha256": sha256(completed.stderr.encode("utf-8", "replace")),
|
||||||
|
"duration_ms": int((time.monotonic() - started) * 1000),
|
||||||
|
}
|
||||||
|
if completed.returncode != 0:
|
||||||
|
raise IsolatedExecutorError(
|
||||||
|
"COMMAND_FAILED:" + result["label"] + ":" + str(completed.returncode)
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def apply_changes(request: Mapping[str, Any], root: Path) -> list[str]:
|
||||||
|
changes = request.get("change_set")
|
||||||
|
if not isinstance(changes, list) or not changes:
|
||||||
|
raise IsolatedExecutorError("CHANGE_SET_INVALID")
|
||||||
|
changed: list[str] = []
|
||||||
|
for change in changes:
|
||||||
|
if not isinstance(change, Mapping):
|
||||||
|
raise IsolatedExecutorError("CHANGE_INVALID")
|
||||||
|
relative = safe_relative(change.get("path"))
|
||||||
|
path = resolve_under(root, relative)
|
||||||
|
current = path.read_bytes() if path.is_file() else None
|
||||||
|
expected = change.get("expected_sha256")
|
||||||
|
if expected is not None and (current is None or sha256(current) != expected):
|
||||||
|
raise IsolatedExecutorError("SOURCE_DRIFT:" + relative)
|
||||||
|
operation = change.get("operation")
|
||||||
|
if operation == "write_file":
|
||||||
|
content = change.get("content")
|
||||||
|
if not isinstance(content, str):
|
||||||
|
raise IsolatedExecutorError("WRITE_CONTENT_INVALID")
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text(content, encoding="utf-8")
|
||||||
|
elif operation == "replace_text":
|
||||||
|
if current is None:
|
||||||
|
raise IsolatedExecutorError("REPLACE_SOURCE_MISSING")
|
||||||
|
old = str(change.get("old"))
|
||||||
|
new = str(change.get("new"))
|
||||||
|
count = int(change.get("count", 1))
|
||||||
|
text = current.decode("utf-8")
|
||||||
|
if text.count(old) != count:
|
||||||
|
raise IsolatedExecutorError("REPLACE_COUNT_INVALID:" + relative)
|
||||||
|
path.write_text(text.replace(old, new), encoding="utf-8")
|
||||||
|
elif operation == "delete_file":
|
||||||
|
if path.exists():
|
||||||
|
path.unlink()
|
||||||
|
else:
|
||||||
|
raise IsolatedExecutorError("CHANGE_OPERATION_INVALID")
|
||||||
|
changed.append(relative)
|
||||||
|
return sorted(set(changed))
|
||||||
|
|
||||||
|
|
||||||
|
def changed_bundle(root: Path, changed: list[str]) -> list[dict[str, Any]]:
|
||||||
|
files = []
|
||||||
|
for relative in changed:
|
||||||
|
path = resolve_under(root, relative)
|
||||||
|
if path.exists():
|
||||||
|
data = path.read_bytes()
|
||||||
|
if len(data) > MAX_FILE_BYTES:
|
||||||
|
raise IsolatedExecutorError("CHANGED_FILE_TOO_LARGE:" + relative)
|
||||||
|
files.append({
|
||||||
|
"path": relative,
|
||||||
|
"deleted": False,
|
||||||
|
"sha256": sha256(data),
|
||||||
|
"content_b64": base64.b64encode(data).decode("ascii"),
|
||||||
|
})
|
||||||
|
else:
|
||||||
|
files.append({"path": relative, "deleted": True, "sha256": None, "content_b64": None})
|
||||||
|
return files
|
||||||
|
|
||||||
|
|
||||||
|
def execute_request(request_path: Path, result_path: Path) -> None:
|
||||||
|
assert_uncredentialed()
|
||||||
|
request = json.loads(request_path.read_text(encoding="utf-8"))
|
||||||
|
if request.get("schema") != "aios.isolated-execution.request.v1":
|
||||||
|
raise IsolatedExecutorError("REQUEST_SCHEMA_INVALID")
|
||||||
|
request_id = request.get("request_id")
|
||||||
|
if not isinstance(request_id, str) or not REQUEST_ID_RE.fullmatch(request_id):
|
||||||
|
raise IsolatedExecutorError("REQUEST_ID_INVALID")
|
||||||
|
source_archive = Path(str(request.get("source_archive") or ""))
|
||||||
|
if not source_archive.is_absolute():
|
||||||
|
raise IsolatedExecutorError("SOURCE_ARCHIVE_PATH_INVALID")
|
||||||
|
expected_archive_sha = str(request.get("source_archive_sha256") or "")
|
||||||
|
if sha256(source_archive.read_bytes()) != expected_archive_sha:
|
||||||
|
raise IsolatedExecutorError("SOURCE_ARCHIVE_SHA_MISMATCH")
|
||||||
|
with tempfile.TemporaryDirectory(prefix="aios-isolated-executor-") as temporary:
|
||||||
|
root = Path(temporary) / "workspace"
|
||||||
|
root.mkdir()
|
||||||
|
extract_source(source_archive, root)
|
||||||
|
changed = apply_changes(request, root)
|
||||||
|
commands = [
|
||||||
|
run_command(item, root)
|
||||||
|
for item in request.get("verification", [])
|
||||||
|
if isinstance(item, Mapping)
|
||||||
|
]
|
||||||
|
result = {
|
||||||
|
"schema": "aios.isolated-execution.result.v1",
|
||||||
|
"request_id": request_id,
|
||||||
|
"request_sha256": sha256(canonical(request)),
|
||||||
|
"changed_files": changed_bundle(root, changed),
|
||||||
|
"verification": commands,
|
||||||
|
"credential_isolation": {
|
||||||
|
"credential_env_absent": True,
|
||||||
|
"credential_mount_inaccessible": True,
|
||||||
|
"governed_bridge_unreachable": True,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
result["result_sha256"] = sha256(canonical(result))
|
||||||
|
temporary_result = result_path.with_suffix(".tmp")
|
||||||
|
temporary_result.write_text(json.dumps(result, sort_keys=True, separators=(",", ":")), encoding="utf-8")
|
||||||
|
os.replace(temporary_result, result_path)
|
||||||
|
|
||||||
|
|
||||||
|
def serve(root: Path) -> None:
|
||||||
|
assert_uncredentialed()
|
||||||
|
requests = root / "requests"
|
||||||
|
results = root / "results"
|
||||||
|
failed = root / "failed"
|
||||||
|
for path in (requests, results, failed):
|
||||||
|
path.mkdir(parents=True, exist_ok=True)
|
||||||
|
while True:
|
||||||
|
for request in sorted(requests.glob("*.json")):
|
||||||
|
if request.is_symlink():
|
||||||
|
continue
|
||||||
|
result = results / request.name
|
||||||
|
try:
|
||||||
|
execute_request(request, result)
|
||||||
|
request.unlink()
|
||||||
|
except Exception as exc:
|
||||||
|
(failed / request.name).write_text(
|
||||||
|
json.dumps({"error": str(exc) or type(exc).__name__}, sort_keys=True),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
request.unlink(missing_ok=True)
|
||||||
|
time.sleep(0.5)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
import argparse
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--root", default="/var/lib/aios-online-agent-executor")
|
||||||
|
parser.add_argument("--request")
|
||||||
|
parser.add_argument("--result")
|
||||||
|
args = parser.parse_args()
|
||||||
|
if args.request and args.result:
|
||||||
|
execute_request(Path(args.request), Path(args.result))
|
||||||
|
else:
|
||||||
|
serve(Path(args.root))
|
||||||
@@ -3,6 +3,15 @@ from __future__ import annotations
|
|||||||
import hashlib, hmac, json, os, re, subprocess, sys, time, urllib.parse, urllib.request, uuid
|
import hashlib, hmac, json, os, re, subprocess, sys, time, urllib.parse, urllib.request, uuid
|
||||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from aios_caller_admission_projection import (
|
||||||
|
CALLER,
|
||||||
|
SCOPE,
|
||||||
|
GOVERNED_TASK_SCOPE,
|
||||||
|
PROJECTED_SCOPES,
|
||||||
|
CallerProjectionClient,
|
||||||
|
CallerProjectionError,
|
||||||
|
)
|
||||||
from baseline_authority import BaselineAuthority, BaselineAuthorityError
|
from baseline_authority import BaselineAuthority, BaselineAuthorityError
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
@@ -89,6 +98,11 @@ SILENT_RELEASE_PUBLIC_KEYS = Path(os.getenv(
|
|||||||
'AIOS_SILENT_RELEASE_PUBLIC_KEYS',
|
'AIOS_SILENT_RELEASE_PUBLIC_KEYS',
|
||||||
'/etc/aios-gitops/silent-release-owner-public-keys.json',
|
'/etc/aios-gitops/silent-release-owner-public-keys.json',
|
||||||
))
|
))
|
||||||
|
CALLER_PROJECTION_SOCKET = Path(os.getenv(
|
||||||
|
'AIOS_OWNER_AUTH_BROKER_SOCKET',
|
||||||
|
'/run/aios-owner-authorization-broker/broker.sock',
|
||||||
|
))
|
||||||
|
CALLER_PROJECTION_CLIENT = CallerProjectionClient(CALLER_PROJECTION_SOCKET)
|
||||||
monitor = ActionMonitor()
|
monitor = ActionMonitor()
|
||||||
|
|
||||||
BASE_MODELS = [
|
BASE_MODELS = [
|
||||||
@@ -112,6 +126,9 @@ READ_ONLY_GET_PATHS = frozenset({
|
|||||||
'/admin/ai-evolution/api/data',
|
'/admin/ai-evolution/api/data',
|
||||||
'/api/release-consistency', '/api/tag-publication-status',
|
'/api/release-consistency', '/api/tag-publication-status',
|
||||||
'/api/release-acceptance',
|
'/api/release-acceptance',
|
||||||
|
'/api/governed-write-capabilities', '/api/control-plane-snapshot',
|
||||||
|
'/api/caller-capabilities',
|
||||||
|
'/api/online-agent-task', '/api/online-agent-task-events',
|
||||||
'/getReleaseCandidate', '/getReleasePlan', '/getReleaseAcceptance',
|
'/getReleaseCandidate', '/getReleasePlan', '/getReleaseAcceptance',
|
||||||
'/getApprovalEnvelope', '/getReleaseSaga', '/getCapabilityStatus',
|
'/getApprovalEnvelope', '/getReleaseSaga', '/getCapabilityStatus',
|
||||||
'/getRecoveryStatus',
|
'/getRecoveryStatus',
|
||||||
@@ -124,6 +141,25 @@ SILENT_RELEASE_POST_PATHS = frozenset({
|
|||||||
'/startDeployment',
|
'/startDeployment',
|
||||||
})
|
})
|
||||||
|
|
||||||
|
GOVERNED_EXECUTION_CAPABILITIES = frozenset({
|
||||||
|
"governed.write.branch",
|
||||||
|
"governed.write.file",
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def requires_governed_execution(value: Mapping[str, Any]) -> bool:
|
||||||
|
capabilities = value.get("required_capabilities")
|
||||||
|
if capabilities is None:
|
||||||
|
return False
|
||||||
|
if not isinstance(capabilities, list):
|
||||||
|
return True
|
||||||
|
return bool(
|
||||||
|
GOVERNED_EXECUTION_CAPABILITIES.intersection(
|
||||||
|
{str(item) for item in capabilities}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def reject_non_read_http(handler, method: str, path: str) -> bool:
|
def reject_non_read_http(handler, method: str, path: str) -> bool:
|
||||||
if method == 'GET':
|
if method == 'GET':
|
||||||
return False
|
return False
|
||||||
@@ -407,14 +443,113 @@ class H(BaseHTTPRequestHandler):
|
|||||||
except Exception: return {'_raw':raw}
|
except Exception: return {'_raw':raw}
|
||||||
def actor(self):
|
def actor(self):
|
||||||
auth=self.headers.get('Authorization',''); tok=auth[7:].strip() if auth.startswith('Bearer ') else self.headers.get('X-AIOS-Token','')
|
auth=self.headers.get('Authorization',''); tok=auth[7:].strip() if auth.startswith('Bearer ') else self.headers.get('X-AIOS-Token','')
|
||||||
rec=authenticate(tok)
|
try:
|
||||||
if rec: return {'ok':True,'account':safe(rec.get('account') or rec.get('name') or 'token-user'),'role':rec.get('role','user'),'scopes':set(rec.get('scopes') or [])}
|
rec=authenticate(tok)
|
||||||
return {'ok':False,'account':'anonymous','role':'anonymous','scopes':set()}
|
except Exception:
|
||||||
def need(self, scope):
|
rec=None
|
||||||
a=self.actor()
|
if not isinstance(rec, dict) or not rec:
|
||||||
if not a['ok']: self.sendj({'error':'unauthorized','hint':'Bearer token required'},401); return None
|
return {
|
||||||
if '*' not in a['scopes'] and scope not in a['scopes']: self.sendj({'error':'forbidden','required_scope':scope,'account':a['account']},403); return None
|
'ok': False,
|
||||||
|
'account': 'anonymous',
|
||||||
|
'role': 'anonymous',
|
||||||
|
'scopes': set(),
|
||||||
|
'static_scopes': set(),
|
||||||
|
'projection': None,
|
||||||
|
'credential_loaded': False,
|
||||||
|
}
|
||||||
|
scopes = set(rec.get('scopes') or [])
|
||||||
|
return {
|
||||||
|
'ok': True,
|
||||||
|
'account': safe(rec.get('account') or rec.get('name') or 'token-user'),
|
||||||
|
'role': rec.get('role', 'user'),
|
||||||
|
'scopes': scopes,
|
||||||
|
'static_scopes': set(scopes),
|
||||||
|
'projection': None,
|
||||||
|
'credential_loaded': False,
|
||||||
|
}
|
||||||
|
def need(self, scope, *, projection_request=None):
|
||||||
|
a = self.actor()
|
||||||
|
if not a['ok']:
|
||||||
|
self.sendj({'error': 'unauthorized', 'hint': 'Bearer token required'}, 401)
|
||||||
|
return None
|
||||||
|
if a['account'] == CALLER and scope in PROJECTED_SCOPES:
|
||||||
|
try:
|
||||||
|
result = CALLER_PROJECTION_CLIENT.rehydrate(
|
||||||
|
a['account'],
|
||||||
|
scope,
|
||||||
|
projection_request,
|
||||||
|
)
|
||||||
|
except CallerProjectionError as exc:
|
||||||
|
self.sendj(
|
||||||
|
{
|
||||||
|
'error': 'caller_projection_denied',
|
||||||
|
'projection_state': 'DENIED',
|
||||||
|
'detail': exc.code,
|
||||||
|
'account': a['account'],
|
||||||
|
},
|
||||||
|
exc.status,
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
projection = result.public
|
||||||
|
a['scopes'].update(projection.get('effective_scopes') or [])
|
||||||
|
a['projection'] = projection
|
||||||
|
a['credential_loaded'] = result.credential_loaded
|
||||||
|
if '*' not in a['scopes'] and scope not in a['scopes']:
|
||||||
|
self.sendj(
|
||||||
|
{
|
||||||
|
'error': 'forbidden',
|
||||||
|
'required_scope': scope,
|
||||||
|
'account': a['account'],
|
||||||
|
},
|
||||||
|
403,
|
||||||
|
)
|
||||||
|
return None
|
||||||
return a
|
return a
|
||||||
|
def caller_capabilities(self):
|
||||||
|
actor = self.actor()
|
||||||
|
if not actor['ok']:
|
||||||
|
return self.sendj(
|
||||||
|
{'error': 'unauthorized', 'hint': 'Bearer token required'},
|
||||||
|
401,
|
||||||
|
)
|
||||||
|
effective = set(actor['static_scopes'])
|
||||||
|
projection_state = 'NOT_REQUIRED'
|
||||||
|
credential_state = 'NOT_REQUIRED'
|
||||||
|
rehydration_state = 'NOT_APPLICABLE'
|
||||||
|
admitted = True
|
||||||
|
projection_error = None
|
||||||
|
if actor['account'] == CALLER:
|
||||||
|
try:
|
||||||
|
result = CALLER_PROJECTION_CLIENT.rehydrate(
|
||||||
|
actor['account'],
|
||||||
|
GOVERNED_TASK_SCOPE,
|
||||||
|
{},
|
||||||
|
)
|
||||||
|
except CallerProjectionError as exc:
|
||||||
|
admitted = False
|
||||||
|
projection_state = 'DENIED'
|
||||||
|
credential_state = 'NOT_REHYDRATED'
|
||||||
|
rehydration_state = 'FAILED'
|
||||||
|
projection_error = exc.code
|
||||||
|
else:
|
||||||
|
effective.update(result.public.get('effective_scopes') or [])
|
||||||
|
projection_state = result.public['projection_state']
|
||||||
|
credential_state = result.public['credential_state']
|
||||||
|
rehydration_state = result.public['rehydration_state']
|
||||||
|
return self.sendj(
|
||||||
|
{
|
||||||
|
'schema': 'aios.caller-capability-introspection.v1',
|
||||||
|
'caller_identity': actor['account'],
|
||||||
|
'admitted': admitted,
|
||||||
|
'effective_scopes': sorted(effective),
|
||||||
|
'projection_state': projection_state,
|
||||||
|
'credential_state': credential_state,
|
||||||
|
'rehydration_state': rehydration_state,
|
||||||
|
'projection_error': projection_error,
|
||||||
|
'secret_exposed': False,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
def bridge_release(self, path, data):
|
def bridge_release(self, path, data):
|
||||||
route = {
|
route = {
|
||||||
'/createTagApprovalRequest': '/v2/tag/approval-request',
|
'/createTagApprovalRequest': '/v2/tag/approval-request',
|
||||||
@@ -446,24 +581,183 @@ class H(BaseHTTPRequestHandler):
|
|||||||
return self.sendj(body, exc.code)
|
return self.sendj(body, exc.code)
|
||||||
except (urllib.error.URLError, TimeoutError) as exc:
|
except (urllib.error.URLError, TimeoutError) as exc:
|
||||||
return self.sendj({'error': 'GOVERNED_BRIDGE_UNAVAILABLE', 'detail': type(exc).__name__}, 503)
|
return self.sendj({'error': 'GOVERNED_BRIDGE_UNAVAILABLE', 'detail': type(exc).__name__}, 503)
|
||||||
def do_POST(self):
|
def bridge_v2_json(self, route, data=None, method='POST'):
|
||||||
path = urllib.parse.urlparse(self.path).path
|
base = os.getenv('AIOS_BRIDGE_V2_URL', 'http://127.0.0.1:18892').rstrip('/')
|
||||||
if path in {
|
payload = None if data is None else json.dumps(
|
||||||
'/createTagApprovalRequest', '/requestOrCreateTag', '/getTagPublicationStatus',
|
data, sort_keys=True, separators=(',', ':')
|
||||||
'/buildProductionArtifact', '/executeApprovedProductionRelease',
|
).encode('utf-8')
|
||||||
'/getReleaseConsistency', '/runReleaseAcceptance',
|
request = urllib.request.Request(
|
||||||
}:
|
base + route,
|
||||||
actor = self.need('tag:create')
|
data=payload,
|
||||||
if not actor:
|
headers={'Content-Type': 'application/json'} if payload is not None else {},
|
||||||
return
|
method=method,
|
||||||
return self.bridge_release(path, self.body())
|
)
|
||||||
if reject_non_read_http(self, 'POST', path):
|
try:
|
||||||
return
|
with urllib.request.urlopen(request, timeout=30) as response:
|
||||||
return self.sendj({'error': 'not_found', 'path': path}, 404)
|
result = json.loads(response.read().decode('utf-8'))
|
||||||
|
return self.sendj(result, response.status)
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
try:
|
||||||
|
body = json.loads(exc.read().decode('utf-8'))
|
||||||
|
except Exception:
|
||||||
|
body = {'error': 'bridge_http_error'}
|
||||||
|
return self.sendj(body, exc.code)
|
||||||
|
except (urllib.error.URLError, TimeoutError) as exc:
|
||||||
|
return self.sendj(
|
||||||
|
{'error': 'GOVERNED_BRIDGE_UNAVAILABLE', 'detail': type(exc).__name__},
|
||||||
|
503,
|
||||||
|
)
|
||||||
|
|
||||||
|
def control_plane_snapshot(self):
|
||||||
|
"""Build a non-sensitive snapshot locally when Bridge v2 has no snapshot HTTP route."""
|
||||||
|
base = os.getenv('AIOS_BRIDGE_V2_URL', 'http://127.0.0.1:18892').rstrip('/')
|
||||||
|
|
||||||
|
def bridge_get(route):
|
||||||
|
request = urllib.request.Request(
|
||||||
|
base + route,
|
||||||
|
headers={'Accept': 'application/json'},
|
||||||
|
method='GET',
|
||||||
|
)
|
||||||
|
with urllib.request.urlopen(request, timeout=15) as response:
|
||||||
|
if response.status != 200:
|
||||||
|
raise RuntimeError('bridge_read_http_' + str(response.status))
|
||||||
|
value = json.loads(response.read().decode('utf-8'))
|
||||||
|
if not isinstance(value, dict):
|
||||||
|
raise RuntimeError('bridge_read_not_object')
|
||||||
|
return value
|
||||||
|
|
||||||
|
try:
|
||||||
|
runtime = bridge_get('/v2/runtime')
|
||||||
|
capabilities = bridge_get('/v2/capabilities')
|
||||||
|
repository = repository_snapshot(
|
||||||
|
git_dir=AUTHORITATIVE_GIT_DIR,
|
||||||
|
repository_identity=REPOSITORY_IDENTITY,
|
||||||
|
)
|
||||||
|
if not repository.get('repository_available'):
|
||||||
|
return self.sendj({
|
||||||
|
'error': 'AUTHORITATIVE_REPOSITORY_UNAVAILABLE',
|
||||||
|
'query_error': repository.get('query_error'),
|
||||||
|
}, 503)
|
||||||
|
baseline = dict(BASELINE_AUTHORITY.read())
|
||||||
|
except BaselineAuthorityError as exc:
|
||||||
|
return self.sendj({
|
||||||
|
'error': 'RECONCILIATION_REQUIRED',
|
||||||
|
'detail': str(exc),
|
||||||
|
}, 503)
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
return self.sendj({
|
||||||
|
'error': 'GOVERNED_BRIDGE_UNAVAILABLE',
|
||||||
|
'detail': 'HTTP_' + str(exc.code),
|
||||||
|
}, 503)
|
||||||
|
except (urllib.error.URLError, TimeoutError, OSError, ValueError, RuntimeError, json.JSONDecodeError) as exc:
|
||||||
|
return self.sendj({
|
||||||
|
'error': 'GOVERNED_BRIDGE_UNAVAILABLE',
|
||||||
|
'detail': type(exc).__name__,
|
||||||
|
}, 503)
|
||||||
|
|
||||||
|
legacy = capabilities.get('legacy_writes_enabled')
|
||||||
|
if legacy is None:
|
||||||
|
legacy = capabilities.get('legacy_global_writes_enabled')
|
||||||
|
payload = {
|
||||||
|
'schema': 'aios.control-plane-snapshot.aggregate.v1',
|
||||||
|
'snapshot_source': 'model-api-local-aggregate',
|
||||||
|
'version': {
|
||||||
|
'target_release': runtime.get('control_plane_release_version'),
|
||||||
|
},
|
||||||
|
'runtime': {
|
||||||
|
'bridge_schema_version': runtime.get('bridge_schema_version'),
|
||||||
|
'control_plane_release_version': runtime.get('control_plane_release_version'),
|
||||||
|
'runtime_release_commit': runtime.get('runtime_release_commit'),
|
||||||
|
'runtime_path': runtime.get('runtime_path'),
|
||||||
|
'artifact_sha256': runtime.get('artifact_sha256'),
|
||||||
|
'drift_status': runtime.get('drift_status'),
|
||||||
|
'deployment_state': runtime.get('deployment_state'),
|
||||||
|
},
|
||||||
|
'repository': {
|
||||||
|
'identity': runtime.get('repository_identity') or runtime.get('repository') or REPOSITORY_IDENTITY,
|
||||||
|
'authority_git_dir': runtime.get('authoritative_git_dir') or repository.get('authoritative_git_dir'),
|
||||||
|
'authority_main_oid': repository.get('authoritative_head_oid'),
|
||||||
|
},
|
||||||
|
'production': {
|
||||||
|
'baseline_version': baseline.get('current_baseline'),
|
||||||
|
'target_oid': baseline.get('target_oid'),
|
||||||
|
'deployment_state': baseline.get('status'),
|
||||||
|
'baseline_generation': baseline.get('generation'),
|
||||||
|
'baseline_sha256': baseline.get('authority_sha256'),
|
||||||
|
},
|
||||||
|
'task': {},
|
||||||
|
'capabilities': {
|
||||||
|
'legacy_writes_enabled': legacy,
|
||||||
|
'governed_writes_enabled': capabilities.get('governed_writes_enabled'),
|
||||||
|
'force_push': capabilities.get('force_push'),
|
||||||
|
'main_write': capabilities.get('main_write'),
|
||||||
|
'tag_write': capabilities.get('tag_write'),
|
||||||
|
'baseline_write': capabilities.get('baseline_write'),
|
||||||
|
'production_write': capabilities.get('production_write'),
|
||||||
|
},
|
||||||
|
'credential_broker': {
|
||||||
|
'status': 'SERVER_SIDE_ONLY',
|
||||||
|
'provider': 'BridgeV2RuntimeCredentialProjection',
|
||||||
|
},
|
||||||
|
'secret_material_exposed': False,
|
||||||
|
}
|
||||||
|
return self.sendj(payload)
|
||||||
|
|
||||||
|
def agent_task_control(self, method, route, data=None):
|
||||||
|
token_path = Path(os.getenv(
|
||||||
|
'AIOS_AGENT_TASK_CONTROL_TOKEN_FILE',
|
||||||
|
'/run/credentials/aios-gitops-model-api-v2r2c1-staging.service/agent_task_control_token',
|
||||||
|
))
|
||||||
|
if token_path.is_symlink() or not token_path.is_file():
|
||||||
|
return self.sendj({'error': 'AGENT_TASK_CONTROL_CREDENTIAL_UNAVAILABLE'}, 503)
|
||||||
|
token = token_path.read_text(encoding='utf-8').strip()
|
||||||
|
if len(token) < 32:
|
||||||
|
return self.sendj({'error': 'AGENT_TASK_CONTROL_CREDENTIAL_INVALID'}, 503)
|
||||||
|
base = os.getenv('AIOS_AGENT_TASK_API_URL', 'http://127.0.0.1:18940').rstrip('/')
|
||||||
|
payload = None if data is None else json.dumps(
|
||||||
|
data, sort_keys=True, separators=(',', ':')
|
||||||
|
).encode('utf-8')
|
||||||
|
headers = {'Authorization': 'Bearer ' + token, 'Accept': 'application/json'}
|
||||||
|
if payload is not None:
|
||||||
|
headers['Content-Type'] = 'application/json'
|
||||||
|
request = urllib.request.Request(
|
||||||
|
base + route, data=payload, headers=headers, method=method
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(request, timeout=30) as response:
|
||||||
|
result = json.loads(response.read().decode('utf-8'))
|
||||||
|
return self.sendj(result, response.status)
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
try:
|
||||||
|
body = json.loads(exc.read().decode('utf-8'))
|
||||||
|
except Exception:
|
||||||
|
body = {'error': 'agent_task_http_error'}
|
||||||
|
return self.sendj(body, exc.code)
|
||||||
|
except (urllib.error.URLError, TimeoutError) as exc:
|
||||||
|
return self.sendj(
|
||||||
|
{'error': 'AGENT_TASK_API_UNAVAILABLE', 'detail': type(exc).__name__},
|
||||||
|
503,
|
||||||
|
)
|
||||||
|
|
||||||
def do_GET(self):
|
def do_GET(self):
|
||||||
u=urllib.parse.urlparse(self.path); path=u.path.rstrip('/') if u.path != '/' else u.path
|
u=urllib.parse.urlparse(self.path); path=u.path.rstrip('/') if u.path != '/' else u.path
|
||||||
if path not in READ_ONLY_GET_PATHS: return self.sendj({'error':'not_found','path':path},404)
|
if path not in READ_ONLY_GET_PATHS: return self.sendj({'error':'not_found','path':path},404)
|
||||||
q=urllib.parse.parse_qs(u.query)
|
q=urllib.parse.parse_qs(u.query)
|
||||||
|
if path == '/api/governed-write-capabilities':
|
||||||
|
return self.bridge_v2_json('/v2/capabilities', method='GET')
|
||||||
|
if path == '/api/control-plane-snapshot':
|
||||||
|
return self.control_plane_snapshot()
|
||||||
|
if path == '/api/caller-capabilities':
|
||||||
|
return self.caller_capabilities()
|
||||||
|
if path in ('/api/online-agent-task', '/api/online-agent-task-events'):
|
||||||
|
actor = self.need('agent-task:control')
|
||||||
|
if not actor:
|
||||||
|
return
|
||||||
|
task_id = (q.get('task_id') or [''])[0]
|
||||||
|
if not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._:-]{2,179}', task_id):
|
||||||
|
return self.sendj({'error': 'task_id_invalid'}, 400)
|
||||||
|
suffix = '/events' if path.endswith('-events') else ''
|
||||||
|
return self.agent_task_control('GET', '/v1/tasks/' + task_id + suffix)
|
||||||
try:
|
try:
|
||||||
if path in ('/health','/api/health'):
|
if path in ('/health','/api/health'):
|
||||||
return self.sendj({
|
return self.sendj({
|
||||||
@@ -700,12 +994,68 @@ class H(BaseHTTPRequestHandler):
|
|||||||
return self.sendj({'error':'not_found','path':path},404)
|
return self.sendj({'error':'not_found','path':path},404)
|
||||||
def do_POST(self):
|
def do_POST(self):
|
||||||
path=urllib.parse.urlparse(self.path).path.rstrip('/')
|
path=urllib.parse.urlparse(self.path).path.rstrip('/')
|
||||||
|
governed_routes = {
|
||||||
|
'/createGovernedTask': ('/v2/governed-task', 'branch:create'),
|
||||||
|
'/acquireWriterLease': ('/v2/writer-lease', 'file:write'),
|
||||||
|
'/commitFileGoverned': ('/v2/commit-file', 'file:write'),
|
||||||
|
'/commitFilesGoverned': ('/v2/commit-files', 'file:write'),
|
||||||
|
'/getGitOpsJob': ('/v2/job', 'file:write'),
|
||||||
|
'/resumeGitOpsJob': ('/v2/job/resume', 'file:write'),
|
||||||
|
'/reconcileGitOpsJob': ('/v2/job/reconcile', 'file:write'),
|
||||||
|
'/createTagApprovalRequest': ('/v2/tag/approval-request', 'tag:create'),
|
||||||
|
'/requestOrCreateTag': ('/v2/tag/publish', 'tag:create'),
|
||||||
|
'/getTagPublicationStatus': ('/v2/tag/status', 'tag:create'),
|
||||||
|
'/buildProductionArtifact': ('/v2/release/artifact', 'tag:create'),
|
||||||
|
'/executeApprovedProductionRelease': ('/v2/release/execute', 'tag:create'),
|
||||||
|
'/getReleaseConsistency': ('/v2/release/consistency', 'tag:create'),
|
||||||
|
'/runReleaseAcceptance': ('/v2/release/acceptance', 'tag:create'),
|
||||||
|
}
|
||||||
|
if path in governed_routes:
|
||||||
|
route, scope = governed_routes[path]
|
||||||
|
if path == '/createGovernedTask' and self.actor()['account'] == CALLER:
|
||||||
|
scope = GOVERNED_TASK_SCOPE
|
||||||
|
actor = self.need(scope)
|
||||||
|
if not actor:
|
||||||
|
return
|
||||||
|
return self.bridge_v2_json(route, self.body(), method='POST')
|
||||||
|
if path in {
|
||||||
|
'/createOnlineAgentTask', '/registerOnlineAgent',
|
||||||
|
'/verifyOnlineAgentTask', '/cancelOnlineAgentTask',
|
||||||
|
}:
|
||||||
|
actor = self.need('agent-task:control')
|
||||||
|
if not actor:
|
||||||
|
return
|
||||||
|
data = self.body()
|
||||||
|
if path == '/createOnlineAgentTask':
|
||||||
|
if requires_governed_execution(data):
|
||||||
|
return self.sendj(
|
||||||
|
{
|
||||||
|
'error': 'governed_execution_binding_required',
|
||||||
|
'hint': 'use createGovernedTask',
|
||||||
|
},
|
||||||
|
409,
|
||||||
|
)
|
||||||
|
return self.agent_task_control('POST', '/v1/tasks', data)
|
||||||
|
if path == '/registerOnlineAgent':
|
||||||
|
return self.agent_task_control('POST', '/v1/agents/register', data)
|
||||||
|
task_id = str(data.pop('task_id', '') or '')
|
||||||
|
if not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._:-]{2,179}', task_id):
|
||||||
|
return self.sendj({'error': 'task_id_invalid'}, 400)
|
||||||
|
suffix = '/verify' if path == '/verifyOnlineAgentTask' else '/cancel'
|
||||||
|
return self.agent_task_control('POST', '/v1/tasks/' + task_id + suffix, data)
|
||||||
if reject_non_read_http(self, 'POST', path): return
|
if reject_non_read_http(self, 'POST', path): return
|
||||||
data=self.body()
|
data=self.body()
|
||||||
try:
|
try:
|
||||||
if path == '/createReleaseCandidate':
|
if path == '/createReleaseCandidate':
|
||||||
a=self.need('release:candidate')
|
a = self.need(
|
||||||
if not a: return
|
'release:candidate',
|
||||||
|
projection_request={
|
||||||
|
'source_commit': str(data.get('source_commit') or ''),
|
||||||
|
'version': str(data.get('version') or ''),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if not a:
|
||||||
|
return
|
||||||
return self.sendj(
|
return self.sendj(
|
||||||
silent_release_service().create_release_candidate(data)
|
silent_release_service().create_release_candidate(data)
|
||||||
)
|
)
|
||||||
@@ -996,7 +1346,7 @@ def api_spec():
|
|||||||
return {
|
return {
|
||||||
'version': 'V2R2C1-IMMUTABLE-STAGING',
|
'version': 'V2R2C1-IMMUTABLE-STAGING',
|
||||||
'openapi': '/api/openapi.yaml',
|
'openapi': '/api/openapi.yaml',
|
||||||
'read': ['GET /api/base','GET /api/ver','GET /api/diff','GET /api/patch','GET /api/model','GET /api/run','GET /getReleaseCandidate','GET /getApprovalEnvelope','GET /getReleaseSaga','GET /admin/ai-evolution'],
|
'read': ['GET /api/caller-capabilities','GET /api/base','GET /api/ver','GET /api/diff','GET /api/patch','GET /api/model','GET /api/run','GET /getReleaseCandidate','GET /getApprovalEnvelope','GET /getReleaseSaga','GET /admin/ai-evolution'],
|
||||||
'write': ['POST /api/branch','POST /api/file','POST /api/commit','POST /api/patch','POST /api/tag','POST /api/rollback','POST /api/model','PUT /api/model-priority','POST /api/nl','POST /createReleaseCandidate','POST /submitOwnerApproval','POST /startDeployment'],
|
'write': ['POST /api/branch','POST /api/file','POST /api/commit','POST /api/patch','POST /api/tag','POST /api/rollback','POST /api/model','PUT /api/model-priority','POST /api/nl','POST /createReleaseCandidate','POST /submitOwnerApproval','POST /startDeployment'],
|
||||||
'policy': {
|
'policy': {
|
||||||
'bearer_required_for_write': True,
|
'bearer_required_for_write': True,
|
||||||
@@ -1011,6 +1361,9 @@ def api_spec():
|
|||||||
'silent_release_shadow_default': True,
|
'silent_release_shadow_default': True,
|
||||||
'runtime_mode': 'IMMUTABLE_STAGING',
|
'runtime_mode': 'IMMUTABLE_STAGING',
|
||||||
'governed_release_routing': 'bridge-v2',
|
'governed_release_routing': 'bridge-v2',
|
||||||
|
'caller_admission_projection': 'durable-owner-policy-server-side-ack',
|
||||||
|
'caller_projection_scope': SCOPE,
|
||||||
|
'caller_projection_caller': CALLER,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,232 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Root-owned Model Governance configuration reconciler.
|
||||||
|
|
||||||
|
The config contains provider/model/policy metadata only. Raw credentials are rejected by
|
||||||
|
aios_model_governance and never stored in the governance database.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Mapping
|
||||||
|
|
||||||
|
from aios_model_governance import (
|
||||||
|
ModelGovernanceError,
|
||||||
|
ModelGovernanceStore,
|
||||||
|
ModelRecord,
|
||||||
|
ProviderRecord,
|
||||||
|
ScopePolicy,
|
||||||
|
)
|
||||||
|
|
||||||
|
SCHEMA = "aios.model-governance.bootstrap.v1"
|
||||||
|
MAX_CONFIG_BYTES = 512 * 1024
|
||||||
|
|
||||||
|
|
||||||
|
def canonical(value: object) -> bytes:
|
||||||
|
return json.dumps(
|
||||||
|
value, ensure_ascii=False, sort_keys=True, separators=(",", ":")
|
||||||
|
).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def fail(code: str) -> None:
|
||||||
|
raise SystemExit(code)
|
||||||
|
|
||||||
|
|
||||||
|
def _load(path: Path) -> dict[str, Any]:
|
||||||
|
if path.is_symlink() or not path.is_file() or path.stat().st_size > MAX_CONFIG_BYTES:
|
||||||
|
fail("MODEL_GOVERNANCE_CONFIG_INVALID")
|
||||||
|
try:
|
||||||
|
value = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
fail("MODEL_GOVERNANCE_CONFIG_INVALID")
|
||||||
|
if not isinstance(value, Mapping) or value.get("schema") != SCHEMA:
|
||||||
|
fail("MODEL_GOVERNANCE_CONFIG_SCHEMA_INVALID")
|
||||||
|
return dict(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _provider_desired(value: Mapping[str, Any]) -> ProviderRecord:
|
||||||
|
return ProviderRecord(
|
||||||
|
provider_id=str(value.get("provider_id") or ""),
|
||||||
|
credential_ref=value.get("credential_ref"),
|
||||||
|
enabled=bool(value.get("enabled", True)),
|
||||||
|
failure_domain=str(value.get("failure_domain") or "default"),
|
||||||
|
max_failures=int(value.get("max_failures") or 3),
|
||||||
|
cooldown_seconds=int(value.get("cooldown_seconds") or 60),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _model_desired(value: Mapping[str, Any]) -> ModelRecord:
|
||||||
|
return ModelRecord(
|
||||||
|
model_id=str(value.get("model_id") or ""),
|
||||||
|
provider_id=str(value.get("provider_id") or ""),
|
||||||
|
version=str(value.get("version") or ""),
|
||||||
|
capabilities=tuple(value.get("capabilities") or ()),
|
||||||
|
input_cost_micros_per_1k=int(value.get("input_cost_micros_per_1k") or 0),
|
||||||
|
output_cost_micros_per_1k=int(value.get("output_cost_micros_per_1k") or 0),
|
||||||
|
max_input_tokens=int(value.get("max_input_tokens") or 0),
|
||||||
|
max_output_tokens=int(value.get("max_output_tokens") or 0),
|
||||||
|
evaluation_state=str(value.get("evaluation_state") or "APPROVED"),
|
||||||
|
evaluation_sha256=str(value.get("evaluation_sha256") or ""),
|
||||||
|
enabled=bool(value.get("enabled", True)),
|
||||||
|
priority=int(value.get("priority") or 100),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _policy_desired(value: Mapping[str, Any]) -> ScopePolicy:
|
||||||
|
return ScopePolicy(
|
||||||
|
scope_type=str(value.get("scope_type") or ""),
|
||||||
|
tenant_id=value.get("tenant_id"),
|
||||||
|
project_id=value.get("project_id"),
|
||||||
|
allowed_models=tuple(value.get("allowed_models") or ()),
|
||||||
|
denied_providers=tuple(value.get("denied_providers") or ()),
|
||||||
|
required_capabilities=tuple(value.get("required_capabilities") or ()),
|
||||||
|
pinned_model_id=value.get("pinned_model_id"),
|
||||||
|
pinned_model_version=value.get("pinned_model_version"),
|
||||||
|
max_input_tokens_per_request=int(value.get("max_input_tokens_per_request") or 0),
|
||||||
|
max_output_tokens_per_request=int(value.get("max_output_tokens_per_request") or 0),
|
||||||
|
daily_token_budget=int(value.get("daily_token_budget") or 0),
|
||||||
|
daily_cost_budget_micros=int(value.get("daily_cost_budget_micros") or 0),
|
||||||
|
fallback_limit=int(value.get("fallback_limit") or 0),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _row_dict(row: object) -> dict[str, Any] | None:
|
||||||
|
return dict(row) if row is not None else None
|
||||||
|
|
||||||
|
|
||||||
|
def _provider_matches(row: Mapping[str, Any], desired: ProviderRecord) -> bool:
|
||||||
|
return (
|
||||||
|
row["credential_ref"] == desired.credential_ref
|
||||||
|
and bool(row["enabled"]) == desired.enabled
|
||||||
|
and row["failure_domain"] == desired.failure_domain
|
||||||
|
and int(row["max_failures"]) == desired.max_failures
|
||||||
|
and int(row["cooldown_seconds"]) == desired.cooldown_seconds
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _model_matches(row: Mapping[str, Any], desired: ModelRecord) -> bool:
|
||||||
|
return (
|
||||||
|
row["provider_id"] == desired.provider_id
|
||||||
|
and row["version"] == desired.version
|
||||||
|
and tuple(json.loads(row["capabilities_json"])) == tuple(sorted(set(desired.capabilities)))
|
||||||
|
and int(row["input_cost_micros_per_1k"]) == desired.input_cost_micros_per_1k
|
||||||
|
and int(row["output_cost_micros_per_1k"]) == desired.output_cost_micros_per_1k
|
||||||
|
and int(row["max_input_tokens"]) == desired.max_input_tokens
|
||||||
|
and int(row["max_output_tokens"]) == desired.max_output_tokens
|
||||||
|
and row["evaluation_state"] == desired.evaluation_state
|
||||||
|
and row["evaluation_sha256"] == desired.evaluation_sha256
|
||||||
|
and bool(row["enabled"]) == desired.enabled
|
||||||
|
and int(row["priority"]) == desired.priority
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _policy_matches(row: Mapping[str, Any], desired: ScopePolicy) -> bool:
|
||||||
|
return (
|
||||||
|
row["scope_type"] == desired.scope_type
|
||||||
|
and row["tenant_id"] == desired.tenant_id
|
||||||
|
and row["project_id"] == desired.project_id
|
||||||
|
and tuple(json.loads(row["allowed_models_json"])) == tuple(sorted(set(desired.allowed_models)))
|
||||||
|
and tuple(json.loads(row["denied_providers_json"])) == tuple(sorted(set(desired.denied_providers)))
|
||||||
|
and tuple(json.loads(row["required_capabilities_json"])) == tuple(sorted(set(desired.required_capabilities)))
|
||||||
|
and row["pinned_model_id"] == desired.pinned_model_id
|
||||||
|
and row["pinned_model_version"] == desired.pinned_model_version
|
||||||
|
and int(row["max_input_tokens_per_request"]) == desired.max_input_tokens_per_request
|
||||||
|
and int(row["max_output_tokens_per_request"]) == desired.max_output_tokens_per_request
|
||||||
|
and int(row["daily_token_budget"]) == desired.daily_token_budget
|
||||||
|
and int(row["daily_cost_budget_micros"]) == desired.daily_cost_budget_micros
|
||||||
|
and int(row["fallback_limit"]) == desired.fallback_limit
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def apply_config(db_path: Path, config_path: Path) -> dict[str, Any]:
|
||||||
|
config = _load(config_path)
|
||||||
|
config_sha = hashlib.sha256(canonical(config)).hexdigest()
|
||||||
|
providers = config.get("providers") or []
|
||||||
|
models = config.get("models") or []
|
||||||
|
policies = config.get("policies") or []
|
||||||
|
if not all(isinstance(value, list) for value in (providers, models, policies)):
|
||||||
|
fail("MODEL_GOVERNANCE_CONFIG_COLLECTION_INVALID")
|
||||||
|
|
||||||
|
store = ModelGovernanceStore(db_path)
|
||||||
|
changes = {"providers": 0, "models": 0, "policies": 0, "unchanged": 0}
|
||||||
|
try:
|
||||||
|
for item in providers:
|
||||||
|
if not isinstance(item, Mapping):
|
||||||
|
fail("MODEL_GOVERNANCE_PROVIDER_CONFIG_INVALID")
|
||||||
|
desired = _provider_desired(item)
|
||||||
|
row = store.conn.execute(
|
||||||
|
"SELECT * FROM model_governance_providers WHERE provider_id=?",
|
||||||
|
(desired.provider_id,),
|
||||||
|
).fetchone()
|
||||||
|
if row is not None and _provider_matches(row, desired):
|
||||||
|
changes["unchanged"] += 1
|
||||||
|
continue
|
||||||
|
expected = int(row["revision"]) if row is not None else 0
|
||||||
|
store.register_provider(desired, expected_revision=expected)
|
||||||
|
changes["providers"] += 1
|
||||||
|
|
||||||
|
for item in models:
|
||||||
|
if not isinstance(item, Mapping):
|
||||||
|
fail("MODEL_GOVERNANCE_MODEL_CONFIG_INVALID")
|
||||||
|
desired = _model_desired(item)
|
||||||
|
row = store.conn.execute(
|
||||||
|
"SELECT * FROM model_governance_models WHERE model_id=?",
|
||||||
|
(desired.model_id,),
|
||||||
|
).fetchone()
|
||||||
|
if row is not None and _model_matches(row, desired):
|
||||||
|
changes["unchanged"] += 1
|
||||||
|
continue
|
||||||
|
expected = int(row["revision"]) if row is not None else 0
|
||||||
|
store.register_model(desired, expected_revision=expected)
|
||||||
|
changes["models"] += 1
|
||||||
|
|
||||||
|
for item in policies:
|
||||||
|
if not isinstance(item, Mapping):
|
||||||
|
fail("MODEL_GOVERNANCE_POLICY_CONFIG_INVALID")
|
||||||
|
desired = _policy_desired(item)
|
||||||
|
row = store.conn.execute(
|
||||||
|
"SELECT * FROM model_governance_scope_policies WHERE scope_key=?",
|
||||||
|
(desired.scope_key,),
|
||||||
|
).fetchone()
|
||||||
|
if row is not None and _policy_matches(row, desired):
|
||||||
|
changes["unchanged"] += 1
|
||||||
|
continue
|
||||||
|
expected = int(row["revision"]) if row is not None else 0
|
||||||
|
store.set_scope_policy(desired, expected_revision=expected)
|
||||||
|
changes["policies"] += 1
|
||||||
|
|
||||||
|
if not store.verify_audit_chain():
|
||||||
|
fail("MODEL_GOVERNANCE_AUDIT_CHAIN_INVALID")
|
||||||
|
return {
|
||||||
|
"schema": "aios.model-governance.bootstrap-result.v1",
|
||||||
|
"database": str(db_path),
|
||||||
|
"config_sha256": config_sha,
|
||||||
|
"changes": changes,
|
||||||
|
"audit_chain_valid": True,
|
||||||
|
}
|
||||||
|
finally:
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--db", required=True)
|
||||||
|
parser.add_argument("--config", required=True)
|
||||||
|
parser.add_argument("--output")
|
||||||
|
args = parser.parse_args()
|
||||||
|
try:
|
||||||
|
result = apply_config(Path(args.db), Path(args.config))
|
||||||
|
except ModelGovernanceError as exc:
|
||||||
|
print(json.dumps({"status": "FAILED", "error": exc.code}, sort_keys=True))
|
||||||
|
return 20
|
||||||
|
data = json.dumps(result, indent=2, sort_keys=True) + "\n"
|
||||||
|
if args.output:
|
||||||
|
Path(args.output).write_text(data, encoding="utf-8")
|
||||||
|
print(data, end="")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,302 @@
|
|||||||
|
"""Durable checkpoint journal for the generic Online Agent worker.
|
||||||
|
|
||||||
|
The JSON task state remains a fast cache. This SQLite journal is the durable,
|
||||||
|
append-only evidence used for restart/resume/replay decisions.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sqlite3
|
||||||
|
from typing import Any, Mapping
|
||||||
|
|
||||||
|
PHASE_ORDER = {
|
||||||
|
"claimed": 10,
|
||||||
|
"workspace_prepared": 20,
|
||||||
|
"verification_passed": 30,
|
||||||
|
"agent_runtime_done": 40,
|
||||||
|
"source_capture_done": 40,
|
||||||
|
"commit_done": 40,
|
||||||
|
"reconciliation_required": 45,
|
||||||
|
"result_pending": 50,
|
||||||
|
"result_submitted": 60,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class CheckpointError(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def canonical(value: object) -> bytes:
|
||||||
|
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, default=str).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def sha256_json(value: object) -> str:
|
||||||
|
return hashlib.sha256(canonical(value)).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def now() -> str:
|
||||||
|
return datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class CheckpointBinding:
|
||||||
|
task_id: str
|
||||||
|
attempt: int
|
||||||
|
execution_digest: str
|
||||||
|
job_id: str
|
||||||
|
repository: str
|
||||||
|
base_oid: str
|
||||||
|
expected_branch_oid: str
|
||||||
|
target_branch: str
|
||||||
|
generation: int
|
||||||
|
|
||||||
|
def material(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"task_id": self.task_id,
|
||||||
|
"attempt": self.attempt,
|
||||||
|
"execution_digest": self.execution_digest,
|
||||||
|
"job_id": self.job_id,
|
||||||
|
"repository": self.repository,
|
||||||
|
"base_oid": self.base_oid,
|
||||||
|
"expected_branch_oid": self.expected_branch_oid,
|
||||||
|
"target_branch": self.target_branch,
|
||||||
|
"generation": self.generation,
|
||||||
|
}
|
||||||
|
|
||||||
|
@property
|
||||||
|
def binding_sha256(self) -> str:
|
||||||
|
return sha256_json(self.material())
|
||||||
|
|
||||||
|
|
||||||
|
class CheckpointStore:
|
||||||
|
def __init__(self, path: Path) -> None:
|
||||||
|
self.path = Path(path)
|
||||||
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with self._connect() as connection:
|
||||||
|
connection.executescript(
|
||||||
|
"""
|
||||||
|
CREATE TABLE IF NOT EXISTS checkpoints(
|
||||||
|
task_id TEXT NOT NULL,
|
||||||
|
attempt INTEGER NOT NULL,
|
||||||
|
sequence INTEGER NOT NULL,
|
||||||
|
phase TEXT NOT NULL,
|
||||||
|
phase_order INTEGER NOT NULL,
|
||||||
|
execution_digest TEXT NOT NULL,
|
||||||
|
job_id TEXT NOT NULL,
|
||||||
|
repository TEXT NOT NULL,
|
||||||
|
base_oid TEXT NOT NULL,
|
||||||
|
expected_branch_oid TEXT NOT NULL,
|
||||||
|
target_branch TEXT NOT NULL,
|
||||||
|
generation INTEGER NOT NULL,
|
||||||
|
binding_sha256 TEXT NOT NULL,
|
||||||
|
payload_json TEXT NOT NULL,
|
||||||
|
payload_sha256 TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL,
|
||||||
|
PRIMARY KEY(task_id, attempt, sequence)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_checkpoints_task_latest
|
||||||
|
ON checkpoints(task_id, attempt DESC, sequence DESC);
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_checkpoints_exact_replay
|
||||||
|
ON checkpoints(task_id, attempt, phase, payload_sha256);
|
||||||
|
CREATE TABLE IF NOT EXISTS checkpoint_heads(
|
||||||
|
task_id TEXT PRIMARY KEY,
|
||||||
|
attempt INTEGER NOT NULL,
|
||||||
|
sequence INTEGER NOT NULL,
|
||||||
|
phase TEXT NOT NULL,
|
||||||
|
phase_order INTEGER NOT NULL,
|
||||||
|
execution_digest TEXT NOT NULL,
|
||||||
|
binding_sha256 TEXT NOT NULL,
|
||||||
|
payload_sha256 TEXT NOT NULL,
|
||||||
|
updated_at TEXT NOT NULL
|
||||||
|
);
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
def _connect(self) -> sqlite3.Connection:
|
||||||
|
connection = sqlite3.connect(self.path, timeout=30.0)
|
||||||
|
connection.row_factory = sqlite3.Row
|
||||||
|
connection.execute("PRAGMA foreign_keys=ON")
|
||||||
|
connection.execute("PRAGMA journal_mode=WAL")
|
||||||
|
connection.execute("PRAGMA synchronous=FULL")
|
||||||
|
connection.execute("PRAGMA busy_timeout=30000")
|
||||||
|
return connection
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _phase_order(phase: str) -> int:
|
||||||
|
try:
|
||||||
|
return PHASE_ORDER[phase]
|
||||||
|
except KeyError as exc:
|
||||||
|
raise CheckpointError("CHECKPOINT_PHASE_INVALID") from exc
|
||||||
|
|
||||||
|
def append(
|
||||||
|
self,
|
||||||
|
binding: CheckpointBinding,
|
||||||
|
phase: str,
|
||||||
|
payload: Mapping[str, Any],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
order = self._phase_order(phase)
|
||||||
|
payload_value = dict(payload)
|
||||||
|
payload_sha = sha256_json(payload_value)
|
||||||
|
with self._connect() as connection:
|
||||||
|
connection.execute("BEGIN IMMEDIATE")
|
||||||
|
replay = connection.execute(
|
||||||
|
"""
|
||||||
|
SELECT * FROM checkpoints
|
||||||
|
WHERE task_id=? AND attempt=? AND phase=? AND payload_sha256=?
|
||||||
|
""",
|
||||||
|
(binding.task_id, binding.attempt, phase, payload_sha),
|
||||||
|
).fetchone()
|
||||||
|
if replay is not None:
|
||||||
|
connection.commit()
|
||||||
|
return dict(replay)
|
||||||
|
|
||||||
|
head = connection.execute(
|
||||||
|
"SELECT * FROM checkpoint_heads WHERE task_id=?",
|
||||||
|
(binding.task_id,),
|
||||||
|
).fetchone()
|
||||||
|
if head is not None:
|
||||||
|
head_attempt = int(head["attempt"])
|
||||||
|
head_order = int(head["phase_order"])
|
||||||
|
if binding.attempt < head_attempt:
|
||||||
|
raise CheckpointError("CHECKPOINT_ATTEMPT_REGRESSION")
|
||||||
|
if binding.attempt == head_attempt:
|
||||||
|
if head["execution_digest"] != binding.execution_digest:
|
||||||
|
raise CheckpointError("CHECKPOINT_EXECUTION_DIGEST_CONFLICT")
|
||||||
|
if head["binding_sha256"] != binding.binding_sha256:
|
||||||
|
raise CheckpointError("CHECKPOINT_SCOPE_MISMATCH_DENY")
|
||||||
|
if order < head_order and phase != "reconciliation_required":
|
||||||
|
raise CheckpointError("CHECKPOINT_PHASE_REGRESSION")
|
||||||
|
sequence = int(head["sequence"]) + 1
|
||||||
|
else:
|
||||||
|
sequence = 1
|
||||||
|
else:
|
||||||
|
sequence = 1
|
||||||
|
|
||||||
|
created_at = now()
|
||||||
|
connection.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO checkpoints(
|
||||||
|
task_id,attempt,sequence,phase,phase_order,execution_digest,job_id,
|
||||||
|
repository,base_oid,expected_branch_oid,target_branch,generation,
|
||||||
|
binding_sha256,payload_json,payload_sha256,created_at
|
||||||
|
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
binding.task_id,
|
||||||
|
binding.attempt,
|
||||||
|
sequence,
|
||||||
|
phase,
|
||||||
|
order,
|
||||||
|
binding.execution_digest,
|
||||||
|
binding.job_id,
|
||||||
|
binding.repository,
|
||||||
|
binding.base_oid,
|
||||||
|
binding.expected_branch_oid,
|
||||||
|
binding.target_branch,
|
||||||
|
binding.generation,
|
||||||
|
binding.binding_sha256,
|
||||||
|
canonical(payload_value).decode("utf-8"),
|
||||||
|
payload_sha,
|
||||||
|
created_at,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
connection.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO checkpoint_heads(
|
||||||
|
task_id,attempt,sequence,phase,phase_order,execution_digest,
|
||||||
|
binding_sha256,payload_sha256,updated_at
|
||||||
|
) VALUES (?,?,?,?,?,?,?,?,?)
|
||||||
|
ON CONFLICT(task_id) DO UPDATE SET
|
||||||
|
attempt=excluded.attempt,
|
||||||
|
sequence=excluded.sequence,
|
||||||
|
phase=excluded.phase,
|
||||||
|
phase_order=excluded.phase_order,
|
||||||
|
execution_digest=excluded.execution_digest,
|
||||||
|
binding_sha256=excluded.binding_sha256,
|
||||||
|
payload_sha256=excluded.payload_sha256,
|
||||||
|
updated_at=excluded.updated_at
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
binding.task_id,
|
||||||
|
binding.attempt,
|
||||||
|
sequence,
|
||||||
|
phase,
|
||||||
|
order,
|
||||||
|
binding.execution_digest,
|
||||||
|
binding.binding_sha256,
|
||||||
|
payload_sha,
|
||||||
|
created_at,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
connection.commit()
|
||||||
|
row = connection.execute(
|
||||||
|
"SELECT * FROM checkpoints WHERE task_id=? AND attempt=? AND sequence=?",
|
||||||
|
(binding.task_id, binding.attempt, sequence),
|
||||||
|
).fetchone()
|
||||||
|
return dict(row)
|
||||||
|
|
||||||
|
def latest(self, task_id: str) -> dict[str, Any] | None:
|
||||||
|
with self._connect() as connection:
|
||||||
|
row = connection.execute(
|
||||||
|
"""
|
||||||
|
SELECT * FROM checkpoints
|
||||||
|
WHERE task_id=?
|
||||||
|
ORDER BY attempt DESC, sequence DESC
|
||||||
|
LIMIT 1
|
||||||
|
""",
|
||||||
|
(task_id,),
|
||||||
|
).fetchone()
|
||||||
|
if row is None:
|
||||||
|
return None
|
||||||
|
value = dict(row)
|
||||||
|
value["payload"] = json.loads(value.pop("payload_json"))
|
||||||
|
return value
|
||||||
|
|
||||||
|
def latest_for_attempt(self, task_id: str, attempt: int) -> dict[str, Any] | None:
|
||||||
|
with self._connect() as connection:
|
||||||
|
row = connection.execute(
|
||||||
|
"""
|
||||||
|
SELECT * FROM checkpoints
|
||||||
|
WHERE task_id=? AND attempt=?
|
||||||
|
ORDER BY sequence DESC
|
||||||
|
LIMIT 1
|
||||||
|
""",
|
||||||
|
(task_id, attempt),
|
||||||
|
).fetchone()
|
||||||
|
if row is None:
|
||||||
|
return None
|
||||||
|
value = dict(row)
|
||||||
|
value["payload"] = json.loads(value.pop("payload_json"))
|
||||||
|
return value
|
||||||
|
|
||||||
|
def verify_latest_binding(self, binding: CheckpointBinding) -> dict[str, Any] | None:
|
||||||
|
value = self.latest(binding.task_id)
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
if int(value["attempt"]) > binding.attempt:
|
||||||
|
raise CheckpointError("CHECKPOINT_FUTURE_ATTEMPT")
|
||||||
|
# A previous attempt may be reused only when the immutable execution/scope binding matches.
|
||||||
|
if value["execution_digest"] != binding.execution_digest:
|
||||||
|
return None
|
||||||
|
expected = CheckpointBinding(
|
||||||
|
task_id=value["task_id"],
|
||||||
|
attempt=int(value["attempt"]),
|
||||||
|
execution_digest=value["execution_digest"],
|
||||||
|
job_id=value["job_id"],
|
||||||
|
repository=value["repository"],
|
||||||
|
base_oid=value["base_oid"],
|
||||||
|
expected_branch_oid=value["expected_branch_oid"],
|
||||||
|
target_branch=value["target_branch"],
|
||||||
|
generation=int(value["generation"]),
|
||||||
|
)
|
||||||
|
current_scope = binding.material()
|
||||||
|
historical_scope = expected.material()
|
||||||
|
current_scope.pop("attempt", None)
|
||||||
|
historical_scope.pop("attempt", None)
|
||||||
|
if sha256_json(current_scope) != sha256_json(historical_scope):
|
||||||
|
raise CheckpointError("CHECKPOINT_SCOPE_MISMATCH_DENY")
|
||||||
|
return value
|
||||||
@@ -0,0 +1,463 @@
|
|||||||
|
"""Bounded read-only repository interface map for governed Online Agent analysis.
|
||||||
|
|
||||||
|
The compact map intentionally returns symbols and short source excerpts rather than a
|
||||||
|
source archive. It is designed for control-plane consumers that cannot safely ingest a
|
||||||
|
multi-megabyte repository snapshot.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import ast
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
from typing import Any, Iterable, Mapping, Sequence
|
||||||
|
|
||||||
|
|
||||||
|
SCHEMA = "aios.engineering.interface-map.v1"
|
||||||
|
MAX_RESULT_BYTES = 120 * 1024
|
||||||
|
TARGET_RESULT_BYTES = 108 * 1024
|
||||||
|
MAX_MODULES = 40
|
||||||
|
MAX_SYMBOLS = 80
|
||||||
|
MAX_TESTS = 40
|
||||||
|
MAX_EXCERPTS = 25
|
||||||
|
MAX_EXCERPT_CHARS = 1200
|
||||||
|
MAX_FILE_BYTES = 2 * 1024 * 1024
|
||||||
|
|
||||||
|
ROOTS = (
|
||||||
|
"active-runtime/app",
|
||||||
|
"runtime/agent-os",
|
||||||
|
"tests",
|
||||||
|
"db/migrations",
|
||||||
|
"migrations",
|
||||||
|
"deploy/systemd",
|
||||||
|
"docs/task-plane",
|
||||||
|
"docs/superpowers/specs",
|
||||||
|
)
|
||||||
|
SUFFIXES = {".py", ".sql", ".service", ".md", ".json", ".yaml", ".yml"}
|
||||||
|
|
||||||
|
DEFAULT_TERMS = {
|
||||||
|
"agent", "worker", "fleet", "scheduler", "scheduling", "task", "claim", "lease",
|
||||||
|
"heartbeat", "capacity", "concurrency", "priority", "fairness", "queue", "retry",
|
||||||
|
"backoff", "circuit", "health", "liveness", "scope", "tenant", "project", "trace",
|
||||||
|
"message", "checkpoint", "recovery", "idempotency", "dispatch", "assignment",
|
||||||
|
}
|
||||||
|
FLEET_CONTRACT_TERMS: dict[str, tuple[str, ...]] = {
|
||||||
|
"FLEET_AGENT_REGISTRATION": ("agent", "register", "worker", "capability", "version"),
|
||||||
|
"FLEET_CAPACITY_SCHEDULING": ("capacity", "slot", "concurrency", "scheduler", "queue"),
|
||||||
|
"FLEET_FAIRNESS": ("fair", "priority", "queue", "tenant", "project", "starvation"),
|
||||||
|
"FLEET_FAULT_ISOLATION": ("failure", "health", "liveness", "timeout", "circuit", "poison"),
|
||||||
|
"FLEET_WORKER_REASSIGNMENT": ("reassign", "assignment", "claim", "lease", "retry", "worker"),
|
||||||
|
"FLEET_SCOPE_ISOLATION": ("tenant", "project", "scope", "conversation", "task", "run"),
|
||||||
|
"FLEET_IDEMPOTENT_DISPATCH": ("idempot", "dispatch", "claim", "lease", "result", "attempt"),
|
||||||
|
"FLEET_FAILURE_RECOVERY": ("checkpoint", "resume", "recover", "retry", "cleanup", "receipt"),
|
||||||
|
}
|
||||||
|
|
||||||
|
MODEL_GOVERNANCE_CONTRACT_TERMS: dict[str, tuple[str, ...]] = {
|
||||||
|
"MODEL_PROVIDER_REGISTRY": ("model", "provider", "registry", "endpoint", "capability"),
|
||||||
|
"MODEL_SELECTION_POLICY": ("model", "select", "route", "policy", "capability"),
|
||||||
|
"MODEL_VERSION_PINNING": ("model", "version", "pin", "revision", "snapshot"),
|
||||||
|
"MODEL_BUDGET_ACCOUNTING": ("budget", "token", "cost", "quota", "usage"),
|
||||||
|
"MODEL_FALLBACK_POLICY": ("fallback", "retry", "route", "provider", "model"),
|
||||||
|
"MODEL_SCOPE_ISOLATION": ("tenant", "project", "scope", "policy", "model"),
|
||||||
|
"MODEL_CREDENTIAL_BOUNDARY": ("credential", "secret", "provider", "tool", "token"),
|
||||||
|
"MODEL_TRACE_AUDIT": ("trace", "audit", "model", "provider", "request"),
|
||||||
|
"MODEL_EVALUATION_GATE": ("eval", "evaluation", "approval", "quality", "gate"),
|
||||||
|
"MODEL_ROUTING_RESILIENCE": ("route", "retry", "fallback", "timeout", "health"),
|
||||||
|
}
|
||||||
|
|
||||||
|
PROFILE_CLASSIFICATION = {
|
||||||
|
"fleet": "P2_INTERFACE_MAP_CAPTURED",
|
||||||
|
"model_governance": "MODEL_GOVERNANCE_INTERFACE_MAP_CAPTURED",
|
||||||
|
}
|
||||||
|
|
||||||
|
MODEL_PROFILE_MARKERS = {
|
||||||
|
"model", "provider", "llm", "inference", "routing", "router", "budget",
|
||||||
|
"token", "cost", "quota", "fallback", "evaluation", "eval", "approval",
|
||||||
|
"version", "pinning",
|
||||||
|
}
|
||||||
|
SENSITIVE_RE = re.compile(
|
||||||
|
r"(?i)(authorization\s*:\s*bearer\s+\S+|"
|
||||||
|
r"(?:password|passwd|secret|token|api[_-]?key)\s*[:=]\s*['\"]?[^'\"\s,;]+)"
|
||||||
|
)
|
||||||
|
SQL_TABLE_RE = re.compile(
|
||||||
|
r"(?is)\bcreate\s+table\s+(?:if\s+not\s+exists\s+)?[\"`[]?([A-Za-z_][A-Za-z0-9_.-]*)"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class CompactInterfaceMapError(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _sha(data: bytes) -> str:
|
||||||
|
return hashlib.sha256(data).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _bounded_excerpt(text: str, start_line: int, end_line: int) -> str:
|
||||||
|
lines = text.splitlines()
|
||||||
|
start = max(0, start_line - 2)
|
||||||
|
end = min(len(lines), max(end_line + 1, start + 1))
|
||||||
|
excerpt = "\n".join(lines[start:end])
|
||||||
|
excerpt = SENSITIVE_RE.sub("<redacted-sensitive-source-pattern>", excerpt)
|
||||||
|
if len(excerpt) > MAX_EXCERPT_CHARS:
|
||||||
|
excerpt = excerpt[: MAX_EXCERPT_CHARS - 1] + "…"
|
||||||
|
return excerpt
|
||||||
|
|
||||||
|
|
||||||
|
def _signal_terms(signals: Sequence[str]) -> set[str]:
|
||||||
|
terms = set(DEFAULT_TERMS)
|
||||||
|
for signal in signals:
|
||||||
|
if not isinstance(signal, str):
|
||||||
|
continue
|
||||||
|
terms.update(
|
||||||
|
token.lower()
|
||||||
|
for token in re.findall(r"[A-Za-z][A-Za-z0-9_.-]{2,63}", signal)
|
||||||
|
)
|
||||||
|
return set(sorted(terms)[:96])
|
||||||
|
|
||||||
|
|
||||||
|
def _profile(source_signals: Sequence[str]) -> str:
|
||||||
|
tokens: set[str] = set()
|
||||||
|
for signal in source_signals:
|
||||||
|
if not isinstance(signal, str):
|
||||||
|
continue
|
||||||
|
tokens.update(
|
||||||
|
token.lower()
|
||||||
|
for token in re.findall(r"[A-Za-z][A-Za-z0-9_.-]{2,63}", signal)
|
||||||
|
)
|
||||||
|
model_score = len(tokens & MODEL_PROFILE_MARKERS)
|
||||||
|
return "model_governance" if model_score >= 2 else "fleet"
|
||||||
|
|
||||||
|
|
||||||
|
def _contract_terms(profile: str) -> Mapping[str, tuple[str, ...]]:
|
||||||
|
if profile == "model_governance":
|
||||||
|
return MODEL_GOVERNANCE_CONTRACT_TERMS
|
||||||
|
if profile == "fleet":
|
||||||
|
return FLEET_CONTRACT_TERMS
|
||||||
|
raise CompactInterfaceMapError("INTERFACE_MAP_PROFILE_INVALID")
|
||||||
|
|
||||||
|
|
||||||
|
def _candidate_files(root: Path, terms: set[str]) -> list[dict[str, Any]]:
|
||||||
|
root = root.resolve()
|
||||||
|
candidates: list[dict[str, Any]] = []
|
||||||
|
for relative_root in ROOTS:
|
||||||
|
directory = (root / relative_root).resolve(strict=False)
|
||||||
|
if directory == root or root not in directory.parents:
|
||||||
|
raise CompactInterfaceMapError("INTERFACE_MAP_SCOPE_INVALID")
|
||||||
|
if not directory.is_dir() or directory.is_symlink():
|
||||||
|
continue
|
||||||
|
for path in sorted(directory.rglob("*")):
|
||||||
|
if path.is_symlink() or not path.is_file() or path.suffix.lower() not in SUFFIXES:
|
||||||
|
continue
|
||||||
|
resolved = path.resolve(strict=True)
|
||||||
|
if root not in resolved.parents:
|
||||||
|
raise CompactInterfaceMapError("INTERFACE_MAP_PATH_ESCAPE")
|
||||||
|
data = path.read_bytes()
|
||||||
|
if len(data) > MAX_FILE_BYTES:
|
||||||
|
continue
|
||||||
|
relative = path.relative_to(root).as_posix()
|
||||||
|
text = data.decode("utf-8", "ignore")
|
||||||
|
searchable = (relative + "\n" + text[:384000]).lower()
|
||||||
|
matched = sorted(term for term in terms if term in searchable)
|
||||||
|
if not matched:
|
||||||
|
continue
|
||||||
|
path_bonus = sum(3 for term in terms if term in relative.lower())
|
||||||
|
test_bonus = 3 if relative.startswith("tests/") else 0
|
||||||
|
score = min(250, len(matched) + path_bonus + test_bonus)
|
||||||
|
candidates.append(
|
||||||
|
{
|
||||||
|
"path": relative,
|
||||||
|
"sha256": _sha(data),
|
||||||
|
"bytes": len(data),
|
||||||
|
"score": score,
|
||||||
|
"text": text,
|
||||||
|
"suffix": path.suffix.lower(),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
candidates.sort(key=lambda item: (-item["score"], item["path"]))
|
||||||
|
return candidates
|
||||||
|
|
||||||
|
|
||||||
|
def _python_symbols(item: Mapping[str, Any], terms: set[str]) -> list[dict[str, Any]]:
|
||||||
|
try:
|
||||||
|
tree = ast.parse(str(item["text"]), filename=str(item["path"]))
|
||||||
|
except SyntaxError:
|
||||||
|
return []
|
||||||
|
result: list[dict[str, Any]] = []
|
||||||
|
|
||||||
|
class Visitor(ast.NodeVisitor):
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.stack: list[str] = []
|
||||||
|
|
||||||
|
def _record(self, node: ast.AST, kind: str, name: str) -> None:
|
||||||
|
qualified = ".".join([*self.stack, name]) if self.stack else name
|
||||||
|
haystack = f"{item['path']} {qualified}".lower()
|
||||||
|
score = sum(3 if term in haystack else 0 for term in terms)
|
||||||
|
if name.startswith("test_"):
|
||||||
|
score += 12
|
||||||
|
result.append(
|
||||||
|
{
|
||||||
|
"path": item["path"],
|
||||||
|
"symbol": qualified,
|
||||||
|
"kind": kind,
|
||||||
|
"line": int(getattr(node, "lineno", 1)),
|
||||||
|
"end_line": int(getattr(node, "end_lineno", getattr(node, "lineno", 1))),
|
||||||
|
"score": score + int(item["score"]),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
def visit_ClassDef(self, node: ast.ClassDef) -> Any:
|
||||||
|
self._record(node, "class", node.name)
|
||||||
|
self.stack.append(node.name)
|
||||||
|
self.generic_visit(node)
|
||||||
|
self.stack.pop()
|
||||||
|
|
||||||
|
def visit_FunctionDef(self, node: ast.FunctionDef) -> Any:
|
||||||
|
self._record(node, "function", node.name)
|
||||||
|
self.stack.append(node.name)
|
||||||
|
self.generic_visit(node)
|
||||||
|
self.stack.pop()
|
||||||
|
|
||||||
|
def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> Any:
|
||||||
|
self._record(node, "async_function", node.name)
|
||||||
|
self.stack.append(node.name)
|
||||||
|
self.generic_visit(node)
|
||||||
|
self.stack.pop()
|
||||||
|
|
||||||
|
Visitor().visit(tree)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _surface_ref(symbol: Mapping[str, Any]) -> str:
|
||||||
|
return f"{symbol['path']}::{symbol['symbol']}"
|
||||||
|
|
||||||
|
|
||||||
|
def _contract_map(
|
||||||
|
symbols: Sequence[Mapping[str, Any]],
|
||||||
|
modules: Sequence[Mapping[str, Any]],
|
||||||
|
contract_terms: Mapping[str, tuple[str, ...]],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
result: dict[str, Any] = {}
|
||||||
|
module_paths = [str(item["path"]) for item in modules]
|
||||||
|
for contract, terms in contract_terms.items():
|
||||||
|
scored: list[tuple[int, str]] = []
|
||||||
|
for symbol in symbols:
|
||||||
|
haystack = _surface_ref(symbol).lower()
|
||||||
|
score = sum(1 for term in terms if term in haystack)
|
||||||
|
if score:
|
||||||
|
scored.append((score, _surface_ref(symbol)))
|
||||||
|
scored.sort(key=lambda item: (-item[0], item[1]))
|
||||||
|
surfaces = [value for _, value in scored[:6]]
|
||||||
|
if not surfaces:
|
||||||
|
for path in module_paths:
|
||||||
|
score = sum(1 for term in terms if term in path.lower())
|
||||||
|
if score:
|
||||||
|
surfaces.append(path)
|
||||||
|
if len(surfaces) == 6:
|
||||||
|
break
|
||||||
|
result[contract] = {
|
||||||
|
"status": "PARTIAL" if surfaces else "GAP",
|
||||||
|
"surfaces": surfaces,
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _gap_evidence(
|
||||||
|
symbols: Sequence[Mapping[str, Any]],
|
||||||
|
profile: str,
|
||||||
|
) -> tuple[list[str], list[str], list[str]]:
|
||||||
|
haystack = "\n".join(_surface_ref(item).lower() for item in symbols)
|
||||||
|
|
||||||
|
if profile == "model_governance":
|
||||||
|
invariants = [
|
||||||
|
"Online Task claim/heartbeat remains the authoritative worker ownership boundary.",
|
||||||
|
"Provider/model credentials remain outside uncredentialed planner/executor processes.",
|
||||||
|
"P0-A tenant/project scope and P0-C trace/audit lineage must constrain model routing.",
|
||||||
|
"P1 retry/checkpoint semantics remain authoritative for model-call execution recovery.",
|
||||||
|
]
|
||||||
|
failure_windows: list[str] = []
|
||||||
|
if "provider" not in haystack and "model" not in haystack:
|
||||||
|
failure_windows.append("No first-class model/provider registry surface detected.")
|
||||||
|
if "budget" not in haystack and "cost" not in haystack and "quota" not in haystack:
|
||||||
|
failure_windows.append("No explicit durable model budget/cost/quota accounting surface detected.")
|
||||||
|
if "fallback" not in haystack and "route" not in haystack:
|
||||||
|
failure_windows.append("No explicit model fallback/routing policy surface detected.")
|
||||||
|
if "eval" not in haystack and "approval" not in haystack:
|
||||||
|
failure_windows.append("No explicit evaluation/approval gate surface detected for model changes.")
|
||||||
|
if "version" not in haystack and "pin" not in haystack:
|
||||||
|
failure_windows.append("No explicit model-version pinning surface detected.")
|
||||||
|
desired = (
|
||||||
|
("trace", "P0-C trace/audit lineage"),
|
||||||
|
("scope", "P0-A tenant/project scope isolation"),
|
||||||
|
("credential", "P0-C credential governance"),
|
||||||
|
("checkpoint", "P1 durable execution recovery"),
|
||||||
|
("message", "P0-B MessageEnvelope/handoff"),
|
||||||
|
("retry", "P1 retry/idempotency"),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
invariants = [
|
||||||
|
"Online Task claim/heartbeat remains the authoritative worker ownership boundary.",
|
||||||
|
"Governed mutation remains separated from read-only analysis and requires WriterLease.",
|
||||||
|
"P1 attempt-aware result idempotency and durable checkpoint recovery are reusable fleet primitives.",
|
||||||
|
"Scope/trace/message lineage must remain attached when work is scheduled or reassigned.",
|
||||||
|
]
|
||||||
|
failure_windows = []
|
||||||
|
if "scheduler" not in haystack:
|
||||||
|
failure_windows.append("No first-class durable fleet scheduler symbol detected; worker-local polling may still be the scheduling mechanism.")
|
||||||
|
if "capacity" not in haystack and "slot" not in haystack:
|
||||||
|
failure_windows.append("No explicit durable worker-capacity/slot primitive detected.")
|
||||||
|
if "fair" not in haystack and "starvation" not in haystack:
|
||||||
|
failure_windows.append("No explicit fairness/starvation policy symbol detected.")
|
||||||
|
if "circuit" not in haystack:
|
||||||
|
failure_windows.append("No explicit circuit-breaker primitive detected for repeated worker/task failures.")
|
||||||
|
if "reassign" not in haystack:
|
||||||
|
failure_windows.append("No explicit worker-reassignment primitive detected beyond lease expiry/retry semantics.")
|
||||||
|
desired = (
|
||||||
|
("checkpoint", "P1 durable checkpoint/resume"),
|
||||||
|
("message", "P0-B MessageEnvelope/handoff"),
|
||||||
|
("trace", "P0-C trace correlation"),
|
||||||
|
("scope", "P0-A scope isolation"),
|
||||||
|
("lease", "claim/WriterLease ownership"),
|
||||||
|
("heartbeat", "worker liveness"),
|
||||||
|
("idempot", "retry-safe dispatch/result"),
|
||||||
|
)
|
||||||
|
|
||||||
|
reuse_points: list[str] = []
|
||||||
|
for token, label in desired:
|
||||||
|
match = next((item for item in symbols if token in _surface_ref(item).lower()), None)
|
||||||
|
if match is not None:
|
||||||
|
reuse_points.append(f"{label}: {_surface_ref(match)}")
|
||||||
|
return invariants, failure_windows, reuse_points
|
||||||
|
|
||||||
|
|
||||||
|
def _trim_to_budget(value: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
def size() -> int:
|
||||||
|
return len(json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8"))
|
||||||
|
|
||||||
|
while size() > TARGET_RESULT_BYTES and value["bounded_excerpts"]:
|
||||||
|
value["bounded_excerpts"].pop()
|
||||||
|
while size() > TARGET_RESULT_BYTES and len(value["implementation_map"]["functions"]) > 20:
|
||||||
|
value["implementation_map"]["functions"].pop()
|
||||||
|
while size() > TARGET_RESULT_BYTES and len(value["implementation_map"]["classes"]) > 15:
|
||||||
|
value["implementation_map"]["classes"].pop()
|
||||||
|
while size() > TARGET_RESULT_BYTES and len(value["implementation_map"]["tests"]) > 15:
|
||||||
|
value["implementation_map"]["tests"].pop()
|
||||||
|
while size() > TARGET_RESULT_BYTES and len(value["implementation_map"]["modules"]) > 20:
|
||||||
|
value["implementation_map"]["modules"].pop()
|
||||||
|
final_size = size()
|
||||||
|
if final_size > MAX_RESULT_BYTES:
|
||||||
|
raise CompactInterfaceMapError("INTERFACE_MAP_RESULT_TOO_LARGE")
|
||||||
|
value["result_bytes"] = final_size
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def build_compact_interface_map(
|
||||||
|
worktree: Path,
|
||||||
|
source_signals: Sequence[str],
|
||||||
|
*,
|
||||||
|
source_commit: str,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
root = Path(worktree)
|
||||||
|
if not root.is_dir() or root.is_symlink():
|
||||||
|
raise CompactInterfaceMapError("INTERFACE_MAP_WORKTREE_INVALID")
|
||||||
|
if re.fullmatch(r"[0-9a-f]{40}", source_commit) is None:
|
||||||
|
raise CompactInterfaceMapError("INTERFACE_MAP_SOURCE_COMMIT_INVALID")
|
||||||
|
|
||||||
|
profile = _profile(source_signals)
|
||||||
|
terms = _signal_terms(source_signals)
|
||||||
|
candidates = _candidate_files(root, terms)
|
||||||
|
if not candidates:
|
||||||
|
raise CompactInterfaceMapError("INTERFACE_MAP_EMPTY")
|
||||||
|
|
||||||
|
module_candidates = candidates[:MAX_MODULES]
|
||||||
|
modules = [
|
||||||
|
{
|
||||||
|
"path": item["path"],
|
||||||
|
"sha256": item["sha256"],
|
||||||
|
"bytes": item["bytes"],
|
||||||
|
"score": item["score"],
|
||||||
|
}
|
||||||
|
for item in module_candidates
|
||||||
|
]
|
||||||
|
|
||||||
|
symbols: list[dict[str, Any]] = []
|
||||||
|
tables: list[dict[str, Any]] = []
|
||||||
|
systemd_units: list[dict[str, Any]] = []
|
||||||
|
for item in candidates:
|
||||||
|
if item["suffix"] == ".py":
|
||||||
|
symbols.extend(_python_symbols(item, terms))
|
||||||
|
elif item["suffix"] == ".sql":
|
||||||
|
for table in SQL_TABLE_RE.findall(str(item["text"])):
|
||||||
|
tables.append({"path": item["path"], "table": table, "sha256": item["sha256"]})
|
||||||
|
elif item["suffix"] == ".service":
|
||||||
|
systemd_units.append({"path": item["path"], "unit": Path(str(item["path"])).name, "sha256": item["sha256"]})
|
||||||
|
|
||||||
|
symbols.sort(key=lambda item: (-int(item["score"]), str(item["path"]), str(item["symbol"])))
|
||||||
|
all_symbols = list(symbols)
|
||||||
|
test_symbols = [
|
||||||
|
item for item in all_symbols
|
||||||
|
if str(item["path"]).startswith("tests/") or str(item["symbol"]).split(".")[-1].startswith("test_")
|
||||||
|
][: min(MAX_TESTS, 24)]
|
||||||
|
non_test_symbols = [
|
||||||
|
item for item in all_symbols
|
||||||
|
if item not in test_symbols
|
||||||
|
][: max(0, MAX_SYMBOLS - len(test_symbols))]
|
||||||
|
symbols = [*test_symbols, *non_test_symbols]
|
||||||
|
symbols.sort(key=lambda item: (-int(item["score"]), str(item["path"]), str(item["symbol"])))
|
||||||
|
classes = [
|
||||||
|
{k: item[k] for k in ("path", "symbol", "line", "end_line")}
|
||||||
|
for item in symbols
|
||||||
|
if item["kind"] == "class"
|
||||||
|
]
|
||||||
|
functions = [
|
||||||
|
{k: item[k] for k in ("path", "symbol", "line", "end_line")}
|
||||||
|
for item in symbols
|
||||||
|
if item["kind"] in {"function", "async_function"} and not str(item["symbol"]).split(".")[-1].startswith("test_")
|
||||||
|
]
|
||||||
|
tests = [
|
||||||
|
{k: item[k] for k in ("path", "symbol", "line", "end_line")}
|
||||||
|
for item in symbols
|
||||||
|
if str(item["path"]).startswith("tests/") or str(item["symbol"]).split(".")[-1].startswith("test_")
|
||||||
|
][:MAX_TESTS]
|
||||||
|
|
||||||
|
excerpts: list[dict[str, Any]] = []
|
||||||
|
by_path = {str(item["path"]): item for item in candidates}
|
||||||
|
for symbol in symbols:
|
||||||
|
if len(excerpts) == MAX_EXCERPTS:
|
||||||
|
break
|
||||||
|
item = by_path.get(str(symbol["path"]))
|
||||||
|
if item is None:
|
||||||
|
continue
|
||||||
|
excerpt = _bounded_excerpt(str(item["text"]), int(symbol["line"]), int(symbol["end_line"]))
|
||||||
|
if not excerpt.strip():
|
||||||
|
continue
|
||||||
|
excerpts.append(
|
||||||
|
{
|
||||||
|
"path": symbol["path"],
|
||||||
|
"symbol": symbol["symbol"],
|
||||||
|
"sha256": item["sha256"],
|
||||||
|
"excerpt": excerpt,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
invariants, failure_windows, reuse_points = _gap_evidence(all_symbols, profile)
|
||||||
|
result = {
|
||||||
|
"schema": SCHEMA,
|
||||||
|
"profile": profile,
|
||||||
|
"classification": PROFILE_CLASSIFICATION[profile],
|
||||||
|
"source_commit": source_commit,
|
||||||
|
"implementation_map": {
|
||||||
|
"modules": modules,
|
||||||
|
"classes": classes[:MAX_SYMBOLS],
|
||||||
|
"functions": functions[:MAX_SYMBOLS],
|
||||||
|
"tables": tables[:40],
|
||||||
|
"systemd_units": systemd_units[:20],
|
||||||
|
"tests": tests,
|
||||||
|
"current_invariants": invariants,
|
||||||
|
"failure_windows": failure_windows,
|
||||||
|
"reuse_points": reuse_points,
|
||||||
|
},
|
||||||
|
"bounded_excerpts": excerpts,
|
||||||
|
"contracts": _contract_map(all_symbols, modules, _contract_terms(profile)),
|
||||||
|
}
|
||||||
|
return _trim_to_budget(result)
|
||||||
@@ -0,0 +1,482 @@
|
|||||||
|
"""Canonical governed execution projection for Online Agent Tasks.
|
||||||
|
|
||||||
|
The durable governed binding is authoritative. Natural-language task fields and
|
||||||
|
worker configuration are never allowed to invent repository, ref, generation, or
|
||||||
|
idempotency coordinates.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import asdict, dataclass
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Mapping, Sequence
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
|
||||||
|
EXECUTION_SCHEMA = "aios.online-agent.execution.v2"
|
||||||
|
PROJECTION_SCHEMA = "aios.online-agent.task-projection.v1"
|
||||||
|
OID_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||||
|
TASK_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{2,179}$")
|
||||||
|
BRANCH_RE = re.compile(r"^agent/[A-Za-z0-9][A-Za-z0-9._/-]{2,179}$")
|
||||||
|
KEY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{7,159}$")
|
||||||
|
PROTECTED_PREFIXES = ("main", "master", "refs/", "tags/", "tag/", "release/", "baseline/", "production/")
|
||||||
|
MUTATION_ACTIONS = frozenset({"deterministic"})
|
||||||
|
READ_ONLY_ACTIONS = frozenset({"source_capture", "agent_runtime"})
|
||||||
|
SUPPORTED_ACTIONS = frozenset({"deterministic", "source_capture", "agent_runtime"})
|
||||||
|
AGENT_RUNTIME_ADAPTERS = frozenset({"hermes", "openclaw"})
|
||||||
|
MAX_AGENT_RUNTIME_INPUT_BYTES = 64 * 1024
|
||||||
|
|
||||||
|
|
||||||
|
class ExecutionProjectionError(RuntimeError):
|
||||||
|
"""A canonical fail-closed projection or validation error."""
|
||||||
|
|
||||||
|
|
||||||
|
class OnlineTaskProjectionClient:
|
||||||
|
"""Official HTTP projection client using only a control credential reference."""
|
||||||
|
|
||||||
|
def __init__(self, api_base: str, control_token_path: Path) -> None:
|
||||||
|
self.api_base = api_base.rstrip("/")
|
||||||
|
self.control_token_path = Path(control_token_path)
|
||||||
|
|
||||||
|
def _token(self) -> str:
|
||||||
|
path = self.control_token_path
|
||||||
|
if path.is_symlink() or not path.is_file():
|
||||||
|
_error("ONLINE_TASK_PROJECTION_CONTROL_UNAVAILABLE")
|
||||||
|
value = path.read_text(encoding="utf-8").strip()
|
||||||
|
if len(value) < 32:
|
||||||
|
_error("ONLINE_TASK_PROJECTION_CONTROL_INVALID")
|
||||||
|
return value
|
||||||
|
|
||||||
|
def create(
|
||||||
|
self,
|
||||||
|
projected_task: Mapping[str, Any],
|
||||||
|
*,
|
||||||
|
binding: "GovernedExecutionBinding",
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
body = dict(projected_task)
|
||||||
|
payload = _canonical(body).encode("utf-8")
|
||||||
|
request = urllib.request.Request(
|
||||||
|
self.api_base + "/v1/tasks",
|
||||||
|
method="POST",
|
||||||
|
data=payload,
|
||||||
|
headers={
|
||||||
|
"Authorization": "Bearer " + self._token(),
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"Accept": "application/json",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(request, timeout=15) as response:
|
||||||
|
result = json.loads(response.read().decode("utf-8"))
|
||||||
|
status = response.status
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
raw = exc.read(8192).decode("utf-8", "replace")
|
||||||
|
try:
|
||||||
|
error = json.loads(raw).get("error")
|
||||||
|
except (AttributeError, json.JSONDecodeError):
|
||||||
|
error = "HTTP_ERROR"
|
||||||
|
raise ExecutionProjectionError(f"ONLINE_TASK_PROJECTION_FAILED:{exc.code}:{error}") from exc
|
||||||
|
except (OSError, urllib.error.URLError, TimeoutError) as exc:
|
||||||
|
raise ExecutionProjectionError("ONLINE_TASK_PROJECTION_UNAVAILABLE") from exc
|
||||||
|
if status not in {200, 201} or not isinstance(result, dict):
|
||||||
|
_error("ONLINE_TASK_PROJECTION_RESPONSE_INVALID")
|
||||||
|
if result.get("task_id") != binding.task_id or result.get("status") not in {"QUEUED", "VERIFYING", "SUCCEEDED"}:
|
||||||
|
_error("ONLINE_TASK_PROJECTION_RESPONSE_INVALID")
|
||||||
|
metadata = result.get("metadata")
|
||||||
|
if not isinstance(metadata, Mapping):
|
||||||
|
_error("ONLINE_TASK_PROJECTION_RESPONSE_INVALID")
|
||||||
|
returned_execution = validate_execution_envelope(
|
||||||
|
_object(metadata.get("execution"), "ONLINE_TASK_PROJECTION_RESPONSE_INVALID"),
|
||||||
|
binding=binding,
|
||||||
|
)
|
||||||
|
submitted_metadata = _object(body.get("metadata"), "ONLINE_TASK_PROJECTION_REQUEST_INVALID")
|
||||||
|
submitted_execution = validate_execution_envelope(
|
||||||
|
_object(submitted_metadata.get("execution"), "ONLINE_TASK_PROJECTION_REQUEST_INVALID"),
|
||||||
|
binding=binding,
|
||||||
|
)
|
||||||
|
if _canonical(returned_execution) != _canonical(submitted_execution):
|
||||||
|
_error("ONLINE_TASK_PROJECTION_RESPONSE_BINDING_INVALID")
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _error(code: str) -> None:
|
||||||
|
raise ExecutionProjectionError(code)
|
||||||
|
|
||||||
|
|
||||||
|
def _object(value: object, code: str) -> dict[str, Any]:
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
_error(code)
|
||||||
|
return dict(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _string(value: object, code: str) -> str:
|
||||||
|
if not isinstance(value, str) or not value.strip():
|
||||||
|
_error(code)
|
||||||
|
return value.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _task_id(value: object) -> str:
|
||||||
|
value = _string(value, "TASK_ID_INVALID")
|
||||||
|
if TASK_RE.fullmatch(value) is None:
|
||||||
|
_error("TASK_ID_INVALID")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _oid(value: object, code: str) -> str:
|
||||||
|
value = _string(value, code)
|
||||||
|
if OID_RE.fullmatch(value) is None:
|
||||||
|
_error(code)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _branch(value: object) -> str:
|
||||||
|
value = _string(value, "TARGET_BRANCH_INVALID")
|
||||||
|
if BRANCH_RE.fullmatch(value) is None or value.startswith(PROTECTED_PREFIXES):
|
||||||
|
_error("TARGET_BRANCH_INVALID")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _generation(value: object) -> int:
|
||||||
|
if not isinstance(value, int) or isinstance(value, bool) or value < 1:
|
||||||
|
_error("GENERATION_INVALID")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _canonical(value: object) -> str:
|
||||||
|
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||||
|
|
||||||
|
|
||||||
|
def _digest(value: object) -> str:
|
||||||
|
return hashlib.sha256(_canonical(value).encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _source_signals(value: object) -> tuple[str, ...]:
|
||||||
|
"""Normalize untrusted natural-language hints before they enter a signed envelope."""
|
||||||
|
if value is None:
|
||||||
|
return ()
|
||||||
|
if not isinstance(value, Sequence) or isinstance(value, (str, bytes)):
|
||||||
|
_error("SOURCE_CAPTURE_SIGNALS_INVALID")
|
||||||
|
result: list[str] = []
|
||||||
|
for item in value:
|
||||||
|
if not isinstance(item, str):
|
||||||
|
_error("SOURCE_CAPTURE_SIGNALS_INVALID")
|
||||||
|
text = " ".join(item.split()).strip()
|
||||||
|
if not text:
|
||||||
|
continue
|
||||||
|
# Projection may accept long task prose, but execution envelopes stay bounded.
|
||||||
|
text = text[:512].rstrip()
|
||||||
|
if text not in result:
|
||||||
|
result.append(text)
|
||||||
|
if len(result) == 16:
|
||||||
|
break
|
||||||
|
return tuple(result)
|
||||||
|
|
||||||
|
|
||||||
|
def _agent_runtime_input(value: object) -> dict[str, Any]:
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
_error("AGENT_RUNTIME_INPUT_REQUIRED")
|
||||||
|
raw = dict(value)
|
||||||
|
allowed = {"goal", "instructions", "success_criteria", "constraints"}
|
||||||
|
if set(raw) - allowed:
|
||||||
|
_error("AGENT_RUNTIME_INPUT_FIELDS_INVALID")
|
||||||
|
goal = _string(raw.get("goal"), "AGENT_RUNTIME_GOAL_REQUIRED")
|
||||||
|
instructions = raw.get("instructions", "")
|
||||||
|
if not isinstance(instructions, str):
|
||||||
|
_error("AGENT_RUNTIME_INSTRUCTIONS_INVALID")
|
||||||
|
criteria = raw.get("success_criteria", [])
|
||||||
|
if not isinstance(criteria, list) or len(criteria) > 32:
|
||||||
|
_error("AGENT_RUNTIME_SUCCESS_CRITERIA_INVALID")
|
||||||
|
normalized_criteria: list[dict[str, str]] = []
|
||||||
|
for item in criteria:
|
||||||
|
if not isinstance(item, Mapping):
|
||||||
|
_error("AGENT_RUNTIME_SUCCESS_CRITERIA_INVALID")
|
||||||
|
criterion_id = _string(item.get("id"), "AGENT_RUNTIME_SUCCESS_CRITERIA_INVALID")
|
||||||
|
description = _string(
|
||||||
|
item.get("description"), "AGENT_RUNTIME_SUCCESS_CRITERIA_INVALID"
|
||||||
|
)
|
||||||
|
normalized_criteria.append(
|
||||||
|
{"id": criterion_id[:120], "description": description[:1000]}
|
||||||
|
)
|
||||||
|
constraints = raw.get("constraints", {})
|
||||||
|
if not isinstance(constraints, Mapping):
|
||||||
|
_error("AGENT_RUNTIME_CONSTRAINTS_INVALID")
|
||||||
|
sensitive_markers = ("token", "secret", "password", "credential", "authorization", "api_key", "apikey")
|
||||||
|
for key in constraints:
|
||||||
|
lowered = str(key).lower()
|
||||||
|
if any(marker in lowered for marker in sensitive_markers):
|
||||||
|
_error("AGENT_RUNTIME_SECRET_FIELD_DENY")
|
||||||
|
normalized = {
|
||||||
|
"goal": goal[:12000],
|
||||||
|
"instructions": instructions[:32000],
|
||||||
|
"success_criteria": normalized_criteria,
|
||||||
|
"constraints": dict(constraints),
|
||||||
|
}
|
||||||
|
if len(_canonical(normalized).encode("utf-8")) > MAX_AGENT_RUNTIME_INPUT_BYTES:
|
||||||
|
_error("AGENT_RUNTIME_INPUT_TOO_LARGE")
|
||||||
|
return normalized
|
||||||
|
|
||||||
|
|
||||||
|
def _capabilities(value: object) -> tuple[str, ...]:
|
||||||
|
if value is None:
|
||||||
|
return ()
|
||||||
|
if not isinstance(value, Sequence) or isinstance(value, (str, bytes)):
|
||||||
|
_error("CAPABILITY_SCOPE_INVALID")
|
||||||
|
result = tuple(sorted({_string(item, "CAPABILITY_SCOPE_INVALID") for item in value}))
|
||||||
|
if any(len(item) > 120 for item in result):
|
||||||
|
_error("CAPABILITY_SCOPE_INVALID")
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class GovernedExecutionBinding:
|
||||||
|
task_id: str
|
||||||
|
job_id: str
|
||||||
|
repository: str
|
||||||
|
base_oid: str
|
||||||
|
expected_branch_oid: str
|
||||||
|
target_branch: str
|
||||||
|
branch_mode: str
|
||||||
|
generation: int
|
||||||
|
idempotency_key: str
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_mapping(cls, value: Mapping[str, Any]) -> "GovernedExecutionBinding":
|
||||||
|
data = dict(value)
|
||||||
|
task_id = _task_id(data.get("task_id"))
|
||||||
|
job_id = _string(data.get("job_id"), "JOB_ID_REQUIRED")
|
||||||
|
if TASK_RE.fullmatch(job_id) is None:
|
||||||
|
_error("JOB_ID_INVALID")
|
||||||
|
repository = _string(data.get("repository"), "REPOSITORY_INVALID")
|
||||||
|
if repository != "aios/aios":
|
||||||
|
_error("WRONG_REPOSITORY_ENVELOPE_DENY")
|
||||||
|
base_oid = _oid(data.get("base_oid"), "BASE_OID_INVALID")
|
||||||
|
expected = _oid(data.get("expected_branch_oid"), "EXPECTED_BRANCH_OID_INVALID")
|
||||||
|
target = _branch(data.get("target_branch"))
|
||||||
|
mode = _string(data.get("branch_mode"), "BRANCH_MODE_MISSING")
|
||||||
|
if mode not in {"CREATE_NEW", "CONTINUE_EXISTING"}:
|
||||||
|
_error("BRANCH_MODE_INVALID")
|
||||||
|
generation = _generation(data.get("generation"))
|
||||||
|
if mode == "CREATE_NEW" and expected != base_oid:
|
||||||
|
_error("EXPECTED_BASE_CONFLICT")
|
||||||
|
key = _string(data.get("idempotency_key"), "IDEMPOTENCY_KEY_INVALID")
|
||||||
|
if KEY_RE.fullmatch(key) is None:
|
||||||
|
_error("IDEMPOTENCY_KEY_INVALID")
|
||||||
|
return cls(task_id, job_id, repository, base_oid, expected, target, mode, generation, key)
|
||||||
|
|
||||||
|
def public(self) -> dict[str, Any]:
|
||||||
|
return asdict(self)
|
||||||
|
|
||||||
|
|
||||||
|
def binding_from_bootstrap(value: Mapping[str, Any]) -> GovernedExecutionBinding:
|
||||||
|
return GovernedExecutionBinding.from_mapping(value)
|
||||||
|
|
||||||
|
|
||||||
|
def validate_execution_envelope(
|
||||||
|
value: Mapping[str, Any],
|
||||||
|
*,
|
||||||
|
binding: GovernedExecutionBinding | None = None,
|
||||||
|
worker_capabilities: Sequence[str] | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
envelope = dict(value)
|
||||||
|
if envelope.get("schema") != EXECUTION_SCHEMA:
|
||||||
|
_error("EXECUTION_ENVELOPE_SCHEMA_VALIDATION")
|
||||||
|
candidate = GovernedExecutionBinding.from_mapping(envelope)
|
||||||
|
if binding is not None and candidate != binding:
|
||||||
|
comparisons = (
|
||||||
|
(candidate.task_id, binding.task_id, "TASK_BINDING_CONFLICT"),
|
||||||
|
(candidate.job_id, binding.job_id, "JOB_BINDING_CONFLICT"),
|
||||||
|
(candidate.repository, binding.repository, "WRONG_REPOSITORY_ENVELOPE_DENY"),
|
||||||
|
(candidate.base_oid, binding.base_oid, "WRONG_BASE_ENVELOPE_DENY"),
|
||||||
|
(candidate.expected_branch_oid, binding.expected_branch_oid, "WRONG_EXPECTED_HEAD_ENVELOPE_DENY"),
|
||||||
|
(candidate.target_branch, binding.target_branch, "WRONG_BRANCH_ENVELOPE_DENY"),
|
||||||
|
(candidate.generation, binding.generation, "WRONG_GENERATION_ENVELOPE_DENY"),
|
||||||
|
(candidate.branch_mode, binding.branch_mode, "BRANCH_MODE_BINDING_CONFLICT"),
|
||||||
|
(candidate.idempotency_key, binding.idempotency_key, "IDEMPOTENCY_BINDING_CONFLICT"),
|
||||||
|
)
|
||||||
|
for actual, expected, code in comparisons:
|
||||||
|
if actual != expected:
|
||||||
|
_error(code)
|
||||||
|
action = _string(envelope.get("action"), "EXECUTION_ACTION_MISSING")
|
||||||
|
if action not in SUPPORTED_ACTIONS:
|
||||||
|
_error("EXECUTION_ACTION_UNSUPPORTED")
|
||||||
|
capability_scope = _capabilities(envelope.get("capability_scope"))
|
||||||
|
if worker_capabilities is not None and not set(capability_scope).issubset(set(worker_capabilities)):
|
||||||
|
_error("CAPABILITY_MISMATCH_DENY")
|
||||||
|
workspace = _object(envelope.get("workspace"), "WORKSPACE_POLICY_REQUIRED")
|
||||||
|
if workspace.get("authority_git_dir") != "/data/aios-gitops/repo.git" or workspace.get("read_only_repository") is not True:
|
||||||
|
_error("WORKSPACE_POLICY_DENY")
|
||||||
|
governance = _object(envelope.get("governance"), "GOVERNANCE_POLICY_REQUIRED")
|
||||||
|
governed_coordinates = (
|
||||||
|
(governance.get("task_id"), candidate.task_id, "TASK_BINDING_CONFLICT"),
|
||||||
|
(governance.get("job_id"), candidate.job_id, "JOB_BINDING_CONFLICT"),
|
||||||
|
(governance.get("idempotency_key"), candidate.idempotency_key, "IDEMPOTENCY_BINDING_CONFLICT"),
|
||||||
|
)
|
||||||
|
for actual_coordinate, expected_coordinate, code in governed_coordinates:
|
||||||
|
if actual_coordinate != expected_coordinate:
|
||||||
|
_error(code)
|
||||||
|
commit_key = _string(governance.get("commit_idempotency_key"), "COMMIT_IDEMPOTENCY_KEY_INVALID")
|
||||||
|
if KEY_RE.fullmatch(commit_key) is None:
|
||||||
|
_error("COMMIT_IDEMPOTENCY_KEY_INVALID")
|
||||||
|
if governance.get("writer_lease") is not (action in MUTATION_ACTIONS):
|
||||||
|
_error("WRITER_LEASE_POLICY_DENY")
|
||||||
|
for field in ("main_write", "production_write", "tag_write", "baseline_write", "force_push", "protected_ref_bypass"):
|
||||||
|
if governance.get(field, False) is not False:
|
||||||
|
_error("POLICY_DENY")
|
||||||
|
changes = envelope.get("change_set")
|
||||||
|
verification = envelope.get("verification")
|
||||||
|
if not isinstance(changes, list) or not isinstance(verification, list):
|
||||||
|
_error("EXECUTION_OPERATIONS_INVALID")
|
||||||
|
if action == "source_capture":
|
||||||
|
if changes or verification:
|
||||||
|
_error("SOURCE_CAPTURE_MUTATION_DENY")
|
||||||
|
signals = list(_source_signals(envelope.get("source_signals") or []))
|
||||||
|
if envelope.get("source_signals") != signals:
|
||||||
|
_error("SOURCE_CAPTURE_SIGNALS_INVALID")
|
||||||
|
if envelope.get("runtime_input") not in (None, {}):
|
||||||
|
_error("SOURCE_CAPTURE_RUNTIME_INPUT_DENY")
|
||||||
|
elif action == "agent_runtime":
|
||||||
|
if changes or verification:
|
||||||
|
_error("AGENT_RUNTIME_MUTATION_DENY")
|
||||||
|
adapter = str(envelope.get("adapter") or "")
|
||||||
|
if adapter not in AGENT_RUNTIME_ADAPTERS:
|
||||||
|
_error("AGENT_RUNTIME_ADAPTER_INVALID")
|
||||||
|
runtime_input = _agent_runtime_input(envelope.get("runtime_input"))
|
||||||
|
if envelope.get("runtime_input") != runtime_input:
|
||||||
|
_error("AGENT_RUNTIME_INPUT_NOT_CANONICAL")
|
||||||
|
required_runtime_capability = f"agent.runtime.{adapter}"
|
||||||
|
if required_runtime_capability not in capability_scope:
|
||||||
|
_error("AGENT_RUNTIME_CAPABILITY_REQUIRED")
|
||||||
|
if envelope.get("source_signals") not in (None, []):
|
||||||
|
_error("AGENT_RUNTIME_SOURCE_SIGNALS_DENY")
|
||||||
|
elif not changes:
|
||||||
|
_error("CHANGE_SET_INVALID")
|
||||||
|
declared = str(envelope.get("execution_digest") or "")
|
||||||
|
material = {key: item for key, item in envelope.items() if key != "execution_digest"}
|
||||||
|
actual = _digest(material)
|
||||||
|
if declared != actual:
|
||||||
|
_error("EXECUTION_DIGEST_INVALID")
|
||||||
|
return {**material, "execution_digest": actual}
|
||||||
|
|
||||||
|
|
||||||
|
def project_execution_envelope(
|
||||||
|
binding: GovernedExecutionBinding,
|
||||||
|
execution_request: Mapping[str, Any] | None = None,
|
||||||
|
*,
|
||||||
|
required_capabilities: Sequence[str] = (),
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
request = dict(execution_request or {})
|
||||||
|
action = str(request.get("action") or "source_capture")
|
||||||
|
if action not in SUPPORTED_ACTIONS:
|
||||||
|
_error("EXECUTION_ACTION_UNSUPPORTED")
|
||||||
|
capabilities = _capabilities(request.get("capability_scope", required_capabilities))
|
||||||
|
governance = {
|
||||||
|
"task_id": binding.task_id,
|
||||||
|
"job_id": binding.job_id,
|
||||||
|
"idempotency_key": binding.idempotency_key,
|
||||||
|
"commit_idempotency_key": str(request.get("commit_idempotency_key") or f"{binding.task_id}-commit-g{binding.generation}"),
|
||||||
|
"writer_lease": action in MUTATION_ACTIONS,
|
||||||
|
"main_write": False,
|
||||||
|
"production_write": False,
|
||||||
|
"tag_write": False,
|
||||||
|
"baseline_write": False,
|
||||||
|
"force_push": False,
|
||||||
|
"protected_ref_bypass": False,
|
||||||
|
}
|
||||||
|
supplied_governance = request.get("governance")
|
||||||
|
if supplied_governance is not None:
|
||||||
|
governance.update(_object(supplied_governance, "GOVERNANCE_POLICY_REQUIRED"))
|
||||||
|
material: dict[str, Any] = {
|
||||||
|
"schema": EXECUTION_SCHEMA,
|
||||||
|
**binding.public(),
|
||||||
|
"action": action,
|
||||||
|
"adapter": str(
|
||||||
|
request.get("adapter")
|
||||||
|
or ("deterministic" if action == "deterministic" else "source_capture")
|
||||||
|
),
|
||||||
|
"capability_scope": list(capabilities),
|
||||||
|
"change_set": list(request.get("change_set") or []),
|
||||||
|
"verification": list(request.get("verification") or []),
|
||||||
|
"source_signals": list(_source_signals(request.get("source_signals") or [])) if action == "source_capture" else [],
|
||||||
|
"runtime_input": _agent_runtime_input(request.get("runtime_input")) if action == "agent_runtime" else {},
|
||||||
|
"governance": governance,
|
||||||
|
"workspace": {
|
||||||
|
"authority_git_dir": "/data/aios-gitops/repo.git",
|
||||||
|
"read_only_repository": True,
|
||||||
|
"cleanup_policy": "always",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
envelope = {**material, "execution_digest": _digest(material)}
|
||||||
|
return validate_execution_envelope(envelope, binding=binding)
|
||||||
|
|
||||||
|
|
||||||
|
def project_online_task(
|
||||||
|
binding: GovernedExecutionBinding,
|
||||||
|
task_request: Mapping[str, Any],
|
||||||
|
execution_request: Mapping[str, Any] | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
request = dict(task_request)
|
||||||
|
required = _capabilities(request.get("required_capabilities"))
|
||||||
|
execution_value = dict(execution_request or {})
|
||||||
|
if str(execution_value.get("action") or "source_capture") == "agent_runtime":
|
||||||
|
adapter = str(execution_value.get("adapter") or "")
|
||||||
|
if adapter not in AGENT_RUNTIME_ADAPTERS:
|
||||||
|
_error("AGENT_RUNTIME_ADAPTER_INVALID")
|
||||||
|
runtime_input = execution_value.get("runtime_input")
|
||||||
|
if runtime_input is None:
|
||||||
|
runtime_input = {
|
||||||
|
"goal": request.get("goal"),
|
||||||
|
"instructions": request.get("instructions") or "",
|
||||||
|
"success_criteria": request.get("success_criteria") or [],
|
||||||
|
"constraints": request.get("constraints") or {},
|
||||||
|
}
|
||||||
|
execution_value["runtime_input"] = runtime_input
|
||||||
|
execution = project_execution_envelope(
|
||||||
|
binding,
|
||||||
|
execution_value,
|
||||||
|
required_capabilities=required,
|
||||||
|
)
|
||||||
|
metadata = dict(request.get("metadata") or {})
|
||||||
|
if "execution" in metadata:
|
||||||
|
validate_execution_envelope(_object(metadata["execution"], "EXECUTION_ENVELOPE_INVALID"), binding=binding)
|
||||||
|
if metadata["execution"] != execution:
|
||||||
|
_error("EXECUTION_PROJECTION_CONFLICT")
|
||||||
|
metadata["execution"] = execution
|
||||||
|
metadata["execution_projection"] = {
|
||||||
|
"schema": PROJECTION_SCHEMA,
|
||||||
|
"task_id": binding.task_id,
|
||||||
|
"job_id": binding.job_id,
|
||||||
|
"execution_digest": execution["execution_digest"],
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
**request,
|
||||||
|
"task_id": binding.task_id,
|
||||||
|
"metadata": metadata,
|
||||||
|
"required_capabilities": list(required),
|
||||||
|
"idempotency_key": str(request.get("idempotency_key") or f"online-task:{binding.idempotency_key}"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_task_execution(
|
||||||
|
task: Mapping[str, Any],
|
||||||
|
binding: GovernedExecutionBinding,
|
||||||
|
*,
|
||||||
|
worker_capabilities: Sequence[str] | None = None,
|
||||||
|
) -> tuple[dict[str, Any], str]:
|
||||||
|
metadata = task.get("metadata")
|
||||||
|
if isinstance(metadata, Mapping) and "execution" in metadata:
|
||||||
|
value = validate_execution_envelope(
|
||||||
|
_object(metadata["execution"], "EXECUTION_ENVELOPE_INVALID"),
|
||||||
|
binding=binding,
|
||||||
|
worker_capabilities=worker_capabilities,
|
||||||
|
)
|
||||||
|
return value, "persisted"
|
||||||
|
projected = project_execution_envelope(
|
||||||
|
binding,
|
||||||
|
{"action": "source_capture", "capability_scope": task.get("required_capabilities") or []},
|
||||||
|
required_capabilities=task.get("required_capabilities") or [],
|
||||||
|
)
|
||||||
|
validate_execution_envelope(projected, binding=binding, worker_capabilities=worker_capabilities)
|
||||||
|
return projected, "recovered_projection"
|
||||||
@@ -0,0 +1,187 @@
|
|||||||
|
"""Durable, authority-bound deterministic execution plans for Online Agent Tasks."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import fcntl
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Mapping
|
||||||
|
|
||||||
|
from aios_online_agent_execution import (
|
||||||
|
GovernedExecutionBinding,
|
||||||
|
project_execution_envelope,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ExecutionPlanError(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _canonical(value: object) -> bytes:
|
||||||
|
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _digest(value: object) -> str:
|
||||||
|
return hashlib.sha256(_canonical(value)).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _task_key(task_id: str) -> str:
|
||||||
|
readable = "".join(ch if ch.isalnum() or ch in "._-" else "-" for ch in task_id).strip("-")[:72] or "task"
|
||||||
|
return readable + "-" + hashlib.sha256(task_id.encode("utf-8")).hexdigest()[:24]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ExecutionPlanRecord:
|
||||||
|
task_id: str
|
||||||
|
job_id: str
|
||||||
|
source_head_oid: str
|
||||||
|
generation: int
|
||||||
|
idempotency_key: str
|
||||||
|
plan_digest: str
|
||||||
|
execution: dict[str, Any]
|
||||||
|
|
||||||
|
def public(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"schema": "aios.online-agent.plan-record.v1",
|
||||||
|
"task_id": self.task_id,
|
||||||
|
"job_id": self.job_id,
|
||||||
|
"source_head_oid": self.source_head_oid,
|
||||||
|
"generation": self.generation,
|
||||||
|
"idempotency_key": self.idempotency_key,
|
||||||
|
"plan_digest": self.plan_digest,
|
||||||
|
"execution": dict(self.execution),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class ExecutionPlanStore:
|
||||||
|
def __init__(self, root: Path) -> None:
|
||||||
|
self.root = Path(root)
|
||||||
|
self.root.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
def _path(self, task_id: str) -> Path:
|
||||||
|
return self.root / (_task_key(task_id) + ".json")
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def _lock(self, task_id: str):
|
||||||
|
path = self.root / (_task_key(task_id) + ".lock")
|
||||||
|
with path.open("a+b") as handle:
|
||||||
|
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
|
||||||
|
try:
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _source_head(binding: GovernedExecutionBinding) -> str:
|
||||||
|
return binding.expected_branch_oid if binding.branch_mode == "CONTINUE_EXISTING" else binding.base_oid
|
||||||
|
|
||||||
|
def _read(self, task_id: str) -> ExecutionPlanRecord | None:
|
||||||
|
path = self._path(task_id)
|
||||||
|
if not path.is_file() or path.is_symlink():
|
||||||
|
return None
|
||||||
|
value = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
if not isinstance(value, Mapping) or value.get("schema") != "aios.online-agent.plan-record.v1":
|
||||||
|
raise ExecutionPlanError("EXECUTION_PLAN_RECORD_INVALID")
|
||||||
|
execution = value.get("execution")
|
||||||
|
if not isinstance(execution, Mapping):
|
||||||
|
raise ExecutionPlanError("EXECUTION_PLAN_RECORD_INVALID")
|
||||||
|
return ExecutionPlanRecord(
|
||||||
|
task_id=str(value.get("task_id") or ""),
|
||||||
|
job_id=str(value.get("job_id") or ""),
|
||||||
|
source_head_oid=str(value.get("source_head_oid") or ""),
|
||||||
|
generation=int(value.get("generation") or 0),
|
||||||
|
idempotency_key=str(value.get("idempotency_key") or ""),
|
||||||
|
plan_digest=str(value.get("plan_digest") or ""),
|
||||||
|
execution=dict(execution),
|
||||||
|
)
|
||||||
|
|
||||||
|
def register(
|
||||||
|
self,
|
||||||
|
binding: GovernedExecutionBinding,
|
||||||
|
request: Mapping[str, Any],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
with self._lock(binding.task_id):
|
||||||
|
return self._register_locked(binding, request)
|
||||||
|
|
||||||
|
def _register_locked(
|
||||||
|
self,
|
||||||
|
binding: GovernedExecutionBinding,
|
||||||
|
request: Mapping[str, Any],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if request.get("schema") != "aios.online-agent.plan.v1":
|
||||||
|
raise ExecutionPlanError("EXECUTION_PLAN_SCHEMA_INVALID")
|
||||||
|
if request.get("task_id") != binding.task_id or request.get("job_id") != binding.job_id:
|
||||||
|
raise ExecutionPlanError("EXECUTION_PLAN_BINDING_CONFLICT")
|
||||||
|
if int(request.get("generation") or 0) != binding.generation:
|
||||||
|
raise ExecutionPlanError("EXECUTION_PLAN_GENERATION_CONFLICT")
|
||||||
|
source_head = str(request.get("source_head_oid") or "")
|
||||||
|
if source_head != self._source_head(binding):
|
||||||
|
raise ExecutionPlanError("EXECUTION_PLAN_SOURCE_HEAD_CONFLICT")
|
||||||
|
key = str(request.get("idempotency_key") or "")
|
||||||
|
if len(key) < 8:
|
||||||
|
raise ExecutionPlanError("EXECUTION_PLAN_IDEMPOTENCY_INVALID")
|
||||||
|
execution_request = request.get("execution")
|
||||||
|
if not isinstance(execution_request, Mapping) or execution_request.get("action") != "deterministic":
|
||||||
|
raise ExecutionPlanError("EXECUTION_PLAN_DETERMINISTIC_REQUIRED")
|
||||||
|
envelope = project_execution_envelope(binding, execution_request)
|
||||||
|
material = {
|
||||||
|
"task_id": binding.task_id,
|
||||||
|
"job_id": binding.job_id,
|
||||||
|
"source_head_oid": source_head,
|
||||||
|
"generation": binding.generation,
|
||||||
|
"idempotency_key": key,
|
||||||
|
"execution": dict(execution_request),
|
||||||
|
}
|
||||||
|
plan_digest = _digest(material)
|
||||||
|
existing = self._read(binding.task_id)
|
||||||
|
expected_previous = request.get("expected_previous_plan_digest")
|
||||||
|
if existing is not None:
|
||||||
|
if existing.idempotency_key == key:
|
||||||
|
if existing.plan_digest != plan_digest:
|
||||||
|
raise ExecutionPlanError("EXECUTION_PLAN_IDEMPOTENCY_CONFLICT")
|
||||||
|
return {**existing.public(), "idempotent": True, "execution_digest": envelope["execution_digest"]}
|
||||||
|
if expected_previous != existing.plan_digest:
|
||||||
|
raise ExecutionPlanError("EXECUTION_PLAN_REPLACEMENT_CAS_MISMATCH")
|
||||||
|
if existing.source_head_oid != source_head:
|
||||||
|
raise ExecutionPlanError("EXECUTION_PLAN_STALE_REPLACEMENT_DENY")
|
||||||
|
elif expected_previous not in (None, ""):
|
||||||
|
raise ExecutionPlanError("EXECUTION_PLAN_REPLACEMENT_CAS_MISMATCH")
|
||||||
|
|
||||||
|
record = ExecutionPlanRecord(
|
||||||
|
task_id=binding.task_id,
|
||||||
|
job_id=binding.job_id,
|
||||||
|
source_head_oid=source_head,
|
||||||
|
generation=binding.generation,
|
||||||
|
idempotency_key=key,
|
||||||
|
plan_digest=plan_digest,
|
||||||
|
execution=dict(execution_request),
|
||||||
|
)
|
||||||
|
path = self._path(binding.task_id)
|
||||||
|
temporary = path.with_suffix(".tmp")
|
||||||
|
temporary.write_text(
|
||||||
|
json.dumps(record.public(), sort_keys=True, separators=(",", ":")),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
temporary.replace(path)
|
||||||
|
return {**record.public(), "idempotent": False, "execution_digest": envelope["execution_digest"]}
|
||||||
|
|
||||||
|
def resolve(self, binding: GovernedExecutionBinding) -> dict[str, Any] | None:
|
||||||
|
record = self._read(binding.task_id)
|
||||||
|
if record is None:
|
||||||
|
return None
|
||||||
|
if (
|
||||||
|
record.task_id != binding.task_id
|
||||||
|
or record.job_id != binding.job_id
|
||||||
|
or record.generation != binding.generation
|
||||||
|
):
|
||||||
|
raise ExecutionPlanError("EXECUTION_PLAN_BINDING_CONFLICT")
|
||||||
|
if record.source_head_oid != self._source_head(binding):
|
||||||
|
raise ExecutionPlanError("EXECUTION_PLAN_SOURCE_HEAD_DRIFT")
|
||||||
|
envelope = project_execution_envelope(binding, record.execution)
|
||||||
|
return {
|
||||||
|
"execution": dict(record.execution),
|
||||||
|
"plan_digest": record.plan_digest,
|
||||||
|
"execution_digest": envelope["execution_digest"],
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,82 @@
|
|||||||
|
"""Immutable release-lineage contracts used by activation and recovery."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import asdict, dataclass
|
||||||
|
from typing import Any, Mapping
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ReleaseLineage:
|
||||||
|
task_id: str
|
||||||
|
candidate_id: str
|
||||||
|
source_commit: str
|
||||||
|
version: str
|
||||||
|
required_capabilities: tuple[str, ...]
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, Any]:
|
||||||
|
return asdict(self)
|
||||||
|
|
||||||
|
|
||||||
|
FIXED_V2R2C2 = ReleaseLineage(
|
||||||
|
task_id="r42-release-candidate-v2r2c2-live-20260819",
|
||||||
|
candidate_id="r42-release-candidate-v2r2c2-20260819",
|
||||||
|
source_commit="71e736b05fab13efc06af7a4fe2dc022c1cf9a85",
|
||||||
|
version="V2R2C2",
|
||||||
|
required_capabilities=("release:candidate",),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ReleaseLineageError(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def validate_fixed_v2r2c2_task(task: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
|
"""Validate identity/source/version without mutating Task state.
|
||||||
|
|
||||||
|
This contract intentionally does not bind status/attempt/revision, because
|
||||||
|
those are lifecycle state and may change through governed retry/recovery.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if str(task.get("task_id") or "") != FIXED_V2R2C2.task_id:
|
||||||
|
raise ReleaseLineageError("FIXED_TASK_ID_DRIFT")
|
||||||
|
|
||||||
|
capabilities = task.get("required_capabilities")
|
||||||
|
if (
|
||||||
|
not isinstance(capabilities, list)
|
||||||
|
or tuple(capabilities) != FIXED_V2R2C2.required_capabilities
|
||||||
|
):
|
||||||
|
raise ReleaseLineageError("FIXED_TASK_CAPABILITY_DRIFT")
|
||||||
|
|
||||||
|
constraints = task.get("constraints")
|
||||||
|
if not isinstance(constraints, Mapping):
|
||||||
|
raise ReleaseLineageError("FIXED_TASK_CONSTRAINTS_MISSING")
|
||||||
|
release = constraints.get("release_candidate")
|
||||||
|
if not isinstance(release, Mapping):
|
||||||
|
raise ReleaseLineageError("FIXED_RELEASE_BINDING_MISSING")
|
||||||
|
|
||||||
|
actual = {
|
||||||
|
"candidate_id": str(release.get("candidate_id") or ""),
|
||||||
|
"source_commit": str(release.get("source_commit") or ""),
|
||||||
|
"version": str(release.get("version") or ""),
|
||||||
|
}
|
||||||
|
expected = {
|
||||||
|
"candidate_id": FIXED_V2R2C2.candidate_id,
|
||||||
|
"source_commit": FIXED_V2R2C2.source_commit,
|
||||||
|
"version": FIXED_V2R2C2.version,
|
||||||
|
}
|
||||||
|
if actual != expected:
|
||||||
|
raise ReleaseLineageError(
|
||||||
|
"FIXED_RELEASE_LINEAGE_DRIFT:"
|
||||||
|
+ repr({"expected": expected, "actual": actual})
|
||||||
|
)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"schema": "aios.release-lineage.validation.v1",
|
||||||
|
"status": "VERIFIED",
|
||||||
|
"lineage": FIXED_V2R2C2.as_dict(),
|
||||||
|
"task_status": task.get("status"),
|
||||||
|
"attempt": task.get("attempt"),
|
||||||
|
"max_attempts": task.get("max_attempts"),
|
||||||
|
"revision": task.get("revision"),
|
||||||
|
}
|
||||||
@@ -0,0 +1,714 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Persistent release-preparation Online Agent worker.
|
||||||
|
|
||||||
|
The worker is intentionally narrower than the generic engineering worker:
|
||||||
|
it can only consume tasks whose required capability set is exactly
|
||||||
|
{"release:candidate"} and whose payload asks for immutable release-candidate
|
||||||
|
creation. It never receives Owner signing, deployment, tag, baseline or Git
|
||||||
|
write credentials.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import socket
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from typing import Any, Mapping
|
||||||
|
from aios_autonomous_control_plane import (
|
||||||
|
classify_failure,
|
||||||
|
repair_for_failure,
|
||||||
|
runtime_admission,
|
||||||
|
)
|
||||||
|
|
||||||
|
AGENT_ID = os.environ.get(
|
||||||
|
"AIOS_RELEASE_WORKER_AGENT_ID",
|
||||||
|
"online-release-worker-cn-network-mkt-20260819",
|
||||||
|
).strip()
|
||||||
|
AGENT_VERSION = os.environ.get(
|
||||||
|
"AIOS_RELEASE_WORKER_AGENT_VERSION",
|
||||||
|
"2026-08-19.1-release-candidate",
|
||||||
|
).strip()
|
||||||
|
TASK_API = os.environ.get(
|
||||||
|
"AIOS_AGENT_TASK_API_URL",
|
||||||
|
"http://127.0.0.1:18940",
|
||||||
|
).rstrip("/")
|
||||||
|
RELEASE_API = os.environ.get("AIOS_RELEASE_API_URL", "").rstrip("/")
|
||||||
|
POLL_SECONDS = int(os.environ.get("AIOS_RELEASE_WORKER_POLL_SECONDS", "3"))
|
||||||
|
LEASE_SECONDS = int(os.environ.get("AIOS_RELEASE_WORKER_LEASE_SECONDS", "60"))
|
||||||
|
HEARTBEAT_SECONDS = int(
|
||||||
|
os.environ.get("AIOS_RELEASE_WORKER_HEARTBEAT_SECONDS", "15")
|
||||||
|
)
|
||||||
|
STATE_ROOT = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_RELEASE_WORKER_STATE_ROOT",
|
||||||
|
"/var/lib/aios-online-agent-release",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
EXPECTED_CAPABILITIES = frozenset({"release:candidate"})
|
||||||
|
LIVENESS_PATH = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_WORKER_LIVENESS_PATH",
|
||||||
|
str(STATE_ROOT / "worker-liveness.json"),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
CANDIDATE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{7,255}$")
|
||||||
|
OID_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||||
|
VERSION_RE = re.compile(r"^V[0-9]+R[0-9]+C[0-9]+$")
|
||||||
|
TASK_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{2,179}$")
|
||||||
|
SECRET_MARKERS = (
|
||||||
|
"authorization",
|
||||||
|
"token",
|
||||||
|
"secret",
|
||||||
|
"credential",
|
||||||
|
"password",
|
||||||
|
"private_key",
|
||||||
|
"signature",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ReleaseWorkerError(RuntimeError):
|
||||||
|
def __init__(self, code: str, *, phase: str) -> None:
|
||||||
|
super().__init__(code)
|
||||||
|
self.code = code
|
||||||
|
self.phase = phase
|
||||||
|
|
||||||
|
|
||||||
|
def now() -> str:
|
||||||
|
return datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def canonical(value: object) -> bytes:
|
||||||
|
return json.dumps(
|
||||||
|
value,
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def sha256(value: bytes) -> str:
|
||||||
|
return hashlib.sha256(value).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def sanitized(value: Any) -> Any:
|
||||||
|
if isinstance(value, Mapping):
|
||||||
|
result: dict[str, Any] = {}
|
||||||
|
for key, inner in value.items():
|
||||||
|
lowered = str(key).lower()
|
||||||
|
if any(marker in lowered for marker in SECRET_MARKERS):
|
||||||
|
result[str(key)] = "[REDACTED]"
|
||||||
|
else:
|
||||||
|
result[str(key)] = sanitized(inner)
|
||||||
|
return result
|
||||||
|
if isinstance(value, list):
|
||||||
|
return [sanitized(item) for item in value]
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def audit(event: str, **fields: Any) -> None:
|
||||||
|
STATE_ROOT.mkdir(parents=True, exist_ok=True, mode=0o750)
|
||||||
|
path = STATE_ROOT / "events.jsonl"
|
||||||
|
record = {
|
||||||
|
"schema": "aios.release-worker.event.v1",
|
||||||
|
"event": event,
|
||||||
|
"created_at": now(),
|
||||||
|
"agent_id": AGENT_ID,
|
||||||
|
**sanitized(fields),
|
||||||
|
}
|
||||||
|
with path.open("a", encoding="utf-8") as handle:
|
||||||
|
handle.write(json.dumps(record, sort_keys=True) + "\n")
|
||||||
|
os.chmod(path, 0o640)
|
||||||
|
|
||||||
|
|
||||||
|
def sd_notify(message: str) -> None:
|
||||||
|
target = os.environ.get("NOTIFY_SOCKET", "")
|
||||||
|
if not target:
|
||||||
|
return
|
||||||
|
if target.startswith("@"):
|
||||||
|
address = "\0" + target[1:]
|
||||||
|
else:
|
||||||
|
address = target
|
||||||
|
sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||||
|
try:
|
||||||
|
sock.connect(address)
|
||||||
|
sock.sendall(message.encode("utf-8"))
|
||||||
|
except OSError:
|
||||||
|
# Liveness JSON remains the diagnostic source; systemd will enforce
|
||||||
|
# WatchdogSec if notify transport is persistently broken.
|
||||||
|
pass
|
||||||
|
finally:
|
||||||
|
sock.close()
|
||||||
|
|
||||||
|
|
||||||
|
def write_liveness(
|
||||||
|
state: str,
|
||||||
|
*,
|
||||||
|
task_id: str | None = None,
|
||||||
|
stage: str | None = None,
|
||||||
|
error: str | None = None,
|
||||||
|
visible_count: int | None = None,
|
||||||
|
) -> None:
|
||||||
|
STATE_ROOT.mkdir(parents=True, exist_ok=True, mode=0o750)
|
||||||
|
value = {
|
||||||
|
"schema": "aios.online-agent.worker-liveness.v1",
|
||||||
|
"agent_id": AGENT_ID,
|
||||||
|
"agent_type": "release-preparation-worker",
|
||||||
|
"agent_version": AGENT_VERSION,
|
||||||
|
"pid": os.getpid(),
|
||||||
|
"state": state,
|
||||||
|
"last_seen_epoch": time.time(),
|
||||||
|
"last_seen_at": now(),
|
||||||
|
"poll_seconds": POLL_SECONDS,
|
||||||
|
"heartbeat_seconds": HEARTBEAT_SECONDS,
|
||||||
|
"api_base": TASK_API,
|
||||||
|
"capabilities": sorted(EXPECTED_CAPABILITIES),
|
||||||
|
"task_id": task_id,
|
||||||
|
"stage": stage,
|
||||||
|
"error": error,
|
||||||
|
"visible_count": visible_count,
|
||||||
|
}
|
||||||
|
LIVENESS_PATH.parent.mkdir(parents=True, exist_ok=True, mode=0o750)
|
||||||
|
temporary = LIVENESS_PATH.with_name(
|
||||||
|
"." + LIVENESS_PATH.name + f".{os.getpid()}.tmp"
|
||||||
|
)
|
||||||
|
temporary.write_text(
|
||||||
|
json.dumps(value, ensure_ascii=False, sort_keys=True) + "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
os.chmod(temporary, 0o640)
|
||||||
|
os.replace(temporary, LIVENESS_PATH)
|
||||||
|
|
||||||
|
|
||||||
|
def load_credential(name: str) -> str:
|
||||||
|
directory = Path(os.environ.get("CREDENTIALS_DIRECTORY", ""))
|
||||||
|
path = directory / name
|
||||||
|
if path.is_symlink() or not path.is_file():
|
||||||
|
raise ReleaseWorkerError(
|
||||||
|
"CREDENTIAL_UNAVAILABLE:" + name,
|
||||||
|
phase="startup",
|
||||||
|
)
|
||||||
|
value = path.read_text(encoding="utf-8").strip()
|
||||||
|
if len(value) < 24 or any(ch.isspace() for ch in value):
|
||||||
|
raise ReleaseWorkerError(
|
||||||
|
"CREDENTIAL_INVALID:" + name,
|
||||||
|
phase="startup",
|
||||||
|
)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def http_json(
|
||||||
|
base: str,
|
||||||
|
method: str,
|
||||||
|
route: str,
|
||||||
|
token: str,
|
||||||
|
body: Mapping[str, Any] | None = None,
|
||||||
|
*,
|
||||||
|
timeout: int = 30,
|
||||||
|
) -> tuple[int, dict[str, Any]]:
|
||||||
|
data = None if body is None else canonical(body)
|
||||||
|
headers = {"Accept": "application/json"}
|
||||||
|
if token:
|
||||||
|
headers["Authorization"] = "Bearer " + token
|
||||||
|
if data is not None:
|
||||||
|
headers["Content-Type"] = "application/json"
|
||||||
|
request = urllib.request.Request(
|
||||||
|
base + route,
|
||||||
|
method=method,
|
||||||
|
data=data,
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||||
|
raw, status = response.read(), response.status
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
raw, status = exc.read(), exc.code
|
||||||
|
except (urllib.error.URLError, TimeoutError) as exc:
|
||||||
|
raise ReleaseWorkerError(
|
||||||
|
"HTTP_TRANSPORT_UNAVAILABLE",
|
||||||
|
phase="network",
|
||||||
|
) from exc
|
||||||
|
try:
|
||||||
|
value = json.loads(raw.decode())
|
||||||
|
except Exception as exc:
|
||||||
|
raise ReleaseWorkerError(
|
||||||
|
"HTTP_RESPONSE_INVALID",
|
||||||
|
phase="network",
|
||||||
|
) from exc
|
||||||
|
return status, value if isinstance(value, dict) else {}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ReleaseRequest:
|
||||||
|
candidate_id: str
|
||||||
|
source_commit: str
|
||||||
|
version: str
|
||||||
|
projection_id: str | None
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_task(cls, task: Mapping[str, Any]) -> "ReleaseRequest":
|
||||||
|
if not TASK_RE.fullmatch(str(task.get("task_id") or "")):
|
||||||
|
raise ReleaseWorkerError("TASK_ID_INVALID", phase="validation")
|
||||||
|
caps = task.get("required_capabilities")
|
||||||
|
if (
|
||||||
|
not isinstance(caps, list)
|
||||||
|
or frozenset(caps) != EXPECTED_CAPABILITIES
|
||||||
|
or len(caps) != 1
|
||||||
|
):
|
||||||
|
raise ReleaseWorkerError(
|
||||||
|
"TASK_CAPABILITY_SCOPE_INVALID",
|
||||||
|
phase="validation",
|
||||||
|
)
|
||||||
|
metadata = task.get("metadata")
|
||||||
|
if not isinstance(metadata, Mapping):
|
||||||
|
metadata = {}
|
||||||
|
constraints = task.get("constraints")
|
||||||
|
if not isinstance(constraints, Mapping):
|
||||||
|
constraints = {}
|
||||||
|
release = constraints.get("release_candidate")
|
||||||
|
if not isinstance(release, Mapping):
|
||||||
|
release = metadata.get("release_candidate")
|
||||||
|
if not isinstance(release, Mapping):
|
||||||
|
release = task.get("release_candidate")
|
||||||
|
if not isinstance(release, Mapping):
|
||||||
|
raise ReleaseWorkerError(
|
||||||
|
"RELEASE_CANDIDATE_PAYLOAD_MISSING",
|
||||||
|
phase="validation",
|
||||||
|
)
|
||||||
|
allowed = {
|
||||||
|
"candidate_id",
|
||||||
|
"source_commit",
|
||||||
|
"version",
|
||||||
|
"projection_id",
|
||||||
|
}
|
||||||
|
if set(release) - allowed:
|
||||||
|
raise ReleaseWorkerError(
|
||||||
|
"RELEASE_CANDIDATE_PAYLOAD_FIELDS_INVALID",
|
||||||
|
phase="validation",
|
||||||
|
)
|
||||||
|
candidate_id = str(release.get("candidate_id") or "")
|
||||||
|
source_commit = str(release.get("source_commit") or "")
|
||||||
|
version = str(release.get("version") or "")
|
||||||
|
projection_id = release.get("projection_id")
|
||||||
|
if CANDIDATE_RE.fullmatch(candidate_id) is None:
|
||||||
|
raise ReleaseWorkerError(
|
||||||
|
"RELEASE_CANDIDATE_ID_INVALID",
|
||||||
|
phase="validation",
|
||||||
|
)
|
||||||
|
if OID_RE.fullmatch(source_commit) is None:
|
||||||
|
raise ReleaseWorkerError(
|
||||||
|
"RELEASE_SOURCE_COMMIT_INVALID",
|
||||||
|
phase="validation",
|
||||||
|
)
|
||||||
|
if VERSION_RE.fullmatch(version) is None:
|
||||||
|
raise ReleaseWorkerError(
|
||||||
|
"RELEASE_VERSION_INVALID",
|
||||||
|
phase="validation",
|
||||||
|
)
|
||||||
|
if projection_id is not None:
|
||||||
|
projection_id = str(projection_id)
|
||||||
|
if not re.fullmatch(r"projection-[0-9a-f]{32}", projection_id):
|
||||||
|
raise ReleaseWorkerError(
|
||||||
|
"RELEASE_PROJECTION_ID_INVALID",
|
||||||
|
phase="validation",
|
||||||
|
)
|
||||||
|
return cls(
|
||||||
|
candidate_id=candidate_id,
|
||||||
|
source_commit=source_commit,
|
||||||
|
version=version,
|
||||||
|
projection_id=projection_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
def body(self) -> dict[str, str]:
|
||||||
|
return {
|
||||||
|
"candidate_id": self.candidate_id,
|
||||||
|
"source_commit": self.source_commit,
|
||||||
|
"version": self.version,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class Heartbeat:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
agent_token: str,
|
||||||
|
task_id: str,
|
||||||
|
lease_token: str,
|
||||||
|
) -> None:
|
||||||
|
self.agent_token = agent_token
|
||||||
|
self.task_id = task_id
|
||||||
|
self.lease_token = lease_token
|
||||||
|
self.stage = "claimed"
|
||||||
|
self.failure: str | None = None
|
||||||
|
self.stop = threading.Event()
|
||||||
|
self.thread = threading.Thread(
|
||||||
|
target=self._run,
|
||||||
|
name="release-worker-heartbeat",
|
||||||
|
daemon=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
def first(self) -> None:
|
||||||
|
status, value = http_json(
|
||||||
|
TASK_API,
|
||||||
|
"POST",
|
||||||
|
f"/v1/tasks/{self.task_id}/heartbeat",
|
||||||
|
self.agent_token,
|
||||||
|
{
|
||||||
|
"lease_token": self.lease_token,
|
||||||
|
"lease_seconds": LEASE_SECONDS,
|
||||||
|
"progress": {
|
||||||
|
"stage": self.stage,
|
||||||
|
"worker_version": AGENT_VERSION,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if status != 200:
|
||||||
|
raise ReleaseWorkerError(
|
||||||
|
"TASK_HEARTBEAT_FAILED",
|
||||||
|
phase="heartbeat",
|
||||||
|
)
|
||||||
|
audit(
|
||||||
|
"TASK_HEARTBEAT",
|
||||||
|
task_id=self.task_id,
|
||||||
|
stage=self.stage,
|
||||||
|
api_status=status,
|
||||||
|
)
|
||||||
|
write_liveness(
|
||||||
|
"BUSY",
|
||||||
|
task_id=self.task_id,
|
||||||
|
stage=self.stage,
|
||||||
|
)
|
||||||
|
sd_notify("WATCHDOG=1")
|
||||||
|
|
||||||
|
def _run(self) -> None:
|
||||||
|
while not self.stop.wait(HEARTBEAT_SECONDS):
|
||||||
|
try:
|
||||||
|
status, value = http_json(
|
||||||
|
TASK_API,
|
||||||
|
"POST",
|
||||||
|
f"/v1/tasks/{self.task_id}/heartbeat",
|
||||||
|
self.agent_token,
|
||||||
|
{
|
||||||
|
"lease_token": self.lease_token,
|
||||||
|
"lease_seconds": LEASE_SECONDS,
|
||||||
|
"progress": {
|
||||||
|
"stage": self.stage,
|
||||||
|
"worker_version": AGENT_VERSION,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if status != 200:
|
||||||
|
self.failure = str(value.get("error") or status)
|
||||||
|
return
|
||||||
|
audit(
|
||||||
|
"TASK_HEARTBEAT",
|
||||||
|
task_id=self.task_id,
|
||||||
|
stage=self.stage,
|
||||||
|
api_status=status,
|
||||||
|
)
|
||||||
|
write_liveness(
|
||||||
|
"BUSY",
|
||||||
|
task_id=self.task_id,
|
||||||
|
stage=self.stage,
|
||||||
|
)
|
||||||
|
sd_notify("WATCHDOG=1")
|
||||||
|
except Exception as exc:
|
||||||
|
self.failure = type(exc).__name__
|
||||||
|
return
|
||||||
|
|
||||||
|
def ensure(self) -> None:
|
||||||
|
if self.failure:
|
||||||
|
raise ReleaseWorkerError(
|
||||||
|
"TASK_LEASE_LOST",
|
||||||
|
phase="heartbeat",
|
||||||
|
)
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
self.stop.set()
|
||||||
|
if self.thread.ident is not None:
|
||||||
|
self.thread.join(timeout=2)
|
||||||
|
|
||||||
|
|
||||||
|
def release_admission(
|
||||||
|
task: Mapping[str, Any],
|
||||||
|
*,
|
||||||
|
release_token: str,
|
||||||
|
):
|
||||||
|
required = task.get("required_capabilities")
|
||||||
|
if not isinstance(required, list):
|
||||||
|
required = []
|
||||||
|
|
||||||
|
def release_api_health() -> bool:
|
||||||
|
status, value = http_json(
|
||||||
|
RELEASE_API,
|
||||||
|
"GET",
|
||||||
|
"/health",
|
||||||
|
"",
|
||||||
|
timeout=10,
|
||||||
|
)
|
||||||
|
return status == 200 and str(value.get("status") or "").lower() in {"ok", "healthy"}
|
||||||
|
|
||||||
|
def authenticated_probe() -> bool:
|
||||||
|
status, _ = http_json(
|
||||||
|
RELEASE_API,
|
||||||
|
"POST",
|
||||||
|
"/createReleaseCandidate",
|
||||||
|
release_token,
|
||||||
|
{},
|
||||||
|
timeout=10,
|
||||||
|
)
|
||||||
|
# Authentication + credential projection succeeded; the empty release
|
||||||
|
# request is intentionally rejected by request validation.
|
||||||
|
return status == 400
|
||||||
|
|
||||||
|
probes = {
|
||||||
|
"credential_projection": authenticated_probe,
|
||||||
|
"trusted_builder": release_api_health,
|
||||||
|
"authenticated_probe": authenticated_probe,
|
||||||
|
"release_api_health": release_api_health,
|
||||||
|
"worker_liveness": lambda: True,
|
||||||
|
}
|
||||||
|
return runtime_admission(
|
||||||
|
worker_capabilities=EXPECTED_CAPABILITIES,
|
||||||
|
required_capabilities=required,
|
||||||
|
requested_operations=("prepare_release",),
|
||||||
|
probes=probes,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def result_payload(
|
||||||
|
task_id: str,
|
||||||
|
release: ReleaseRequest,
|
||||||
|
builder: Mapping[str, Any],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
builder_candidate = builder.get("candidate_id")
|
||||||
|
if builder_candidate not in {None, release.candidate_id}:
|
||||||
|
raise ReleaseWorkerError(
|
||||||
|
"BUILDER_CANDIDATE_BINDING_INVALID",
|
||||||
|
phase="release",
|
||||||
|
)
|
||||||
|
immutable = {
|
||||||
|
"candidate_id": release.candidate_id,
|
||||||
|
"source_commit": release.source_commit,
|
||||||
|
"version": release.version,
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
"status": "SUCCEEDED",
|
||||||
|
"classification": "RELEASE_CANDIDATE_CREATED",
|
||||||
|
"execution_mode": "release_candidate",
|
||||||
|
"adapter": "release_candidate",
|
||||||
|
"task_id": task_id,
|
||||||
|
"agent_id": AGENT_ID,
|
||||||
|
"release_candidate": immutable,
|
||||||
|
"release_candidate_request_sha256": sha256(canonical(immutable)),
|
||||||
|
"builder_response": sanitized(dict(builder)),
|
||||||
|
"projection_id": release.projection_id,
|
||||||
|
"writer_lease": "NONE",
|
||||||
|
"changed_files": [],
|
||||||
|
"direct_main_write": False,
|
||||||
|
"production_write": False,
|
||||||
|
"tag_write": False,
|
||||||
|
"baseline_write": False,
|
||||||
|
"runtime_switch": False,
|
||||||
|
"owner_approval_submitted": False,
|
||||||
|
"credential_material_exposed": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def execute_task(
|
||||||
|
task: Mapping[str, Any],
|
||||||
|
*,
|
||||||
|
agent_token: str,
|
||||||
|
release_token: str,
|
||||||
|
) -> bool:
|
||||||
|
task_id = str(task.get("task_id") or "")
|
||||||
|
release = ReleaseRequest.from_task(task)
|
||||||
|
admission = release_admission(
|
||||||
|
task,
|
||||||
|
release_token=release_token,
|
||||||
|
)
|
||||||
|
if not admission.admitted:
|
||||||
|
audit(
|
||||||
|
"TASK_ADMISSION_BLOCKED",
|
||||||
|
task_id=task_id,
|
||||||
|
admission=admission.as_dict(),
|
||||||
|
)
|
||||||
|
write_liveness(
|
||||||
|
"BLOCKED",
|
||||||
|
task_id=task_id,
|
||||||
|
stage="execution_ready",
|
||||||
|
error=admission.failure_class.value if admission.failure_class else "ADMISSION_FAILED",
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
audit(
|
||||||
|
"TASK_ADMISSION_PASSED",
|
||||||
|
task_id=task_id,
|
||||||
|
admission=admission.as_dict(),
|
||||||
|
)
|
||||||
|
status, claimed = http_json(
|
||||||
|
TASK_API,
|
||||||
|
"POST",
|
||||||
|
f"/v1/tasks/{task_id}/claim",
|
||||||
|
agent_token,
|
||||||
|
{"lease_seconds": LEASE_SECONDS},
|
||||||
|
)
|
||||||
|
if status in {409, 423}:
|
||||||
|
return False
|
||||||
|
if status != 200:
|
||||||
|
raise ReleaseWorkerError(
|
||||||
|
"TASK_CLAIM_FAILED",
|
||||||
|
phase="claim",
|
||||||
|
)
|
||||||
|
lease_token = str(claimed.get("lease_token") or "")
|
||||||
|
if len(lease_token) < 16:
|
||||||
|
raise ReleaseWorkerError(
|
||||||
|
"TASK_LEASE_TOKEN_INVALID",
|
||||||
|
phase="claim",
|
||||||
|
)
|
||||||
|
attempt = int(claimed.get("attempt") or 0)
|
||||||
|
audit(
|
||||||
|
"TASK_CLAIMED",
|
||||||
|
task_id=task_id,
|
||||||
|
attempt=attempt,
|
||||||
|
)
|
||||||
|
|
||||||
|
heartbeat = Heartbeat(agent_token, task_id, lease_token)
|
||||||
|
heartbeat.first()
|
||||||
|
heartbeat.thread.start()
|
||||||
|
try:
|
||||||
|
heartbeat.stage = "release_candidate"
|
||||||
|
heartbeat.ensure()
|
||||||
|
status, builder = http_json(
|
||||||
|
RELEASE_API,
|
||||||
|
"POST",
|
||||||
|
"/createReleaseCandidate",
|
||||||
|
release_token,
|
||||||
|
release.body(),
|
||||||
|
timeout=120,
|
||||||
|
)
|
||||||
|
if status not in {200, 201, 202}:
|
||||||
|
raise ReleaseWorkerError(
|
||||||
|
"RELEASE_CANDIDATE_CREATE_FAILED:"
|
||||||
|
+ str(builder.get("error") or status),
|
||||||
|
phase="release",
|
||||||
|
)
|
||||||
|
heartbeat.ensure()
|
||||||
|
result = result_payload(task_id, release, builder)
|
||||||
|
idempotency_key = (
|
||||||
|
f"{task_id}-attempt-{attempt}-release-result-v1"
|
||||||
|
)
|
||||||
|
status, submitted = http_json(
|
||||||
|
TASK_API,
|
||||||
|
"POST",
|
||||||
|
f"/v1/tasks/{task_id}/result",
|
||||||
|
agent_token,
|
||||||
|
{
|
||||||
|
"lease_token": lease_token,
|
||||||
|
"idempotency_key": idempotency_key,
|
||||||
|
"result": result,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if status not in {200, 201, 202}:
|
||||||
|
raise ReleaseWorkerError(
|
||||||
|
"TASK_RESULT_SUBMIT_FAILED",
|
||||||
|
phase="result",
|
||||||
|
)
|
||||||
|
audit(
|
||||||
|
"RESULT_SUBMITTED",
|
||||||
|
task_id=task_id,
|
||||||
|
attempt=attempt,
|
||||||
|
release_candidate_id=release.candidate_id,
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
finally:
|
||||||
|
heartbeat.close()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
if not RELEASE_API:
|
||||||
|
raise SystemExit("FATAL:AIOS_RELEASE_API_URL_REQUIRED")
|
||||||
|
agent_token = load_credential("agent_token")
|
||||||
|
release_token = load_credential("release_candidate_token")
|
||||||
|
STATE_ROOT.mkdir(parents=True, exist_ok=True, mode=0o750)
|
||||||
|
audit(
|
||||||
|
"WORKER_STARTED",
|
||||||
|
capabilities=sorted(EXPECTED_CAPABILITIES),
|
||||||
|
release_api=RELEASE_API,
|
||||||
|
)
|
||||||
|
write_liveness("STARTING", stage="startup")
|
||||||
|
sd_notify("READY=1\nSTATUS=release worker ready")
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
status, response = http_json(
|
||||||
|
TASK_API,
|
||||||
|
"POST",
|
||||||
|
"/v1/tasks/poll",
|
||||||
|
agent_token,
|
||||||
|
{"limit": 100},
|
||||||
|
)
|
||||||
|
if status != 200:
|
||||||
|
raise ReleaseWorkerError(
|
||||||
|
"TASK_POLL_FAILED",
|
||||||
|
phase="poll",
|
||||||
|
)
|
||||||
|
tasks = response.get("tasks")
|
||||||
|
if not isinstance(tasks, list):
|
||||||
|
tasks = []
|
||||||
|
write_liveness(
|
||||||
|
"IDLE",
|
||||||
|
stage="poll",
|
||||||
|
visible_count=len(tasks),
|
||||||
|
)
|
||||||
|
sd_notify("WATCHDOG=1\nSTATUS=release worker poll ok")
|
||||||
|
handled = False
|
||||||
|
for raw in tasks:
|
||||||
|
if not isinstance(raw, Mapping):
|
||||||
|
continue
|
||||||
|
caps = raw.get("required_capabilities")
|
||||||
|
if (
|
||||||
|
not isinstance(caps, list)
|
||||||
|
or frozenset(caps) != EXPECTED_CAPABILITIES
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
handled = execute_task(
|
||||||
|
raw,
|
||||||
|
agent_token=agent_token,
|
||||||
|
release_token=release_token,
|
||||||
|
) or handled
|
||||||
|
except ReleaseWorkerError as exc:
|
||||||
|
failure = classify_failure(exc.code, phase=exc.phase)
|
||||||
|
repair = repair_for_failure(failure)
|
||||||
|
audit(
|
||||||
|
"TASK_EXECUTION_FAILED",
|
||||||
|
task_id=raw.get("task_id"),
|
||||||
|
phase=exc.phase,
|
||||||
|
error=exc.code,
|
||||||
|
failure_class=failure.value,
|
||||||
|
repair=repair.as_dict(),
|
||||||
|
)
|
||||||
|
time.sleep(0 if handled else POLL_SECONDS)
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
return 0
|
||||||
|
except Exception as exc:
|
||||||
|
audit(
|
||||||
|
"WORKER_LOOP_ERROR",
|
||||||
|
error=type(exc).__name__,
|
||||||
|
)
|
||||||
|
write_liveness(
|
||||||
|
"DEGRADED",
|
||||||
|
stage="loop",
|
||||||
|
error=type(exc).__name__,
|
||||||
|
)
|
||||||
|
time.sleep(POLL_SECONDS)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,287 @@
|
|||||||
|
"""Credential-isolated durable coordinator for the release-preparation supervisor.
|
||||||
|
|
||||||
|
The coordinator owns only its Online Task agent credential. It never forwards
|
||||||
|
that credential to Hermes/OpenClaw and never submits a Task result while the
|
||||||
|
supervision goal is incomplete. Its job is deliberately small: claim the one
|
||||||
|
fixed supervisor Task after an exact coordinator projection and keep a durable
|
||||||
|
heartbeat lease while later phases observe and repair through governed APIs.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
from typing import Any, Mapping
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
|
||||||
|
API_BASE = os.environ.get(
|
||||||
|
"AIOS_AGENT_TASK_API_URL", "http://127.0.0.1:18940"
|
||||||
|
).rstrip("/")
|
||||||
|
AGENT_ID = os.environ.get(
|
||||||
|
"AIOS_AGENT_ID",
|
||||||
|
"online-engineering-worker-cn-network-mkt-autonomous-20260815",
|
||||||
|
)
|
||||||
|
TASK_ID = os.environ.get(
|
||||||
|
"AIOS_RELEASE_PREPARATION_SUPERVISOR_TASK_ID",
|
||||||
|
"r42-release-preparation-autonomous-supervisor-20260820",
|
||||||
|
)
|
||||||
|
EXPECTED_EXECUTION_DIGEST = os.environ.get(
|
||||||
|
"AIOS_RELEASE_PREPARATION_SUPERVISOR_EXECUTION_DIGEST", ""
|
||||||
|
)
|
||||||
|
RUNTIME_CAPABILITY = "agent.runtime.openclaw"
|
||||||
|
COORDINATOR_CAPABILITY = "engineering.autonomous"
|
||||||
|
REQUIRED_CAPABILITIES = [RUNTIME_CAPABILITY, COORDINATOR_CAPABILITY]
|
||||||
|
POLL_SECONDS = int(os.environ.get("AIOS_SUPERVISOR_COORDINATOR_POLL_SECONDS", "5"))
|
||||||
|
HEARTBEAT_SECONDS = int(
|
||||||
|
os.environ.get("AIOS_SUPERVISOR_COORDINATOR_HEARTBEAT_SECONDS", "10")
|
||||||
|
)
|
||||||
|
LEASE_SECONDS = int(os.environ.get("AIOS_SUPERVISOR_COORDINATOR_LEASE_SECONDS", "300"))
|
||||||
|
STATE_ROOT = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_SUPERVISOR_COORDINATOR_STATE_ROOT",
|
||||||
|
"/var/lib/aios-release-preparation-supervisor-coordinator",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class CoordinatorError(RuntimeError):
|
||||||
|
def __init__(self, code: str) -> None:
|
||||||
|
super().__init__(code)
|
||||||
|
self.code = code
|
||||||
|
|
||||||
|
|
||||||
|
def now() -> str:
|
||||||
|
return datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def canonical(value: object) -> bytes:
|
||||||
|
return json.dumps(
|
||||||
|
value, sort_keys=True, separators=(",", ":"), ensure_ascii=False
|
||||||
|
).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def atomic_json(path: Path, value: Mapping[str, Any]) -> None:
|
||||||
|
path.parent.mkdir(parents=True, mode=0o750, exist_ok=True)
|
||||||
|
if path.parent.is_symlink() or path.is_symlink():
|
||||||
|
raise CoordinatorError("COORDINATOR_STATE_SYMLINK_REJECTED")
|
||||||
|
fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=str(path.parent))
|
||||||
|
try:
|
||||||
|
os.fchmod(fd, 0o640)
|
||||||
|
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||||
|
json.dump(dict(value), handle, ensure_ascii=False, sort_keys=True)
|
||||||
|
handle.write("\n")
|
||||||
|
handle.flush()
|
||||||
|
os.fsync(handle.fileno())
|
||||||
|
os.replace(temporary, path)
|
||||||
|
finally:
|
||||||
|
if os.path.exists(temporary):
|
||||||
|
os.unlink(temporary)
|
||||||
|
|
||||||
|
|
||||||
|
def audit(event_type: str, **fields: Any) -> None:
|
||||||
|
record = {
|
||||||
|
"event_type": event_type,
|
||||||
|
"created_at": now(),
|
||||||
|
"agent_id": AGENT_ID,
|
||||||
|
"task_id": TASK_ID,
|
||||||
|
"credential_material_exposed": False,
|
||||||
|
"planner_credential_material_exposed": False,
|
||||||
|
"lease_material_exposed": False,
|
||||||
|
**fields,
|
||||||
|
}
|
||||||
|
STATE_ROOT.mkdir(parents=True, mode=0o750, exist_ok=True)
|
||||||
|
with (STATE_ROOT / "events.jsonl").open("a", encoding="utf-8") as handle:
|
||||||
|
handle.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n")
|
||||||
|
atomic_json(STATE_ROOT / "liveness.json", record)
|
||||||
|
|
||||||
|
|
||||||
|
def api_call(
|
||||||
|
method: str,
|
||||||
|
route: str,
|
||||||
|
token: str,
|
||||||
|
body: Mapping[str, Any] | None = None,
|
||||||
|
) -> tuple[int, dict[str, Any]]:
|
||||||
|
payload = None if body is None else canonical(dict(body))
|
||||||
|
headers = {"Authorization": "Bearer " + token, "Accept": "application/json"}
|
||||||
|
if payload is not None:
|
||||||
|
headers["Content-Type"] = "application/json"
|
||||||
|
request = urllib.request.Request(
|
||||||
|
API_BASE + route, data=payload, headers=headers, method=method
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(request, timeout=20) as response:
|
||||||
|
value = json.loads(response.read().decode("utf-8"))
|
||||||
|
return response.status, value if isinstance(value, dict) else {}
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
try:
|
||||||
|
value = json.loads(exc.read(4096).decode("utf-8", "replace"))
|
||||||
|
except Exception:
|
||||||
|
value = {"error": "HTTP_ERROR"}
|
||||||
|
return exc.code, value if isinstance(value, dict) else {"error": "HTTP_ERROR"}
|
||||||
|
except (urllib.error.URLError, TimeoutError):
|
||||||
|
return 599, {"error": "AGENT_TASK_API_UNAVAILABLE"}
|
||||||
|
|
||||||
|
|
||||||
|
def load_agent_token() -> str:
|
||||||
|
path = Path(os.environ.get("CREDENTIALS_DIRECTORY", "")) / "agent_token"
|
||||||
|
if path.is_symlink() or not path.is_file():
|
||||||
|
raise CoordinatorError("COORDINATOR_AGENT_CREDENTIAL_UNAVAILABLE")
|
||||||
|
token = path.read_text(encoding="utf-8").strip()
|
||||||
|
if len(token) < 24 or any(character.isspace() for character in token):
|
||||||
|
raise CoordinatorError("COORDINATOR_AGENT_CREDENTIAL_INVALID")
|
||||||
|
return token
|
||||||
|
|
||||||
|
|
||||||
|
def validate_task(task: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
|
if task.get("task_id") != TASK_ID or task.get("status") != "QUEUED":
|
||||||
|
raise CoordinatorError("SUPERVISOR_TASK_STATE_INVALID")
|
||||||
|
if task.get("required_capabilities") != REQUIRED_CAPABILITIES:
|
||||||
|
raise CoordinatorError("SUPERVISOR_COORDINATOR_CAPABILITIES_INVALID")
|
||||||
|
metadata = task.get("metadata")
|
||||||
|
if not isinstance(metadata, Mapping):
|
||||||
|
raise CoordinatorError("SUPERVISOR_EXECUTION_METADATA_MISSING")
|
||||||
|
execution = metadata.get("execution")
|
||||||
|
if not isinstance(execution, Mapping):
|
||||||
|
raise CoordinatorError("SUPERVISOR_EXECUTION_METADATA_MISSING")
|
||||||
|
governance = execution.get("governance")
|
||||||
|
if (
|
||||||
|
execution.get("action") != "agent_runtime"
|
||||||
|
or execution.get("adapter") != "openclaw"
|
||||||
|
or execution.get("capability_scope") != REQUIRED_CAPABILITIES
|
||||||
|
or execution.get("execution_digest") != EXPECTED_EXECUTION_DIGEST
|
||||||
|
or execution.get("change_set") != []
|
||||||
|
or execution.get("verification") != []
|
||||||
|
or not isinstance(governance, Mapping)
|
||||||
|
or governance.get("writer_lease") is not False
|
||||||
|
or any(
|
||||||
|
governance.get(field) is not False
|
||||||
|
for field in (
|
||||||
|
"main_write",
|
||||||
|
"production_write",
|
||||||
|
"tag_write",
|
||||||
|
"baseline_write",
|
||||||
|
"force_push",
|
||||||
|
"protected_ref_bypass",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
):
|
||||||
|
raise CoordinatorError("SUPERVISOR_COORDINATOR_ENVELOPE_INVALID")
|
||||||
|
return dict(execution)
|
||||||
|
|
||||||
|
|
||||||
|
def heartbeat_progress(sequence: int) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"stage": "supervisor_coordinator_active",
|
||||||
|
"phase": "external_caller_projection",
|
||||||
|
"state": "PENDING_EXTERNAL_CONNECTOR_ACK",
|
||||||
|
"sequence": sequence,
|
||||||
|
"control_plane_access_granted": True,
|
||||||
|
"external_planner_role": "UNPRIVILEGED_PLANNER",
|
||||||
|
"credential_forwarded_to_planner": False,
|
||||||
|
"main_write": False,
|
||||||
|
"production_write": False,
|
||||||
|
"owner_approval_submitted": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def coordinate_claim(
|
||||||
|
claim: Mapping[str, Any],
|
||||||
|
token: str,
|
||||||
|
*,
|
||||||
|
max_heartbeats: int | None = None,
|
||||||
|
) -> int:
|
||||||
|
lease_token = claim.get("lease_token")
|
||||||
|
if not isinstance(lease_token, str) or len(lease_token) < 24:
|
||||||
|
raise CoordinatorError("COORDINATOR_LEASE_TOKEN_MISSING")
|
||||||
|
sequence = 0
|
||||||
|
while max_heartbeats is None or sequence < max_heartbeats:
|
||||||
|
sequence += 1
|
||||||
|
status, response = api_call(
|
||||||
|
"POST",
|
||||||
|
f"/v1/tasks/{TASK_ID}/heartbeat",
|
||||||
|
token,
|
||||||
|
{
|
||||||
|
"lease_token": lease_token,
|
||||||
|
"lease_seconds": LEASE_SECONDS,
|
||||||
|
"progress": heartbeat_progress(sequence),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if status != 200:
|
||||||
|
raise CoordinatorError(
|
||||||
|
"COORDINATOR_HEARTBEAT_FAILED_" + str(response.get("error") or status)
|
||||||
|
)
|
||||||
|
audit(
|
||||||
|
"SUPERVISOR_COORDINATOR_HEARTBEAT",
|
||||||
|
sequence=sequence,
|
||||||
|
task_status=response.get("status"),
|
||||||
|
lease_expires_at=response.get("lease_expires_at"),
|
||||||
|
control_plane_access_granted=True,
|
||||||
|
external_planner_role="UNPRIVILEGED_PLANNER",
|
||||||
|
)
|
||||||
|
if max_heartbeats is None or sequence < max_heartbeats:
|
||||||
|
time.sleep(HEARTBEAT_SECONDS)
|
||||||
|
return sequence
|
||||||
|
|
||||||
|
|
||||||
|
def worker_once(token: str) -> bool:
|
||||||
|
status, poll = api_call("POST", "/v1/tasks/poll", token, {"limit": 100})
|
||||||
|
if status != 200:
|
||||||
|
audit("SUPERVISOR_COORDINATOR_POLL_FAILED", error=poll.get("error") or status)
|
||||||
|
return False
|
||||||
|
task = next(
|
||||||
|
(
|
||||||
|
item
|
||||||
|
for item in poll.get("tasks", [])
|
||||||
|
if isinstance(item, Mapping) and item.get("task_id") == TASK_ID
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if task is None:
|
||||||
|
audit("SUPERVISOR_COORDINATOR_WAITING_FOR_TASK")
|
||||||
|
return False
|
||||||
|
validate_task(task)
|
||||||
|
status, claim = api_call(
|
||||||
|
"POST", f"/v1/tasks/{TASK_ID}/claim", token, {"lease_seconds": LEASE_SECONDS}
|
||||||
|
)
|
||||||
|
if status != 200:
|
||||||
|
audit("SUPERVISOR_COORDINATOR_CLAIM_FAILED", error=claim.get("error") or status)
|
||||||
|
return False
|
||||||
|
audit(
|
||||||
|
"SUPERVISOR_COORDINATOR_CLAIMED",
|
||||||
|
attempt=claim.get("attempt"),
|
||||||
|
revision=claim.get("revision"),
|
||||||
|
claim_mode="server_atomic",
|
||||||
|
control_plane_access_granted=True,
|
||||||
|
)
|
||||||
|
coordinate_claim(claim, token)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
if not EXPECTED_EXECUTION_DIGEST or len(EXPECTED_EXECUTION_DIGEST) != 64:
|
||||||
|
raise CoordinatorError("SUPERVISOR_EXECUTION_DIGEST_CONFIG_INVALID")
|
||||||
|
token = load_agent_token()
|
||||||
|
audit(
|
||||||
|
"SUPERVISOR_COORDINATOR_READY",
|
||||||
|
required_capabilities=REQUIRED_CAPABILITIES,
|
||||||
|
control_plane_access_granted=True,
|
||||||
|
external_planner_role="UNPRIVILEGED_PLANNER",
|
||||||
|
)
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
worker_once(token)
|
||||||
|
except Exception as exc:
|
||||||
|
audit(
|
||||||
|
"SUPERVISOR_COORDINATOR_LOOP_ERROR",
|
||||||
|
error=exc.code if isinstance(exc, CoordinatorError) else type(exc).__name__,
|
||||||
|
)
|
||||||
|
time.sleep(POLL_SECONDS)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,462 @@
|
|||||||
|
"""Self-healing availability guard for persistent Online Agent workers.
|
||||||
|
|
||||||
|
The guard never claims Tasks and never writes Task/Job/Git databases. It only:
|
||||||
|
- ensures required worker/executor systemd units are enabled + active;
|
||||||
|
- checks worker-owned liveness JSON written by the real poll/heartbeat loop;
|
||||||
|
- restarts an active worker only when its liveness is stale;
|
||||||
|
- honors an explicit maintenance inhibit file;
|
||||||
|
- writes a bounded status JSON under /run for observability.
|
||||||
|
|
||||||
|
Idle liveness is the successful poll loop itself. Busy liveness is refreshed by the
|
||||||
|
Task lease-heartbeat thread, so long-running Tasks are not mistaken for hung workers.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import asdict, dataclass
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import shlex
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
from typing import Any, Callable, Mapping, Sequence
|
||||||
|
|
||||||
|
SCHEMA = "aios.worker-availability.status.v1"
|
||||||
|
DEFAULT_WORKER_UNITS = (
|
||||||
|
"aios-online-agent-worker-20260815.service",
|
||||||
|
"aios-hermes-online-agent-worker.service",
|
||||||
|
"aios-openclaw-online-agent-worker.service",
|
||||||
|
)
|
||||||
|
DEFAULT_DEPENDENCY_UNITS = (
|
||||||
|
"aios-external-agent-executor.service",
|
||||||
|
)
|
||||||
|
DEFAULT_RUN_ROOT = Path("/run/aios-worker-availability")
|
||||||
|
DEFAULT_STATUS = DEFAULT_RUN_ROOT / "status.json"
|
||||||
|
DEFAULT_RECOVERY = DEFAULT_RUN_ROOT / "recovery.json"
|
||||||
|
DEFAULT_INHIBIT = DEFAULT_RUN_ROOT / "maintenance.lock"
|
||||||
|
DEFAULT_MAX_STALE_SECONDS = 90
|
||||||
|
DEFAULT_RESTART_COOLDOWN_SECONDS = 60
|
||||||
|
DEFAULT_STARTUP_WAIT_SECONDS = 10
|
||||||
|
ACCEPTED_ENABLED_STATES = frozenset({"enabled", "enabled-runtime", "static", "indirect", "generated"})
|
||||||
|
|
||||||
|
|
||||||
|
class AvailabilityGuardError(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class UnitSnapshot:
|
||||||
|
unit: str
|
||||||
|
role: str
|
||||||
|
load_state: str
|
||||||
|
active_state: str
|
||||||
|
unit_file_state: str
|
||||||
|
enabled_ok: bool
|
||||||
|
liveness_path: str | None = None
|
||||||
|
liveness_state: str | None = None
|
||||||
|
last_seen_epoch: float | None = None
|
||||||
|
liveness_age_seconds: float | None = None
|
||||||
|
fresh: bool | None = None
|
||||||
|
action: str = "NONE"
|
||||||
|
detail: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class Systemd:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
runner: Callable[..., subprocess.CompletedProcess[str]] | None = None,
|
||||||
|
) -> None:
|
||||||
|
self.runner = runner or subprocess.run
|
||||||
|
|
||||||
|
def _run(self, args: Sequence[str], *, check: bool = False) -> subprocess.CompletedProcess[str]:
|
||||||
|
result = self.runner(
|
||||||
|
["/usr/bin/systemctl", *args],
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
timeout=20,
|
||||||
|
)
|
||||||
|
if check and result.returncode != 0:
|
||||||
|
raise AvailabilityGuardError(
|
||||||
|
"SYSTEMD_COMMAND_FAILED:"
|
||||||
|
+ ":".join(args)
|
||||||
|
+ ":"
|
||||||
|
+ str(result.returncode)
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
|
def property(self, unit: str, name: str) -> str:
|
||||||
|
result = self._run(["show", unit, f"--property={name}", "--value"])
|
||||||
|
if result.returncode != 0:
|
||||||
|
return ""
|
||||||
|
return result.stdout.strip()
|
||||||
|
|
||||||
|
def environment(self, unit: str) -> dict[str, str]:
|
||||||
|
text = self.property(unit, "Environment")
|
||||||
|
result: dict[str, str] = {}
|
||||||
|
for item in shlex.split(text):
|
||||||
|
if "=" not in item:
|
||||||
|
continue
|
||||||
|
key, value = item.split("=", 1)
|
||||||
|
result[key] = value
|
||||||
|
return result
|
||||||
|
|
||||||
|
def enable(self, unit: str) -> None:
|
||||||
|
self._run(["enable", unit], check=True)
|
||||||
|
|
||||||
|
def start(self, unit: str) -> None:
|
||||||
|
self._run(["start", unit], check=True)
|
||||||
|
|
||||||
|
def restart(self, unit: str) -> None:
|
||||||
|
self._run(["restart", unit], check=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _atomic_json(path: Path, value: Mapping[str, Any]) -> None:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True, mode=0o755)
|
||||||
|
if path.parent.is_symlink() or path.is_symlink():
|
||||||
|
raise AvailabilityGuardError("STATUS_PATH_SYMLINK_REJECTED")
|
||||||
|
fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
|
||||||
|
try:
|
||||||
|
os.fchmod(fd, 0o644)
|
||||||
|
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||||
|
json.dump(value, handle, ensure_ascii=False, sort_keys=True, indent=2)
|
||||||
|
handle.write("\n")
|
||||||
|
handle.flush()
|
||||||
|
os.fsync(handle.fileno())
|
||||||
|
os.replace(temporary, path)
|
||||||
|
os.chmod(path, 0o644)
|
||||||
|
finally:
|
||||||
|
if os.path.exists(temporary):
|
||||||
|
os.unlink(temporary)
|
||||||
|
|
||||||
|
|
||||||
|
def _load_json(path: Path) -> dict[str, Any] | None:
|
||||||
|
if path.is_symlink() or not path.is_file():
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
value = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, json.JSONDecodeError):
|
||||||
|
return None
|
||||||
|
return dict(value) if isinstance(value, Mapping) else None
|
||||||
|
|
||||||
|
|
||||||
|
def _state_root(environment: Mapping[str, str], unit: str) -> Path:
|
||||||
|
explicit = environment.get("AIOS_WORKER_STATE_ROOT")
|
||||||
|
if explicit:
|
||||||
|
return Path(explicit)
|
||||||
|
fallbacks = {
|
||||||
|
"aios-online-agent-worker-20260815.service": Path(
|
||||||
|
"/var/lib/aios-online-agent-worker-20260815"
|
||||||
|
),
|
||||||
|
"aios-hermes-online-agent-worker.service": Path(
|
||||||
|
"/var/lib/aios-online-agent-hermes"
|
||||||
|
),
|
||||||
|
"aios-openclaw-online-agent-worker.service": Path(
|
||||||
|
"/var/lib/aios-online-agent-openclaw"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
if unit not in fallbacks:
|
||||||
|
raise AvailabilityGuardError("WORKER_STATE_ROOT_UNAVAILABLE:" + unit)
|
||||||
|
return fallbacks[unit]
|
||||||
|
|
||||||
|
|
||||||
|
def _liveness_path(environment: Mapping[str, str], unit: str) -> Path:
|
||||||
|
explicit = environment.get("AIOS_WORKER_LIVENESS_PATH")
|
||||||
|
if explicit:
|
||||||
|
return Path(explicit)
|
||||||
|
return _state_root(environment, unit) / "worker-liveness.json"
|
||||||
|
|
||||||
|
|
||||||
|
class AvailabilityGuard:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
systemd: Systemd | None = None,
|
||||||
|
worker_units: Sequence[str] = DEFAULT_WORKER_UNITS,
|
||||||
|
dependency_units: Sequence[str] = DEFAULT_DEPENDENCY_UNITS,
|
||||||
|
status_path: Path = DEFAULT_STATUS,
|
||||||
|
recovery_path: Path = DEFAULT_RECOVERY,
|
||||||
|
inhibit_path: Path = DEFAULT_INHIBIT,
|
||||||
|
max_stale_seconds: int = DEFAULT_MAX_STALE_SECONDS,
|
||||||
|
restart_cooldown_seconds: int = DEFAULT_RESTART_COOLDOWN_SECONDS,
|
||||||
|
startup_wait_seconds: int = DEFAULT_STARTUP_WAIT_SECONDS,
|
||||||
|
now_fn: Callable[[], float] = time.time,
|
||||||
|
sleep_fn: Callable[[float], None] = time.sleep,
|
||||||
|
) -> None:
|
||||||
|
self.systemd = systemd or Systemd()
|
||||||
|
self.worker_units = tuple(worker_units)
|
||||||
|
self.dependency_units = tuple(dependency_units)
|
||||||
|
self.status_path = Path(status_path)
|
||||||
|
self.recovery_path = Path(recovery_path)
|
||||||
|
self.inhibit_path = Path(inhibit_path)
|
||||||
|
self.max_stale_seconds = int(max_stale_seconds)
|
||||||
|
self.restart_cooldown_seconds = int(restart_cooldown_seconds)
|
||||||
|
self.startup_wait_seconds = int(startup_wait_seconds)
|
||||||
|
self.now_fn = now_fn
|
||||||
|
self.sleep_fn = sleep_fn
|
||||||
|
if self.max_stale_seconds < 15:
|
||||||
|
raise AvailabilityGuardError("MAX_STALE_SECONDS_TOO_SMALL")
|
||||||
|
if self.restart_cooldown_seconds < 15:
|
||||||
|
raise AvailabilityGuardError("RESTART_COOLDOWN_TOO_SMALL")
|
||||||
|
if self.startup_wait_seconds < 0 or self.startup_wait_seconds > 60:
|
||||||
|
raise AvailabilityGuardError("STARTUP_WAIT_INVALID")
|
||||||
|
|
||||||
|
def _recovery_state(self) -> dict[str, float]:
|
||||||
|
value = _load_json(self.recovery_path) or {}
|
||||||
|
raw = value.get("last_restart_epoch")
|
||||||
|
if not isinstance(raw, Mapping):
|
||||||
|
return {}
|
||||||
|
result: dict[str, float] = {}
|
||||||
|
for unit, epoch in raw.items():
|
||||||
|
try:
|
||||||
|
result[str(unit)] = float(epoch)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
continue
|
||||||
|
return result
|
||||||
|
|
||||||
|
def _save_recovery_state(self, state: Mapping[str, float]) -> None:
|
||||||
|
_atomic_json(
|
||||||
|
self.recovery_path,
|
||||||
|
{
|
||||||
|
"schema": "aios.worker-availability.recovery.v1",
|
||||||
|
"last_restart_epoch": dict(state),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def _basic_snapshot(self, unit: str, role: str) -> tuple[str, str, str, bool]:
|
||||||
|
load = self.systemd.property(unit, "LoadState")
|
||||||
|
active = self.systemd.property(unit, "ActiveState")
|
||||||
|
enabled = self.systemd.property(unit, "UnitFileState")
|
||||||
|
return load, active, enabled, enabled in ACCEPTED_ENABLED_STATES
|
||||||
|
|
||||||
|
def _ensure_enabled_active(self, unit: str, role: str) -> tuple[str, bool]:
|
||||||
|
load, active, enabled, enabled_ok = self._basic_snapshot(unit, role)
|
||||||
|
if load != "loaded":
|
||||||
|
raise AvailabilityGuardError("REQUIRED_UNIT_NOT_LOADED:" + unit)
|
||||||
|
action = "NONE"
|
||||||
|
if not enabled_ok:
|
||||||
|
self.systemd.enable(unit)
|
||||||
|
action = "ENABLE"
|
||||||
|
if active != "active":
|
||||||
|
self.systemd.start(unit)
|
||||||
|
action = "START" if action == "NONE" else "ENABLE_START"
|
||||||
|
return action, active == "active"
|
||||||
|
|
||||||
|
def _wait_for_liveness(self, path: Path) -> dict[str, Any] | None:
|
||||||
|
deadline = self.now_fn() + self.startup_wait_seconds
|
||||||
|
while True:
|
||||||
|
value = _load_json(path)
|
||||||
|
if value is not None:
|
||||||
|
try:
|
||||||
|
epoch = float(value.get("last_seen_epoch"))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
epoch = 0.0
|
||||||
|
if epoch > 0 and self.now_fn() - epoch <= self.max_stale_seconds:
|
||||||
|
return value
|
||||||
|
if self.now_fn() >= deadline:
|
||||||
|
return value
|
||||||
|
self.sleep_fn(0.5)
|
||||||
|
|
||||||
|
def run(self, *, heal: bool) -> dict[str, Any]:
|
||||||
|
now = self.now_fn()
|
||||||
|
if self.inhibit_path.exists():
|
||||||
|
result = {
|
||||||
|
"schema": SCHEMA,
|
||||||
|
"status": "MAINTENANCE_INHIBITED",
|
||||||
|
"generated_at_epoch": now,
|
||||||
|
"maintenance_inhibit": str(self.inhibit_path),
|
||||||
|
"workers": [],
|
||||||
|
"dependencies": [],
|
||||||
|
"all_fresh": False,
|
||||||
|
"actions": [],
|
||||||
|
}
|
||||||
|
_atomic_json(self.status_path, result)
|
||||||
|
return result
|
||||||
|
|
||||||
|
actions: list[dict[str, str]] = []
|
||||||
|
dependencies: list[UnitSnapshot] = []
|
||||||
|
for unit in self.dependency_units:
|
||||||
|
load, active, enabled, enabled_ok = self._basic_snapshot(unit, "dependency")
|
||||||
|
action = "NONE"
|
||||||
|
if heal:
|
||||||
|
if load != "loaded":
|
||||||
|
raise AvailabilityGuardError("REQUIRED_UNIT_NOT_LOADED:" + unit)
|
||||||
|
if not enabled_ok:
|
||||||
|
self.systemd.enable(unit)
|
||||||
|
action = "ENABLE"
|
||||||
|
if active != "active":
|
||||||
|
self.systemd.start(unit)
|
||||||
|
action = "START" if action == "NONE" else "ENABLE_START"
|
||||||
|
if action != "NONE":
|
||||||
|
actions.append({"unit": unit, "action": action})
|
||||||
|
load, active, enabled, enabled_ok = self._basic_snapshot(unit, "dependency")
|
||||||
|
dependencies.append(
|
||||||
|
UnitSnapshot(
|
||||||
|
unit=unit,
|
||||||
|
role="dependency",
|
||||||
|
load_state=load,
|
||||||
|
active_state=active,
|
||||||
|
unit_file_state=enabled,
|
||||||
|
enabled_ok=enabled_ok,
|
||||||
|
fresh=(active == "active"),
|
||||||
|
action=action,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
recovery = self._recovery_state()
|
||||||
|
workers: list[UnitSnapshot] = []
|
||||||
|
for unit in self.worker_units:
|
||||||
|
load, active, enabled, enabled_ok = self._basic_snapshot(unit, "worker")
|
||||||
|
environment = self.systemd.environment(unit) if load == "loaded" else {}
|
||||||
|
path = _liveness_path(environment, unit) if load == "loaded" else None
|
||||||
|
action = "NONE"
|
||||||
|
|
||||||
|
if heal:
|
||||||
|
if load != "loaded":
|
||||||
|
raise AvailabilityGuardError("REQUIRED_UNIT_NOT_LOADED:" + unit)
|
||||||
|
if not enabled_ok:
|
||||||
|
self.systemd.enable(unit)
|
||||||
|
action = "ENABLE"
|
||||||
|
if active != "active":
|
||||||
|
self.systemd.start(unit)
|
||||||
|
action = "START" if action == "NONE" else "ENABLE_START"
|
||||||
|
actions.append({"unit": unit, "action": action})
|
||||||
|
if path is not None:
|
||||||
|
self._wait_for_liveness(path)
|
||||||
|
load, active, enabled, enabled_ok = self._basic_snapshot(unit, "worker")
|
||||||
|
|
||||||
|
live = _load_json(path) if path is not None else None
|
||||||
|
last_seen: float | None = None
|
||||||
|
state: str | None = None
|
||||||
|
age: float | None = None
|
||||||
|
fresh = False
|
||||||
|
detail: str | None = None
|
||||||
|
if live is not None:
|
||||||
|
state = str(live.get("state") or "")
|
||||||
|
try:
|
||||||
|
last_seen = float(live.get("last_seen_epoch"))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
last_seen = None
|
||||||
|
if last_seen is not None:
|
||||||
|
age = max(0.0, now - last_seen)
|
||||||
|
fresh = age <= self.max_stale_seconds
|
||||||
|
if live.get("agent_id") in (None, ""):
|
||||||
|
fresh = False
|
||||||
|
detail = "LIVENESS_AGENT_ID_MISSING"
|
||||||
|
else:
|
||||||
|
detail = "LIVENESS_MISSING"
|
||||||
|
|
||||||
|
if heal and active == "active" and not fresh:
|
||||||
|
previous = float(recovery.get(unit, 0.0))
|
||||||
|
if now - previous >= self.restart_cooldown_seconds:
|
||||||
|
self.systemd.restart(unit)
|
||||||
|
recovery[unit] = now
|
||||||
|
action = "RESTART_STALE"
|
||||||
|
actions.append({"unit": unit, "action": action})
|
||||||
|
if path is not None:
|
||||||
|
live = self._wait_for_liveness(path)
|
||||||
|
load, active, enabled, enabled_ok = self._basic_snapshot(unit, "worker")
|
||||||
|
if live is not None:
|
||||||
|
state = str(live.get("state") or "")
|
||||||
|
try:
|
||||||
|
last_seen = float(live.get("last_seen_epoch"))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
last_seen = None
|
||||||
|
if last_seen is not None:
|
||||||
|
age = max(0.0, self.now_fn() - last_seen)
|
||||||
|
fresh = age <= self.max_stale_seconds
|
||||||
|
detail = None if fresh else "LIVENESS_STALE_AFTER_RESTART"
|
||||||
|
else:
|
||||||
|
detail = "RESTART_COOLDOWN_ACTIVE"
|
||||||
|
|
||||||
|
workers.append(
|
||||||
|
UnitSnapshot(
|
||||||
|
unit=unit,
|
||||||
|
role="worker",
|
||||||
|
load_state=load,
|
||||||
|
active_state=active,
|
||||||
|
unit_file_state=enabled,
|
||||||
|
enabled_ok=enabled_ok,
|
||||||
|
liveness_path=str(path) if path is not None else None,
|
||||||
|
liveness_state=state,
|
||||||
|
last_seen_epoch=last_seen,
|
||||||
|
liveness_age_seconds=age,
|
||||||
|
fresh=bool(active == "active" and enabled_ok and fresh),
|
||||||
|
action=action,
|
||||||
|
detail=detail,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if heal:
|
||||||
|
self._save_recovery_state(recovery)
|
||||||
|
|
||||||
|
all_fresh = (
|
||||||
|
all(item.active_state == "active" and item.enabled_ok and item.fresh for item in workers)
|
||||||
|
and all(item.active_state == "active" for item in dependencies)
|
||||||
|
)
|
||||||
|
result = {
|
||||||
|
"schema": SCHEMA,
|
||||||
|
"status": "HEALTHY" if all_fresh else "DEGRADED",
|
||||||
|
"generated_at_epoch": self.now_fn(),
|
||||||
|
"max_stale_seconds": self.max_stale_seconds,
|
||||||
|
"maintenance_inhibit": str(self.inhibit_path),
|
||||||
|
"workers": [asdict(item) for item in workers],
|
||||||
|
"dependencies": [asdict(item) for item in dependencies],
|
||||||
|
"all_fresh": all_fresh,
|
||||||
|
"actions": actions,
|
||||||
|
}
|
||||||
|
_atomic_json(self.status_path, result)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
mode = parser.add_mutually_exclusive_group(required=True)
|
||||||
|
mode.add_argument("--heal", action="store_true")
|
||||||
|
mode.add_argument("--check", action="store_true")
|
||||||
|
parser.add_argument("--status-file", default=str(DEFAULT_STATUS))
|
||||||
|
parser.add_argument("--recovery-file", default=str(DEFAULT_RECOVERY))
|
||||||
|
parser.add_argument("--inhibit-file", default=str(DEFAULT_INHIBIT))
|
||||||
|
parser.add_argument(
|
||||||
|
"--max-stale-seconds",
|
||||||
|
type=int,
|
||||||
|
default=int(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_WORKER_AVAILABILITY_MAX_STALE_SECONDS",
|
||||||
|
str(DEFAULT_MAX_STALE_SECONDS),
|
||||||
|
)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--restart-cooldown-seconds",
|
||||||
|
type=int,
|
||||||
|
default=int(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_WORKER_AVAILABILITY_RESTART_COOLDOWN_SECONDS",
|
||||||
|
str(DEFAULT_RESTART_COOLDOWN_SECONDS),
|
||||||
|
)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
guard = AvailabilityGuard(
|
||||||
|
status_path=Path(args.status_file),
|
||||||
|
recovery_path=Path(args.recovery_file),
|
||||||
|
inhibit_path=Path(args.inhibit_file),
|
||||||
|
max_stale_seconds=args.max_stale_seconds,
|
||||||
|
restart_cooldown_seconds=args.restart_cooldown_seconds,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
result = guard.run(heal=args.heal)
|
||||||
|
except AvailabilityGuardError as exc:
|
||||||
|
print(json.dumps({"status": "FAILED", "error": str(exc)}, sort_keys=True))
|
||||||
|
return 20
|
||||||
|
print(json.dumps(result, ensure_ascii=False, sort_keys=True))
|
||||||
|
if args.check and not result.get("all_fresh"):
|
||||||
|
return 10
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,911 @@
|
|||||||
|
"""Durable multi-agent fleet scheduling for TaskPlane.
|
||||||
|
|
||||||
|
The fleet layer decides *which* eligible worker should receive *which* queued task.
|
||||||
|
It never replaces TaskPlane claim/lease ownership: callers must still perform the
|
||||||
|
authoritative TaskPlane claim CAS after a reservation is returned.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import math
|
||||||
|
import sqlite3
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from typing import Any, Iterable, Mapping, Sequence
|
||||||
|
|
||||||
|
DEFAULT_RESERVATION_SECONDS = 30
|
||||||
|
DEFAULT_CIRCUIT_THRESHOLD = 3
|
||||||
|
DEFAULT_CIRCUIT_SECONDS = 30
|
||||||
|
MAX_CIRCUIT_SECONDS = 15 * 60
|
||||||
|
MAX_SLOTS = 128
|
||||||
|
|
||||||
|
|
||||||
|
class FleetError(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class FleetConflict(FleetError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _canonical(value: object) -> bytes:
|
||||||
|
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _sha(value: object) -> str:
|
||||||
|
return hashlib.sha256(_canonical(value)).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _text(value: object, default: str = "") -> str:
|
||||||
|
if value is None:
|
||||||
|
return default
|
||||||
|
return str(value).strip() or default
|
||||||
|
|
||||||
|
|
||||||
|
def _int(value: object, default: int, minimum: int, maximum: int) -> int:
|
||||||
|
if value is None:
|
||||||
|
return default
|
||||||
|
if isinstance(value, bool):
|
||||||
|
raise FleetError("FLEET_INTEGER_INVALID")
|
||||||
|
try:
|
||||||
|
result = int(value)
|
||||||
|
except (TypeError, ValueError) as exc:
|
||||||
|
raise FleetError("FLEET_INTEGER_INVALID") from exc
|
||||||
|
if result < minimum or result > maximum:
|
||||||
|
raise FleetError("FLEET_INTEGER_OUT_OF_RANGE")
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _float(value: object, default: float, minimum: float, maximum: float) -> float:
|
||||||
|
if value is None:
|
||||||
|
return default
|
||||||
|
try:
|
||||||
|
result = float(value)
|
||||||
|
except (TypeError, ValueError) as exc:
|
||||||
|
raise FleetError("FLEET_NUMBER_INVALID") from exc
|
||||||
|
if not math.isfinite(result) or result < minimum or result > maximum:
|
||||||
|
raise FleetError("FLEET_NUMBER_OUT_OF_RANGE")
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _mapping(value: object) -> Mapping[str, Any]:
|
||||||
|
if isinstance(value, Mapping):
|
||||||
|
return value
|
||||||
|
if hasattr(value, "to_dict"):
|
||||||
|
data = value.to_dict()
|
||||||
|
if isinstance(data, Mapping):
|
||||||
|
return data
|
||||||
|
if hasattr(value, "__dict__"):
|
||||||
|
return vars(value)
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def _attr(value: object, name: str, default: Any = None) -> Any:
|
||||||
|
if isinstance(value, Mapping):
|
||||||
|
return value.get(name, default)
|
||||||
|
return getattr(value, name, default)
|
||||||
|
|
||||||
|
|
||||||
|
def _task_plan(task: object) -> object:
|
||||||
|
return _attr(task, "task_plan", _attr(task, "plan", task))
|
||||||
|
|
||||||
|
|
||||||
|
def _task_id(task: object) -> str:
|
||||||
|
return _text(_attr(task, "task_id", _attr(_task_plan(task), "task_id", "")))
|
||||||
|
|
||||||
|
|
||||||
|
def _task_status(task: object) -> str:
|
||||||
|
return _text(_attr(task, "status", "")).upper()
|
||||||
|
|
||||||
|
|
||||||
|
def _task_revision(task: object) -> int:
|
||||||
|
return _int(_attr(task, "revision", 0), 0, 0, 2**31 - 1)
|
||||||
|
|
||||||
|
|
||||||
|
def _task_created_at(task: object) -> float:
|
||||||
|
value = _attr(task, "created_at", _attr(task, "updated_at", 0.0))
|
||||||
|
try:
|
||||||
|
return float(value or 0.0)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def _task_required_capabilities(task: object) -> frozenset[str]:
|
||||||
|
plan = _task_plan(task)
|
||||||
|
value = _attr(plan, "required_capabilities", ())
|
||||||
|
if not isinstance(value, Sequence) or isinstance(value, (str, bytes)):
|
||||||
|
return frozenset()
|
||||||
|
return frozenset(_text(item) for item in value if _text(item))
|
||||||
|
|
||||||
|
|
||||||
|
def _task_metadata(task: object) -> Mapping[str, Any]:
|
||||||
|
plan = _task_plan(task)
|
||||||
|
value = _attr(plan, "metadata", {})
|
||||||
|
return value if isinstance(value, Mapping) else {}
|
||||||
|
|
||||||
|
|
||||||
|
def _task_scope_key(task: object) -> str:
|
||||||
|
plan = _task_plan(task)
|
||||||
|
scope_type = _text(_attr(plan, "scope_type", "legacy"), "legacy")
|
||||||
|
tenant = _text(_attr(plan, "tenant_id", ""))
|
||||||
|
project = _text(_attr(plan, "project_id", ""))
|
||||||
|
user = _text(_attr(plan, "user_id", ""))
|
||||||
|
if scope_type == "tenant":
|
||||||
|
return "tenant:" + tenant + "|project:" + (project or "*")
|
||||||
|
if scope_type == "system":
|
||||||
|
return "system"
|
||||||
|
return "legacy:" + (user or "*")
|
||||||
|
|
||||||
|
|
||||||
|
def _worker_id(worker: object) -> str:
|
||||||
|
return _text(_attr(worker, "worker_id", ""))
|
||||||
|
|
||||||
|
|
||||||
|
def _worker_capabilities(worker: object) -> frozenset[str]:
|
||||||
|
value = _attr(worker, "capabilities", ())
|
||||||
|
if isinstance(value, Mapping):
|
||||||
|
value = value.keys()
|
||||||
|
if not isinstance(value, Iterable) or isinstance(value, (str, bytes)):
|
||||||
|
return frozenset()
|
||||||
|
return frozenset(_text(item) for item in value if _text(item))
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class FleetWorkerPolicy:
|
||||||
|
worker_id: str
|
||||||
|
max_slots: int = 1
|
||||||
|
failure_domain: str = "default"
|
||||||
|
weight: float = 1.0
|
||||||
|
labels: Mapping[str, str] | None = None
|
||||||
|
revision: int = 1
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class FleetDispatchDecision:
|
||||||
|
reservation_id: str
|
||||||
|
task_id: str
|
||||||
|
worker_id: str
|
||||||
|
task_revision: int
|
||||||
|
scope_key: str
|
||||||
|
failure_domain: str
|
||||||
|
reserved_until: float
|
||||||
|
score: tuple[float, float, float, str]
|
||||||
|
|
||||||
|
def to_dict(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"reservation_id": self.reservation_id,
|
||||||
|
"task_id": self.task_id,
|
||||||
|
"worker_id": self.worker_id,
|
||||||
|
"task_revision": self.task_revision,
|
||||||
|
"scope_key": self.scope_key,
|
||||||
|
"failure_domain": self.failure_domain,
|
||||||
|
"reserved_until": self.reserved_until,
|
||||||
|
"score": list(self.score),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class FleetStateStore:
|
||||||
|
"""Small durable fleet journal colocated in the existing TaskPlane SQLite database."""
|
||||||
|
|
||||||
|
def __init__(self, connection: sqlite3.Connection, lock: threading.RLock | None = None) -> None:
|
||||||
|
self.conn = connection
|
||||||
|
self.lock = lock or threading.RLock()
|
||||||
|
self._savepoint_counter = 0
|
||||||
|
self._ensure_schema()
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def _transaction(self):
|
||||||
|
with self.lock:
|
||||||
|
self._savepoint_counter += 1
|
||||||
|
name = f"fleet_{threading.get_ident()}_{self._savepoint_counter}"
|
||||||
|
self.conn.execute(f"SAVEPOINT {name}")
|
||||||
|
try:
|
||||||
|
yield
|
||||||
|
except Exception:
|
||||||
|
self.conn.execute(f"ROLLBACK TO {name}")
|
||||||
|
self.conn.execute(f"RELEASE {name}")
|
||||||
|
raise
|
||||||
|
else:
|
||||||
|
self.conn.execute(f"RELEASE {name}")
|
||||||
|
|
||||||
|
def _ensure_schema(self) -> None:
|
||||||
|
with self.lock:
|
||||||
|
self.conn.executescript(
|
||||||
|
"""
|
||||||
|
CREATE TABLE IF NOT EXISTS fleet_worker_capacity (
|
||||||
|
worker_id TEXT PRIMARY KEY,
|
||||||
|
max_slots INTEGER NOT NULL,
|
||||||
|
failure_domain TEXT NOT NULL,
|
||||||
|
weight REAL NOT NULL,
|
||||||
|
labels_json TEXT NOT NULL,
|
||||||
|
revision INTEGER NOT NULL,
|
||||||
|
updated_at REAL NOT NULL
|
||||||
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS fleet_scope_state (
|
||||||
|
scope_key TEXT PRIMARY KEY,
|
||||||
|
virtual_finish REAL NOT NULL,
|
||||||
|
dispatch_count INTEGER NOT NULL,
|
||||||
|
last_dispatched_at REAL,
|
||||||
|
revision INTEGER NOT NULL
|
||||||
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS fleet_dispatches (
|
||||||
|
reservation_id TEXT PRIMARY KEY,
|
||||||
|
task_id TEXT NOT NULL UNIQUE,
|
||||||
|
worker_id TEXT NOT NULL,
|
||||||
|
task_revision INTEGER NOT NULL,
|
||||||
|
scope_key TEXT NOT NULL,
|
||||||
|
failure_domain TEXT NOT NULL,
|
||||||
|
state TEXT NOT NULL,
|
||||||
|
reserved_until REAL NOT NULL,
|
||||||
|
claim_id TEXT,
|
||||||
|
created_at REAL NOT NULL,
|
||||||
|
updated_at REAL NOT NULL
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS ix_fleet_dispatch_worker_state
|
||||||
|
ON fleet_dispatches(worker_id, state, reserved_until);
|
||||||
|
CREATE TABLE IF NOT EXISTS fleet_worker_failures (
|
||||||
|
worker_id TEXT PRIMARY KEY,
|
||||||
|
consecutive_failures INTEGER NOT NULL,
|
||||||
|
circuit_open_until REAL,
|
||||||
|
last_failure_class TEXT,
|
||||||
|
revision INTEGER NOT NULL,
|
||||||
|
updated_at REAL NOT NULL
|
||||||
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS fleet_task_worker_failures (
|
||||||
|
task_id TEXT NOT NULL,
|
||||||
|
worker_id TEXT NOT NULL,
|
||||||
|
failure_count INTEGER NOT NULL,
|
||||||
|
blocked_until REAL,
|
||||||
|
last_failure_class TEXT,
|
||||||
|
revision INTEGER NOT NULL,
|
||||||
|
updated_at REAL NOT NULL,
|
||||||
|
PRIMARY KEY(task_id, worker_id)
|
||||||
|
);
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
def configure_worker(
|
||||||
|
self,
|
||||||
|
worker_id: str,
|
||||||
|
*,
|
||||||
|
max_slots: int = 1,
|
||||||
|
failure_domain: str = "default",
|
||||||
|
weight: float = 1.0,
|
||||||
|
labels: Mapping[str, str] | None = None,
|
||||||
|
expected_revision: int | None = None,
|
||||||
|
now: float | None = None,
|
||||||
|
) -> FleetWorkerPolicy:
|
||||||
|
worker_id = _text(worker_id)
|
||||||
|
if not worker_id:
|
||||||
|
raise FleetError("FLEET_WORKER_ID_REQUIRED")
|
||||||
|
max_slots = _int(max_slots, 1, 1, MAX_SLOTS)
|
||||||
|
weight = _float(weight, 1.0, 0.05, 100.0)
|
||||||
|
failure_domain = _text(failure_domain, "default")
|
||||||
|
normalized_labels = {
|
||||||
|
_text(k): _text(v)
|
||||||
|
for k, v in dict(labels or {}).items()
|
||||||
|
if _text(k) and _text(v)
|
||||||
|
}
|
||||||
|
now = float(time.time() if now is None else now)
|
||||||
|
with self._transaction():
|
||||||
|
row = self.conn.execute(
|
||||||
|
"SELECT revision FROM fleet_worker_capacity WHERE worker_id=?",
|
||||||
|
(worker_id,),
|
||||||
|
).fetchone()
|
||||||
|
current = int(row[0]) if row else 0
|
||||||
|
if expected_revision is not None and current != int(expected_revision):
|
||||||
|
raise FleetConflict("FLEET_WORKER_POLICY_CAS_MISMATCH")
|
||||||
|
revision = current + 1
|
||||||
|
self.conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO fleet_worker_capacity(
|
||||||
|
worker_id,max_slots,failure_domain,weight,labels_json,revision,updated_at
|
||||||
|
) VALUES (?,?,?,?,?,?,?)
|
||||||
|
ON CONFLICT(worker_id) DO UPDATE SET
|
||||||
|
max_slots=excluded.max_slots,
|
||||||
|
failure_domain=excluded.failure_domain,
|
||||||
|
weight=excluded.weight,
|
||||||
|
labels_json=excluded.labels_json,
|
||||||
|
revision=excluded.revision,
|
||||||
|
updated_at=excluded.updated_at
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
worker_id,
|
||||||
|
max_slots,
|
||||||
|
failure_domain,
|
||||||
|
weight,
|
||||||
|
json.dumps(normalized_labels, sort_keys=True, separators=(",", ":")),
|
||||||
|
revision,
|
||||||
|
now,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return FleetWorkerPolicy(
|
||||||
|
worker_id, max_slots, failure_domain, weight, normalized_labels, revision
|
||||||
|
)
|
||||||
|
|
||||||
|
def worker_policy(self, worker_id: str) -> FleetWorkerPolicy:
|
||||||
|
worker_id = _text(worker_id)
|
||||||
|
with self.lock:
|
||||||
|
row = self.conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT worker_id,max_slots,failure_domain,weight,labels_json,revision
|
||||||
|
FROM fleet_worker_capacity WHERE worker_id=?
|
||||||
|
""",
|
||||||
|
(worker_id,),
|
||||||
|
).fetchone()
|
||||||
|
if row is None:
|
||||||
|
return FleetWorkerPolicy(worker_id=worker_id)
|
||||||
|
return FleetWorkerPolicy(
|
||||||
|
worker_id=str(row[0]),
|
||||||
|
max_slots=int(row[1]),
|
||||||
|
failure_domain=str(row[2]),
|
||||||
|
weight=float(row[3]),
|
||||||
|
labels=json.loads(row[4] or "{}"),
|
||||||
|
revision=int(row[5]),
|
||||||
|
)
|
||||||
|
|
||||||
|
def _cleanup_expired(self, now: float) -> None:
|
||||||
|
self.conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE fleet_dispatches
|
||||||
|
SET state='EXPIRED', updated_at=?
|
||||||
|
WHERE state='RESERVED' AND reserved_until<=?
|
||||||
|
""",
|
||||||
|
(now, now),
|
||||||
|
)
|
||||||
|
|
||||||
|
def reserved_slots(self, worker_id: str, now: float) -> int:
|
||||||
|
with self.lock:
|
||||||
|
self._cleanup_expired(now)
|
||||||
|
row = self.conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT COUNT(*) FROM fleet_dispatches
|
||||||
|
WHERE worker_id=? AND state='RESERVED' AND reserved_until>?
|
||||||
|
""",
|
||||||
|
(worker_id, now),
|
||||||
|
).fetchone()
|
||||||
|
return int(row[0] if row else 0)
|
||||||
|
|
||||||
|
def scope_virtual_finish(self, scope_key: str) -> float:
|
||||||
|
with self.lock:
|
||||||
|
row = self.conn.execute(
|
||||||
|
"SELECT virtual_finish FROM fleet_scope_state WHERE scope_key=?",
|
||||||
|
(scope_key,),
|
||||||
|
).fetchone()
|
||||||
|
return float(row[0]) if row else 0.0
|
||||||
|
|
||||||
|
def reserve(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
task_id: str,
|
||||||
|
worker_id: str,
|
||||||
|
task_revision: int,
|
||||||
|
scope_key: str,
|
||||||
|
failure_domain: str,
|
||||||
|
score: tuple[float, float, float, str],
|
||||||
|
reservation_seconds: int,
|
||||||
|
now: float,
|
||||||
|
scope_weight: float,
|
||||||
|
) -> FleetDispatchDecision:
|
||||||
|
reservation_seconds = _int(
|
||||||
|
reservation_seconds, DEFAULT_RESERVATION_SECONDS, 1, 300
|
||||||
|
)
|
||||||
|
reservation_id = "fleet-" + _sha(
|
||||||
|
{
|
||||||
|
"task_id": task_id,
|
||||||
|
"worker_id": worker_id,
|
||||||
|
"task_revision": task_revision,
|
||||||
|
"scope_key": scope_key,
|
||||||
|
}
|
||||||
|
)[:32]
|
||||||
|
reserved_until = now + reservation_seconds
|
||||||
|
with self._transaction():
|
||||||
|
self._cleanup_expired(now)
|
||||||
|
existing = self.conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT reservation_id,worker_id,task_revision,scope_key,failure_domain,
|
||||||
|
reserved_until,state
|
||||||
|
FROM fleet_dispatches WHERE task_id=?
|
||||||
|
""",
|
||||||
|
(task_id,),
|
||||||
|
).fetchone()
|
||||||
|
# Capacity is rechecked inside the same durable reservation transaction.
|
||||||
|
# This closes the cross-thread/process race between eligibility scoring and reserve.
|
||||||
|
policy_row = self.conn.execute(
|
||||||
|
"SELECT max_slots FROM fleet_worker_capacity WHERE worker_id=?",
|
||||||
|
(worker_id,),
|
||||||
|
).fetchone()
|
||||||
|
max_slots = int(policy_row[0]) if policy_row else 1
|
||||||
|
if existing and str(existing[6]) in {"RESERVED", "CLAIMED"} and float(existing[5]) > now:
|
||||||
|
if str(existing[1]) != worker_id or int(existing[2]) != task_revision:
|
||||||
|
raise FleetConflict("FLEET_TASK_ALREADY_RESERVED")
|
||||||
|
return FleetDispatchDecision(
|
||||||
|
str(existing[0]), task_id, worker_id, task_revision, str(existing[3]),
|
||||||
|
str(existing[4]), float(existing[5]), score
|
||||||
|
)
|
||||||
|
|
||||||
|
claim_columns = {
|
||||||
|
str(row[1])
|
||||||
|
for row in self.conn.execute("PRAGMA table_info(claims)").fetchall()
|
||||||
|
}
|
||||||
|
active_claims = 0
|
||||||
|
if {"worker_id", "status"}.issubset(claim_columns):
|
||||||
|
sql = "SELECT COUNT(*) FROM claims WHERE worker_id=? AND status='ACTIVE'"
|
||||||
|
params: list[Any] = [worker_id]
|
||||||
|
if "lease_expires_at" in claim_columns:
|
||||||
|
sql += " AND lease_expires_at>?"
|
||||||
|
params.append(now)
|
||||||
|
row = self.conn.execute(sql, tuple(params)).fetchone()
|
||||||
|
active_claims = int(row[0] if row else 0)
|
||||||
|
row = self.conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT COUNT(*) FROM fleet_dispatches
|
||||||
|
WHERE worker_id=? AND task_id<>?
|
||||||
|
AND state='RESERVED' AND reserved_until>?
|
||||||
|
""",
|
||||||
|
(worker_id, task_id, now),
|
||||||
|
).fetchone()
|
||||||
|
active_reservations = int(row[0] if row else 0)
|
||||||
|
if active_claims + active_reservations >= max_slots:
|
||||||
|
raise FleetConflict("FLEET_WORKER_CAPACITY_EXHAUSTED")
|
||||||
|
|
||||||
|
self.conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO fleet_dispatches(
|
||||||
|
reservation_id,task_id,worker_id,task_revision,scope_key,failure_domain,
|
||||||
|
state,reserved_until,claim_id,created_at,updated_at
|
||||||
|
) VALUES (?,?,?,?,?,?,'RESERVED',?,NULL,?,?)
|
||||||
|
ON CONFLICT(task_id) DO UPDATE SET
|
||||||
|
reservation_id=excluded.reservation_id,
|
||||||
|
worker_id=excluded.worker_id,
|
||||||
|
task_revision=excluded.task_revision,
|
||||||
|
scope_key=excluded.scope_key,
|
||||||
|
failure_domain=excluded.failure_domain,
|
||||||
|
state='RESERVED',
|
||||||
|
reserved_until=excluded.reserved_until,
|
||||||
|
claim_id=NULL,
|
||||||
|
updated_at=excluded.updated_at
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
reservation_id, task_id, worker_id, task_revision, scope_key,
|
||||||
|
failure_domain, reserved_until, now, now,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
row = self.conn.execute(
|
||||||
|
"SELECT virtual_finish,dispatch_count,revision FROM fleet_scope_state WHERE scope_key=?",
|
||||||
|
(scope_key,),
|
||||||
|
).fetchone()
|
||||||
|
current_finish = float(row[0]) if row else 0.0
|
||||||
|
dispatch_count = int(row[1]) if row else 0
|
||||||
|
revision = int(row[2]) if row else 0
|
||||||
|
next_finish = max(current_finish, 0.0) + 1.0 / max(scope_weight, 0.05)
|
||||||
|
self.conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO fleet_scope_state(
|
||||||
|
scope_key,virtual_finish,dispatch_count,last_dispatched_at,revision
|
||||||
|
) VALUES (?,?,?,?,?)
|
||||||
|
ON CONFLICT(scope_key) DO UPDATE SET
|
||||||
|
virtual_finish=excluded.virtual_finish,
|
||||||
|
dispatch_count=excluded.dispatch_count,
|
||||||
|
last_dispatched_at=excluded.last_dispatched_at,
|
||||||
|
revision=excluded.revision
|
||||||
|
""",
|
||||||
|
(scope_key, next_finish, dispatch_count + 1, now, revision + 1),
|
||||||
|
)
|
||||||
|
return FleetDispatchDecision(
|
||||||
|
reservation_id, task_id, worker_id, task_revision, scope_key,
|
||||||
|
failure_domain, reserved_until, score
|
||||||
|
)
|
||||||
|
|
||||||
|
def bind_claim(self, reservation_id: str, claim_id: str, *, now: float | None = None) -> None:
|
||||||
|
now = float(time.time() if now is None else now)
|
||||||
|
with self._transaction():
|
||||||
|
row = self.conn.execute(
|
||||||
|
"SELECT state FROM fleet_dispatches WHERE reservation_id=?",
|
||||||
|
(reservation_id,),
|
||||||
|
).fetchone()
|
||||||
|
if row is None:
|
||||||
|
raise FleetError("FLEET_RESERVATION_NOT_FOUND")
|
||||||
|
if str(row[0]) not in {"RESERVED", "CLAIMED"}:
|
||||||
|
raise FleetConflict("FLEET_RESERVATION_NOT_ACTIVE")
|
||||||
|
self.conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE fleet_dispatches
|
||||||
|
SET state='CLAIMED', claim_id=?, updated_at=?
|
||||||
|
WHERE reservation_id=?
|
||||||
|
""",
|
||||||
|
(claim_id, now, reservation_id),
|
||||||
|
)
|
||||||
|
|
||||||
|
def release(
|
||||||
|
self,
|
||||||
|
task_id: str,
|
||||||
|
*,
|
||||||
|
state: str = "RELEASED",
|
||||||
|
now: float | None = None,
|
||||||
|
) -> None:
|
||||||
|
if state not in {"RELEASED", "COMPLETED", "FAILED", "EXPIRED"}:
|
||||||
|
raise FleetError("FLEET_RESERVATION_STATE_INVALID")
|
||||||
|
now = float(time.time() if now is None else now)
|
||||||
|
with self._transaction():
|
||||||
|
self.conn.execute(
|
||||||
|
"UPDATE fleet_dispatches SET state=?,updated_at=? WHERE task_id=?",
|
||||||
|
(state, now, task_id),
|
||||||
|
)
|
||||||
|
|
||||||
|
def worker_circuit_open(self, worker_id: str, now: float) -> bool:
|
||||||
|
with self.lock:
|
||||||
|
row = self.conn.execute(
|
||||||
|
"SELECT circuit_open_until FROM fleet_worker_failures WHERE worker_id=?",
|
||||||
|
(worker_id,),
|
||||||
|
).fetchone()
|
||||||
|
return bool(row and row[0] is not None and float(row[0]) > now)
|
||||||
|
|
||||||
|
def task_worker_blocked(self, task_id: str, worker_id: str, now: float) -> bool:
|
||||||
|
with self.lock:
|
||||||
|
row = self.conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT blocked_until FROM fleet_task_worker_failures
|
||||||
|
WHERE task_id=? AND worker_id=?
|
||||||
|
""",
|
||||||
|
(task_id, worker_id),
|
||||||
|
).fetchone()
|
||||||
|
return bool(row and row[0] is not None and float(row[0]) > now)
|
||||||
|
|
||||||
|
def record_worker_failure(
|
||||||
|
self,
|
||||||
|
worker_id: str,
|
||||||
|
failure_class: str,
|
||||||
|
*,
|
||||||
|
threshold: int = DEFAULT_CIRCUIT_THRESHOLD,
|
||||||
|
base_circuit_seconds: int = DEFAULT_CIRCUIT_SECONDS,
|
||||||
|
now: float | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
now = float(time.time() if now is None else now)
|
||||||
|
threshold = _int(threshold, DEFAULT_CIRCUIT_THRESHOLD, 1, 20)
|
||||||
|
base_circuit_seconds = _int(base_circuit_seconds, DEFAULT_CIRCUIT_SECONDS, 1, 3600)
|
||||||
|
with self._transaction():
|
||||||
|
row = self.conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT consecutive_failures,revision FROM fleet_worker_failures
|
||||||
|
WHERE worker_id=?
|
||||||
|
""",
|
||||||
|
(worker_id,),
|
||||||
|
).fetchone()
|
||||||
|
failures = (int(row[0]) if row else 0) + 1
|
||||||
|
revision = (int(row[1]) if row else 0) + 1
|
||||||
|
open_until = None
|
||||||
|
if failures >= threshold:
|
||||||
|
exponent = min(failures - threshold, 8)
|
||||||
|
open_until = now + min(
|
||||||
|
MAX_CIRCUIT_SECONDS, base_circuit_seconds * (2**exponent)
|
||||||
|
)
|
||||||
|
self.conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO fleet_worker_failures(
|
||||||
|
worker_id,consecutive_failures,circuit_open_until,last_failure_class,
|
||||||
|
revision,updated_at
|
||||||
|
) VALUES (?,?,?,?,?,?)
|
||||||
|
ON CONFLICT(worker_id) DO UPDATE SET
|
||||||
|
consecutive_failures=excluded.consecutive_failures,
|
||||||
|
circuit_open_until=excluded.circuit_open_until,
|
||||||
|
last_failure_class=excluded.last_failure_class,
|
||||||
|
revision=excluded.revision,
|
||||||
|
updated_at=excluded.updated_at
|
||||||
|
""",
|
||||||
|
(worker_id, failures, open_until, _text(failure_class, "unknown"), revision, now),
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"worker_id": worker_id,
|
||||||
|
"consecutive_failures": failures,
|
||||||
|
"circuit_open_until": open_until,
|
||||||
|
"revision": revision,
|
||||||
|
}
|
||||||
|
|
||||||
|
def record_worker_success(self, worker_id: str, *, now: float | None = None) -> None:
|
||||||
|
now = float(time.time() if now is None else now)
|
||||||
|
with self._transaction():
|
||||||
|
self.conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO fleet_worker_failures(
|
||||||
|
worker_id,consecutive_failures,circuit_open_until,last_failure_class,
|
||||||
|
revision,updated_at
|
||||||
|
) VALUES (?,0,NULL,NULL,1,?)
|
||||||
|
ON CONFLICT(worker_id) DO UPDATE SET
|
||||||
|
consecutive_failures=0,
|
||||||
|
circuit_open_until=NULL,
|
||||||
|
last_failure_class=NULL,
|
||||||
|
revision=revision+1,
|
||||||
|
updated_at=excluded.updated_at
|
||||||
|
""",
|
||||||
|
(worker_id, now),
|
||||||
|
)
|
||||||
|
|
||||||
|
def record_task_worker_failure(
|
||||||
|
self,
|
||||||
|
task_id: str,
|
||||||
|
worker_id: str,
|
||||||
|
failure_class: str,
|
||||||
|
*,
|
||||||
|
quarantine_seconds: int = 60,
|
||||||
|
now: float | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
now = float(time.time() if now is None else now)
|
||||||
|
quarantine_seconds = _int(quarantine_seconds, 60, 1, 3600)
|
||||||
|
with self._transaction():
|
||||||
|
row = self.conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT failure_count,revision FROM fleet_task_worker_failures
|
||||||
|
WHERE task_id=? AND worker_id=?
|
||||||
|
""",
|
||||||
|
(task_id, worker_id),
|
||||||
|
).fetchone()
|
||||||
|
count = (int(row[0]) if row else 0) + 1
|
||||||
|
revision = (int(row[1]) if row else 0) + 1
|
||||||
|
blocked_until = now + quarantine_seconds
|
||||||
|
self.conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO fleet_task_worker_failures(
|
||||||
|
task_id,worker_id,failure_count,blocked_until,last_failure_class,
|
||||||
|
revision,updated_at
|
||||||
|
) VALUES (?,?,?,?,?,?,?)
|
||||||
|
ON CONFLICT(task_id,worker_id) DO UPDATE SET
|
||||||
|
failure_count=excluded.failure_count,
|
||||||
|
blocked_until=excluded.blocked_until,
|
||||||
|
last_failure_class=excluded.last_failure_class,
|
||||||
|
revision=excluded.revision,
|
||||||
|
updated_at=excluded.updated_at
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
task_id, worker_id, count, blocked_until,
|
||||||
|
_text(failure_class, "unknown"), revision, now,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"task_id": task_id,
|
||||||
|
"worker_id": worker_id,
|
||||||
|
"failure_count": count,
|
||||||
|
"blocked_until": blocked_until,
|
||||||
|
"revision": revision,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class FleetScheduler:
|
||||||
|
"""Deterministic capacity/fairness scheduler over existing TaskPlane records."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
store: Any,
|
||||||
|
workers: Any,
|
||||||
|
*,
|
||||||
|
now_fn: Any = time.time,
|
||||||
|
reservation_seconds: int = DEFAULT_RESERVATION_SECONDS,
|
||||||
|
) -> None:
|
||||||
|
self.store = store
|
||||||
|
self.workers = workers
|
||||||
|
self.now_fn = now_fn
|
||||||
|
self.reservation_seconds = _int(
|
||||||
|
reservation_seconds, DEFAULT_RESERVATION_SECONDS, 1, 300
|
||||||
|
)
|
||||||
|
connection = getattr(store, "conn", None)
|
||||||
|
if not isinstance(connection, sqlite3.Connection):
|
||||||
|
raise FleetError("FLEET_TASKPLANE_SQLITE_CONNECTION_REQUIRED")
|
||||||
|
lock = getattr(store, "_lock", None)
|
||||||
|
self.state = FleetStateStore(connection, lock)
|
||||||
|
|
||||||
|
def configure_worker(self, worker_id: str, **kwargs: Any) -> FleetWorkerPolicy:
|
||||||
|
return self.state.configure_worker(worker_id, **kwargs)
|
||||||
|
|
||||||
|
def _online(self, worker_id: str, now: float) -> bool:
|
||||||
|
method = getattr(self.workers, "is_online", None)
|
||||||
|
if not callable(method):
|
||||||
|
raise FleetError("FLEET_WORKER_REGISTRY_ONLINE_CHECK_REQUIRED")
|
||||||
|
try:
|
||||||
|
return bool(method(worker_id, now=now))
|
||||||
|
except TypeError:
|
||||||
|
return bool(method(worker_id))
|
||||||
|
|
||||||
|
def _worker_record(self, worker_id: str) -> object:
|
||||||
|
getter = getattr(self.store, "get_worker", None)
|
||||||
|
if not callable(getter):
|
||||||
|
raise FleetError("FLEET_WORKER_LOOKUP_REQUIRED")
|
||||||
|
return getter(worker_id)
|
||||||
|
|
||||||
|
def _active_claims(self, worker_id: str, now: float) -> int:
|
||||||
|
# The TaskPlane store owns one SQLite connection in the current runtime.
|
||||||
|
# Fleet reads must share its lock just like fleet writes do, otherwise two
|
||||||
|
# concurrent pullers can issue overlapping operations on the same handle.
|
||||||
|
with self.state.lock:
|
||||||
|
columns = {
|
||||||
|
str(row[1])
|
||||||
|
for row in self.store.conn.execute("PRAGMA table_info(claims)").fetchall()
|
||||||
|
}
|
||||||
|
required = {"worker_id", "status"}
|
||||||
|
if not required.issubset(columns):
|
||||||
|
raise FleetError("FLEET_CLAIMS_SCHEMA_UNSUPPORTED")
|
||||||
|
sql = "SELECT COUNT(*) FROM claims WHERE worker_id=? AND status='ACTIVE'"
|
||||||
|
params: list[Any] = [worker_id]
|
||||||
|
if "lease_expires_at" in columns:
|
||||||
|
sql += " AND lease_expires_at>?"
|
||||||
|
params.append(now)
|
||||||
|
row = self.store.conn.execute(sql, tuple(params)).fetchone()
|
||||||
|
return int(row[0] if row else 0)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _labels_match(policy: FleetWorkerPolicy, task: object) -> bool:
|
||||||
|
metadata = _task_metadata(task)
|
||||||
|
affinity = metadata.get("fleet_affinity_labels") or {}
|
||||||
|
if affinity and not isinstance(affinity, Mapping):
|
||||||
|
return False
|
||||||
|
labels = dict(policy.labels or {})
|
||||||
|
for key, value in dict(affinity).items():
|
||||||
|
if labels.get(_text(key)) != _text(value):
|
||||||
|
return False
|
||||||
|
avoid = metadata.get("fleet_avoid_failure_domains") or []
|
||||||
|
if isinstance(avoid, Sequence) and not isinstance(avoid, (str, bytes)):
|
||||||
|
if policy.failure_domain in {_text(item) for item in avoid}:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _eligible_worker(self, worker_id: str, task: object, now: float) -> tuple[bool, int, FleetWorkerPolicy]:
|
||||||
|
if not self._online(worker_id, now):
|
||||||
|
return False, 0, self.state.worker_policy(worker_id)
|
||||||
|
worker = self._worker_record(worker_id)
|
||||||
|
policy = self.state.worker_policy(worker_id)
|
||||||
|
if self.state.worker_circuit_open(worker_id, now):
|
||||||
|
return False, 0, policy
|
||||||
|
if self.state.task_worker_blocked(_task_id(task), worker_id, now):
|
||||||
|
return False, 0, policy
|
||||||
|
if not self._labels_match(policy, task):
|
||||||
|
return False, 0, policy
|
||||||
|
required = _task_required_capabilities(task)
|
||||||
|
capabilities = _worker_capabilities(worker)
|
||||||
|
if required and not required.issubset(capabilities):
|
||||||
|
return False, 0, policy
|
||||||
|
active = self._active_claims(worker_id, now)
|
||||||
|
reserved = self.state.reserved_slots(worker_id, now)
|
||||||
|
load = active + reserved
|
||||||
|
if load >= policy.max_slots:
|
||||||
|
return False, load, policy
|
||||||
|
return True, load, policy
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _task_priority(task: object, now: float) -> float:
|
||||||
|
metadata = _task_metadata(task)
|
||||||
|
base = _int(metadata.get("fleet_priority"), 0, -100, 100)
|
||||||
|
created = _task_created_at(task)
|
||||||
|
waited = max(0.0, now - created) if created > 0 else 0.0
|
||||||
|
aging = min(100.0, waited / 60.0)
|
||||||
|
return float(base) + aging
|
||||||
|
|
||||||
|
def _task_score(self, task: object, now: float) -> tuple[float, float, float, str]:
|
||||||
|
scope = _task_scope_key(task)
|
||||||
|
virtual_finish = self.state.scope_virtual_finish(scope)
|
||||||
|
priority = self._task_priority(task, now)
|
||||||
|
return (virtual_finish, -priority, _task_created_at(task), _task_id(task))
|
||||||
|
|
||||||
|
def _tasks(self, actor_context: Any = None) -> list[object]:
|
||||||
|
lister = getattr(self.store, "list_tasks", None)
|
||||||
|
if not callable(lister):
|
||||||
|
raise FleetError("FLEET_TASK_LIST_REQUIRED")
|
||||||
|
try:
|
||||||
|
tasks = lister(actor_context)
|
||||||
|
except TypeError:
|
||||||
|
tasks = lister()
|
||||||
|
return [
|
||||||
|
task for task in tasks
|
||||||
|
if _task_status(task) in {"CREATED", "QUEUED"}
|
||||||
|
and _task_id(task)
|
||||||
|
]
|
||||||
|
|
||||||
|
def rank_for_worker(
|
||||||
|
self,
|
||||||
|
worker_id: str,
|
||||||
|
*,
|
||||||
|
actor_context: Any = None,
|
||||||
|
now: float | None = None,
|
||||||
|
) -> list[object]:
|
||||||
|
"""Return fleet-ranked tasks without reserving or claiming anything."""
|
||||||
|
now = float(self.now_fn() if now is None else now)
|
||||||
|
worker_id = _text(worker_id)
|
||||||
|
if not worker_id:
|
||||||
|
return []
|
||||||
|
ranked: list[object] = []
|
||||||
|
for task in self._tasks(actor_context):
|
||||||
|
eligible, _, _ = self._eligible_worker(worker_id, task, now)
|
||||||
|
if eligible:
|
||||||
|
ranked.append(task)
|
||||||
|
return sorted(ranked, key=lambda task: self._task_score(task, now))
|
||||||
|
|
||||||
|
def reserve_for_worker(
|
||||||
|
self,
|
||||||
|
worker_id: str,
|
||||||
|
task: object,
|
||||||
|
*,
|
||||||
|
now: float | None = None,
|
||||||
|
) -> FleetDispatchDecision:
|
||||||
|
"""Reserve exactly one already-ranked task, rechecking eligibility and capacity."""
|
||||||
|
now = float(self.now_fn() if now is None else now)
|
||||||
|
worker_id = _text(worker_id)
|
||||||
|
eligible, _, policy = self._eligible_worker(worker_id, task, now)
|
||||||
|
if not eligible:
|
||||||
|
raise FleetConflict("FLEET_WORKER_NOT_ELIGIBLE")
|
||||||
|
metadata = _task_metadata(task)
|
||||||
|
scope_weight = _float(metadata.get("fleet_scope_weight"), 1.0, 0.05, 100.0)
|
||||||
|
return self.state.reserve(
|
||||||
|
task_id=_task_id(task),
|
||||||
|
worker_id=worker_id,
|
||||||
|
task_revision=_task_revision(task),
|
||||||
|
scope_key=_task_scope_key(task),
|
||||||
|
failure_domain=policy.failure_domain,
|
||||||
|
score=self._task_score(task, now),
|
||||||
|
reservation_seconds=self.reservation_seconds,
|
||||||
|
now=now,
|
||||||
|
scope_weight=scope_weight,
|
||||||
|
)
|
||||||
|
|
||||||
|
def select(
|
||||||
|
self,
|
||||||
|
worker_ids: Sequence[str],
|
||||||
|
*,
|
||||||
|
actor_context: Any = None,
|
||||||
|
limit: int = 1,
|
||||||
|
now: float | None = None,
|
||||||
|
) -> list[FleetDispatchDecision]:
|
||||||
|
now = float(self.now_fn() if now is None else now)
|
||||||
|
limit = _int(limit, 1, 1, 100)
|
||||||
|
worker_ids = tuple(dict.fromkeys(_text(item) for item in worker_ids if _text(item)))
|
||||||
|
tasks = sorted(self._tasks(actor_context), key=lambda task: self._task_score(task, now))
|
||||||
|
decisions: list[FleetDispatchDecision] = []
|
||||||
|
local_load: dict[str, int] = {}
|
||||||
|
for task in tasks:
|
||||||
|
if len(decisions) >= limit:
|
||||||
|
break
|
||||||
|
candidates: list[tuple[int, float, str, FleetWorkerPolicy]] = []
|
||||||
|
for worker_id in worker_ids:
|
||||||
|
eligible, load, policy = self._eligible_worker(worker_id, task, now)
|
||||||
|
if not eligible:
|
||||||
|
continue
|
||||||
|
effective_load = local_load.get(worker_id, load)
|
||||||
|
if effective_load >= policy.max_slots:
|
||||||
|
continue
|
||||||
|
candidates.append(
|
||||||
|
(
|
||||||
|
effective_load,
|
||||||
|
-policy.weight,
|
||||||
|
worker_id,
|
||||||
|
policy,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if not candidates:
|
||||||
|
continue
|
||||||
|
candidates.sort(key=lambda item: (item[0], item[1], item[2]))
|
||||||
|
_, _, worker_id, policy = candidates[0]
|
||||||
|
score = self._task_score(task, now)
|
||||||
|
metadata = _task_metadata(task)
|
||||||
|
scope_weight = _float(metadata.get("fleet_scope_weight"), 1.0, 0.05, 100.0)
|
||||||
|
try:
|
||||||
|
decision = self.state.reserve(
|
||||||
|
task_id=_task_id(task),
|
||||||
|
worker_id=worker_id,
|
||||||
|
task_revision=_task_revision(task),
|
||||||
|
scope_key=_task_scope_key(task),
|
||||||
|
failure_domain=policy.failure_domain,
|
||||||
|
score=score,
|
||||||
|
reservation_seconds=self.reservation_seconds,
|
||||||
|
now=now,
|
||||||
|
scope_weight=scope_weight,
|
||||||
|
)
|
||||||
|
except FleetConflict:
|
||||||
|
continue
|
||||||
|
local_load[worker_id] = local_load.get(
|
||||||
|
worker_id,
|
||||||
|
self._active_claims(worker_id, now)
|
||||||
|
+ self.state.reserved_slots(worker_id, now)
|
||||||
|
- 1,
|
||||||
|
) + 1
|
||||||
|
decisions.append(decision)
|
||||||
|
return decisions
|
||||||
@@ -74,3 +74,203 @@ class TaskScheduler:
|
|||||||
operation_digest=task.task_plan.operation_digest(),
|
operation_digest=task.task_plan.operation_digest(),
|
||||||
)
|
)
|
||||||
return claim
|
return claim
|
||||||
|
|
||||||
|
# R4.2 P2 durable multi-agent fleet scheduling public surface.
|
||||||
|
from .fleet import (
|
||||||
|
FleetConflict,
|
||||||
|
FleetDispatchDecision,
|
||||||
|
FleetError,
|
||||||
|
FleetScheduler,
|
||||||
|
FleetStateStore,
|
||||||
|
FleetWorkerPolicy,
|
||||||
|
)
|
||||||
|
# R4.2 P2-B fleet-aware pull scheduling.
|
||||||
|
#
|
||||||
|
# The original TaskScheduler remains the security oracle and claim-CAS implementation.
|
||||||
|
# Fleet ordering only controls the order presented to that oracle.
|
||||||
|
import inspect as _fleet_inspect
|
||||||
|
import threading as _fleet_threading
|
||||||
|
from collections.abc import Mapping as _FleetMapping
|
||||||
|
|
||||||
|
from .worker_registry import WorkerRegistry as _FleetWorkerRegistry
|
||||||
|
from .errors import NotFoundError as _FleetNotFoundError
|
||||||
|
|
||||||
|
_LegacyTaskScheduler = TaskScheduler
|
||||||
|
_LegacyScheduleParameters = _fleet_inspect.signature(
|
||||||
|
_LegacyTaskScheduler.schedule_for_worker
|
||||||
|
).parameters
|
||||||
|
_LegacyNextParameters = _fleet_inspect.signature(
|
||||||
|
_LegacyTaskScheduler.next_queued_task
|
||||||
|
).parameters
|
||||||
|
|
||||||
|
|
||||||
|
class _FleetOrderedStoreProxy:
|
||||||
|
def __init__(self, delegate, tasks):
|
||||||
|
self._delegate = delegate
|
||||||
|
self._tasks = tuple(tasks)
|
||||||
|
|
||||||
|
def list_tasks(self, *args, **kwargs):
|
||||||
|
return list(self._tasks)
|
||||||
|
|
||||||
|
def __getattr__(self, name):
|
||||||
|
return getattr(self._delegate, name)
|
||||||
|
|
||||||
|
|
||||||
|
def _fleet_claim_field(claim, name, default=None):
|
||||||
|
if isinstance(claim, _FleetMapping):
|
||||||
|
return claim.get(name, default)
|
||||||
|
return getattr(claim, name, default)
|
||||||
|
|
||||||
|
|
||||||
|
class TaskScheduler(_LegacyTaskScheduler):
|
||||||
|
"""Fleet-integrated pull scheduler preserving legacy security and claim semantics."""
|
||||||
|
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
store = getattr(self, "store", args[0] if args else None)
|
||||||
|
if store is None:
|
||||||
|
raise FleetError("FLEET_SCHEDULER_STORE_REQUIRED")
|
||||||
|
registry = getattr(self, "workers", None)
|
||||||
|
if registry is None:
|
||||||
|
registry = _FleetWorkerRegistry(store)
|
||||||
|
self.fleet_scheduler = FleetScheduler(store, registry)
|
||||||
|
self._fleet_local = _fleet_threading.local()
|
||||||
|
|
||||||
|
def _fleet_legacy_next(self, worker_id, tasks, *args, **kwargs):
|
||||||
|
proxy = _FleetOrderedStoreProxy(self.store, tasks)
|
||||||
|
legacy = _LegacyTaskScheduler(proxy)
|
||||||
|
legacy_kwargs = dict(kwargs)
|
||||||
|
if "actor_context" not in _LegacyNextParameters:
|
||||||
|
legacy_kwargs.pop("actor_context", None)
|
||||||
|
return legacy.next_queued_task(worker_id, *args, **legacy_kwargs)
|
||||||
|
|
||||||
|
def next_queued_task(self, worker_id, *args, **kwargs):
|
||||||
|
actor_context = kwargs.get(
|
||||||
|
"actor_context", getattr(self._fleet_local, "actor_context", None)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Preserve legacy fail-closed worker checks before fleet ordering.
|
||||||
|
probe_kwargs = dict(kwargs)
|
||||||
|
if "actor_context" not in _LegacyNextParameters:
|
||||||
|
probe_kwargs.pop("actor_context", None)
|
||||||
|
legacy_probe = _LegacyTaskScheduler(self.store).next_queued_task(
|
||||||
|
worker_id, *args, **probe_kwargs
|
||||||
|
)
|
||||||
|
if legacy_probe is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
ranked = self.fleet_scheduler.rank_for_worker(
|
||||||
|
worker_id, actor_context=actor_context
|
||||||
|
)
|
||||||
|
# Preserve FleetScheduler fairness order. The legacy scheduler is used
|
||||||
|
# only as a fail-closed security oracle: present one fleet-ranked task
|
||||||
|
# at a time so its internal sort cannot reorder across scopes.
|
||||||
|
# Keep the legacy priority dimension ahead of Fleet fairness for
|
||||||
|
# compatibility with existing TaskPlane scheduling contracts; the
|
||||||
|
# Fleet order remains authoritative among equal-priority tasks.
|
||||||
|
def _legacy_priority(task):
|
||||||
|
plan = getattr(task, "task_plan", task)
|
||||||
|
metadata = getattr(plan, "metadata", {}) or {}
|
||||||
|
if not isinstance(metadata, _FleetMapping):
|
||||||
|
return 0
|
||||||
|
try:
|
||||||
|
return int(metadata.get("priority", 0))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return 0
|
||||||
|
|
||||||
|
ranked = sorted(ranked, key=lambda task: -_legacy_priority(task))
|
||||||
|
for ranked_candidate in ranked:
|
||||||
|
try:
|
||||||
|
candidate = self._fleet_legacy_next(
|
||||||
|
worker_id, [ranked_candidate], *args, **kwargs
|
||||||
|
)
|
||||||
|
except _FleetNotFoundError:
|
||||||
|
continue
|
||||||
|
if candidate is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
ranked_id = getattr(
|
||||||
|
ranked_candidate,
|
||||||
|
"task_id",
|
||||||
|
getattr(getattr(ranked_candidate, "task_plan", None), "task_id", None),
|
||||||
|
)
|
||||||
|
candidate_id = getattr(
|
||||||
|
candidate,
|
||||||
|
"task_id",
|
||||||
|
getattr(getattr(candidate, "task_plan", None), "task_id", None),
|
||||||
|
)
|
||||||
|
if candidate_id != ranked_id:
|
||||||
|
raise FleetConflict("FLEET_LEGACY_ORACLE_ORDER_VIOLATION")
|
||||||
|
|
||||||
|
try:
|
||||||
|
decision = self.fleet_scheduler.reserve_for_worker(
|
||||||
|
worker_id, candidate
|
||||||
|
)
|
||||||
|
except FleetConflict:
|
||||||
|
continue
|
||||||
|
self._fleet_local.pending = decision
|
||||||
|
return candidate
|
||||||
|
return None
|
||||||
|
|
||||||
|
def schedule_for_worker(self, worker_id, *args, **kwargs):
|
||||||
|
actor_context = kwargs.get("actor_context")
|
||||||
|
legacy_kwargs = dict(kwargs)
|
||||||
|
if "actor_context" not in _LegacyScheduleParameters:
|
||||||
|
legacy_kwargs.pop("actor_context", None)
|
||||||
|
self._fleet_local.actor_context = actor_context
|
||||||
|
self._fleet_local.pending = None
|
||||||
|
try:
|
||||||
|
claim = _LegacyTaskScheduler.schedule_for_worker(
|
||||||
|
self, worker_id, *args, **legacy_kwargs
|
||||||
|
)
|
||||||
|
decision = getattr(self._fleet_local, "pending", None)
|
||||||
|
if claim is None:
|
||||||
|
if decision is not None:
|
||||||
|
self.fleet_scheduler.state.release(
|
||||||
|
decision.task_id, state="RELEASED"
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
if decision is not None:
|
||||||
|
claim_id = _fleet_claim_field(claim, "claim_id")
|
||||||
|
claim_task_id = _fleet_claim_field(claim, "task_id", decision.task_id)
|
||||||
|
if not claim_id:
|
||||||
|
self.fleet_scheduler.state.release(
|
||||||
|
decision.task_id, state="RELEASED"
|
||||||
|
)
|
||||||
|
raise FleetError("FLEET_CLAIM_ID_REQUIRED")
|
||||||
|
if claim_task_id != decision.task_id:
|
||||||
|
self.fleet_scheduler.state.release(
|
||||||
|
decision.task_id, state="RELEASED"
|
||||||
|
)
|
||||||
|
raise FleetConflict("FLEET_CLAIM_TASK_MISMATCH")
|
||||||
|
self.fleet_scheduler.state.bind_claim(
|
||||||
|
decision.reservation_id, str(claim_id)
|
||||||
|
)
|
||||||
|
return claim
|
||||||
|
except Exception:
|
||||||
|
decision = getattr(self._fleet_local, "pending", None)
|
||||||
|
if decision is not None:
|
||||||
|
self.fleet_scheduler.state.release(
|
||||||
|
decision.task_id, state="RELEASED"
|
||||||
|
)
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
self._fleet_local.pending = None
|
||||||
|
self._fleet_local.actor_context = None
|
||||||
|
|
||||||
|
def record_worker_failure(self, worker_id, failure_class, **kwargs):
|
||||||
|
return self.fleet_scheduler.state.record_worker_failure(
|
||||||
|
worker_id, failure_class, **kwargs
|
||||||
|
)
|
||||||
|
|
||||||
|
def record_worker_success(self, worker_id, **kwargs):
|
||||||
|
return self.fleet_scheduler.state.record_worker_success(
|
||||||
|
worker_id, **kwargs
|
||||||
|
)
|
||||||
|
|
||||||
|
def quarantine_task_worker(
|
||||||
|
self, task_id, worker_id, failure_class, **kwargs
|
||||||
|
):
|
||||||
|
return self.fleet_scheduler.state.record_task_worker_failure(
|
||||||
|
task_id, worker_id, failure_class, **kwargs
|
||||||
|
)
|
||||||
|
|||||||
@@ -0,0 +1,9 @@
|
|||||||
|
providers:
|
||||||
|
oneapi:
|
||||||
|
name: oneapi
|
||||||
|
base_url: http://172.18.0.2:3000/v1
|
||||||
|
key_env: OPENAI_API_KEY
|
||||||
|
default_model: deepseek-v4-flash
|
||||||
|
model:
|
||||||
|
provider: oneapi
|
||||||
|
name: deepseek-v4-flash
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
{
|
||||||
|
"argv": [
|
||||||
|
"/usr/local/bin/hermes",
|
||||||
|
"run",
|
||||||
|
"--non-interactive"
|
||||||
|
],
|
||||||
|
"input_mode": "stdin_text",
|
||||||
|
"max_output_bytes": 131072,
|
||||||
|
"output_mode": "stdout_text",
|
||||||
|
"runtime": "hermes",
|
||||||
|
"schema": "aios.external-agent-runtime.profile.v1",
|
||||||
|
"timeout_seconds": 900
|
||||||
|
}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
{
|
||||||
|
"argv": [
|
||||||
|
"/opt/aios-hermes-agent/hermes-agent/.venv/bin/hermes",
|
||||||
|
"-z",
|
||||||
|
"{prompt}",
|
||||||
|
"--provider",
|
||||||
|
"oneapi",
|
||||||
|
"-m",
|
||||||
|
"{model_id}",
|
||||||
|
"--no-restore-cwd"
|
||||||
|
],
|
||||||
|
"credential_env": {
|
||||||
|
"OPENAI_API_KEY": {
|
||||||
|
"credential": "aios-external-agent-oneapi-env",
|
||||||
|
"format": "dotenv",
|
||||||
|
"key": "ONEAPI_REGULAR_TOKEN"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"environment": {
|
||||||
|
"HERMES_HOME": "/etc/aios/external-agent-runtime.d/hermes-home"
|
||||||
|
},
|
||||||
|
"input_mode": "stdin_text",
|
||||||
|
"max_output_bytes": 131072,
|
||||||
|
"model_governance": {
|
||||||
|
"max_output_tokens": 512,
|
||||||
|
"model_capabilities": [
|
||||||
|
"text"
|
||||||
|
],
|
||||||
|
"provider_ids": [
|
||||||
|
"oneapi"
|
||||||
|
],
|
||||||
|
"required": true
|
||||||
|
},
|
||||||
|
"output_mode": "stdout_text",
|
||||||
|
"runtime": "hermes",
|
||||||
|
"schema": "aios.external-agent-runtime.profile.v1",
|
||||||
|
"timeout_seconds": 180
|
||||||
|
}
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
{
|
||||||
|
"tools": {
|
||||||
|
"profile": "minimal"
|
||||||
|
},
|
||||||
|
"memory": {
|
||||||
|
"search": {
|
||||||
|
"enabled": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"agents": {
|
||||||
|
"defaults": {
|
||||||
|
"workspace": "/var/lib/aios-external-agent-executor/openclaw-workspace",
|
||||||
|
"model": {
|
||||||
|
"primary": "openai/deepseek-v4-flash"
|
||||||
|
},
|
||||||
|
"thinkingDefault": "off"
|
||||||
|
},
|
||||||
|
"entries": {
|
||||||
|
"main": {
|
||||||
|
"default": true,
|
||||||
|
"workspace": "/var/lib/aios-external-agent-executor/openclaw-workspace"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"models": {
|
||||||
|
"providers": {
|
||||||
|
"openai": {
|
||||||
|
"baseUrl": "http://172.18.0.2:3000/v1",
|
||||||
|
"api": "openai-completions",
|
||||||
|
"models": [
|
||||||
|
{
|
||||||
|
"id": "deepseek-v4-flash",
|
||||||
|
"name": "deepseek-v4-flash",
|
||||||
|
"contextWindow": 32768,
|
||||||
|
"maxTokens": 256
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
{
|
||||||
|
"argv": [
|
||||||
|
"/usr/local/bin/openclaw",
|
||||||
|
"run",
|
||||||
|
"--non-interactive"
|
||||||
|
],
|
||||||
|
"input_mode": "stdin_text",
|
||||||
|
"max_output_bytes": 131072,
|
||||||
|
"output_mode": "stdout_text",
|
||||||
|
"runtime": "openclaw",
|
||||||
|
"schema": "aios.external-agent-runtime.profile.v1",
|
||||||
|
"timeout_seconds": 900
|
||||||
|
}
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
{
|
||||||
|
"argv": [
|
||||||
|
"/usr/bin/node",
|
||||||
|
"/opt/aios/openclaw-engineering/v2026.7.2-beta.7/openclaw.mjs",
|
||||||
|
"agent",
|
||||||
|
"--local",
|
||||||
|
"--json",
|
||||||
|
"--agent",
|
||||||
|
"main",
|
||||||
|
"--thinking",
|
||||||
|
"off",
|
||||||
|
"--message",
|
||||||
|
"{prompt}",
|
||||||
|
"--model",
|
||||||
|
"{model_id}",
|
||||||
|
"--timeout",
|
||||||
|
"180"
|
||||||
|
],
|
||||||
|
"credential_env": {
|
||||||
|
"OPENAI_API_KEY": {
|
||||||
|
"credential": "aios-external-agent-oneapi-env",
|
||||||
|
"format": "dotenv",
|
||||||
|
"key": "ONEAPI_REGULAR_TOKEN"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"environment": {
|
||||||
|
"OPENCLAW_CONFIG_PATH": "/etc/aios/external-agent-runtime.d/openclaw-config.json",
|
||||||
|
"OPENCLAW_HOME": "/var/lib/aios-external-agent-executor/openclaw-home",
|
||||||
|
"OPENCLAW_STATE_DIR": "/var/lib/aios-external-agent-executor/openclaw-state"
|
||||||
|
},
|
||||||
|
"input_mode": "stdin_text",
|
||||||
|
"max_output_bytes": 131072,
|
||||||
|
"model_governance": {
|
||||||
|
"input_reservation_floor_tokens": 16384,
|
||||||
|
"max_output_tokens": 256,
|
||||||
|
"model_capabilities": [
|
||||||
|
"text"
|
||||||
|
],
|
||||||
|
"provider_ids": [
|
||||||
|
"openai"
|
||||||
|
],
|
||||||
|
"required": true
|
||||||
|
},
|
||||||
|
"output_mode": "stdout_json",
|
||||||
|
"runtime": "openclaw",
|
||||||
|
"schema": "aios.external-agent-runtime.profile.v1",
|
||||||
|
"timeout_seconds": 240
|
||||||
|
}
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
{
|
||||||
|
"models": [
|
||||||
|
{
|
||||||
|
"capabilities": [
|
||||||
|
"agent.runtime.hermes",
|
||||||
|
"text"
|
||||||
|
],
|
||||||
|
"enabled": true,
|
||||||
|
"evaluation_sha256": "43ae0db1640daeb6d33e2787769732844f366d36070065d3a2239905bd9a823b",
|
||||||
|
"evaluation_state": "APPROVED",
|
||||||
|
"input_cost_micros_per_1k": 0,
|
||||||
|
"max_input_tokens": 32768,
|
||||||
|
"max_output_tokens": 512,
|
||||||
|
"model_id": "deepseek-v4-flash",
|
||||||
|
"output_cost_micros_per_1k": 0,
|
||||||
|
"priority": 10,
|
||||||
|
"provider_id": "oneapi",
|
||||||
|
"version": "profile-sha:54615e533bfdb6a2e8b870ae74bb8cd7fc45177495bcf6e8c45a50837cefb85a"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"capabilities": [
|
||||||
|
"agent.runtime.openclaw",
|
||||||
|
"text"
|
||||||
|
],
|
||||||
|
"enabled": true,
|
||||||
|
"evaluation_sha256": "7b1c89634eb9c0e7cb5a3d75323d904ebbd91b3ae5494ce938bd8316a4cf5b12",
|
||||||
|
"evaluation_state": "APPROVED",
|
||||||
|
"input_cost_micros_per_1k": 0,
|
||||||
|
"max_input_tokens": 32768,
|
||||||
|
"max_output_tokens": 256,
|
||||||
|
"model_id": "openai/deepseek-v4-flash",
|
||||||
|
"output_cost_micros_per_1k": 0,
|
||||||
|
"priority": 10,
|
||||||
|
"provider_id": "openai",
|
||||||
|
"version": "profile-sha:2a31c480226aa78d44ed9692bddcedc99c3aa03c23c0dd5f031a5cbbb8bbd7ba"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"policies": [
|
||||||
|
{
|
||||||
|
"allowed_models": [
|
||||||
|
"deepseek-v4-flash",
|
||||||
|
"openai/deepseek-v4-flash"
|
||||||
|
],
|
||||||
|
"daily_cost_budget_micros": 0,
|
||||||
|
"daily_token_budget": 250000,
|
||||||
|
"denied_providers": [],
|
||||||
|
"fallback_limit": 1,
|
||||||
|
"max_input_tokens_per_request": 16384,
|
||||||
|
"max_output_tokens_per_request": 512,
|
||||||
|
"project_id": null,
|
||||||
|
"required_capabilities": [
|
||||||
|
"text"
|
||||||
|
],
|
||||||
|
"scope_type": "legacy",
|
||||||
|
"tenant_id": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"allowed_models": [
|
||||||
|
"deepseek-v4-flash",
|
||||||
|
"openai/deepseek-v4-flash"
|
||||||
|
],
|
||||||
|
"daily_cost_budget_micros": 0,
|
||||||
|
"daily_token_budget": 250000,
|
||||||
|
"denied_providers": [],
|
||||||
|
"fallback_limit": 1,
|
||||||
|
"max_input_tokens_per_request": 16384,
|
||||||
|
"max_output_tokens_per_request": 512,
|
||||||
|
"project_id": "staging",
|
||||||
|
"required_capabilities": [
|
||||||
|
"text"
|
||||||
|
],
|
||||||
|
"scope_type": "tenant",
|
||||||
|
"tenant_id": "model-governance-live"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"providers": [
|
||||||
|
{
|
||||||
|
"cooldown_seconds": 60,
|
||||||
|
"credential_ref": "runtime-profile:hermes",
|
||||||
|
"enabled": true,
|
||||||
|
"failure_domain": "oneapi",
|
||||||
|
"max_failures": 2,
|
||||||
|
"provider_id": "oneapi"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cooldown_seconds": 60,
|
||||||
|
"credential_ref": "runtime-profile:openclaw",
|
||||||
|
"enabled": true,
|
||||||
|
"failure_domain": "oneapi",
|
||||||
|
"max_failures": 2,
|
||||||
|
"provider_id": "openai"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"schema": "aios.model-governance.bootstrap.v1"
|
||||||
|
}
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS owner-controlled Task execution-rebind API
|
||||||
|
After=aios-agent-task-api.service
|
||||||
|
Wants=aios-agent-task-api.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=aios-agent-task
|
||||||
|
Group=aios-agent-task
|
||||||
|
LoadCredential=agent_task_control_token:/etc/aios-gitops/agent-task-control.token
|
||||||
|
EnvironmentFile=/etc/aios-gitops/agent-task-execution-rebind.env
|
||||||
|
Environment=AIOS_AGENT_TASK_DB=/var/lib/aios-agent-task/state.sqlite3
|
||||||
|
Environment=AIOS_AGENT_TASK_CONTROL_TOKEN_FILE=%d/agent_task_control_token
|
||||||
|
Environment=AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892
|
||||||
|
Environment=AIOS_TASK_EXECUTION_REBIND_HOST=127.0.0.1
|
||||||
|
Environment=AIOS_TASK_EXECUTION_REBIND_PORT=18942
|
||||||
|
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_agent_task_execution_rebind.py
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=2
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
ProtectKernelTunables=true
|
||||||
|
ProtectKernelModules=true
|
||||||
|
ProtectControlGroups=true
|
||||||
|
LockPersonality=true
|
||||||
|
RestrictSUIDSGID=true
|
||||||
|
RestrictRealtime=true
|
||||||
|
RestrictNamespaces=true
|
||||||
|
MemoryDenyWriteExecute=true
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET
|
||||||
|
ReadWritePaths=/var/lib/aios-agent-task
|
||||||
|
UMask=0077
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS owner-controlled Task initial execution projection API
|
||||||
|
After=aios-agent-task-api.service aios-chatgpt-bridge-v2-staging.service
|
||||||
|
Wants=aios-agent-task-api.service aios-chatgpt-bridge-v2-staging.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=aios-agent-task
|
||||||
|
Group=aios-agent-task
|
||||||
|
LoadCredential=agent_task_control_token:/etc/aios-gitops/agent-task-control.token
|
||||||
|
Environment=AIOS_AGENT_TASK_DB=/var/lib/aios-agent-task/state.sqlite3
|
||||||
|
Environment=AIOS_AGENT_TASK_CONTROL_TOKEN_FILE=%d/agent_task_control_token
|
||||||
|
Environment=AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892
|
||||||
|
Environment=AIOS_AGENT_TASK_INITIAL_EXECUTION_PROJECTION_AUTHORIZATION_ID=owner-initial-execution-projection-recovery-20260820
|
||||||
|
Environment=AIOS_TASK_INITIAL_EXECUTION_PROJECTION_HOST=127.0.0.1
|
||||||
|
Environment=AIOS_TASK_INITIAL_EXECUTION_PROJECTION_PORT=18943
|
||||||
|
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_agent_task_initial_execution_projection.py
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=2
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
ProtectKernelTunables=true
|
||||||
|
ProtectKernelModules=true
|
||||||
|
ProtectControlGroups=true
|
||||||
|
LockPersonality=true
|
||||||
|
RestrictSUIDSGID=true
|
||||||
|
RestrictRealtime=true
|
||||||
|
RestrictNamespaces=true
|
||||||
|
MemoryDenyWriteExecute=true
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET
|
||||||
|
ReadWritePaths=/var/lib/aios-agent-task
|
||||||
|
UMask=0077
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS owner-controlled Task retry-extension API
|
||||||
|
After=aios-agent-task-api.service
|
||||||
|
Wants=aios-agent-task-api.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=aios-agent-task
|
||||||
|
Group=aios-agent-task
|
||||||
|
LoadCredential=agent_task_control_token:/etc/aios-gitops/agent-task-control.token
|
||||||
|
EnvironmentFile=/etc/aios-gitops/agent-task-retry-extension.env
|
||||||
|
Environment=AIOS_AGENT_TASK_DB=/var/lib/aios-agent-task/state.sqlite3
|
||||||
|
Environment=AIOS_AGENT_TASK_CONTROL_TOKEN_FILE=%d/agent_task_control_token
|
||||||
|
Environment=AIOS_TASK_RETRY_HOST=127.0.0.1
|
||||||
|
Environment=AIOS_TASK_RETRY_PORT=18941
|
||||||
|
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_agent_task_retry_extension.py
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=2
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
ProtectKernelTunables=true
|
||||||
|
ProtectKernelModules=true
|
||||||
|
ProtectControlGroups=true
|
||||||
|
LockPersonality=true
|
||||||
|
RestrictSUIDSGID=true
|
||||||
|
RestrictRealtime=true
|
||||||
|
RestrictNamespaces=true
|
||||||
|
MemoryDenyWriteExecute=true
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET
|
||||||
|
ReadWritePaths=/var/lib/aios-agent-task
|
||||||
|
UMask=0077
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS owner-controlled supervisor coordinator projection API
|
||||||
|
After=aios-agent-task-api.service aios-chatgpt-bridge-v2-staging.service
|
||||||
|
Wants=aios-agent-task-api.service aios-chatgpt-bridge-v2-staging.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=aios-agent-task
|
||||||
|
Group=aios-agent-task
|
||||||
|
LoadCredential=agent_task_control_token:/etc/aios-gitops/agent-task-control.token
|
||||||
|
Environment=AIOS_AGENT_TASK_DB=/var/lib/aios-agent-task/state.sqlite3
|
||||||
|
Environment=AIOS_AGENT_TASK_CONTROL_TOKEN_FILE=%d/agent_task_control_token
|
||||||
|
Environment=AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892
|
||||||
|
Environment=AIOS_AGENT_TASK_INITIAL_EXECUTION_PROJECTION_AUTHORIZATION_ID=owner-supervisor-coordinator-projection-recovery-20260820
|
||||||
|
Environment=AIOS_SUPERVISOR_COORDINATOR_PROJECTION_HOST=127.0.0.1
|
||||||
|
Environment=AIOS_SUPERVISOR_COORDINATOR_PROJECTION_PORT=18944
|
||||||
|
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_agent_task_supervisor_coordinator_projection.py
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=2
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
ProtectKernelTunables=true
|
||||||
|
ProtectKernelModules=true
|
||||||
|
ProtectControlGroups=true
|
||||||
|
LockPersonality=true
|
||||||
|
RestrictSUIDSGID=true
|
||||||
|
RestrictRealtime=true
|
||||||
|
RestrictNamespaces=true
|
||||||
|
MemoryDenyWriteExecute=true
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET
|
||||||
|
ReadWritePaths=/var/lib/aios-agent-task
|
||||||
|
UMask=0077
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS credential-isolated external Agent runtime executor
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=aios-engineering-executor
|
||||||
|
Group=aios-executor-exchange
|
||||||
|
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_external_agent_runtime.py --root /var/lib/aios-external-agent-executor --profile-root /etc/aios/external-agent-runtime.d
|
||||||
|
Restart=always
|
||||||
|
RestartSec=2
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
ProtectKernelTunables=true
|
||||||
|
ProtectKernelModules=true
|
||||||
|
ProtectControlGroups=true
|
||||||
|
LockPersonality=true
|
||||||
|
RestrictSUIDSGID=true
|
||||||
|
RestrictRealtime=true
|
||||||
|
RestrictNamespaces=true
|
||||||
|
MemoryDenyWriteExecute=true
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
||||||
|
IPAddressDeny=127.0.0.0/8
|
||||||
|
IPAddressDeny=::1/128
|
||||||
|
IPAddressAllow=172.18.0.2/32
|
||||||
|
LoadCredential=aios-external-agent-oneapi-env:/data/agent/secrets/oneapi-model-ready.env
|
||||||
|
InaccessiblePaths=/etc/aios-gitops
|
||||||
|
InaccessiblePaths=/data/aios-gitops
|
||||||
|
InaccessiblePaths=/data/agent/secrets
|
||||||
|
ReadOnlyPaths=/etc/aios/external-agent-runtime.d
|
||||||
|
ReadOnlyPaths=/opt/aios-hermes-agent
|
||||||
|
ReadOnlyPaths=/opt/aios/openclaw-engineering
|
||||||
|
Environment=AIOS_MODEL_GOVERNANCE_DB=/var/lib/aios-model-governance/state.sqlite3
|
||||||
|
Environment=AIOS_MODEL_GOVERNANCE_ENABLED=1
|
||||||
|
ReadWritePaths=/var/lib/aios-external-agent-executor
|
||||||
|
ReadWritePaths=/var/lib/aios-model-governance
|
||||||
|
UMask=0027
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS disabled-by-default HTTPS edge for GPT read-only Online Task ingress
|
||||||
|
After=network-online.target aios-gpt-online-task-ingress.service
|
||||||
|
Wants=network-online.target aios-gpt-online-task-ingress.service
|
||||||
|
ConditionPathExists=/etc/aios-gitops/gpt-https-edge.approved
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
DynamicUser=yes
|
||||||
|
LoadCredential=tls_cert:/etc/aios-gitops/gpt-https-edge-tls.crt
|
||||||
|
LoadCredential=tls_key:/etc/aios-gitops/gpt-https-edge-tls.key
|
||||||
|
Environment=AIOS_GPT_EDGE_TLS_CERT_FILE=%d/tls_cert
|
||||||
|
Environment=AIOS_GPT_EDGE_TLS_KEY_FILE=%d/tls_key
|
||||||
|
Environment=AIOS_GPT_EDGE_HOST=127.0.0.1
|
||||||
|
Environment=AIOS_GPT_EDGE_PORT=18443
|
||||||
|
Environment=AIOS_GPT_EDGE_PUBLIC_BIND_APPROVED=0
|
||||||
|
Environment=AIOS_GPT_EDGE_PUBLIC_DOMAIN=
|
||||||
|
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_gpt_https_edge.py
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=2
|
||||||
|
NoNewPrivileges=yes
|
||||||
|
PrivateTmp=yes
|
||||||
|
PrivateDevices=yes
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=yes
|
||||||
|
ProtectKernelTunables=yes
|
||||||
|
ProtectKernelModules=yes
|
||||||
|
ProtectControlGroups=yes
|
||||||
|
ProtectKernelLogs=yes
|
||||||
|
ProtectClock=yes
|
||||||
|
LockPersonality=yes
|
||||||
|
RestrictSUIDSGID=yes
|
||||||
|
RestrictRealtime=yes
|
||||||
|
RestrictNamespaces=yes
|
||||||
|
MemoryDenyWriteExecute=yes
|
||||||
|
RestrictAddressFamilies=AF_INET AF_INET6
|
||||||
|
InaccessiblePaths=/data/aios-gitops /etc/aios-gitops /home /var/lib/aios-agent-task
|
||||||
|
UMask=0077
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS external GPT read-only Online Task ingress
|
||||||
|
After=network.target aios-chatgpt-bridge-v2-staging.service aios-agent-task-api.service
|
||||||
|
Wants=aios-chatgpt-bridge-v2-staging.service aios-agent-task-api.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
DynamicUser=yes
|
||||||
|
StateDirectory=aios-gpt-online-task-ingress
|
||||||
|
StateDirectoryMode=0750
|
||||||
|
LoadCredential=gpt_ingress_token:/etc/aios-gitops/gpt-online-task-ingress.token
|
||||||
|
LoadCredential=agent_task_control_token:/etc/aios-gitops/agent-task-control.token
|
||||||
|
Environment=AIOS_GPT_INGRESS_TOKEN_FILE=%d/gpt_ingress_token
|
||||||
|
Environment=AIOS_AGENT_TASK_CONTROL_TOKEN_FILE=%d/agent_task_control_token
|
||||||
|
Environment=AIOS_GPT_INGRESS_STATE_ROOT=%S/aios-gpt-online-task-ingress
|
||||||
|
Environment=AIOS_GPT_INGRESS_HOST=127.0.0.1
|
||||||
|
Environment=AIOS_GPT_INGRESS_PORT=18945
|
||||||
|
Environment=AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892
|
||||||
|
Environment=AIOS_AGENT_TASK_API_URL=http://127.0.0.1:18940
|
||||||
|
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_gpt_online_task_ingress.py
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=2
|
||||||
|
NoNewPrivileges=yes
|
||||||
|
PrivateTmp=yes
|
||||||
|
PrivateDevices=yes
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=yes
|
||||||
|
ProtectKernelTunables=yes
|
||||||
|
ProtectKernelModules=yes
|
||||||
|
ProtectControlGroups=yes
|
||||||
|
ProtectKernelLogs=yes
|
||||||
|
ProtectClock=yes
|
||||||
|
LockPersonality=yes
|
||||||
|
RestrictSUIDSGID=yes
|
||||||
|
RestrictRealtime=yes
|
||||||
|
RestrictNamespaces=yes
|
||||||
|
MemoryDenyWriteExecute=yes
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET
|
||||||
|
IPAddressDeny=any
|
||||||
|
IPAddressAllow=127.0.0.0/8
|
||||||
|
InaccessiblePaths=/data/aios-gitops /etc/aios-gitops /home
|
||||||
|
UMask=0077
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS Hermes Online Agent Worker
|
||||||
|
After=aios-agent-task-api.service aios-chatgpt-bridge-v2-staging.service aios-external-agent-executor.service
|
||||||
|
Wants=aios-agent-task-api.service aios-chatgpt-bridge-v2-staging.service aios-external-agent-executor.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=aios-engineering-worker
|
||||||
|
Group=aios-engineering-worker
|
||||||
|
SupplementaryGroups=gitops aios-executor-exchange
|
||||||
|
LoadCredentialEncrypted=agent_token:/etc/credstore.encrypted/aios-online-agent-worker-20260815/agent_token
|
||||||
|
Environment=PYTHONUNBUFFERED=1
|
||||||
|
Environment=AIOS_AGENT_TASK_API_URL=http://127.0.0.1:18940
|
||||||
|
Environment=AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892
|
||||||
|
Environment=AIOS_AGENT_ID=online-hermes-worker-cn-network-mkt
|
||||||
|
Environment=AIOS_AGENT_TYPE=external-agent-runtime-worker
|
||||||
|
Environment=AIOS_AGENT_VERSION=2026-08-18.1-hermes-runtime
|
||||||
|
Environment=AIOS_WORKER_CAPABILITIES=agent.runtime.hermes
|
||||||
|
Environment=AIOS_WORKER_ALLOWED_ADAPTERS=hermes
|
||||||
|
Environment=AIOS_AUTHORITY_GIT_DIR=/data/aios-gitops/repo.git
|
||||||
|
Environment=AIOS_EXTERNAL_AGENT_EXECUTOR_ROOT=/var/lib/aios-external-agent-executor
|
||||||
|
Environment=AIOS_WORKER_STATE_ROOT=/var/lib/aios-online-agent-hermes
|
||||||
|
Environment=AIOS_WORKER_CHECKPOINT_DB=/var/lib/aios-online-agent-hermes/checkpoints.sqlite3
|
||||||
|
Environment=AIOS_WORKER_WORKSPACE_ROOT=/var/lib/aios-online-agent-hermes/workspaces
|
||||||
|
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_online_agent_worker.py
|
||||||
|
Restart=always
|
||||||
|
RestartSec=5
|
||||||
|
TimeoutStopSec=20
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
ProtectKernelTunables=true
|
||||||
|
ProtectKernelModules=true
|
||||||
|
ProtectControlGroups=true
|
||||||
|
LockPersonality=true
|
||||||
|
RestrictSUIDSGID=true
|
||||||
|
RestrictRealtime=true
|
||||||
|
RestrictNamespaces=true
|
||||||
|
MemoryDenyWriteExecute=true
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
||||||
|
ReadOnlyPaths=/data/aios-gitops/repo.git
|
||||||
|
ReadWritePaths=/var/lib/aios-online-agent-hermes
|
||||||
|
ReadWritePaths=/var/lib/aios-external-agent-executor
|
||||||
|
UMask=0027
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS uncredentialed deterministic Planner/Executor
|
||||||
|
After=local-fs.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=aios-engineering-executor
|
||||||
|
Group=aios-executor-exchange
|
||||||
|
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_isolated_executor.py --root /var/lib/aios-online-agent-executor
|
||||||
|
Restart=always
|
||||||
|
RestartSec=2
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
PrivateNetwork=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
ProtectKernelTunables=true
|
||||||
|
ProtectKernelModules=true
|
||||||
|
ProtectControlGroups=true
|
||||||
|
LockPersonality=true
|
||||||
|
RestrictSUIDSGID=true
|
||||||
|
RestrictRealtime=true
|
||||||
|
RestrictNamespaces=true
|
||||||
|
MemoryDenyWriteExecute=true
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET
|
||||||
|
IPAddressDeny=any
|
||||||
|
IPAddressAllow=localhost
|
||||||
|
InaccessiblePaths=/run/credentials
|
||||||
|
ReadWritePaths=/var/lib/aios-online-agent-executor
|
||||||
|
UMask=0027
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS governed Online Agent Worker
|
||||||
|
After=aios-agent-task-api.service aios-chatgpt-bridge-v2-staging.service
|
||||||
|
Wants=aios-agent-task-api.service aios-chatgpt-bridge-v2-staging.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=aios-engineering-worker
|
||||||
|
Group=aios-engineering-worker
|
||||||
|
SupplementaryGroups=gitops aios-executor-exchange
|
||||||
|
LoadCredentialEncrypted=agent_token:/etc/credstore.encrypted/aios-online-agent-worker-20260815/agent_token
|
||||||
|
Environment=PYTHONUNBUFFERED=1
|
||||||
|
Environment=AIOS_AGENT_TASK_API_URL=http://127.0.0.1:18940
|
||||||
|
Environment=AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892
|
||||||
|
Environment=AIOS_AGENT_ID=online-engineering-worker
|
||||||
|
Environment=AIOS_AGENT_TYPE=trusted-online-engineering-worker
|
||||||
|
Environment=AIOS_AGENT_VERSION=2026-08-17.2-durable
|
||||||
|
Environment=AIOS_WORKER_CAPABILITIES=git.read,git.test,governed.write.branch,governed.write.file
|
||||||
|
Environment=AIOS_AUTHORITY_GIT_DIR=/data/aios-gitops/repo.git
|
||||||
|
Environment=AIOS_ISOLATED_EXECUTOR_ROOT=/var/lib/aios-online-agent-executor
|
||||||
|
Environment=AIOS_WORKER_STATE_ROOT=/var/lib/aios-online-agent-worker
|
||||||
|
Environment=AIOS_WORKER_CHECKPOINT_DB=/var/lib/aios-online-agent-worker/checkpoints.sqlite3
|
||||||
|
Environment=AIOS_WORKER_WORKSPACE_ROOT=/var/lib/aios-online-agent-worker/workspaces
|
||||||
|
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_online_agent_worker.py
|
||||||
|
Restart=always
|
||||||
|
RestartSec=5
|
||||||
|
TimeoutStopSec=20
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
ProtectKernelTunables=true
|
||||||
|
ProtectKernelModules=true
|
||||||
|
ProtectControlGroups=true
|
||||||
|
LockPersonality=true
|
||||||
|
RestrictSUIDSGID=true
|
||||||
|
RestrictRealtime=true
|
||||||
|
RestrictNamespaces=true
|
||||||
|
MemoryDenyWriteExecute=true
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
||||||
|
ReadOnlyPaths=/data/aios-gitops/repo.git
|
||||||
|
ReadWritePaths=/var/lib/aios-online-agent-worker
|
||||||
|
UMask=0027
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS OpenClaw Online Agent Worker
|
||||||
|
After=aios-agent-task-api.service aios-chatgpt-bridge-v2-staging.service aios-external-agent-executor.service
|
||||||
|
Wants=aios-agent-task-api.service aios-chatgpt-bridge-v2-staging.service aios-external-agent-executor.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=aios-engineering-worker
|
||||||
|
Group=aios-engineering-worker
|
||||||
|
SupplementaryGroups=gitops aios-executor-exchange
|
||||||
|
LoadCredentialEncrypted=agent_token:/etc/credstore.encrypted/aios-online-agent-worker-20260815/agent_token
|
||||||
|
Environment=PYTHONUNBUFFERED=1
|
||||||
|
Environment=AIOS_AGENT_TASK_API_URL=http://127.0.0.1:18940
|
||||||
|
Environment=AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892
|
||||||
|
Environment=AIOS_AGENT_ID=online-openclaw-worker-cn-network-mkt
|
||||||
|
Environment=AIOS_AGENT_TYPE=external-agent-runtime-worker
|
||||||
|
Environment=AIOS_AGENT_VERSION=2026-08-18.1-openclaw-runtime
|
||||||
|
Environment=AIOS_WORKER_CAPABILITIES=agent.runtime.openclaw
|
||||||
|
Environment=AIOS_WORKER_ALLOWED_ADAPTERS=openclaw
|
||||||
|
Environment=AIOS_AUTHORITY_GIT_DIR=/data/aios-gitops/repo.git
|
||||||
|
Environment=AIOS_EXTERNAL_AGENT_EXECUTOR_ROOT=/var/lib/aios-external-agent-executor
|
||||||
|
Environment=AIOS_WORKER_STATE_ROOT=/var/lib/aios-online-agent-openclaw
|
||||||
|
Environment=AIOS_WORKER_CHECKPOINT_DB=/var/lib/aios-online-agent-openclaw/checkpoints.sqlite3
|
||||||
|
Environment=AIOS_WORKER_WORKSPACE_ROOT=/var/lib/aios-online-agent-openclaw/workspaces
|
||||||
|
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_online_agent_worker.py
|
||||||
|
Restart=always
|
||||||
|
RestartSec=5
|
||||||
|
TimeoutStopSec=20
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
ProtectKernelTunables=true
|
||||||
|
ProtectKernelModules=true
|
||||||
|
ProtectControlGroups=true
|
||||||
|
LockPersonality=true
|
||||||
|
RestrictSUIDSGID=true
|
||||||
|
RestrictRealtime=true
|
||||||
|
RestrictNamespaces=true
|
||||||
|
MemoryDenyWriteExecute=true
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
||||||
|
ReadOnlyPaths=/data/aios-gitops/repo.git
|
||||||
|
ReadWritePaths=/var/lib/aios-online-agent-openclaw
|
||||||
|
ReadWritePaths=/var/lib/aios-external-agent-executor
|
||||||
|
UMask=0027
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS durable Owner authorization broker
|
||||||
|
After=network.target aios-chatgpt-bridge-v2-staging.service
|
||||||
|
Wants=aios-chatgpt-bridge-v2-staging.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=root
|
||||||
|
Group=aios-executor-exchange
|
||||||
|
RuntimeDirectory=aios-owner-authorization-broker
|
||||||
|
RuntimeDirectoryMode=0750
|
||||||
|
StateDirectory=aios-owner-authorization-broker
|
||||||
|
StateDirectoryMode=0750
|
||||||
|
LoadCredential=agent_task_control_token:/etc/aios-gitops/agent-task-control.token
|
||||||
|
Environment=AIOS_AGENT_TASK_CONTROL_TOKEN_FILE=%d/agent_task_control_token
|
||||||
|
Environment=AIOS_OWNER_DURABLE_AUTHORIZATION_POLICY=/etc/aios-gitops/owner-durable-authorization.json
|
||||||
|
Environment=AIOS_OWNER_DURABLE_AUTHORIZATION_REVOCATIONS=/etc/aios-gitops/owner-durable-authorization-revocations.json
|
||||||
|
Environment=AIOS_OWNER_AUTH_BROKER_ACCOUNT=chatgpt-governed-implementer-singlehost
|
||||||
|
Environment=AIOS_OWNER_AUTH_BROKER_SOCKET=%t/aios-owner-authorization-broker/broker.sock
|
||||||
|
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_durable_owner_authorization.py
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=2
|
||||||
|
NoNewPrivileges=yes
|
||||||
|
PrivateTmp=yes
|
||||||
|
PrivateDevices=yes
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=yes
|
||||||
|
ProtectKernelTunables=yes
|
||||||
|
ProtectKernelModules=yes
|
||||||
|
ProtectControlGroups=yes
|
||||||
|
ProtectKernelLogs=yes
|
||||||
|
ProtectClock=yes
|
||||||
|
LockPersonality=yes
|
||||||
|
RestrictSUIDSGID=yes
|
||||||
|
RestrictRealtime=yes
|
||||||
|
RestrictNamespaces=yes
|
||||||
|
MemoryDenyWriteExecute=yes
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET
|
||||||
|
IPAddressDeny=any
|
||||||
|
IPAddressAllow=127.0.0.0/8
|
||||||
|
ReadOnlyPaths=/etc/aios-gitops/owner-durable-authorization.json
|
||||||
|
ReadOnlyPaths=/etc/aios-gitops/owner-durable-authorization-revocations.json
|
||||||
|
InaccessiblePaths=/data/aios-gitops /home
|
||||||
|
UMask=0007
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS release-candidate Online Agent worker
|
||||||
|
After=network.target aios-agent-task-api.service
|
||||||
|
Wants=aios-agent-task-api.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=notify
|
||||||
|
NotifyAccess=main
|
||||||
|
WatchdogSec=90
|
||||||
|
User=root
|
||||||
|
Group=root
|
||||||
|
StateDirectory=aios-online-agent-release
|
||||||
|
StateDirectoryMode=0750
|
||||||
|
LoadCredentialEncrypted=agent_token:/etc/credstore.encrypted/aios-release-online-agent-worker/agent_token
|
||||||
|
LoadCredentialEncrypted=release_candidate_token:/etc/credstore.encrypted/aios-release-online-agent-worker/release_candidate_token
|
||||||
|
Environment=AIOS_RELEASE_WORKER_AGENT_ID=online-release-worker-cn-network-mkt-20260819
|
||||||
|
Environment=AIOS_RELEASE_WORKER_AGENT_VERSION=2026-08-19.1-release-candidate
|
||||||
|
Environment=AIOS_AGENT_TASK_API_URL=http://127.0.0.1:18940
|
||||||
|
Environment=AIOS_RELEASE_WORKER_STATE_ROOT=/var/lib/aios-online-agent-release
|
||||||
|
Environment=AIOS_WORKER_STATE_ROOT=/var/lib/aios-online-agent-release
|
||||||
|
Environment=AIOS_RELEASE_WORKER_POLL_SECONDS=3
|
||||||
|
Environment=AIOS_RELEASE_WORKER_LEASE_SECONDS=60
|
||||||
|
Environment=AIOS_RELEASE_WORKER_HEARTBEAT_SECONDS=15
|
||||||
|
Environment=AIOS_RELEASE_API_URL=http://127.0.0.1:18890
|
||||||
|
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_release_online_agent_worker.py
|
||||||
|
Restart=always
|
||||||
|
RestartSec=2
|
||||||
|
NoNewPrivileges=yes
|
||||||
|
PrivateTmp=yes
|
||||||
|
PrivateDevices=yes
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=yes
|
||||||
|
ProtectKernelTunables=yes
|
||||||
|
ProtectKernelModules=yes
|
||||||
|
ProtectControlGroups=yes
|
||||||
|
ProtectKernelLogs=yes
|
||||||
|
ProtectClock=yes
|
||||||
|
LockPersonality=yes
|
||||||
|
RestrictSUIDSGID=yes
|
||||||
|
RestrictRealtime=yes
|
||||||
|
RestrictNamespaces=yes
|
||||||
|
MemoryDenyWriteExecute=yes
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET
|
||||||
|
IPAddressDeny=any
|
||||||
|
IPAddressAllow=127.0.0.0/8
|
||||||
|
InaccessiblePaths=/data/aios-gitops /home
|
||||||
|
UMask=0077
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS credential-isolated release-preparation supervisor coordinator
|
||||||
|
After=network-online.target aios-agent-task-api.service aios-autonomous-agent-register.service
|
||||||
|
Wants=network-online.target aios-agent-task-api.service
|
||||||
|
Requires=aios-autonomous-agent-register.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=aios-autonomous-agent-worker
|
||||||
|
Group=aios-autonomous-agent-worker
|
||||||
|
SupplementaryGroups=aios-agent-runtime
|
||||||
|
LoadCredentialEncrypted=agent_token:/etc/credstore.encrypted/aios-autonomous-agent-worker/agent_token
|
||||||
|
Environment=PYTHONUNBUFFERED=1
|
||||||
|
Environment=AIOS_AGENT_TASK_API_URL=http://127.0.0.1:18940
|
||||||
|
Environment=AIOS_AGENT_ID=online-engineering-worker-cn-network-mkt-autonomous-20260815
|
||||||
|
Environment=AIOS_RELEASE_PREPARATION_SUPERVISOR_TASK_ID=r42-release-preparation-autonomous-supervisor-20260820
|
||||||
|
Environment=AIOS_SUPERVISOR_COORDINATOR_STATE_ROOT=/var/lib/aios-release-preparation-supervisor-coordinator
|
||||||
|
Environment=AIOS_SUPERVISOR_COORDINATOR_POLL_SECONDS=5
|
||||||
|
Environment=AIOS_SUPERVISOR_COORDINATOR_HEARTBEAT_SECONDS=10
|
||||||
|
Environment=AIOS_SUPERVISOR_COORDINATOR_LEASE_SECONDS=300
|
||||||
|
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_release_preparation_supervisor_coordinator.py
|
||||||
|
Restart=always
|
||||||
|
RestartSec=5
|
||||||
|
TimeoutStopSec=20
|
||||||
|
UMask=0027
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
ProtectKernelTunables=true
|
||||||
|
ProtectKernelModules=true
|
||||||
|
ProtectControlGroups=true
|
||||||
|
LockPersonality=true
|
||||||
|
RestrictSUIDSGID=true
|
||||||
|
RestrictRealtime=true
|
||||||
|
RestrictNamespaces=true
|
||||||
|
MemoryDenyWriteExecute=true
|
||||||
|
RestrictAddressFamilies=AF_INET AF_UNIX
|
||||||
|
ReadWritePaths=/var/lib/aios-release-preparation-supervisor-coordinator
|
||||||
|
InaccessiblePaths=-/etc/credstore.encrypted
|
||||||
|
CapabilityBoundingSet=
|
||||||
|
AmbientCapabilities=
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS persistent Online Agent worker availability guard
|
||||||
|
After=local-fs.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
User=root
|
||||||
|
Group=root
|
||||||
|
Environment=PYTHONUNBUFFERED=1
|
||||||
|
Environment=AIOS_WORKER_AVAILABILITY_MAX_STALE_SECONDS=90
|
||||||
|
Environment=AIOS_WORKER_AVAILABILITY_RESTART_COOLDOWN_SECONDS=60
|
||||||
|
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_worker_availability_guard.py --heal
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
ProtectKernelTunables=true
|
||||||
|
ProtectKernelModules=true
|
||||||
|
ProtectControlGroups=true
|
||||||
|
LockPersonality=true
|
||||||
|
RestrictSUIDSGID=true
|
||||||
|
RestrictRealtime=true
|
||||||
|
RestrictNamespaces=true
|
||||||
|
MemoryDenyWriteExecute=true
|
||||||
|
RestrictAddressFamilies=AF_UNIX
|
||||||
|
ReadWritePaths=/run/aios-worker-availability
|
||||||
|
UMask=0022
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Periodically ensure AIOS Online Agent workers remain available
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnBootSec=15s
|
||||||
|
OnUnitActiveSec=30s
|
||||||
|
AccuracySec=5s
|
||||||
|
RandomizedDelaySec=3s
|
||||||
|
Persistent=true
|
||||||
|
Unit=aios-worker-availability-guard.service
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
g aios-executor-exchange -
|
||||||
|
u aios-engineering-executor - "AIOS isolated engineering executor" /var/lib/aios-online-agent-executor /usr/sbin/nologin
|
||||||
|
m aios-engineering-executor aios-executor-exchange
|
||||||
|
m aios-engineering-worker aios-executor-exchange
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
d /var/lib/aios-external-agent-executor 2770 aios-engineering-executor aios-executor-exchange -
|
||||||
|
d /var/lib/aios-external-agent-executor/requests 2770 aios-engineering-worker aios-executor-exchange -
|
||||||
|
d /var/lib/aios-external-agent-executor/results 2770 aios-engineering-executor aios-executor-exchange -
|
||||||
|
d /var/lib/aios-external-agent-executor/failed 2770 aios-engineering-executor aios-executor-exchange -
|
||||||
|
d /var/lib/aios-external-agent-executor/openclaw-workspace 0750 aios-engineering-executor aios-executor-exchange -
|
||||||
|
d /var/lib/aios-external-agent-executor/openclaw-state 0750 aios-engineering-executor aios-executor-exchange -
|
||||||
|
d /var/lib/aios-external-agent-executor/openclaw-home 0750 aios-engineering-executor aios-executor-exchange -
|
||||||
|
d /var/lib/aios-online-agent-hermes 0750 aios-engineering-worker aios-engineering-worker -
|
||||||
|
d /var/lib/aios-online-agent-hermes/workspaces 0750 aios-engineering-worker aios-engineering-worker -
|
||||||
|
d /var/lib/aios-online-agent-openclaw 0750 aios-engineering-worker aios-engineering-worker -
|
||||||
|
d /var/lib/aios-online-agent-openclaw/workspaces 0750 aios-engineering-worker aios-engineering-worker -
|
||||||
|
d /etc/aios/external-agent-runtime.d 0755 root root -
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
d /var/lib/aios-model-governance 0750 aios-engineering-executor aios-executor-exchange -
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
d /var/lib/aios-online-agent-executor 2770 aios-engineering-executor aios-executor-exchange -
|
||||||
|
d /var/lib/aios-online-agent-executor/requests 2770 aios-engineering-worker aios-executor-exchange -
|
||||||
|
d /var/lib/aios-online-agent-executor/results 2770 aios-engineering-executor aios-executor-exchange -
|
||||||
|
d /var/lib/aios-online-agent-executor/failed 2770 aios-engineering-executor aios-executor-exchange -
|
||||||
|
d /var/lib/aios-online-agent-executor/sources 2770 aios-engineering-worker aios-executor-exchange -
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
d /run/aios-worker-availability 0755 root root -
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
# R4.2 P1 — Durable Online Agent Execution
|
||||||
|
|
||||||
|
Status: candidate implementation. This document does not imply protected-main promotion or Production deployment.
|
||||||
|
|
||||||
|
## State model
|
||||||
|
|
||||||
|
The generic worker uses two local durable layers:
|
||||||
|
|
||||||
|
1. `tasks/<task_id>.json`: atomic fast-state cache for compatibility and quick restart.
|
||||||
|
2. `checkpoints.sqlite3`: append-only attempt/phase journal with `synchronous=FULL`.
|
||||||
|
|
||||||
|
Checkpoint identity is bound to:
|
||||||
|
|
||||||
|
- task_id
|
||||||
|
- attempt
|
||||||
|
- execution_digest
|
||||||
|
- job_id
|
||||||
|
- repository
|
||||||
|
- base_oid
|
||||||
|
- expected_branch_oid
|
||||||
|
- target_branch
|
||||||
|
- generation
|
||||||
|
|
||||||
|
A checkpoint from another repository/branch/generation is never reused.
|
||||||
|
|
||||||
|
Phases are monotonic within an attempt:
|
||||||
|
|
||||||
|
`claimed → workspace_prepared → verification_passed → source_capture_done|commit_done → result_pending → result_submitted`
|
||||||
|
|
||||||
|
`reconciliation_required` is an explicit recovery phase.
|
||||||
|
|
||||||
|
## Attempt-aware result idempotency
|
||||||
|
|
||||||
|
Result submission idempotency keys include the official Task attempt and the execution digest. The same
|
||||||
|
attempt reuses the same key after a timeout/restart; a later official attempt cannot collide with an
|
||||||
|
earlier result.
|
||||||
|
|
||||||
|
Before submitting a result the worker durably writes `result_pending`, including the result hash and
|
||||||
|
idempotency key. After a successful or exact-idempotency-reconciled API response it writes
|
||||||
|
`result_submitted`.
|
||||||
|
|
||||||
|
A 409 is accepted only if the Task API already contains the exact same result body and the Task is
|
||||||
|
VERIFYING/SUCCEEDED. A conflicting result remains a hard failure.
|
||||||
|
|
||||||
|
## Resume rules
|
||||||
|
|
||||||
|
- `source_capture_done` and `result_pending` may be reused across later official attempts only when the
|
||||||
|
execution/scope binding is identical. This avoids repeating read-only source capture.
|
||||||
|
- A deterministic task that crashed before governed commit replays from a new clean detached worktree.
|
||||||
|
This is safe because there is no external side effect yet.
|
||||||
|
- `commit_done` is never recommitted. Recovery requires exact Job receipt OID convergence plus exactly
|
||||||
|
one ACTIVE WriterLease whose `current_head_oid` equals the governed commit.
|
||||||
|
- delayed Git visibility remains `RECONCILIATION_REQUIRED`, preserving an explicit recovery entry.
|
||||||
|
|
||||||
|
## Crash cleanup
|
||||||
|
|
||||||
|
On worker startup, stale `task-*` workspaces under the configured workspace root are removed before
|
||||||
|
polling begins. The path boundary is verified before deletion.
|
||||||
|
|
||||||
|
## Contracts
|
||||||
|
|
||||||
|
P1 gates include:
|
||||||
|
|
||||||
|
- CHECKPOINT_DURABLE_WRITE
|
||||||
|
- RESUME_FROM_CHECKPOINT
|
||||||
|
- RETRY_IDEMPOTENT_NO_DUPLICATE_SIDE_EFFECT
|
||||||
|
- CRASH_RECOVERY_CONVERGENCE
|
||||||
|
- WORKER_RESTART_RESUME
|
||||||
|
- CHECKPOINT_SCOPE_ISOLATION
|
||||||
|
- RESULT_SUBMISSION_REPLAY_SAFE
|
||||||
|
- WORKSPACE_CLEANUP_RECOVERY
|
||||||
|
- LEASE_RENEWAL_RECOVERY
|
||||||
|
|
||||||
|
WriterLease and governed Job authority remain in the existing Bridge/GitOps durable stores; the worker
|
||||||
|
checkpoint DB is not a competing authority for Git state.
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
# R4.2 P2 Agent Fleet / Multi-Agent Scheduling / Fault Isolation
|
||||||
|
|
||||||
|
P2 adds a durable fleet decision layer inside TaskPlane without replacing TaskPlane claim/lease
|
||||||
|
ownership. Worker registration/heartbeat remains authoritative for liveness. TaskPlane claim CAS
|
||||||
|
remains authoritative for ownership. Probe-before-reassignment remains mandatory after an expired
|
||||||
|
lease. P1 checkpoint/result replay remains authoritative after worker restart.
|
||||||
|
|
||||||
|
`FleetScheduler` adds durable worker capacity slots, weighted scope fairness, starvation aging,
|
||||||
|
affinity/anti-affinity labels, worker circuit breakers, per-task/worker quarantine and idempotent
|
||||||
|
dispatch reservations. Fleet state is colocated in the existing TaskPlane SQLite database and is
|
||||||
|
created idempotently; no new daemon or database is introduced.
|
||||||
|
|
||||||
|
A fleet reservation is only a scheduling decision. It does not claim a Task. Callers must perform
|
||||||
|
the normal TaskPlane claim and may then bind the returned claim ID to the reservation. This preserves
|
||||||
|
the existing ownership and recovery invariants.
|
||||||
|
|
||||||
|
Failure recovery never bypasses probe evidence. When TaskPlane recovery places work back in a
|
||||||
|
claimable state, P2 may quarantine the failed worker and prefer another worker/failure domain.
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
# R4.2 P2 Compact Interface Map Runtime
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
The persistent Online Agent already supports the canonical read-only `source_capture` action, but
|
||||||
|
its `SourceCaptureExecutor` always returns an `aios.engineering.source-snapshot.v2` tar.gz+base64
|
||||||
|
payload. That is correct for implementation source transfer, but too large for control-plane
|
||||||
|
consumers that only need a bounded implementation map.
|
||||||
|
|
||||||
|
P2 adds a new **adapter**, not a new execution action:
|
||||||
|
|
||||||
|
```text
|
||||||
|
action=source_capture
|
||||||
|
adapter=compact_interface_map
|
||||||
|
```
|
||||||
|
|
||||||
|
This preserves the existing execution-v2 schema and Bridge policy:
|
||||||
|
|
||||||
|
- read-only repository;
|
||||||
|
- no change set;
|
||||||
|
- no verification commands;
|
||||||
|
- `writer_lease=false`;
|
||||||
|
- no branch or commit;
|
||||||
|
- existing capability and execution-digest validation.
|
||||||
|
|
||||||
|
## Compact map contract
|
||||||
|
|
||||||
|
`aios_online_agent_compact_map.py` scans only approved repository roots and source suffixes. It
|
||||||
|
returns:
|
||||||
|
|
||||||
|
- at most 40 relevant modules;
|
||||||
|
- at most 80 selected Python symbols in total;
|
||||||
|
- at most 40 test symbols;
|
||||||
|
- at most 25 excerpts;
|
||||||
|
- at most 1200 characters per excerpt;
|
||||||
|
- SQL table names;
|
||||||
|
- systemd unit names;
|
||||||
|
- current invariants, failure windows and reuse points;
|
||||||
|
- P2 fleet contract-to-surface mappings.
|
||||||
|
|
||||||
|
The result is hard-bounded to 120 KiB and contains no source archive or `payload_b64`.
|
||||||
|
|
||||||
|
Source excerpts apply a conservative sensitive-pattern redaction before result construction.
|
||||||
|
|
||||||
|
## P2 contracts
|
||||||
|
|
||||||
|
The map supplies evidence surfaces for:
|
||||||
|
|
||||||
|
- `FLEET_AGENT_REGISTRATION`
|
||||||
|
- `FLEET_CAPACITY_SCHEDULING`
|
||||||
|
- `FLEET_FAIRNESS`
|
||||||
|
- `FLEET_FAULT_ISOLATION`
|
||||||
|
- `FLEET_WORKER_REASSIGNMENT`
|
||||||
|
- `FLEET_SCOPE_ISOLATION`
|
||||||
|
- `FLEET_IDEMPOTENT_DISPATCH`
|
||||||
|
- `FLEET_FAILURE_RECOVERY`
|
||||||
|
|
||||||
|
Missing first-class scheduling/fleet primitives are reported as gaps rather than invented.
|
||||||
|
|
||||||
|
## Result
|
||||||
|
|
||||||
|
When the adapter is selected, the worker returns:
|
||||||
|
|
||||||
|
```text
|
||||||
|
status=SUCCEEDED
|
||||||
|
classification=P2_INTERFACE_MAP_CAPTURED
|
||||||
|
execution_mode=compact_interface_map
|
||||||
|
writer_lease=NONE
|
||||||
|
```
|
||||||
|
|
||||||
|
The existing `source_capture_done` / P1 checkpoint/retry path is reused, so the compact analysis
|
||||||
|
inherits attempt-aware result idempotency and restart-safe result submission.
|
||||||
|
|
||||||
|
## Governance
|
||||||
|
|
||||||
|
This capability is runtime infrastructure only. The implementation candidate may use a governed
|
||||||
|
branch/WriterLease for its own code change, but any task executed through `compact_interface_map`
|
||||||
|
is read-only and must not create a WriterLease, branch, commit, protected-ref write, or Production
|
||||||
|
side effect.
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
# R4.2 P2-B Fleet Order Preservation Repair
|
||||||
|
|
||||||
|
P2-C live acceptance exposed a real integration defect in the verified P2-B candidate. The fleet
|
||||||
|
scheduler correctly produced durable cross-scope fairness order, but the integration passed the
|
||||||
|
entire ranked list to the legacy scheduler. The legacy scheduler then applied its own priority /
|
||||||
|
created_at / task_id sort and silently discarded fleet ordering.
|
||||||
|
|
||||||
|
The repair preserves the legacy scheduler as the fail-closed security oracle while changing how it
|
||||||
|
is called: one fleet-ranked candidate is presented at a time. A candidate rejected by legacy
|
||||||
|
security/policy advances to the next fleet candidate. A candidate returned by the oracle must have
|
||||||
|
the same task ID as the candidate presented, otherwise execution fails closed.
|
||||||
|
|
||||||
|
Task ownership remains unchanged: the original schedule_for_worker/claim_task CAS remains
|
||||||
|
authoritative, and a fleet reservation is still only bound after the claim succeeds.
|
||||||
|
|
||||||
|
This repair does not modify FleetScheduler state semantics, capacity, WriterLease, checkpoint,
|
||||||
|
protected refs or Production behavior.
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
# R4.2 P2-B Fleet Runtime Integration
|
||||||
|
|
||||||
|
P2-B connects the verified P2 fleet primitives to the existing TaskPlane pull scheduler without
|
||||||
|
replacing its security or ownership logic.
|
||||||
|
|
||||||
|
The integration uses two stages. `FleetScheduler.rank_for_worker()` performs only read-only
|
||||||
|
capacity/fairness/fault-isolation ranking. The original `TaskScheduler.next_queued_task()` is then
|
||||||
|
run over that ordered list and remains the fail-closed security oracle for worker liveness,
|
||||||
|
capabilities, data classification, execution zone and other existing policy checks. Only a task
|
||||||
|
accepted by the original scheduler receives a short durable fleet reservation.
|
||||||
|
|
||||||
|
The original `TaskScheduler.schedule_for_worker()` remains the sole claim path and continues to call
|
||||||
|
the existing TaskPlane `claim_task` CAS. After a successful claim, the claim ID is bound to the
|
||||||
|
fleet reservation. A claim error releases the reservation. A fleet reservation is never treated as
|
||||||
|
task ownership.
|
||||||
|
|
||||||
|
P2-B also makes the reservation capacity check atomic inside the fleet SQLite transaction so two
|
||||||
|
concurrent pullers cannot oversubscribe the same worker slot. Circuit and task/worker quarantine
|
||||||
|
state are enforced on the real pull path. Explicit execution-failure signals can be recorded through
|
||||||
|
the integrated scheduler's failure/success/quarantine methods; policy/claim failures are not
|
||||||
|
automatically mislabeled as worker crashes.
|
||||||
|
|
||||||
|
No new daemon, database or production write path is introduced.
|
||||||
@@ -0,0 +1,123 @@
|
|||||||
|
# External Agent Attempt-Scoped Request Identity
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
External Agent execution previously derived its durable request ID from:
|
||||||
|
|
||||||
|
```text
|
||||||
|
task_id
|
||||||
|
job_id
|
||||||
|
execution_digest
|
||||||
|
runtime
|
||||||
|
```
|
||||||
|
|
||||||
|
The Online Task attempt was missing.
|
||||||
|
|
||||||
|
That is incorrect for retryable external side effects. A failed Model Governance route reservation is
|
||||||
|
durable and terminal for that request ID. Reusing the same external request ID on a later official
|
||||||
|
Task attempt therefore reuses the old route decision/reservation and can fail settlement with:
|
||||||
|
|
||||||
|
```text
|
||||||
|
MODEL_RESERVATION_STATE_INVALID
|
||||||
|
```
|
||||||
|
|
||||||
|
Archiving an exchange failed marker does not fix the root cause because the Model Governance
|
||||||
|
reservation remains intentionally durable.
|
||||||
|
|
||||||
|
## Contract
|
||||||
|
|
||||||
|
External request identity is now:
|
||||||
|
|
||||||
|
```text
|
||||||
|
hash(
|
||||||
|
task_id,
|
||||||
|
job_id,
|
||||||
|
execution_digest,
|
||||||
|
runtime,
|
||||||
|
attempt
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
The attempt is a **transient execution coordinate**. It is not added to the signed governed execution
|
||||||
|
envelope and does not alter the Task/Job binding or execution digest.
|
||||||
|
|
||||||
|
The worker gets the authoritative attempt from the successful claim response and passes it through:
|
||||||
|
|
||||||
|
```text
|
||||||
|
process_claim
|
||||||
|
→ RuntimeAdapter.execute(..., attempt=N)
|
||||||
|
→ ExternalAgentExecutor
|
||||||
|
→ ExternalAgentExchange
|
||||||
|
→ external request JSON
|
||||||
|
→ external result JSON
|
||||||
|
```
|
||||||
|
|
||||||
|
The external result must bind the same attempt.
|
||||||
|
|
||||||
|
## Idempotency
|
||||||
|
|
||||||
|
This gives the intended two-level behavior:
|
||||||
|
|
||||||
|
```text
|
||||||
|
same Task/Job/execution/runtime + same attempt
|
||||||
|
→ same external request ID
|
||||||
|
→ same external result/reservation can be replayed safely
|
||||||
|
|
||||||
|
same Task/Job/execution/runtime + next official attempt
|
||||||
|
→ different external request ID
|
||||||
|
→ different Model Governance request ID
|
||||||
|
→ new reservation
|
||||||
|
```
|
||||||
|
|
||||||
|
Old FAILED reservations are retained for audit. No Model Governance DB cleanup is required or
|
||||||
|
permitted.
|
||||||
|
|
||||||
|
## Model Governance
|
||||||
|
|
||||||
|
Model Governance already derives its route request ID from:
|
||||||
|
|
||||||
|
```text
|
||||||
|
external_request_id + ":model"
|
||||||
|
```
|
||||||
|
|
||||||
|
Therefore making the external request ID attempt-scoped automatically makes route decisions and
|
||||||
|
reservations attempt-scoped without creating another state authority.
|
||||||
|
|
||||||
|
## Recovery
|
||||||
|
|
||||||
|
For the current Hermes live Task:
|
||||||
|
|
||||||
|
```text
|
||||||
|
attempt 6
|
||||||
|
→ old request/reservation remains FAILED
|
||||||
|
|
||||||
|
official NEEDS_REWORK
|
||||||
|
→ bounded retry extension 6 → 7
|
||||||
|
|
||||||
|
attempt 7
|
||||||
|
→ new external request ID
|
||||||
|
→ new route request ID
|
||||||
|
→ new reservation
|
||||||
|
→ provider execution
|
||||||
|
→ settlement
|
||||||
|
```
|
||||||
|
|
||||||
|
No replacement Task/Job, direct DB rewrite, failed-reservation deletion or marker deletion is
|
||||||
|
required.
|
||||||
|
|
||||||
|
## Worker Availability
|
||||||
|
|
||||||
|
The repair must preserve the already verified Worker Availability runtime. Runtime switches use the
|
||||||
|
maintenance inhibit and must finish with the guard healthy and all three workers fresh.
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
Required regression coverage:
|
||||||
|
|
||||||
|
- request ID stable within one attempt;
|
||||||
|
- request ID different across attempts;
|
||||||
|
- request schema requires a valid attempt;
|
||||||
|
- external result binds attempt;
|
||||||
|
- old failed Model Governance reservation remains FAILED;
|
||||||
|
- next attempt gets a new reservation and can settle;
|
||||||
|
- existing external-agent and Model Governance runtime tests still pass.
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
# Generic Compact Interface-Map Profiles
|
||||||
|
|
||||||
|
The compact interface-map runtime is a reusable read-only control-plane capability. It keeps the
|
||||||
|
canonical `source_capture` action and `compact_interface_map` adapter, but selects a bounded semantic
|
||||||
|
profile from the task's source signals.
|
||||||
|
|
||||||
|
The existing `fleet` profile remains backward compatible and returns
|
||||||
|
`P2_INTERFACE_MAP_CAPTURED` with the `FLEET_*` contracts.
|
||||||
|
|
||||||
|
The new `model_governance` profile is selected when model/provider/routing/budget/evaluation terms
|
||||||
|
are present. It returns `MODEL_GOVERNANCE_INTERFACE_MAP_CAPTURED` and model-specific contracts for
|
||||||
|
provider registry, selection policy, version pinning, budget accounting, fallback, scope isolation,
|
||||||
|
credential boundary, trace/audit, evaluation gates and routing resilience.
|
||||||
|
|
||||||
|
Profile and classification are emitted by the compact-map module and revalidated by the worker
|
||||||
|
before result submission. Unknown or mismatched profile/classification pairs fail closed.
|
||||||
|
|
||||||
|
No new Bridge action, daemon, database, WriterLease, mutation path or credential surface is added.
|
||||||
@@ -0,0 +1,142 @@
|
|||||||
|
# Hermes / OpenClaw Online Agent Runtime
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
Promote Hermes and OpenClaw from shadow-only adapter descriptions to real Online Agent workers
|
||||||
|
without creating a second Task/claim protocol.
|
||||||
|
|
||||||
|
Both workers reuse the existing P1/P2 Online Agent lifecycle:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Online Task projected
|
||||||
|
→ capability-filtered poll
|
||||||
|
→ automatic claim
|
||||||
|
→ heartbeat
|
||||||
|
→ durable checkpoint
|
||||||
|
→ external Agent execution
|
||||||
|
→ attempt-aware result submission
|
||||||
|
→ official verification
|
||||||
|
```
|
||||||
|
|
||||||
|
They do **not** change the existing TaskPlane `OpenClawAdapter/HermesAdapter` shadow contract. Those
|
||||||
|
adapters remain side-effect-free compatibility surfaces.
|
||||||
|
|
||||||
|
## Execution action
|
||||||
|
|
||||||
|
The canonical execution envelope adds a read-only action:
|
||||||
|
|
||||||
|
```text
|
||||||
|
action=agent_runtime
|
||||||
|
adapter=hermes | openclaw
|
||||||
|
```
|
||||||
|
|
||||||
|
The action requires capability `agent.runtime.<adapter>`, has an empty change set, no verification
|
||||||
|
commands, and `writer_lease=false`.
|
||||||
|
|
||||||
|
`project_online_task()` copies only bounded task fields into the signed execution envelope:
|
||||||
|
|
||||||
|
- goal;
|
||||||
|
- instructions;
|
||||||
|
- success criteria;
|
||||||
|
- non-secret constraints.
|
||||||
|
|
||||||
|
The task may not choose a command, executable, URL, credential path, environment, or shell string.
|
||||||
|
|
||||||
|
## Worker separation
|
||||||
|
|
||||||
|
The generic engineering worker defaults to:
|
||||||
|
|
||||||
|
```text
|
||||||
|
AIOS_WORKER_ALLOWED_ADAPTERS=deterministic,source_capture,compact_interface_map,codex
|
||||||
|
```
|
||||||
|
|
||||||
|
so it skips Hermes/OpenClaw tasks before claim.
|
||||||
|
|
||||||
|
Dedicated services advertise:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Hermes:
|
||||||
|
AIOS_WORKER_CAPABILITIES=agent.runtime.hermes
|
||||||
|
AIOS_WORKER_ALLOWED_ADAPTERS=hermes
|
||||||
|
|
||||||
|
OpenClaw:
|
||||||
|
AIOS_WORKER_CAPABILITIES=agent.runtime.openclaw
|
||||||
|
AIOS_WORKER_ALLOWED_ADAPTERS=openclaw
|
||||||
|
```
|
||||||
|
|
||||||
|
This makes adapter selection a pre-claim capability/fleet decision instead of a post-claim failure.
|
||||||
|
|
||||||
|
## Credential-isolated executor
|
||||||
|
|
||||||
|
The Online Agent worker itself still owns only Task API/Bridge coordination credentials. It writes
|
||||||
|
a hash-bound request to `/var/lib/aios-external-agent-executor`.
|
||||||
|
|
||||||
|
`aios-external-agent-executor.service` runs as the existing uncredentialed engineering executor user.
|
||||||
|
It has:
|
||||||
|
|
||||||
|
- one provider-only `LoadCredential` projection for the model-ready OneAPI environment;
|
||||||
|
- `/etc/aios-gitops`, `/data/aios-gitops`, and the original provider-secret directory inaccessible;
|
||||||
|
- no repository path;
|
||||||
|
- no shell execution;
|
||||||
|
- loopback denied, preventing direct access to the AIOS Task/Bridge control plane;
|
||||||
|
- only the staging OneAPI container address allowed for the installed Agent runtime/provider;
|
||||||
|
- only the exchange directory writable.
|
||||||
|
|
||||||
|
The root-owned runtime profile in `/etc/aios/external-agent-runtime.d/<runtime>.json` decides the
|
||||||
|
absolute executable, fixed non-secret environment, provider-only credential mapping, and I/O mode.
|
||||||
|
A task cannot override any of these values. The profile may select `OPENAI_API_KEY` only from the
|
||||||
|
systemd credential named `aios-external-agent-oneapi-env`; it cannot name a credential, inject an
|
||||||
|
environment variable, or carry an inline secret. The runtime executor parses only the selected
|
||||||
|
`ONEAPI_REGULAR_TOKEN` entry and does not expose the source file or other provider tiers to the
|
||||||
|
child.
|
||||||
|
|
||||||
|
## Runtime profile
|
||||||
|
|
||||||
|
A profile supports:
|
||||||
|
|
||||||
|
- `argv`: fixed argv, absolute allowlisted executable;
|
||||||
|
- `input_mode`: `stdin_text`, `stdin_json`, or `file_json`;
|
||||||
|
- `output_mode`: `stdout_text`, `stdout_json`, or `file_json`;
|
||||||
|
- timeout;
|
||||||
|
- bounded output size.
|
||||||
|
|
||||||
|
`{input_json}` and `{output_json}` placeholders are allowed only for file modes. `{prompt}` is a
|
||||||
|
root-profile-only argv placeholder populated from the bounded, signed task projection.
|
||||||
|
|
||||||
|
The repository includes the staging profile templates validated against the installed Hermes
|
||||||
|
v0.20.0 and OpenClaw v2026.7.2-beta.7 CLIs. Before activation, the Agent must re-run the actual
|
||||||
|
`--help`/version probes and install the profiles and static configs root-owned. If the installed
|
||||||
|
CLI differs, update only the root-owned profile/config projection and record the observed command;
|
||||||
|
do not modify the Task contract to compensate.
|
||||||
|
|
||||||
|
The provider projection is staging-only and uses the existing model-ready OneAPI source. It is not
|
||||||
|
an AIOS Task, Bridge, GitOps, SSH, or Writer credential. Live activation remains blocked if the
|
||||||
|
model-ready source cannot be projected through systemd or if the selected provider credential is
|
||||||
|
missing/quota-invalid. This worker never passes API keys through task metadata or arbitrary
|
||||||
|
environment variables.
|
||||||
|
|
||||||
|
## Retry/restart
|
||||||
|
|
||||||
|
The external request ID is deterministic over Task/Job/execution digest/runtime. The exchange keeps
|
||||||
|
the request/result across worker restart. A repeated attempt with the exact execution digest reuses
|
||||||
|
the durable external result instead of invoking the Agent twice.
|
||||||
|
|
||||||
|
The normal P1 result idempotency/checkpoint path remains authoritative for Task result submission.
|
||||||
|
|
||||||
|
## Result
|
||||||
|
|
||||||
|
Successful execution returns:
|
||||||
|
|
||||||
|
```text
|
||||||
|
status=SUCCEEDED
|
||||||
|
classification=EXTERNAL_AGENT_RUNTIME_SUCCEEDED
|
||||||
|
execution_mode=agent_runtime
|
||||||
|
agent_runtime=hermes | openclaw
|
||||||
|
writer_lease=NONE
|
||||||
|
changed_files=[]
|
||||||
|
credential_material_exposed=false
|
||||||
|
control_plane_access_granted_to_runtime=false
|
||||||
|
```
|
||||||
|
|
||||||
|
This first runtime version is intentionally read-only with respect to the repository. Engineering
|
||||||
|
changes continue to use assistant-authored deterministic plans and governed WriterLease/commit.
|
||||||
+109
@@ -0,0 +1,109 @@
|
|||||||
|
# Model Governance Policy / Profile Floor Compatibility
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
The root-owned OpenClaw runtime profile reserves a minimum of 16,384 input tokens so Model
|
||||||
|
Governance can cover OpenClaw-injected workspace, skills and system context before provider
|
||||||
|
execution.
|
||||||
|
|
||||||
|
The staging scope policies still limited input to 16,000 tokens per request. Route selection
|
||||||
|
therefore failed before reservation with:
|
||||||
|
|
||||||
|
```text
|
||||||
|
MODEL_INPUT_TOKEN_LIMIT_DENY
|
||||||
|
```
|
||||||
|
|
||||||
|
This was a configuration invariant violation:
|
||||||
|
|
||||||
|
```text
|
||||||
|
scope max_input_tokens_per_request
|
||||||
|
<
|
||||||
|
root-owned profile input_reservation_floor_tokens
|
||||||
|
```
|
||||||
|
|
||||||
|
The registered OpenClaw model itself supports 32,768 input tokens, so the model capability was not
|
||||||
|
the blocker.
|
||||||
|
|
||||||
|
## Invariant
|
||||||
|
|
||||||
|
For every Model Governance scope that allows a model/runtime whose root-owned profile declares an
|
||||||
|
input reservation floor:
|
||||||
|
|
||||||
|
```text
|
||||||
|
scope.max_input_tokens_per_request
|
||||||
|
>=
|
||||||
|
profile.model_governance.input_reservation_floor_tokens
|
||||||
|
```
|
||||||
|
|
||||||
|
The profile floor remains root-owned. Task payloads cannot alter it.
|
||||||
|
|
||||||
|
## Staging policy
|
||||||
|
|
||||||
|
The two staging policies that allow `openai/deepseek-v4-flash` are updated from:
|
||||||
|
|
||||||
|
```text
|
||||||
|
16000
|
||||||
|
```
|
||||||
|
|
||||||
|
to:
|
||||||
|
|
||||||
|
```text
|
||||||
|
16384
|
||||||
|
```
|
||||||
|
|
||||||
|
This is the minimum compatible value. It intentionally remains below the model's 32,768-token
|
||||||
|
technical input ceiling.
|
||||||
|
|
||||||
|
The daily token budget is unchanged.
|
||||||
|
|
||||||
|
## Governance path
|
||||||
|
|
||||||
|
The live SQLite policy must be changed only through the repository bootstrap and
|
||||||
|
`aios_model_governance_admin.py`.
|
||||||
|
|
||||||
|
The admin reconciler:
|
||||||
|
|
||||||
|
- reads the governed repository config;
|
||||||
|
- uses current policy revision as CAS;
|
||||||
|
- calls `ModelGovernanceStore.set_scope_policy`;
|
||||||
|
- appends normal Model Governance audit evidence;
|
||||||
|
- verifies the audit chain.
|
||||||
|
|
||||||
|
Direct SQL updates remain forbidden.
|
||||||
|
|
||||||
|
## Exhausted OpenClaw recovery
|
||||||
|
|
||||||
|
The fixed OpenClaw live Task has exhausted its normal three attempts.
|
||||||
|
|
||||||
|
After the governed policy candidate is officially verified and the live policy is reconciled:
|
||||||
|
|
||||||
|
```text
|
||||||
|
attempt 3 VERIFYING/FAILED result
|
||||||
|
→ official NEEDS_REWORK
|
||||||
|
→ Task becomes FAILED 3/3
|
||||||
|
→ official bounded retry extension 3 → 4
|
||||||
|
→ same Task / same Job returns QUEUED
|
||||||
|
→ persistent OpenClaw worker auto-claims attempt 4
|
||||||
|
```
|
||||||
|
|
||||||
|
AR1 attempt-scoped external request identity ensures attempt 4 receives a new external request and a
|
||||||
|
new Model Governance reservation.
|
||||||
|
|
||||||
|
No historical reservation is deleted or rewritten.
|
||||||
|
|
||||||
|
## Acceptance
|
||||||
|
|
||||||
|
Attempt 4 must prove:
|
||||||
|
|
||||||
|
```text
|
||||||
|
scope max_input_tokens_per_request=16384
|
||||||
|
profile input_reservation_floor_tokens=16384
|
||||||
|
reserved_estimated_input_tokens>=16384
|
||||||
|
actual input <= reserved input
|
||||||
|
settlement=SETTLED
|
||||||
|
usage_source=runtime
|
||||||
|
audit_chain_valid=true
|
||||||
|
```
|
||||||
|
|
||||||
|
If attempt 4 exposes a new failure, stop. No additional retry extension is authorized by this
|
||||||
|
repair.
|
||||||
@@ -0,0 +1,211 @@
|
|||||||
|
# AIOS Model Governance Runtime
|
||||||
|
|
||||||
|
## Status and purpose
|
||||||
|
|
||||||
|
This component turns the previously `PARTIAL` Model Governance interface map into an enforceable
|
||||||
|
runtime policy for the already verified Hermes/OpenClaw Online Agent path.
|
||||||
|
|
||||||
|
It is intentionally a **single SQLite library**, not a new network service. The existing external
|
||||||
|
Agent executor calls the library immediately before launching a model-backed runtime and settles the
|
||||||
|
same reservation immediately after the result.
|
||||||
|
|
||||||
|
The authoritative execution chain remains:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Online Task scope / trace
|
||||||
|
→ dedicated worker claim + heartbeat
|
||||||
|
→ external Agent executor
|
||||||
|
→ ModelGovernanceStore.route()
|
||||||
|
→ root-owned runtime profile binds provider/model/version
|
||||||
|
→ Hermes / OpenClaw process
|
||||||
|
→ ModelGovernanceStore.record_success() / record_failure()
|
||||||
|
→ durable Online Agent checkpoint/result
|
||||||
|
```
|
||||||
|
|
||||||
|
P0 scope/trace/credential, P1 checkpoint/retry, P2 fleet scheduling and the existing external Agent
|
||||||
|
credential boundary remain authoritative.
|
||||||
|
|
||||||
|
## Ten Model Governance contracts
|
||||||
|
|
||||||
|
### MODEL_PROVIDER_REGISTRY
|
||||||
|
|
||||||
|
`model_governance_providers` stores provider identity, failure domain, circuit policy and an opaque
|
||||||
|
credential reference. Raw credentials are rejected.
|
||||||
|
|
||||||
|
Allowed reference prefixes:
|
||||||
|
|
||||||
|
- `systemd-credential:`
|
||||||
|
- `runtime-profile:`
|
||||||
|
- `credential-broker:`
|
||||||
|
|
||||||
|
### MODEL_SELECTION_POLICY
|
||||||
|
|
||||||
|
`model_governance_scope_policies` constrains eligible models/providers/capabilities. Routing is
|
||||||
|
deterministic: explicit preferred models, then configured priority, then stable provider/model IDs.
|
||||||
|
|
||||||
|
### MODEL_VERSION_PINNING
|
||||||
|
|
||||||
|
A registry row always includes a model version. A scope can additionally pin a model ID + exact
|
||||||
|
version. Every route decision persists both model ID and version plus model/policy revisions.
|
||||||
|
|
||||||
|
### MODEL_BUDGET_ACCOUNTING
|
||||||
|
|
||||||
|
A route first reserves estimated input/output tokens and estimated cost inside `BEGIN IMMEDIATE`.
|
||||||
|
Settled usage and outstanding reservations both count against the daily scope budget, preventing two
|
||||||
|
concurrent requests from oversubscribing the same budget.
|
||||||
|
|
||||||
|
Runtime-reported usage is preferred. When telemetry is absent, the reserved upper bound is charged
|
||||||
|
rather than silently under-counting.
|
||||||
|
|
||||||
|
OpenClaw's verified result shape is supported directly:
|
||||||
|
|
||||||
|
```text
|
||||||
|
meta.agentMeta.usage.input
|
||||||
|
meta.agentMeta.usage.output
|
||||||
|
meta.agentMeta.costUsd
|
||||||
|
```
|
||||||
|
|
||||||
|
Hermes currently has no structured usage in the verified output, so it uses conservative
|
||||||
|
reservation settlement.
|
||||||
|
|
||||||
|
### MODEL_FALLBACK_POLICY
|
||||||
|
|
||||||
|
Provider failures are durable. Repeated failures open a circuit for the configured cooldown. A
|
||||||
|
fallback request excludes the failed provider and increments `fallback_index`; the scope policy
|
||||||
|
bounds the number of fallback attempts.
|
||||||
|
|
||||||
|
### MODEL_SCOPE_ISOLATION
|
||||||
|
|
||||||
|
The signed `agent_runtime` execution envelope now carries authoritative Task scope + trace identity.
|
||||||
|
|
||||||
|
The projection layer always overwrites `runtime_input.scope` and `runtime_input.trace_id` from the
|
||||||
|
Online Task, even if an execution request attempts to supply different values.
|
||||||
|
|
||||||
|
Tenant/project lookup may fall back only to the same tenant's policy. It never falls back to a
|
||||||
|
system or legacy policy. `system` and `legacy` require their own explicit policies, avoiding fake
|
||||||
|
tenant identities.
|
||||||
|
|
||||||
|
### MODEL_CREDENTIAL_BOUNDARY
|
||||||
|
|
||||||
|
Model governance stores credential references only. The root-owned external-runtime profile still
|
||||||
|
owns the actual provider credential projection. Task payloads cannot select executable, argv,
|
||||||
|
environment, provider credential, or credential path.
|
||||||
|
|
||||||
|
The external child process remains unable to reach the AIOS control plane.
|
||||||
|
|
||||||
|
### MODEL_TRACE_AUDIT
|
||||||
|
|
||||||
|
Every route, failure, provider/model/policy update and settlement appends to a SHA-256 hash-chained
|
||||||
|
audit table with task ID, trace ID and scope key. `verify_audit_chain()` detects tampering.
|
||||||
|
|
||||||
|
### MODEL_EVALUATION_GATE
|
||||||
|
|
||||||
|
A model cannot enter the registry unless its state is `APPROVED` and it has a 64-hex evaluation
|
||||||
|
evidence digest.
|
||||||
|
|
||||||
|
The staging bootstrap uses the already verified Hermes/OpenClaw E2E external-result SHA256 values as
|
||||||
|
evaluation evidence, rather than a configuration-file hash.
|
||||||
|
|
||||||
|
### MODEL_ROUTING_RESILIENCE
|
||||||
|
|
||||||
|
Route request IDs are idempotent and bound to a canonical request hash. Conflicting replay is
|
||||||
|
rejected. Provider health, reservations, usage settlement and policy/provider/model updates use
|
||||||
|
transaction/CAS semantics.
|
||||||
|
|
||||||
|
## Runtime profile contract
|
||||||
|
|
||||||
|
A root-owned Hermes/OpenClaw profile may declare:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"model_governance": {
|
||||||
|
"required": true,
|
||||||
|
"provider_ids": ["provider-a"],
|
||||||
|
"model_capabilities": ["text"],
|
||||||
|
"max_output_tokens": 512
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
When `required=true`:
|
||||||
|
|
||||||
|
- `provider_ids` must be non-empty;
|
||||||
|
- `max_output_tokens` must be positive;
|
||||||
|
- argv/environment must contain at least one governance-controlled marker:
|
||||||
|
`{provider_id}`, `{model_id}`, or `{model_version}`;
|
||||||
|
- `AIOS_MODEL_GOVERNANCE_ENABLED` must be enabled;
|
||||||
|
- a matching scope policy/model/provider must exist;
|
||||||
|
- budget reservation must succeed before the child process is launched.
|
||||||
|
|
||||||
|
The task cannot provide these markers or their values.
|
||||||
|
|
||||||
|
## SQLite authority
|
||||||
|
|
||||||
|
Default database:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/var/lib/aios-model-governance/state.sqlite3
|
||||||
|
```
|
||||||
|
|
||||||
|
The executor process already owns the only runtime mutation path. The directory is created by
|
||||||
|
tmpfiles and is the only new writable path added to that executor.
|
||||||
|
|
||||||
|
No direct Task/Job/Git database writes are introduced.
|
||||||
|
|
||||||
|
## Bootstrap/reconciliation
|
||||||
|
|
||||||
|
`aios_model_governance_admin.py` applies a root-owned JSON bootstrap idempotently. It compares desired
|
||||||
|
state to current rows and uses the current revision as the CAS input only when a change is actually
|
||||||
|
required.
|
||||||
|
|
||||||
|
The repository staging bootstrap is not a Production policy. It exists only to reproduce and
|
||||||
|
govern the already verified non-production Hermes/OpenClaw runtimes.
|
||||||
|
|
||||||
|
## Cost caveat
|
||||||
|
|
||||||
|
The current verified OneAPI-backed runtime evidence reports zero authoritative provider cost for
|
||||||
|
OpenClaw and no structured cost for Hermes. The staging bootstrap therefore does **not** invent
|
||||||
|
pricing and sets cost rates/budget to zero/unbounded.
|
||||||
|
|
||||||
|
Token budgets remain enforced. Real monetary budget enforcement becomes effective as soon as
|
||||||
|
authoritative per-model pricing is entered into the root-owned registry.
|
||||||
|
|
||||||
|
## OpenClaw JIT/WASM compatibility
|
||||||
|
|
||||||
|
The previous live acceptance proved that OpenClaw Node/WASM requires
|
||||||
|
`MemoryDenyWriteExecute=no`. Model Governance must preserve that already-evidenced runtime exception;
|
||||||
|
it must not further weaken `NoNewPrivileges`, `ProtectSystem=strict`, `ProtectHome`, loopback deny,
|
||||||
|
credential isolation or repository isolation.
|
||||||
|
|
||||||
|
## Failure/recovery invariants
|
||||||
|
|
||||||
|
- Budget denial occurs before provider execution.
|
||||||
|
- Missing scope policy fails closed.
|
||||||
|
- Missing/disabled governance fails closed when a profile requires it.
|
||||||
|
- Actual usage cannot exceed the reservation.
|
||||||
|
- Provider failure releases that reservation into `FAILED` state before fallback.
|
||||||
|
- Settlement is idempotent.
|
||||||
|
- Route replay is idempotent only for the exact canonical request.
|
||||||
|
- Audit corruption is detectable.
|
||||||
|
- Online Agent checkpoint/result retry remains unchanged and remains the outer recovery authority.
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
Required unit/integration coverage includes:
|
||||||
|
|
||||||
|
- registry/evaluation/CAS;
|
||||||
|
- scope isolation and no system→tenant fallback;
|
||||||
|
- model/version pinning;
|
||||||
|
- atomic budget reservation;
|
||||||
|
- settlement/replay;
|
||||||
|
- missing telemetry conservative settlement;
|
||||||
|
- provider circuit/fallback;
|
||||||
|
- trace/audit tamper detection;
|
||||||
|
- authoritative scope/trace projection;
|
||||||
|
- task runtime-input scope override attack;
|
||||||
|
- root-owned provider/model binding;
|
||||||
|
- OpenClaw nested usage extraction;
|
||||||
|
- real Hermes/OpenClaw staging live tasks with Model Governance evidence.
|
||||||
|
|
||||||
|
The governed implementation is not complete without the unified all-modified-files bundle for the
|
||||||
|
candidate delta and the cumulative lineage bundle.
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
# OpenClaw Input Reservation Floor
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
Model Governance originally estimated input reservation from the user-visible prompt only.
|
||||||
|
|
||||||
|
The verified OpenClaw runtime injects additional root-owned/system-controlled context including
|
||||||
|
workspace, skills and system instructions. In live acceptance the prompt estimate reserved only
|
||||||
|
153 input tokens while OpenClaw reported approximately 7,770 actual input tokens.
|
||||||
|
|
||||||
|
The settlement correctly failed closed with:
|
||||||
|
|
||||||
|
```text
|
||||||
|
MODEL_USAGE_EXCEEDS_RESERVATION
|
||||||
|
```
|
||||||
|
|
||||||
|
The provider call itself had succeeded. The defect was therefore pre-execution reservation
|
||||||
|
underestimation, not provider connectivity.
|
||||||
|
|
||||||
|
## Contract
|
||||||
|
|
||||||
|
A root-owned external Agent profile may now declare:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"model_governance": {
|
||||||
|
"input_reservation_floor_tokens": 16384
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The effective input reservation is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
max(
|
||||||
|
byte-based prompt estimate,
|
||||||
|
profile input_reservation_floor_tokens
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Only the root-owned runtime profile can choose this floor. Online Task payloads cannot lower or
|
||||||
|
override it.
|
||||||
|
|
||||||
|
The OpenClaw staging profile uses 16,384 tokens. This is greater than twice the observed 7,770-token
|
||||||
|
live input high-water mark while remaining below the registered 32,768-token model input limit.
|
||||||
|
|
||||||
|
Hermes has no floor unless explicitly configured and therefore preserves its existing behavior.
|
||||||
|
|
||||||
|
## Fail-closed behavior
|
||||||
|
|
||||||
|
The floor is not an unlimited post-execution top-up.
|
||||||
|
|
||||||
|
If actual runtime usage exceeds the reserved amount, settlement still fails with
|
||||||
|
`MODEL_USAGE_EXCEEDS_RESERVATION`.
|
||||||
|
|
||||||
|
This keeps budget enforcement pre-execution and avoids silently authorizing usage after it has
|
||||||
|
already happened.
|
||||||
|
|
||||||
|
## Evidence
|
||||||
|
|
||||||
|
External results expose:
|
||||||
|
|
||||||
|
```text
|
||||||
|
prompt_estimated_input_tokens
|
||||||
|
input_reservation_floor_tokens
|
||||||
|
reserved_estimated_input_tokens
|
||||||
|
```
|
||||||
|
|
||||||
|
alongside the normal settlement and audit evidence.
|
||||||
|
|
||||||
|
## Retry
|
||||||
|
|
||||||
|
The current OpenClaw attempt 1 failure remains preserved. After this code repair is governed and
|
||||||
|
activated, official `NEEDS_REWORK` returns the same fixed Task to the normal retry path. Because AR1
|
||||||
|
already made external request identity attempt-scoped, attempt 2 receives a new external request ID
|
||||||
|
and a new reservation.
|
||||||
|
|
||||||
|
No retry-extension is required because the Task is currently attempt 1 of 3.
|
||||||
|
|
||||||
|
No Model Governance reservation is deleted or rewritten.
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
# Owner-Controlled Same-Binding Execution Rebind
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
`aios_agent_task_execution_rebind.py` closes a projection/plan race without replacing the Task,
|
||||||
|
Governed Job, target branch, generation, or idempotency lineage.
|
||||||
|
|
||||||
|
The only supported transition is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
official NEEDS_REWORK
|
||||||
|
→ Task QUEUED with no active lease
|
||||||
|
→ persisted source_capture envelope
|
||||||
|
→ canonical Bridge resolves an already-registered deterministic plan
|
||||||
|
→ CAS-bound metadata.execution + execution_projection replacement
|
||||||
|
→ audit event
|
||||||
|
→ normal worker claim
|
||||||
|
```
|
||||||
|
|
||||||
|
The service rejects arbitrary metadata patches. It derives the new envelope from canonical Bridge
|
||||||
|
`/v2/execution-envelope`, validates every governed binding coordinate against the persisted envelope,
|
||||||
|
requires `source_capture → deterministic`, requires exact old/new execution digests supplied by the
|
||||||
|
Owner recovery request, and increments the Task revision under `BEGIN IMMEDIATE`.
|
||||||
|
|
||||||
|
It records `TASK_EXECUTION_REBIND_AUTHORIZED` and an idempotent `task_execution_rebinds` receipt.
|
||||||
|
No Task/Job/branch replacement is created and the worker still performs the real claim/heartbeat,
|
||||||
|
WriterLease and governed commit.
|
||||||
|
|
||||||
|
The endpoint is loopback-only:
|
||||||
|
|
||||||
|
`POST /v1/tasks/{task_id}/execution-rebind`
|
||||||
|
|
||||||
|
and requires the existing Owner control credential plus
|
||||||
|
`AIOS_AGENT_TASK_EXECUTION_REBIND_AUTHORIZATION_ID`.
|
||||||
|
|
||||||
|
Direct/manual Task database edits remain forbidden; the service is the audited mutation boundary.
|
||||||
@@ -0,0 +1,212 @@
|
|||||||
|
# AIOS Worker Availability Runtime
|
||||||
|
|
||||||
|
## Priority
|
||||||
|
|
||||||
|
This runtime exists to make the already verified Online Agent execution path continuously usable
|
||||||
|
without repeated human intervention.
|
||||||
|
|
||||||
|
The system already proved that generic, Hermes and OpenClaw workers can claim and complete real
|
||||||
|
Online Tasks. The remaining availability defect was operational: evidence-freeze workflows left the
|
||||||
|
worker services stopped, while registry `ACTIVE` records remained stale.
|
||||||
|
|
||||||
|
The availability contract therefore separates:
|
||||||
|
|
||||||
|
```text
|
||||||
|
AUTO_CLAIM_MECHANISM=VERIFIED
|
||||||
|
CURRENT_WORKER_AVAILABILITY=<fresh runtime state>
|
||||||
|
```
|
||||||
|
|
||||||
|
A historical successful claim is not current liveness.
|
||||||
|
|
||||||
|
## Source of truth for idle liveness
|
||||||
|
|
||||||
|
An idle worker may have no Task lease heartbeat. Therefore Task heartbeat alone cannot prove that
|
||||||
|
the worker is continuously polling.
|
||||||
|
|
||||||
|
`aios_online_agent_worker.py` now writes:
|
||||||
|
|
||||||
|
```text
|
||||||
|
<AIOS_WORKER_STATE_ROOT>/worker-liveness.json
|
||||||
|
```
|
||||||
|
|
||||||
|
on:
|
||||||
|
|
||||||
|
- process start: `STARTING`;
|
||||||
|
- every successful poll: `POLL_OK`;
|
||||||
|
- poll failure: `POLL_FAILED`;
|
||||||
|
- every successful Task lease heartbeat while executing: `BUSY`.
|
||||||
|
|
||||||
|
The file contains:
|
||||||
|
|
||||||
|
- agent ID/type/version;
|
||||||
|
- process PID;
|
||||||
|
- monotonically increasing local sequence;
|
||||||
|
- `last_seen_epoch`;
|
||||||
|
- poll/heartbeat intervals;
|
||||||
|
- current Task/stage while busy.
|
||||||
|
|
||||||
|
The liveness file is worker-owned observability state only. It does not write or mutate Task, Job,
|
||||||
|
Git, WriterLease or registry databases.
|
||||||
|
|
||||||
|
## Self-healing guard
|
||||||
|
|
||||||
|
`aios-worker-availability-guard.timer` invokes a root-owned oneshot guard every 30 seconds.
|
||||||
|
|
||||||
|
The guard manages:
|
||||||
|
|
||||||
|
```text
|
||||||
|
aios-online-agent-worker-20260815.service
|
||||||
|
aios-hermes-online-agent-worker.service
|
||||||
|
aios-openclaw-online-agent-worker.service
|
||||||
|
```
|
||||||
|
|
||||||
|
and the required external runtime dependency:
|
||||||
|
|
||||||
|
```text
|
||||||
|
aios-external-agent-executor.service
|
||||||
|
```
|
||||||
|
|
||||||
|
For each worker:
|
||||||
|
|
||||||
|
1. missing systemd unit → fail closed;
|
||||||
|
2. disabled → `systemctl enable`;
|
||||||
|
3. inactive → `systemctl start`;
|
||||||
|
4. active but liveness older than 90 seconds → restart;
|
||||||
|
5. repeated stale restart within 60 seconds → do not restart again; report degraded.
|
||||||
|
|
||||||
|
Long-running Tasks are safe because the Task lease-heartbeat thread refreshes `BUSY` liveness.
|
||||||
|
|
||||||
|
The guard never claims a Task and never touches Task/Job/Git databases.
|
||||||
|
|
||||||
|
## Maintenance inhibit
|
||||||
|
|
||||||
|
Planned maintenance creates:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/run/aios-worker-availability/maintenance.lock
|
||||||
|
```
|
||||||
|
|
||||||
|
While this file exists the guard performs no start/restart actions and reports:
|
||||||
|
|
||||||
|
```text
|
||||||
|
MAINTENANCE_INHIBITED
|
||||||
|
```
|
||||||
|
|
||||||
|
Maintenance tooling must remove the file when done. This prevents the self-healer from fighting
|
||||||
|
intentional service changes.
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
The guard writes:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/run/aios-worker-availability/status.json
|
||||||
|
```
|
||||||
|
|
||||||
|
with current worker/dependency states, liveness ages and recovery actions.
|
||||||
|
|
||||||
|
A worker is currently available only when:
|
||||||
|
|
||||||
|
```text
|
||||||
|
systemd load_state=loaded
|
||||||
|
systemd active_state=active
|
||||||
|
unit enabled/static
|
||||||
|
worker-liveness.json present
|
||||||
|
last_seen_age <= 90s
|
||||||
|
```
|
||||||
|
|
||||||
|
Registry `ACTIVE` without fresh poll/busy liveness is not sufficient.
|
||||||
|
|
||||||
|
## Boot/restart behavior
|
||||||
|
|
||||||
|
The timer is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
OnBootSec=15s
|
||||||
|
OnUnitActiveSec=30s
|
||||||
|
Persistent=true
|
||||||
|
```
|
||||||
|
|
||||||
|
Therefore reboot, manual stop left behind by another workflow, process crash, and hung poll loop are
|
||||||
|
all repaired without requiring a human to notice and intervene.
|
||||||
|
|
||||||
|
Normal worker units retain their own `Restart=always`; the availability guard handles the cases that
|
||||||
|
`Restart=always` cannot, especially a deliberate `systemctl stop` left after an evidence workflow.
|
||||||
|
|
||||||
|
## Acceptance
|
||||||
|
|
||||||
|
The live acceptance requires all of the following:
|
||||||
|
|
||||||
|
```text
|
||||||
|
guard timer active/enabled
|
||||||
|
external executor active/enabled
|
||||||
|
generic worker active/enabled + fresh liveness
|
||||||
|
Hermes worker active/enabled + fresh liveness
|
||||||
|
OpenClaw worker active/enabled + fresh liveness
|
||||||
|
```
|
||||||
|
|
||||||
|
Then a new read-only Online Task is projected through canonical Bridge `18892`.
|
||||||
|
|
||||||
|
Success requires:
|
||||||
|
|
||||||
|
```text
|
||||||
|
TASK_CREATED
|
||||||
|
→ generic worker auto claim
|
||||||
|
→ TASK_HEARTBEAT
|
||||||
|
→ RESULT_SUBMITTED
|
||||||
|
→ official TASK_VERIFIED(SUCCEEDED)
|
||||||
|
```
|
||||||
|
|
||||||
|
After the Task, the services must remain active and liveness must continue advancing. The acceptance
|
||||||
|
also stops a worker intentionally once and proves the guard starts it again with a new fresh
|
||||||
|
liveness sequence.
|
||||||
|
|
||||||
|
Only after these gates may current status be upgraded from:
|
||||||
|
|
||||||
|
```text
|
||||||
|
OVERALL=PARTIAL
|
||||||
|
```
|
||||||
|
|
||||||
|
to:
|
||||||
|
|
||||||
|
```text
|
||||||
|
PERSISTENT_WORKER_LIVENESS=VERIFIED
|
||||||
|
NEW_TASK_AUTO_CONSUMPTION=VERIFIED
|
||||||
|
WORKER_AVAILABILITY_RUNTIME=VERIFIED_CANDIDATE
|
||||||
|
```
|
||||||
|
|
||||||
|
Protected main and Production remain out of scope.
|
||||||
|
|
||||||
|
|
||||||
|
## Forced source-capture projection compatibility
|
||||||
|
|
||||||
|
The deployed canonical Bridge currently projects a default `source_capture` envelope even when a
|
||||||
|
governed binding request asks for binding-only behavior. This must not be allowed to make always-on
|
||||||
|
workers consume engineering Tasks before their deterministic plan is registered.
|
||||||
|
|
||||||
|
The worker therefore distinguishes a real read-only source capture from an engineering placeholder:
|
||||||
|
|
||||||
|
```text
|
||||||
|
source_capture
|
||||||
|
+ capability_scope contains governed.write.branch or governed.write.file
|
||||||
|
= PENDING_DETERMINISTIC_PLAN
|
||||||
|
```
|
||||||
|
|
||||||
|
For such a placeholder the worker:
|
||||||
|
|
||||||
|
1. re-resolves through the canonical plan registry **without replaying the stale source-capture
|
||||||
|
envelope**;
|
||||||
|
2. if no deterministic plan exists, emits `TASK_SKIPPED_PENDING_DETERMINISTIC_PLAN` and does not
|
||||||
|
claim;
|
||||||
|
3. if the registered canonical envelope is deterministic and the governed binding still matches,
|
||||||
|
uses that deterministic envelope and claims normally.
|
||||||
|
|
||||||
|
A real read-only source-capture Task with only `git.read`/`git.test` still replays its exact persisted
|
||||||
|
payload and remains claimable.
|
||||||
|
|
||||||
|
This makes persistent workers safe with the currently deployed Bridge while preserving the same
|
||||||
|
Task/Job binding. No Task DB rewrite, cancellation, replacement Task, fake NEEDS_REWORK, or
|
||||||
|
pre-claim metadata mutation is required.
|
||||||
|
|
||||||
|
A later Bridge cleanup may implement true binding-only projection, but Worker Availability does not
|
||||||
|
depend on that cleanup.
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
# Canonical all-modified-files packager and secret scanning
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
The historical all-modified-files evidence packager lived only as a copied handoff script.
|
||||||
|
|
||||||
|
Its private-key scanner matched the fragment:
|
||||||
|
|
||||||
|
```text
|
||||||
|
BEGIN PRIVATE KEY
|
||||||
|
```
|
||||||
|
|
||||||
|
instead of a complete PEM header. A regression test containing that literal therefore caused a
|
||||||
|
false secret finding even though the candidate contained no credential.
|
||||||
|
|
||||||
|
A task had to use a temporary evidence-only compatibility packager to complete its bundle. That is
|
||||||
|
acceptable as an emergency workaround but not as a durable unattended path.
|
||||||
|
|
||||||
|
## Repository source of truth
|
||||||
|
|
||||||
|
The canonical implementation is now:
|
||||||
|
|
||||||
|
```text
|
||||||
|
active-runtime/app/aios_all_modified_files_packager.py
|
||||||
|
```
|
||||||
|
|
||||||
|
It preserves the historical output/bundle schemas:
|
||||||
|
|
||||||
|
```text
|
||||||
|
aios.all-modified-files.bundle.v1
|
||||||
|
aios.all-modified-files.bundle-summary.v1
|
||||||
|
aios.modified-documents.index.v1
|
||||||
|
```
|
||||||
|
|
||||||
|
and the host-visible status lines:
|
||||||
|
|
||||||
|
```text
|
||||||
|
ALL_MODIFIED_FILES_BUNDLE=PASS
|
||||||
|
ALL_MODIFIED_FILES_BUNDLE_PATH=...
|
||||||
|
ALL_MODIFIED_FILES_BUNDLE_SHA256=...
|
||||||
|
```
|
||||||
|
|
||||||
|
Future governed tasks should use this repository version instead of copying an unversioned scanner.
|
||||||
|
|
||||||
|
## Secret scanning invariant
|
||||||
|
|
||||||
|
Private-key detection requires a complete PEM header:
|
||||||
|
|
||||||
|
```text
|
||||||
|
-----BEGIN [RSA|EC|DSA|OPENSSH ]PRIVATE KEY-----
|
||||||
|
```
|
||||||
|
|
||||||
|
The following source/test/document text is not secret material and must remain packageable:
|
||||||
|
|
||||||
|
```text
|
||||||
|
BEGIN PRIVATE KEY
|
||||||
|
PRIVATE KEY
|
||||||
|
PEM private-key header
|
||||||
|
```
|
||||||
|
|
||||||
|
Bearer and key/value credential detection remains fail-closed:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Authorization: Bearer <20+ token chars>
|
||||||
|
password=<12+ non-space chars>
|
||||||
|
passwd=<12+ non-space chars>
|
||||||
|
secret=<12+ non-space chars>
|
||||||
|
api_key=<12+ non-space chars>
|
||||||
|
api-key=<12+ non-space chars>
|
||||||
|
```
|
||||||
|
|
||||||
|
Tests construct the real PEM header dynamically so the scanner's own test source does not contain a
|
||||||
|
literal credential-looking header.
|
||||||
|
|
||||||
|
## Bundle integrity
|
||||||
|
|
||||||
|
The packager continues to enforce:
|
||||||
|
|
||||||
|
- exact base→candidate ancestry;
|
||||||
|
- before/after file capture;
|
||||||
|
- rename/copy/delete handling;
|
||||||
|
- path traversal denial;
|
||||||
|
- execution evidence symlink denial;
|
||||||
|
- secret scan over repository before/after and execution evidence;
|
||||||
|
- document index;
|
||||||
|
- SHA-256 closure;
|
||||||
|
- ZIP without symlinks.
|
||||||
|
|
||||||
|
No secret finding is downgraded to a warning.
|
||||||
|
|
||||||
|
## Migration
|
||||||
|
|
||||||
|
The old handoff script may remain for historical reproducibility, but new handoffs should call the
|
||||||
|
repository packager from the verified candidate lineage.
|
||||||
|
|
||||||
|
A temporary compatibility scanner is no longer needed once this candidate is in the lineage.
|
||||||
@@ -0,0 +1,210 @@
|
|||||||
|
# AIOS Autonomous Control Plane — Admission + Invariant + Repair
|
||||||
|
|
||||||
|
## Why this exists
|
||||||
|
|
||||||
|
Recent Agent registration, Task claim, release-worker, credential and retry failures shared one
|
||||||
|
failure mode:
|
||||||
|
|
||||||
|
> The control plane accepted an intent before the runtime had proved it could execute the intent.
|
||||||
|
|
||||||
|
This design does not add another fleet of daemons. It adds one shared library contract that Task API,
|
||||||
|
workers and governed host recovery consume.
|
||||||
|
|
||||||
|
## Three layers
|
||||||
|
|
||||||
|
```text
|
||||||
|
Intent
|
||||||
|
↓
|
||||||
|
Runtime Admission
|
||||||
|
↓
|
||||||
|
System Invariants
|
||||||
|
↓
|
||||||
|
Repair Policy
|
||||||
|
↓
|
||||||
|
Execution
|
||||||
|
```
|
||||||
|
|
||||||
|
### Runtime Admission
|
||||||
|
|
||||||
|
Admission occurs before Task claim.
|
||||||
|
|
||||||
|
A capability such as `release:candidate` is not considered executable merely because its name
|
||||||
|
matches. The capability contract also binds:
|
||||||
|
|
||||||
|
- supported contract version;
|
||||||
|
- allowed permission;
|
||||||
|
- forbidden operations;
|
||||||
|
- runtime dependencies;
|
||||||
|
- health requirements.
|
||||||
|
|
||||||
|
For `release:candidate`:
|
||||||
|
|
||||||
|
```text
|
||||||
|
permission:
|
||||||
|
prepare_release
|
||||||
|
|
||||||
|
forbidden:
|
||||||
|
production_deploy
|
||||||
|
submit_owner_approval
|
||||||
|
tag_publish
|
||||||
|
baseline_registry_finalize
|
||||||
|
runtime_switch
|
||||||
|
|
||||||
|
dependencies:
|
||||||
|
credential_projection
|
||||||
|
trusted_builder
|
||||||
|
|
||||||
|
health:
|
||||||
|
authenticated_probe
|
||||||
|
release_api_health
|
||||||
|
worker_liveness
|
||||||
|
```
|
||||||
|
|
||||||
|
Only a fully passing decision reaches:
|
||||||
|
|
||||||
|
```text
|
||||||
|
EXECUTION_READY
|
||||||
|
```
|
||||||
|
|
||||||
|
A failed decision remains:
|
||||||
|
|
||||||
|
```text
|
||||||
|
EXECUTION_READY_FAILED
|
||||||
|
```
|
||||||
|
|
||||||
|
and the worker does not call `/claim`.
|
||||||
|
|
||||||
|
Therefore an infrastructure defect discovered by admission does not consume a business attempt.
|
||||||
|
|
||||||
|
### System Invariants
|
||||||
|
|
||||||
|
The same admission checks are exposed as a machine-readable invariant snapshot.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
```text
|
||||||
|
capability_contract_registry
|
||||||
|
task_capabilities_registered
|
||||||
|
worker_capabilities_registered
|
||||||
|
capability_match
|
||||||
|
forbidden_operations_absent
|
||||||
|
credential_projection
|
||||||
|
trusted_builder
|
||||||
|
authenticated_probe
|
||||||
|
release_api_health
|
||||||
|
worker_liveness
|
||||||
|
```
|
||||||
|
|
||||||
|
A runtime can expose:
|
||||||
|
|
||||||
|
```text
|
||||||
|
HEALTHY
|
||||||
|
DEGRADED
|
||||||
|
```
|
||||||
|
|
||||||
|
without relying on a human reconstructing state from logs.
|
||||||
|
|
||||||
|
### Repair Policy
|
||||||
|
|
||||||
|
Failure classification maps to bounded recovery:
|
||||||
|
|
||||||
|
```text
|
||||||
|
DEPENDENCY_UNAVAILABLE → level 0 / auto local
|
||||||
|
RESOURCE_UNAVAILABLE → level 0 / auto local
|
||||||
|
WORKER_CRASH → level 0 / auto local
|
||||||
|
LEASE_LOST → level 0 / auto local
|
||||||
|
|
||||||
|
CAPABILITY_CONTRACT_ERROR → level 1 / same-lineage repair task
|
||||||
|
CREDENTIAL_PROJECTION_ERROR → level 1 / same-lineage repair task
|
||||||
|
UNKNOWN → level 1 / diagnostic repair task
|
||||||
|
|
||||||
|
POLICY_DENIED → level 2 / Owner authorization
|
||||||
|
BUSINESS_PERMANENT → level 3 / terminal/manual decision
|
||||||
|
```
|
||||||
|
|
||||||
|
Only business-permanent failure consumes the business retry budget in this first contract.
|
||||||
|
|
||||||
|
The current Task API does not yet have a first-class `EXECUTION_READY_FAILED` durable state. Until it
|
||||||
|
does, the Release Worker enforces the most important invariant by refusing to claim a Task whose
|
||||||
|
admission fails.
|
||||||
|
|
||||||
|
## Capability Contract Registry
|
||||||
|
|
||||||
|
`aios_capability_contract.py` remains backward-compatible with existing aliases but now validates
|
||||||
|
every alias target against a versioned semantic contract registry.
|
||||||
|
|
||||||
|
Unknown capabilities and wildcard capabilities remain fail-closed.
|
||||||
|
|
||||||
|
Agent registration and Task creation should next be changed to call `registered_capability()` so a
|
||||||
|
capability cannot be accepted at registration and rejected only later by the matcher.
|
||||||
|
|
||||||
|
## Credential rule
|
||||||
|
|
||||||
|
Admission probes credential usability, not merely credential existence.
|
||||||
|
|
||||||
|
For release candidate creation the authenticated probe intentionally sends an invalid empty
|
||||||
|
`/createReleaseCandidate` request.
|
||||||
|
|
||||||
|
Expected:
|
||||||
|
|
||||||
|
```text
|
||||||
|
HTTP 400
|
||||||
|
```
|
||||||
|
|
||||||
|
This proves:
|
||||||
|
|
||||||
|
```text
|
||||||
|
credential readable
|
||||||
|
→ authentication succeeded
|
||||||
|
→ scope accepted
|
||||||
|
→ request validation rejected empty business input
|
||||||
|
```
|
||||||
|
|
||||||
|
A permission failure/401/403/5xx is `EXECUTION_READY_FAILED`.
|
||||||
|
|
||||||
|
## Lineage
|
||||||
|
|
||||||
|
Repair actions are descriptive, not permission grants. Any repair Task must retain the existing
|
||||||
|
root/parent/attempt/authority chain and use governed same-lineage recovery.
|
||||||
|
|
||||||
|
No repair action may silently create a replacement business Task.
|
||||||
|
|
||||||
|
## Authority separation
|
||||||
|
|
||||||
|
Capability answers:
|
||||||
|
|
||||||
|
```text
|
||||||
|
can this runtime perform the operation?
|
||||||
|
```
|
||||||
|
|
||||||
|
Authority answers:
|
||||||
|
|
||||||
|
```text
|
||||||
|
is this operation currently authorized?
|
||||||
|
```
|
||||||
|
|
||||||
|
`release:candidate` never implies Owner approval, deployment, tag publication, baseline finalization
|
||||||
|
or runtime switch.
|
||||||
|
|
||||||
|
## Deployment sequence
|
||||||
|
|
||||||
|
Phase 1 (this candidate):
|
||||||
|
|
||||||
|
- Capability Contract Registry;
|
||||||
|
- shared Autonomous Control Plane library;
|
||||||
|
- Release Worker pre-claim admission;
|
||||||
|
- failure classification + repair evidence.
|
||||||
|
|
||||||
|
Phase 2:
|
||||||
|
|
||||||
|
- Task API uses the same library before returning Tasks from poll and again during claim CAS;
|
||||||
|
- Task schema gains durable `ADMITTED/EXECUTION_READY/EXECUTION_READY_FAILED` evidence without
|
||||||
|
counting pre-side-effect infrastructure failures against business attempts.
|
||||||
|
|
||||||
|
Phase 3:
|
||||||
|
|
||||||
|
- Agent registration validates against the same Capability Contract Registry;
|
||||||
|
- external connector writes only durable Intent/Command Inbox records and leaves execution to the
|
||||||
|
local control plane.
|
||||||
|
|
||||||
|
This sequencing avoids a second controller stack while converging every decision onto one contract.
|
||||||
@@ -0,0 +1,228 @@
|
|||||||
|
# Durable Owner Authorization and capability projection
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
AIOS already supports unattended worker execution after a Task is executable, but Owner intent was
|
||||||
|
not durable at the capability layer. The same Owner repeatedly had to authorize or repair:
|
||||||
|
|
||||||
|
- `governed-task:project`;
|
||||||
|
- `release:candidate`.
|
||||||
|
|
||||||
|
This caused valid one-time Owner intent to be lost between conversations and forced repeated human
|
||||||
|
interaction.
|
||||||
|
|
||||||
|
## Design
|
||||||
|
|
||||||
|
Durable authorization is represented by a root-owned mode `0600` policy outside Git. The policy is
|
||||||
|
persistent until explicitly revoked.
|
||||||
|
|
||||||
|
The first persistable capabilities are deliberately narrow:
|
||||||
|
|
||||||
|
```text
|
||||||
|
governed-task:project
|
||||||
|
release:candidate
|
||||||
|
```
|
||||||
|
|
||||||
|
They cover preparation work only.
|
||||||
|
|
||||||
|
The following capabilities can never be inherited from this durable grant:
|
||||||
|
|
||||||
|
```text
|
||||||
|
submitOwnerApproval
|
||||||
|
startDeployment
|
||||||
|
tag_publish
|
||||||
|
baseline_registry_finalize
|
||||||
|
runtime_switch
|
||||||
|
main_write
|
||||||
|
production_deploy
|
||||||
|
force_push
|
||||||
|
credential_admin
|
||||||
|
```
|
||||||
|
|
||||||
|
Those remain separate high-risk release/Owner-signature gates.
|
||||||
|
|
||||||
|
## Policy
|
||||||
|
|
||||||
|
Example schema:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"schema": "aios.owner-durable-authorization.v1",
|
||||||
|
"grant_id": "owner-permanent-preparation-20260819",
|
||||||
|
"owner_identity": "owner@example",
|
||||||
|
"account": "chatgpt-governed-implementer-singlehost",
|
||||||
|
"capabilities": [
|
||||||
|
"governed-task:project",
|
||||||
|
"release:candidate"
|
||||||
|
],
|
||||||
|
"issued_at": "2026-08-19T05:00:00Z",
|
||||||
|
"not_before": "2026-08-19T05:00:00Z",
|
||||||
|
"persistent_until_revoked": true,
|
||||||
|
"constraints": {
|
||||||
|
"repository": "aios/aios",
|
||||||
|
"allowed_target_prefix": "agent/",
|
||||||
|
"release_version_prefix": "V2R2C",
|
||||||
|
"max_task_risk": "engineering"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The repository never stores the live Owner policy or release credential.
|
||||||
|
|
||||||
|
## Revocation
|
||||||
|
|
||||||
|
A root-owned mode `0600` revocation file lists `revoked_grant_ids`.
|
||||||
|
|
||||||
|
Revocation is checked on every operation, not only at service startup. A running broker therefore
|
||||||
|
fails closed immediately after revocation without needing a new conversation.
|
||||||
|
|
||||||
|
## Broker boundary
|
||||||
|
|
||||||
|
The broker exposes only an AF_UNIX socket:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/run/aios-owner-authorization-broker/broker.sock
|
||||||
|
```
|
||||||
|
|
||||||
|
The socket is mode `0660` and owned by the service's `aios-executor-exchange` group. This avoids introducing another long-lived bearer secret solely for local broker access.
|
||||||
|
|
||||||
|
It exposes only:
|
||||||
|
|
||||||
|
```text
|
||||||
|
GET /v1/capabilities
|
||||||
|
POST /v1/project-governed-task
|
||||||
|
POST /v1/authorize-release-candidate
|
||||||
|
```
|
||||||
|
|
||||||
|
`project-governed-task` calls only the canonical Bridge.
|
||||||
|
|
||||||
|
`authorize-release-candidate` persists an Owner authorization decision only. It deliberately does
|
||||||
|
**not** invent a local trusted-builder endpoint. The external release connector remains responsible
|
||||||
|
for projecting the account's real `release:candidate` scope. Until that external projection exists,
|
||||||
|
the decision state is `AUTHORIZED_PENDING_EXTERNAL_CAPABILITY_PROJECTION`.
|
||||||
|
|
||||||
|
The broker holds only the existing host control credential needed for canonical governed-task
|
||||||
|
projection. It never holds an Owner signing key or a release/deployment credential. The caller also supplies the expected account identifier over the Unix transport; the OS socket permission is the local access boundary.
|
||||||
|
|
||||||
|
|
||||||
|
## Canonical Online Task projection
|
||||||
|
|
||||||
|
A durable `governed-task:project` grant may create a complete canonical Online Task rather than a
|
||||||
|
bare Task. It requires:
|
||||||
|
|
||||||
|
```text
|
||||||
|
require_online_task_projection=true
|
||||||
|
generation=1
|
||||||
|
branch_mode=CREATE_NEW
|
||||||
|
max_attempts<=3
|
||||||
|
risk<=engineering
|
||||||
|
initial action=source_capture
|
||||||
|
```
|
||||||
|
|
||||||
|
Projected task capabilities are limited to:
|
||||||
|
|
||||||
|
```text
|
||||||
|
git.read
|
||||||
|
git.test
|
||||||
|
governed.write.branch
|
||||||
|
governed.write.file
|
||||||
|
```
|
||||||
|
|
||||||
|
A write-capable task still begins as `source_capture`. Existing placeholder-deferral prevents the
|
||||||
|
generic worker from claiming it until a deterministic plan is registered. Repository mutation
|
||||||
|
therefore still requires the normal deterministic plan, WriterLease and governed commit path.
|
||||||
|
|
||||||
|
This fixes the previous split-brain behavior where a bare Task could exist without the canonical
|
||||||
|
governed execution projection.
|
||||||
|
|
||||||
|
|
||||||
|
## External capability projection outbox
|
||||||
|
|
||||||
|
A `release:candidate` authorization now creates a durable machine-readable outbox record:
|
||||||
|
|
||||||
|
```text
|
||||||
|
schema=aios.external-capability-projection.request.v1
|
||||||
|
state=PENDING_EXTERNAL_CONNECTOR
|
||||||
|
capability=release:candidate
|
||||||
|
```
|
||||||
|
|
||||||
|
The projection identifier is deterministic from:
|
||||||
|
|
||||||
|
```text
|
||||||
|
grant_id
|
||||||
|
account
|
||||||
|
capability
|
||||||
|
request_sha256
|
||||||
|
```
|
||||||
|
|
||||||
|
The outbox stores no credentials or Owner signing material. It retains only the already-authorized
|
||||||
|
release-candidate request, policy/request hashes, account, grant id and timestamps.
|
||||||
|
|
||||||
|
The broker exposes read-only Unix-socket endpoints:
|
||||||
|
|
||||||
|
```text
|
||||||
|
GET /v1/projections
|
||||||
|
GET /v1/projections/{projection_id}
|
||||||
|
```
|
||||||
|
|
||||||
|
These endpoints re-check the current Owner policy on every read. If the durable grant has been
|
||||||
|
revoked, enumeration fails closed with `OWNER_GRANT_REVOKED`. The historical outbox record is
|
||||||
|
retained for audit and is not silently deleted.
|
||||||
|
|
||||||
|
The repository intentionally does **not** provide a connector acknowledgement/write endpoint yet.
|
||||||
|
No in-repository code can truthfully assert that the external ChatGPT connector account has gained
|
||||||
|
`release:candidate`. Until the platform consumes the outbox and performs its own identity-scope
|
||||||
|
projection, the only valid state is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
AUTHORIZED_PENDING_EXTERNAL_CAPABILITY_PROJECTION
|
||||||
|
PENDING_EXTERNAL_CONNECTOR
|
||||||
|
```
|
||||||
|
|
||||||
|
A future external projector must preserve:
|
||||||
|
|
||||||
|
```text
|
||||||
|
release:candidate
|
||||||
|
!= submitOwnerApproval
|
||||||
|
!= startDeployment
|
||||||
|
!= tag_publish
|
||||||
|
!= baseline_registry_finalize
|
||||||
|
!= runtime_switch
|
||||||
|
```
|
||||||
|
|
||||||
|
## Idempotency and audit
|
||||||
|
|
||||||
|
Every broker operation requires an idempotency key. A receipt SQLite database stores only:
|
||||||
|
|
||||||
|
- idempotency key;
|
||||||
|
- capability;
|
||||||
|
- request SHA-256;
|
||||||
|
- response SHA-256;
|
||||||
|
- sanitized upstream response;
|
||||||
|
- timestamp.
|
||||||
|
|
||||||
|
An exact retry returns the same receipt. Reusing an idempotency key with different content fails.
|
||||||
|
|
||||||
|
## Unattended invariant
|
||||||
|
|
||||||
|
Once the Owner policy and narrow service credentials are provisioned:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Owner durable grant
|
||||||
|
→ broker decision
|
||||||
|
→ canonical governed task projection / release candidate preparation
|
||||||
|
→ worker / trusted builder
|
||||||
|
```
|
||||||
|
|
||||||
|
No repeated Owner interaction is required for those preparation capabilities.
|
||||||
|
|
||||||
|
Production execution remains:
|
||||||
|
|
||||||
|
```text
|
||||||
|
release candidate
|
||||||
|
→ immutable approval envelope
|
||||||
|
→ Owner signature
|
||||||
|
→ deployment
|
||||||
|
```
|
||||||
|
|
||||||
|
The durable grant never substitutes for the Owner signature.
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
# Owner-approved GPT HTTPS edge activation and rollback
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
This controller is the final host-side control mechanism before the disabled-by-default GPT HTTPS
|
||||||
|
edge may bind publicly.
|
||||||
|
|
||||||
|
Repository implementation and public activation remain separate states:
|
||||||
|
|
||||||
|
```text
|
||||||
|
IMPLEMENTED
|
||||||
|
≠
|
||||||
|
OWNER_APPROVED
|
||||||
|
≠
|
||||||
|
ACTIVATED
|
||||||
|
≠
|
||||||
|
RUNTIME_VERIFIED
|
||||||
|
```
|
||||||
|
|
||||||
|
The controller never creates owner approval, TLS assets, DNS, firewall rules or ingress bearer
|
||||||
|
credentials.
|
||||||
|
|
||||||
|
## Approval binding
|
||||||
|
|
||||||
|
The owner approval file is root-owned mode `0600` and has schema:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"schema": "aios.gpt-https-edge.public-approval.v1",
|
||||||
|
"approval_id": "owner-public-edge-...",
|
||||||
|
"candidate_oid": "<exact governed edge candidate>",
|
||||||
|
"domain": "tasks.example.com",
|
||||||
|
"public_bind": true,
|
||||||
|
"bind_host": "0.0.0.0",
|
||||||
|
"port": 18443,
|
||||||
|
"tls_cert_sha256": "<exact certificate sha256>",
|
||||||
|
"issued_at": "2026-08-19T00:00:00Z",
|
||||||
|
"not_before": "2026-08-19T00:00:00Z",
|
||||||
|
"expires_at": "2026-08-20T00:00:00Z",
|
||||||
|
"owner": "<owner identity>"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Approval lifetime is at most seven days. Candidate/domain/bind/port/certificate are immutable within
|
||||||
|
one approval.
|
||||||
|
|
||||||
|
Changing any of those requires a new owner approval.
|
||||||
|
|
||||||
|
## Check
|
||||||
|
|
||||||
|
```text
|
||||||
|
aios_gpt_https_edge_activation.py check --candidate <GE>
|
||||||
|
```
|
||||||
|
|
||||||
|
Checks:
|
||||||
|
|
||||||
|
- approval ownership/mode/schema/time window;
|
||||||
|
- exact candidate binding;
|
||||||
|
- exact certificate hash;
|
||||||
|
- certificate/private-key pair;
|
||||||
|
- certificate hostname;
|
||||||
|
- certificate remaining lifetime;
|
||||||
|
- loopback GPT ingress service health.
|
||||||
|
|
||||||
|
It performs no public network mutation.
|
||||||
|
|
||||||
|
## Activate
|
||||||
|
|
||||||
|
`activate` additionally:
|
||||||
|
|
||||||
|
- writes one exact systemd public-bind drop-in atomically;
|
||||||
|
- enables/restarts only the HTTPS edge unit;
|
||||||
|
- verifies effective ExecStart points at the immutable candidate release;
|
||||||
|
- verifies effective public-bind/domain environment;
|
||||||
|
- performs a TLS handshake to `127.0.0.1:18443` using the approved public hostname;
|
||||||
|
- writes a durable activation receipt.
|
||||||
|
|
||||||
|
Any failure after drop-in mutation automatically invokes rollback.
|
||||||
|
|
||||||
|
Exact replay of an already-active matching approval returns the durable receipt without restarting
|
||||||
|
or creating duplicate state.
|
||||||
|
|
||||||
|
## Rollback
|
||||||
|
|
||||||
|
`rollback`:
|
||||||
|
|
||||||
|
- stops and disables only the HTTPS edge;
|
||||||
|
- removes only the controller-owned public-bind drop-in;
|
||||||
|
- daemon-reloads systemd;
|
||||||
|
- verifies edge inactive;
|
||||||
|
- writes a durable rollback receipt.
|
||||||
|
|
||||||
|
It does not delete:
|
||||||
|
|
||||||
|
- owner approval;
|
||||||
|
- TLS certificate/private key;
|
||||||
|
- ingress bearer token;
|
||||||
|
- internal loopback ingress;
|
||||||
|
- Task/Job state;
|
||||||
|
- Git branches.
|
||||||
|
|
||||||
|
Rollback is therefore replay-safe and does not require reconstructing secrets.
|
||||||
|
|
||||||
|
## Not covered
|
||||||
|
|
||||||
|
This controller intentionally does not own:
|
||||||
|
|
||||||
|
- DNS records;
|
||||||
|
- host firewall/NAT/security-group changes;
|
||||||
|
- certificate issuance/renewal;
|
||||||
|
- OpenAI Custom GPT domain verification;
|
||||||
|
- public Internet acceptance from outside the host.
|
||||||
|
|
||||||
|
Those remain separate owner/environment gates.
|
||||||
@@ -0,0 +1,120 @@
|
|||||||
|
# Disabled-by-default HTTPS edge for external GPT Online Task access
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
The internal ingress (`127.0.0.1:18945`) remains loopback-only. This edge is the only component
|
||||||
|
allowed to become Internet-facing, and only after a separate owner approval.
|
||||||
|
|
||||||
|
It forwards only:
|
||||||
|
|
||||||
|
```text
|
||||||
|
POST /v1/read-only-tasks
|
||||||
|
GET /v1/read-only-tasks/{request_id}
|
||||||
|
GET /openapi.json
|
||||||
|
GET /health
|
||||||
|
```
|
||||||
|
|
||||||
|
It never proxies:
|
||||||
|
|
||||||
|
```text
|
||||||
|
18892 canonical Bridge
|
||||||
|
18940 Task API
|
||||||
|
WriterLease
|
||||||
|
commit
|
||||||
|
verification
|
||||||
|
release
|
||||||
|
tag
|
||||||
|
baseline
|
||||||
|
arbitrary URLs
|
||||||
|
```
|
||||||
|
|
||||||
|
## Default state
|
||||||
|
|
||||||
|
The systemd unit is disabled by policy unless the owner creates:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/etc/aios-gitops/gpt-https-edge.approved
|
||||||
|
```
|
||||||
|
|
||||||
|
The base unit still binds only to:
|
||||||
|
|
||||||
|
```text
|
||||||
|
127.0.0.1:18443
|
||||||
|
AIOS_GPT_EDGE_PUBLIC_BIND_APPROVED=0
|
||||||
|
```
|
||||||
|
|
||||||
|
Non-loopback bind is rejected by code unless an owner-controlled drop-in sets:
|
||||||
|
|
||||||
|
```text
|
||||||
|
AIOS_GPT_EDGE_PUBLIC_BIND_APPROVED=1
|
||||||
|
AIOS_GPT_EDGE_HOST=0.0.0.0
|
||||||
|
AIOS_GPT_EDGE_PUBLIC_DOMAIN=<approved hostname>
|
||||||
|
```
|
||||||
|
|
||||||
|
This repository candidate does not create that drop-in or approval marker.
|
||||||
|
|
||||||
|
## TLS
|
||||||
|
|
||||||
|
TLS certificate/key are owner-managed systemd credentials:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/etc/aios-gitops/gpt-https-edge-tls.crt
|
||||||
|
/etc/aios-gitops/gpt-https-edge-tls.key
|
||||||
|
```
|
||||||
|
|
||||||
|
The service requires TLS 1.2 or newer, disables TLS compression, and never logs or returns key
|
||||||
|
material.
|
||||||
|
|
||||||
|
Certificate/key issuance and renewal are external owner responsibilities until a separately
|
||||||
|
governed ACME workflow is implemented.
|
||||||
|
|
||||||
|
## Request protections
|
||||||
|
|
||||||
|
The edge provides:
|
||||||
|
|
||||||
|
- 64 KiB request cap;
|
||||||
|
- 256 KiB upstream response cap;
|
||||||
|
- JSON validation before proxying;
|
||||||
|
- fixed upstream `http://127.0.0.1:18945`;
|
||||||
|
- Authorization-only header forwarding;
|
||||||
|
- no Cookie/X-Forwarded-* trust;
|
||||||
|
- route/method allowlist;
|
||||||
|
- per-source-IP in-memory rate limiting;
|
||||||
|
- bounded concurrent request handling;
|
||||||
|
- optional exact Host-header enforcement;
|
||||||
|
- HSTS / no-store / nosniff / no-referrer response headers.
|
||||||
|
|
||||||
|
The rate limiter is intentionally simple and local. Multi-node/global abuse protection is outside
|
||||||
|
the current one-host architecture.
|
||||||
|
|
||||||
|
## Authentication
|
||||||
|
|
||||||
|
The HTTPS edge does not create a new authentication scheme. It forwards the external Bearer
|
||||||
|
credential to the loopback GPT ingress, which verifies `gpt_ingress_token`.
|
||||||
|
|
||||||
|
The edge never receives the AIOS control-plane token.
|
||||||
|
|
||||||
|
## Deployment boundary
|
||||||
|
|
||||||
|
Public activation is a high-risk network operation and requires explicit owner approval. Before
|
||||||
|
activation verify:
|
||||||
|
|
||||||
|
```text
|
||||||
|
host-side GPT ingress E2E=VERIFIED
|
||||||
|
owner ingress token provisioned
|
||||||
|
TLS certificate/key provisioned
|
||||||
|
public hostname owned and DNS configured
|
||||||
|
firewall/network exposure reviewed
|
||||||
|
explicit public-bind approval recorded
|
||||||
|
rollback plan available
|
||||||
|
```
|
||||||
|
|
||||||
|
Only after those gates may an owner-authorized runtime drop-in bind this service publicly.
|
||||||
|
|
||||||
|
Until then:
|
||||||
|
|
||||||
|
```text
|
||||||
|
GPT_HTTPS_EDGE_IMPLEMENTATION=candidate
|
||||||
|
PUBLIC_NETWORK_EXPOSURE=NONE
|
||||||
|
PUBLIC_CUSTOM_GPT_DIRECT_AI0S_ACCESS=NOT_VERIFIED
|
||||||
|
```
|
||||||
@@ -0,0 +1,138 @@
|
|||||||
|
# External GPT → Online Task ingress
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
This component closes the current architectural gap between an external Custom GPT and AIOS
|
||||||
|
without exposing the internal Task API (`18940`) or canonical Bridge (`18892`) directly.
|
||||||
|
|
||||||
|
The first supported public-facing capability is intentionally read-only:
|
||||||
|
|
||||||
|
```text
|
||||||
|
authenticated external intent
|
||||||
|
→ host-side ingress
|
||||||
|
→ canonical /v2/governed-task
|
||||||
|
→ canonical /v2/execution-envelope
|
||||||
|
→ persistent generic worker
|
||||||
|
→ source_capture result
|
||||||
|
→ official control-plane verification
|
||||||
|
```
|
||||||
|
|
||||||
|
The ingress itself never claims, heartbeats, verifies, commits, acquires WriterLease, or writes
|
||||||
|
protected refs.
|
||||||
|
|
||||||
|
## Security boundary
|
||||||
|
|
||||||
|
The service binds only to loopback (`127.0.0.1:18945`). A future owner-approved TLS reverse proxy
|
||||||
|
or tunnel may expose this service externally. Direct non-loopback binding is denied by code.
|
||||||
|
|
||||||
|
Two independent credentials are projected by systemd:
|
||||||
|
|
||||||
|
- `gpt_ingress_token`: authenticates the external caller to this ingress.
|
||||||
|
- `agent_task_control_token`: lets the ingress call the loopback AIOS control plane.
|
||||||
|
|
||||||
|
Neither credential is stored in Git, SQLite, task payloads, results, logs, or API responses.
|
||||||
|
|
||||||
|
The ingress has no repository filesystem access.
|
||||||
|
|
||||||
|
## Caller-controlled fields
|
||||||
|
|
||||||
|
Allowed:
|
||||||
|
|
||||||
|
```text
|
||||||
|
request_id
|
||||||
|
goal
|
||||||
|
instructions
|
||||||
|
success_criteria
|
||||||
|
constraints.max_files
|
||||||
|
constraints.focus_paths
|
||||||
|
constraints.notes
|
||||||
|
```
|
||||||
|
|
||||||
|
Not caller-controlled:
|
||||||
|
|
||||||
|
```text
|
||||||
|
repository
|
||||||
|
base_oid
|
||||||
|
expected_branch_oid
|
||||||
|
target_branch
|
||||||
|
branch_mode
|
||||||
|
generation
|
||||||
|
risk
|
||||||
|
priority
|
||||||
|
max_attempts
|
||||||
|
capabilities
|
||||||
|
action/adapter
|
||||||
|
WriterLease
|
||||||
|
worker identity
|
||||||
|
credentials
|
||||||
|
verification decision
|
||||||
|
main/production/tag/baseline permissions
|
||||||
|
```
|
||||||
|
|
||||||
|
All external tasks are hard-bound to:
|
||||||
|
|
||||||
|
```text
|
||||||
|
repository=aios/aios
|
||||||
|
capabilities=[git.read, git.test]
|
||||||
|
action=source_capture
|
||||||
|
writer_lease=false
|
||||||
|
repository_mutation=forbidden
|
||||||
|
control_plane_side_effects=forbidden
|
||||||
|
credential_access=forbidden
|
||||||
|
```
|
||||||
|
|
||||||
|
## Idempotency
|
||||||
|
|
||||||
|
`request_id` is the durable external idempotency boundary.
|
||||||
|
|
||||||
|
The ingress stores only:
|
||||||
|
|
||||||
|
```text
|
||||||
|
request hash
|
||||||
|
derived AIOS task id
|
||||||
|
canonical job id
|
||||||
|
base oid
|
||||||
|
sanitized receipt
|
||||||
|
state
|
||||||
|
```
|
||||||
|
|
||||||
|
An exact retry returns the same receipt. Reusing a `request_id` with different content returns
|
||||||
|
`IDEMPOTENCY_CONFLICT`.
|
||||||
|
|
||||||
|
A crash after canonical task creation can recover the same binding from the Task API rather than
|
||||||
|
creating a replacement Task.
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
`GET /v1/read-only-tasks/{request_id}` returns only sanitized state:
|
||||||
|
|
||||||
|
```text
|
||||||
|
task/job ids
|
||||||
|
status
|
||||||
|
attempt
|
||||||
|
whether claimed
|
||||||
|
result classification
|
||||||
|
official verification decision/classification
|
||||||
|
source snapshot hash/count when available
|
||||||
|
```
|
||||||
|
|
||||||
|
It does not proxy raw Task metadata, execution envelopes, result text, credentials, or source files.
|
||||||
|
|
||||||
|
## Public exposure
|
||||||
|
|
||||||
|
This candidate does **not** make AIOS publicly reachable. Public exposure requires a separately
|
||||||
|
approved HTTPS ingress/reverse proxy with:
|
||||||
|
|
||||||
|
- TLS;
|
||||||
|
- external authentication lifecycle;
|
||||||
|
- rate limiting / abuse controls;
|
||||||
|
- request size limits;
|
||||||
|
- observability;
|
||||||
|
- explicit owner approval for the network exposure.
|
||||||
|
|
||||||
|
Until that step is verified:
|
||||||
|
|
||||||
|
```text
|
||||||
|
HOST_SIDE_EXTERNAL_GPT_ADAPTER=implemented/staging-candidate
|
||||||
|
PUBLIC_CUSTOM_GPT_DIRECT_AI0S_ACCESS=not verified
|
||||||
|
```
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
# Release candidate capability contract remediation
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
The release-preparation worker can be correctly registered with the raw capability:
|
||||||
|
|
||||||
|
```text
|
||||||
|
release:candidate
|
||||||
|
```
|
||||||
|
|
||||||
|
while the canonical Task API matcher still rejects the worker.
|
||||||
|
|
||||||
|
Before this repair, the capability matrix is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
worker raw = ["release:candidate"]
|
||||||
|
worker effective= []
|
||||||
|
task required = ["exact:release:candidate"]
|
||||||
|
eligible = false
|
||||||
|
```
|
||||||
|
|
||||||
|
The Task therefore remains `QUEUED`, `attempt=0`, and cannot be claimed even though both the Task and
|
||||||
|
worker use the same literal capability.
|
||||||
|
|
||||||
|
## Root cause
|
||||||
|
|
||||||
|
`aios_capability_contract.py` is deliberately fail-closed:
|
||||||
|
|
||||||
|
- task-side unknown names become `exact:<name>`;
|
||||||
|
- worker-side unknown names are discarded;
|
||||||
|
- wildcard worker capabilities are ignored.
|
||||||
|
|
||||||
|
`release:candidate` was absent from both canonical alias tables.
|
||||||
|
|
||||||
|
## Repair
|
||||||
|
|
||||||
|
Add exactly one semantic capability to both vocabularies:
|
||||||
|
|
||||||
|
```text
|
||||||
|
release:candidate → release:candidate
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not add:
|
||||||
|
|
||||||
|
```text
|
||||||
|
release:*
|
||||||
|
submitOwnerApproval
|
||||||
|
startDeployment
|
||||||
|
tag_publish
|
||||||
|
baseline_registry_finalize
|
||||||
|
runtime_switch
|
||||||
|
```
|
||||||
|
|
||||||
|
Unknown names and wildcards must continue to fail closed.
|
||||||
|
|
||||||
|
## Runtime recovery
|
||||||
|
|
||||||
|
After the governed remediation candidate is officially verified:
|
||||||
|
|
||||||
|
1. install the candidate capability-contract module as a higher-precedence Task API overlay;
|
||||||
|
2. restart only `aios-agent-task-api.service`;
|
||||||
|
3. require `/health` success;
|
||||||
|
4. require the fixed existing release Task to remain the same Task/idempotency;
|
||||||
|
5. let the already-active dedicated release worker poll and claim it automatically;
|
||||||
|
6. require heartbeat, trusted-builder result and official verification;
|
||||||
|
7. do not manually claim or create a replacement live Task.
|
||||||
|
|
||||||
|
The fixed V2R2C2 live Task may complete as the acceptance candidate it was originally bound to.
|
||||||
|
Because this remediation is a later source commit, the first release candidate that includes the
|
||||||
|
matcher repair should use the next immutable version (for example `V2R2C3`) rather than rewriting an
|
||||||
|
already-created V2R2C2 candidate.
|
||||||
|
|
||||||
|
## Governance boundary
|
||||||
|
|
||||||
|
This repair affects only Task eligibility. It grants no credential and performs no release action by
|
||||||
|
itself.
|
||||||
|
|
||||||
|
The dedicated release worker still requires its independent `release:candidate` credential, and the
|
||||||
|
following remain separate:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Owner signed approval
|
||||||
|
deployment
|
||||||
|
tag publication
|
||||||
|
baseline finalization
|
||||||
|
runtime switch
|
||||||
|
```
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
# Fixed V2R2C2 release lineage contract
|
||||||
|
|
||||||
|
## Why this contract exists
|
||||||
|
|
||||||
|
AIOS has multiple engineering commits in the same ancestry, and they serve different purposes.
|
||||||
|
|
||||||
|
The fixed V2R2C2 live Task was created by the dedicated Release Worker activation. Its immutable
|
||||||
|
release source is therefore the Release Worker implementation candidate:
|
||||||
|
|
||||||
|
```text
|
||||||
|
71e736b05fab13efc06af7a4fe2dc022c1cf9a85
|
||||||
|
```
|
||||||
|
|
||||||
|
Later engineering commits are descendants but are not retroactive V2R2C2 release sources:
|
||||||
|
|
||||||
|
```text
|
||||||
|
17deb91ca7a187dc81766945b1174e854f86bd25
|
||||||
|
capability-contract remediation
|
||||||
|
|
||||||
|
b4b4416c93fedbe451d655d49e28f915b8fd38e6
|
||||||
|
Autonomous Control Plane engineering candidate
|
||||||
|
```
|
||||||
|
|
||||||
|
Conflating "current engineering base" with "immutable release source" caused ACP activation to
|
||||||
|
fail closed with `FIXED_RELEASE_SOURCE_DRIFT`.
|
||||||
|
|
||||||
|
## Canonical fixed lineage
|
||||||
|
|
||||||
|
```text
|
||||||
|
Task:
|
||||||
|
r42-release-candidate-v2r2c2-live-20260819
|
||||||
|
|
||||||
|
candidate_id:
|
||||||
|
r42-release-candidate-v2r2c2-20260819
|
||||||
|
|
||||||
|
source_commit:
|
||||||
|
71e736b05fab13efc06af7a4fe2dc022c1cf9a85
|
||||||
|
|
||||||
|
version:
|
||||||
|
V2R2C2
|
||||||
|
|
||||||
|
required_capabilities:
|
||||||
|
["release:candidate"]
|
||||||
|
```
|
||||||
|
|
||||||
|
This contract is immutable for V2R2C2.
|
||||||
|
|
||||||
|
## What may change
|
||||||
|
|
||||||
|
The following are lifecycle state, not release identity:
|
||||||
|
|
||||||
|
```text
|
||||||
|
status
|
||||||
|
attempt
|
||||||
|
max_attempts
|
||||||
|
revision
|
||||||
|
claimed_by
|
||||||
|
lease
|
||||||
|
verification
|
||||||
|
```
|
||||||
|
|
||||||
|
They may change through official retry/claim/result/verification without changing the release
|
||||||
|
lineage.
|
||||||
|
|
||||||
|
## What must never be "repaired"
|
||||||
|
|
||||||
|
Do not rewrite the fixed V2R2C2 Task source to a later engineering commit.
|
||||||
|
|
||||||
|
A later engineering source that must itself be released receives a later immutable release version
|
||||||
|
(for example V2R2C3), not a mutation of V2R2C2.
|
||||||
|
|
||||||
|
## Activation invariant
|
||||||
|
|
||||||
|
ACP activation must:
|
||||||
|
|
||||||
|
1. verify its own governed engineering candidate separately;
|
||||||
|
2. read the live fixed V2R2C2 Task;
|
||||||
|
3. validate that Task through `validate_fixed_v2r2c2_task()`;
|
||||||
|
4. never compare the fixed release source to the ACP engineering base/candidate;
|
||||||
|
5. never mutate the fixed Task merely to satisfy activation.
|
||||||
|
|
||||||
|
This makes engineering lineage and release lineage explicit and independently verifiable.
|
||||||
@@ -0,0 +1,142 @@
|
|||||||
|
# Release Preparation Online Agent
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This worker closes the release-preparation execution gap without widening the ChatGPT connector
|
||||||
|
identity.
|
||||||
|
|
||||||
|
The dedicated Agent identity is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
online-release-worker-cn-network-mkt-20260819
|
||||||
|
```
|
||||||
|
|
||||||
|
Its only advertised task capability is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
release:candidate
|
||||||
|
```
|
||||||
|
|
||||||
|
The worker may create an immutable release candidate, but it never submits Owner approval and never
|
||||||
|
starts deployment, publishes a tag, finalizes a baseline, switches runtime or writes Git.
|
||||||
|
|
||||||
|
## Credentials
|
||||||
|
|
||||||
|
The worker uses two independently scoped credentials:
|
||||||
|
|
||||||
|
```text
|
||||||
|
agent_token
|
||||||
|
release_candidate_token
|
||||||
|
```
|
||||||
|
|
||||||
|
`agent_token` is the Online Agent registration credential. It must be encrypted into systemd's
|
||||||
|
credential store and must never be written to Git, Task metadata, evidence or logs.
|
||||||
|
|
||||||
|
`release_candidate_token` is provisioned during Owner-authorized runtime activation. Only its SHA-256 record is stored in the release API token registry; the plaintext is immediately encrypted into systemd credstore and the temporary plaintext is removed. Its only allowed scope is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
release:candidate
|
||||||
|
```
|
||||||
|
|
||||||
|
It is explicitly not an Owner signing key and must not grant:
|
||||||
|
|
||||||
|
```text
|
||||||
|
submitOwnerApproval
|
||||||
|
startDeployment
|
||||||
|
tag_publish
|
||||||
|
baseline_registry_finalize
|
||||||
|
runtime_switch
|
||||||
|
```
|
||||||
|
|
||||||
|
The unit reads both credentials through systemd `LoadCredentialEncrypted`; neither plaintext token is kept in the unit, repository, Task metadata, evidence, or logs.
|
||||||
|
|
||||||
|
## Task contract
|
||||||
|
|
||||||
|
The worker will only claim Tasks whose exact required capability set is:
|
||||||
|
|
||||||
|
```json
|
||||||
|
["release:candidate"]
|
||||||
|
```
|
||||||
|
|
||||||
|
The Task must provide a bounded `constraints.release_candidate` payload (metadata is accepted only for compatibility):
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"candidate_id": "...",
|
||||||
|
"source_commit": "<40 hex>",
|
||||||
|
"version": "V2R2C2",
|
||||||
|
"projection_id": "projection-..."
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
No caller-supplied URL, method, headers, command, shell, environment, credential path, deployment
|
||||||
|
operation or arbitrary release route is accepted.
|
||||||
|
|
||||||
|
The release API base is host configuration, not Task input. The only release mutation invoked by the
|
||||||
|
worker is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
POST /createReleaseCandidate
|
||||||
|
```
|
||||||
|
|
||||||
|
## Lifecycle
|
||||||
|
|
||||||
|
```text
|
||||||
|
poll
|
||||||
|
→ claim
|
||||||
|
→ heartbeat
|
||||||
|
→ createReleaseCandidate
|
||||||
|
→ result submit
|
||||||
|
→ official verifier
|
||||||
|
```
|
||||||
|
|
||||||
|
The result declares:
|
||||||
|
|
||||||
|
```text
|
||||||
|
writer_lease=NONE
|
||||||
|
changed_files=[]
|
||||||
|
direct_main_write=false
|
||||||
|
production_write=false
|
||||||
|
tag_write=false
|
||||||
|
baseline_write=false
|
||||||
|
runtime_switch=false
|
||||||
|
owner_approval_submitted=false
|
||||||
|
```
|
||||||
|
|
||||||
|
## Remaining high-risk gate
|
||||||
|
|
||||||
|
Successful release-candidate creation does not authorize Production.
|
||||||
|
|
||||||
|
The next stage remains:
|
||||||
|
|
||||||
|
```text
|
||||||
|
release candidate
|
||||||
|
→ immutable approval envelope
|
||||||
|
→ Owner signature
|
||||||
|
→ startDeployment
|
||||||
|
```
|
||||||
|
|
||||||
|
That separation is mandatory even when release preparation is fully unattended.
|
||||||
|
|
||||||
|
|
||||||
|
## Unattended availability
|
||||||
|
|
||||||
|
The release worker has its own systemd watchdog rather than modifying the shared worker-availability
|
||||||
|
guard contract.
|
||||||
|
|
||||||
|
Every successful Task poll and Task heartbeat emits:
|
||||||
|
|
||||||
|
```text
|
||||||
|
WATCHDOG=1
|
||||||
|
```
|
||||||
|
|
||||||
|
and the service uses:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Type=notify
|
||||||
|
WatchdogSec=90
|
||||||
|
Restart=always
|
||||||
|
```
|
||||||
|
|
||||||
|
A process that is technically alive but no longer polling/heartbeating therefore gets restarted by
|
||||||
|
systemd. The worker also writes `worker-liveness.json` for operator diagnostics.
|
||||||
@@ -0,0 +1,206 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[3]
|
||||||
|
ACTIVE_RUNTIME = ROOT / "active-runtime"
|
||||||
|
if str(ACTIVE_RUNTIME) not in sys.path:
|
||||||
|
sys.path.insert(0, str(ACTIVE_RUNTIME))
|
||||||
|
|
||||||
|
import sqlite3
|
||||||
|
import threading
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.task_plane.fleet import FleetConflict, FleetScheduler
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Plan:
|
||||||
|
task_id: str
|
||||||
|
required_capabilities: list[str] = field(default_factory=lambda:["code.read"])
|
||||||
|
metadata: dict = field(default_factory=dict)
|
||||||
|
scope_type: str = "tenant"
|
||||||
|
tenant_id: str = "tenant-a"
|
||||||
|
project_id: str | None = "project-a"
|
||||||
|
user_id: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Task:
|
||||||
|
task_id: str
|
||||||
|
revision: int = 1
|
||||||
|
status: str = "QUEUED"
|
||||||
|
created_at: float = 1.0
|
||||||
|
task_plan: Plan | None = None
|
||||||
|
|
||||||
|
def __post_init__(self):
|
||||||
|
if self.task_plan is None:
|
||||||
|
self.task_plan=Plan(self.task_id)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Worker:
|
||||||
|
worker_id: str
|
||||||
|
capabilities: list[str]=field(default_factory=lambda:["code.read"])
|
||||||
|
|
||||||
|
|
||||||
|
class Store:
|
||||||
|
def __init__(self, path=":memory:"):
|
||||||
|
self.conn=sqlite3.connect(path,check_same_thread=False)
|
||||||
|
self._lock=threading.RLock()
|
||||||
|
self.conn.executescript("""
|
||||||
|
CREATE TABLE IF NOT EXISTS claims(
|
||||||
|
claim_id TEXT PRIMARY KEY,task_id TEXT,worker_id TEXT,status TEXT,lease_expires_at REAL
|
||||||
|
);
|
||||||
|
""")
|
||||||
|
self.tasks=[]
|
||||||
|
self.workers={}
|
||||||
|
|
||||||
|
def list_tasks(self,_ctx=None):
|
||||||
|
return list(self.tasks)
|
||||||
|
|
||||||
|
def get_worker(self,worker_id):
|
||||||
|
return self.workers[worker_id]
|
||||||
|
|
||||||
|
|
||||||
|
class Registry:
|
||||||
|
def __init__(self):
|
||||||
|
self.online=set()
|
||||||
|
def is_online(self,worker_id,now=None):
|
||||||
|
return worker_id in self.online
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def fleet():
|
||||||
|
store=Store(); registry=Registry()
|
||||||
|
for wid in ("w-a","w-b","w-c"):
|
||||||
|
store.workers[wid]=Worker(wid)
|
||||||
|
registry.online.add(wid)
|
||||||
|
scheduler=FleetScheduler(store,registry,now_fn=lambda:1000.0)
|
||||||
|
scheduler.configure_worker("w-a",max_slots=1,failure_domain="zone-a",weight=1)
|
||||||
|
scheduler.configure_worker("w-b",max_slots=2,failure_domain="zone-b",weight=1)
|
||||||
|
scheduler.configure_worker("w-c",max_slots=1,failure_domain="zone-c",weight=2)
|
||||||
|
return store,registry,scheduler
|
||||||
|
|
||||||
|
|
||||||
|
def test_FLEET_AGENT_REGISTRATION_capacity_policy_is_durable(fleet):
|
||||||
|
store,_,scheduler=fleet
|
||||||
|
policy=scheduler.state.worker_policy("w-b")
|
||||||
|
assert policy.max_slots==2 and policy.failure_domain=="zone-b"
|
||||||
|
reloaded=type(scheduler)(store, scheduler.workers, now_fn=lambda:1000.0)
|
||||||
|
assert reloaded.state.worker_policy("w-b").max_slots==2
|
||||||
|
|
||||||
|
|
||||||
|
def test_FLEET_CAPACITY_SCHEDULING_does_not_oversubscribe_worker(fleet):
|
||||||
|
store,_,scheduler=fleet
|
||||||
|
store.tasks=[Task("task-1"),Task("task-2"),Task("task-3")]
|
||||||
|
decisions=scheduler.select(["w-a"],limit=3,now=1000.0)
|
||||||
|
assert [d.worker_id for d in decisions]==["w-a"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_FLEET_CAPACITY_SCHEDULING_accounts_for_active_claims(fleet):
|
||||||
|
store,_,scheduler=fleet
|
||||||
|
store.conn.execute(
|
||||||
|
"INSERT INTO claims VALUES ('claim-1','other','w-a','ACTIVE',2000)"
|
||||||
|
)
|
||||||
|
store.tasks=[Task("task-1")]
|
||||||
|
assert scheduler.select(["w-a"],limit=1,now=1000.0)==[]
|
||||||
|
|
||||||
|
|
||||||
|
def test_FLEET_FAIRNESS_scope_virtual_finish_prevents_hot_scope_monopoly(fleet):
|
||||||
|
store,_,scheduler=fleet
|
||||||
|
store.tasks=[
|
||||||
|
Task("a1",created_at=1,task_plan=Plan("a1",tenant_id="tenant-a",project_id="p-a")),
|
||||||
|
Task("a2",created_at=2,task_plan=Plan("a2",tenant_id="tenant-a",project_id="p-a")),
|
||||||
|
Task("b1",created_at=3,task_plan=Plan("b1",tenant_id="tenant-b",project_id="p-b")),
|
||||||
|
]
|
||||||
|
first=scheduler.select(["w-b"],limit=1,now=1000.0)[0]
|
||||||
|
assert first.task_id=="a1"
|
||||||
|
scheduler.state.release("a1",state="COMPLETED",now=1000.1)
|
||||||
|
store.tasks=[task for task in store.tasks if task.task_id!="a1"]
|
||||||
|
second=scheduler.select(["w-b"],limit=1,now=1000.2)[0]
|
||||||
|
assert second.task_id=="b1"
|
||||||
|
|
||||||
|
|
||||||
|
def test_FLEET_FAULT_ISOLATION_opens_worker_circuit_after_threshold(fleet):
|
||||||
|
store,_,scheduler=fleet
|
||||||
|
store.tasks=[Task("task-1")]
|
||||||
|
for _ in range(3):
|
||||||
|
scheduler.state.record_worker_failure("w-a","runtime_crash",now=1000.0)
|
||||||
|
assert scheduler.state.worker_circuit_open("w-a",1001.0)
|
||||||
|
assert scheduler.select(["w-a"],limit=1,now=1001.0)==[]
|
||||||
|
|
||||||
|
|
||||||
|
def test_FLEET_WORKER_REASSIGNMENT_avoids_quarantined_worker(fleet):
|
||||||
|
store,_,scheduler=fleet
|
||||||
|
store.tasks=[Task("task-1")]
|
||||||
|
scheduler.state.record_task_worker_failure(
|
||||||
|
"task-1","w-a","lease_expired",quarantine_seconds=60,now=1000.0
|
||||||
|
)
|
||||||
|
decision=scheduler.select(["w-a","w-b"],limit=1,now=1001.0)[0]
|
||||||
|
assert decision.worker_id=="w-b"
|
||||||
|
|
||||||
|
|
||||||
|
def test_FLEET_SCOPE_ISOLATION_uses_tenant_project_fairness_keys(fleet):
|
||||||
|
store,_,scheduler=fleet
|
||||||
|
store.tasks=[
|
||||||
|
Task("a",task_plan=Plan("a",tenant_id="tenant-a",project_id="p")),
|
||||||
|
Task("b",task_plan=Plan("b",tenant_id="tenant-b",project_id="p")),
|
||||||
|
]
|
||||||
|
decisions=scheduler.select(["w-b"],limit=2,now=1000.0)
|
||||||
|
assert {d.scope_key for d in decisions}=={
|
||||||
|
"tenant:tenant-a|project:p","tenant:tenant-b|project:p"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_FLEET_IDEMPOTENT_DISPATCH_replay_same_task_worker(fleet):
|
||||||
|
store,_,scheduler=fleet
|
||||||
|
store.tasks=[Task("task-1")]
|
||||||
|
one=scheduler.select(["w-b"],limit=1,now=1000.0)[0]
|
||||||
|
# Reservation remains authoritative; a conflicting second worker cannot steal it.
|
||||||
|
with pytest.raises(FleetConflict):
|
||||||
|
scheduler.state.reserve(
|
||||||
|
task_id="task-1",worker_id="w-c",task_revision=1,
|
||||||
|
scope_key=one.scope_key,failure_domain="zone-c",score=one.score,
|
||||||
|
reservation_seconds=30,now=1001.0,scope_weight=1.0,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_FLEET_FAILURE_RECOVERY_worker_success_closes_circuit(fleet):
|
||||||
|
store,_,scheduler=fleet
|
||||||
|
for _ in range(3):
|
||||||
|
scheduler.state.record_worker_failure("w-a","runtime_crash",now=1000.0)
|
||||||
|
scheduler.state.record_worker_success("w-a",now=1001.0)
|
||||||
|
assert not scheduler.state.worker_circuit_open("w-a",1001.1)
|
||||||
|
|
||||||
|
|
||||||
|
def test_affinity_and_failure_domain_anti_affinity(fleet):
|
||||||
|
store,_,scheduler=fleet
|
||||||
|
scheduler.configure_worker("w-b",max_slots=2,failure_domain="zone-b",labels={"gpu":"false"})
|
||||||
|
scheduler.configure_worker("w-c",max_slots=1,failure_domain="zone-c",labels={"gpu":"true"})
|
||||||
|
task=Task("gpu-task")
|
||||||
|
task.task_plan.metadata={
|
||||||
|
"fleet_affinity_labels":{"gpu":"true"},
|
||||||
|
"fleet_avoid_failure_domains":["zone-b"],
|
||||||
|
}
|
||||||
|
store.tasks=[task]
|
||||||
|
decision=scheduler.select(["w-b","w-c"],limit=1,now=1000.0)[0]
|
||||||
|
assert decision.worker_id=="w-c"
|
||||||
|
|
||||||
|
|
||||||
|
def test_FLEET_FAILURE_RECOVERY_state_survives_sqlite_restart(tmp_path):
|
||||||
|
path=tmp_path/"task-plane.sqlite3"
|
||||||
|
first=Store(str(path)); registry=Registry()
|
||||||
|
first.workers["w-a"]=Worker("w-a"); registry.online.add("w-a")
|
||||||
|
scheduler=FleetScheduler(first,registry,now_fn=lambda:1000.0)
|
||||||
|
scheduler.configure_worker("w-a",max_slots=3,failure_domain="zone-a")
|
||||||
|
scheduler.state.record_worker_failure("w-a","runtime_crash",threshold=1,now=1000.0)
|
||||||
|
first.conn.close()
|
||||||
|
|
||||||
|
second=Store(str(path)); second.workers["w-a"]=Worker("w-a")
|
||||||
|
resumed=FleetScheduler(second,registry,now_fn=lambda:1001.0)
|
||||||
|
assert resumed.state.worker_policy("w-a").max_slots==3
|
||||||
|
assert resumed.state.worker_circuit_open("w-a",1001.0)
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[3]
|
||||||
|
ACTIVE_RUNTIME = ROOT / "active-runtime"
|
||||||
|
if str(ACTIVE_RUNTIME) not in sys.path:
|
||||||
|
sys.path.insert(0, str(ACTIVE_RUNTIME))
|
||||||
|
|
||||||
|
from app.task_plane.scheduler import TaskScheduler
|
||||||
|
|
||||||
|
|
||||||
|
def _security_contract():
|
||||||
|
path = Path(__file__).with_name("test_task_plane_security_contract.py")
|
||||||
|
spec = importlib.util.spec_from_file_location("p2b_order_security_contract", path)
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
sys.modules[spec.name] = module
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
def _create_task(service, contract, task_id, tenant, project):
|
||||||
|
return service.create_task(
|
||||||
|
{
|
||||||
|
"task_id": task_id,
|
||||||
|
"issuer": {"type": "agent", "id": contract.ADMIN_IDENTITY},
|
||||||
|
"required_capabilities": ["code.read"],
|
||||||
|
"scope_type": "tenant",
|
||||||
|
"tenant_id": tenant,
|
||||||
|
"project_id": project,
|
||||||
|
"metadata": {"fleet_priority": 0},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _task_id(claim):
|
||||||
|
return getattr(
|
||||||
|
claim,
|
||||||
|
"task_id",
|
||||||
|
claim.get("task_id") if isinstance(claim, dict) else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_P2B_FLEET_ORDER_is_preserved_through_legacy_claim_oracle():
|
||||||
|
contract = _security_contract()
|
||||||
|
service = contract._service()
|
||||||
|
service.register_worker(contract._worker_payload(max_concurrency=3))
|
||||||
|
worker_id = contract.WORKER_ID
|
||||||
|
|
||||||
|
scheduler = TaskScheduler(service.store)
|
||||||
|
scheduler.fleet_scheduler.configure_worker(
|
||||||
|
worker_id, max_slots=3, failure_domain="zone-a"
|
||||||
|
)
|
||||||
|
|
||||||
|
_create_task(service, contract, "p2b-order-a1", "tenant-a", "project-a")
|
||||||
|
_create_task(service, contract, "p2b-order-a2", "tenant-a", "project-a")
|
||||||
|
_create_task(service, contract, "p2b-order-b1", "tenant-b", "project-b")
|
||||||
|
|
||||||
|
first = scheduler.schedule_for_worker(worker_id)
|
||||||
|
second = scheduler.schedule_for_worker(worker_id)
|
||||||
|
|
||||||
|
assert _task_id(first) == "p2b-order-a1"
|
||||||
|
assert _task_id(second) == "p2b-order-b1"
|
||||||
|
|
||||||
|
rows = service.store.conn.execute(
|
||||||
|
"SELECT scope_key,dispatch_count FROM fleet_scope_state ORDER BY scope_key"
|
||||||
|
).fetchall()
|
||||||
|
assert len(rows) >= 2
|
||||||
@@ -0,0 +1,181 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
import importlib.util
|
||||||
|
from pathlib import Path
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[3]
|
||||||
|
ACTIVE_RUNTIME = ROOT / "active-runtime"
|
||||||
|
if str(ACTIVE_RUNTIME) not in sys.path:
|
||||||
|
sys.path.insert(0, str(ACTIVE_RUNTIME))
|
||||||
|
|
||||||
|
from app.task_plane.fleet import FleetConflict, FleetScheduler
|
||||||
|
from app.task_plane.errors import NotFoundError
|
||||||
|
from app.task_plane.scheduler import TaskScheduler
|
||||||
|
|
||||||
|
|
||||||
|
def _security_contract():
|
||||||
|
path = Path(__file__).with_name("test_task_plane_security_contract.py")
|
||||||
|
spec = importlib.util.spec_from_file_location("p2b_security_contract", path)
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
sys.modules[spec.name] = module
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
def test_P2B_REAL_DISPATCH_claim_binds_fleet_reservation_and_circuit():
|
||||||
|
contract = _security_contract()
|
||||||
|
service = contract._service()
|
||||||
|
service.register_worker(contract._worker_payload(max_concurrency=2))
|
||||||
|
worker_id = contract.WORKER_ID
|
||||||
|
|
||||||
|
service.create_task(
|
||||||
|
{
|
||||||
|
"task_id": "task-p2b-real-1",
|
||||||
|
"issuer": {"type": "agent", "id": contract.ADMIN_IDENTITY},
|
||||||
|
"required_capabilities": ["code.read"],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
scheduler = TaskScheduler(service.store)
|
||||||
|
scheduler.fleet_scheduler.configure_worker(
|
||||||
|
worker_id,
|
||||||
|
max_slots=2,
|
||||||
|
failure_domain="zone-real",
|
||||||
|
labels={"runtime": "task-plane"},
|
||||||
|
)
|
||||||
|
first = scheduler.schedule_for_worker(worker_id)
|
||||||
|
assert first is not None
|
||||||
|
assert first.task_id == "task-p2b-real-1"
|
||||||
|
|
||||||
|
row = service.store.conn.execute(
|
||||||
|
"SELECT state,claim_id,worker_id FROM fleet_dispatches WHERE task_id=?",
|
||||||
|
("task-p2b-real-1",),
|
||||||
|
).fetchone()
|
||||||
|
assert tuple(row) == ("CLAIMED", first.claim_id, worker_id)
|
||||||
|
|
||||||
|
# A real circuit blocks the actual scheduler even though one capacity slot remains.
|
||||||
|
failure_now = time.time()
|
||||||
|
scheduler.record_worker_failure(
|
||||||
|
worker_id, "runtime_crash", threshold=1, now=failure_now
|
||||||
|
)
|
||||||
|
service.create_task(
|
||||||
|
{
|
||||||
|
"task_id": "task-p2b-real-2",
|
||||||
|
"issuer": {"type": "agent", "id": contract.ADMIN_IDENTITY},
|
||||||
|
"required_capabilities": ["code.read"],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
with pytest.raises(NotFoundError, match="no queued tasks available"):
|
||||||
|
scheduler.schedule_for_worker(worker_id)
|
||||||
|
|
||||||
|
scheduler.record_worker_success(worker_id, now=failure_now + 1.0)
|
||||||
|
second = scheduler.schedule_for_worker(worker_id)
|
||||||
|
assert second is not None
|
||||||
|
assert second.task_id == "task-p2b-real-2"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Plan:
|
||||||
|
task_id: str
|
||||||
|
required_capabilities: list[str] = field(default_factory=lambda: ["code.read"])
|
||||||
|
metadata: dict = field(default_factory=dict)
|
||||||
|
scope_type: str = "tenant"
|
||||||
|
tenant_id: str = "tenant-a"
|
||||||
|
project_id: str | None = "project-a"
|
||||||
|
user_id: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Task:
|
||||||
|
task_id: str
|
||||||
|
revision: int = 1
|
||||||
|
status: str = "QUEUED"
|
||||||
|
created_at: float = 1.0
|
||||||
|
task_plan: Plan | None = None
|
||||||
|
|
||||||
|
def __post_init__(self):
|
||||||
|
if self.task_plan is None:
|
||||||
|
self.task_plan = Plan(self.task_id)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Worker:
|
||||||
|
worker_id: str
|
||||||
|
capabilities: list[str] = field(default_factory=lambda: ["code.read"])
|
||||||
|
|
||||||
|
|
||||||
|
class Store:
|
||||||
|
def __init__(self):
|
||||||
|
self.conn = sqlite3.connect(":memory:", check_same_thread=False)
|
||||||
|
self._lock = threading.RLock()
|
||||||
|
self.conn.execute(
|
||||||
|
"CREATE TABLE claims("
|
||||||
|
"claim_id TEXT PRIMARY KEY,task_id TEXT,worker_id TEXT,status TEXT,"
|
||||||
|
"lease_expires_at REAL)"
|
||||||
|
)
|
||||||
|
self.tasks = []
|
||||||
|
self.workers = {}
|
||||||
|
|
||||||
|
def list_tasks(self, _ctx=None):
|
||||||
|
return list(self.tasks)
|
||||||
|
|
||||||
|
def get_worker(self, worker_id):
|
||||||
|
return self.workers[worker_id]
|
||||||
|
|
||||||
|
|
||||||
|
class Registry:
|
||||||
|
def __init__(self):
|
||||||
|
self.online = set()
|
||||||
|
|
||||||
|
def is_online(self, worker_id, now=None):
|
||||||
|
return worker_id in self.online
|
||||||
|
|
||||||
|
|
||||||
|
def _fleet():
|
||||||
|
store = Store()
|
||||||
|
registry = Registry()
|
||||||
|
store.workers["w-a"] = Worker("w-a")
|
||||||
|
registry.online.add("w-a")
|
||||||
|
fleet = FleetScheduler(store, registry, now_fn=lambda: 1000.0)
|
||||||
|
fleet.configure_worker("w-a", max_slots=1, failure_domain="zone-a")
|
||||||
|
return store, fleet
|
||||||
|
|
||||||
|
|
||||||
|
def test_P2B_CAPACITY_RESERVATION_is_atomic_under_concurrent_pullers():
|
||||||
|
store, fleet = _fleet()
|
||||||
|
tasks = [Task("task-a"), Task("task-b")]
|
||||||
|
store.tasks = tasks
|
||||||
|
barrier = threading.Barrier(2)
|
||||||
|
outcomes = []
|
||||||
|
|
||||||
|
def reserve(task):
|
||||||
|
barrier.wait()
|
||||||
|
try:
|
||||||
|
fleet.reserve_for_worker("w-a", task, now=1000.0)
|
||||||
|
outcomes.append("reserved")
|
||||||
|
except FleetConflict:
|
||||||
|
outcomes.append("conflict")
|
||||||
|
|
||||||
|
threads = [threading.Thread(target=reserve, args=(task,)) for task in tasks]
|
||||||
|
for thread in threads:
|
||||||
|
thread.start()
|
||||||
|
for thread in threads:
|
||||||
|
thread.join()
|
||||||
|
|
||||||
|
assert sorted(outcomes) == ["conflict", "reserved"]
|
||||||
|
assert fleet.state.reserved_slots("w-a", 1000.0) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_P2B_RANK_is_read_only_until_legacy_security_accepts_candidate():
|
||||||
|
store, fleet = _fleet()
|
||||||
|
store.tasks = [Task("task-a"), Task("task-b")]
|
||||||
|
ranked = fleet.rank_for_worker("w-a", now=1000.0)
|
||||||
|
assert [task.task_id for task in ranked] == ["task-a", "task-b"]
|
||||||
|
assert fleet.state.reserved_slots("w-a", 1000.0) == 0
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0, str(ROOT / "active-runtime/app"))
|
||||||
|
|
||||||
|
from aios_capability_contract import ( # noqa: E402
|
||||||
|
capability_contract,
|
||||||
|
has_capabilities,
|
||||||
|
normalize_required_capabilities,
|
||||||
|
normalize_worker_capabilities,
|
||||||
|
normalized_capability_matrix,
|
||||||
|
registered_capability,
|
||||||
|
validate_capability_contract_registry,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("runtime", ["hermes", "openclaw"])
|
||||||
|
def test_runtime_capability_is_known_to_task_and_worker_vocabulary(runtime: str) -> None:
|
||||||
|
capability = f"agent.runtime.{runtime}"
|
||||||
|
assert normalize_required_capabilities([capability]) == frozenset({capability})
|
||||||
|
assert normalize_worker_capabilities([capability]) == frozenset({capability})
|
||||||
|
assert has_capabilities([capability], [capability]) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_capabilities_are_isolated_between_workers() -> None:
|
||||||
|
assert has_capabilities(["agent.runtime.hermes"], ["agent.runtime.openclaw"]) is False
|
||||||
|
assert has_capabilities(["agent.runtime.openclaw"], ["agent.runtime.hermes"]) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_unknown_runtime_capability_remains_fail_closed() -> None:
|
||||||
|
assert has_capabilities(["agent.runtime.hermes"], ["agent.runtime.unknown"]) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_autonomous_coordinator_capability_is_registered_on_both_sides() -> None:
|
||||||
|
capability = "engineering.autonomous"
|
||||||
|
assert registered_capability(capability) is True
|
||||||
|
assert registered_capability(capability, worker=True) is True
|
||||||
|
assert normalize_required_capabilities([capability]) == frozenset({capability})
|
||||||
|
assert normalize_worker_capabilities([capability]) == frozenset({capability})
|
||||||
|
contract = capability_contract(capability)
|
||||||
|
assert contract is not None
|
||||||
|
assert contract.dependencies == ("agent_task_control_plane",)
|
||||||
|
assert validate_capability_contract_registry()["valid"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_supervisor_coordinator_requires_runtime_and_autonomous_capabilities() -> None:
|
||||||
|
required = ["agent.runtime.openclaw", "engineering.autonomous"]
|
||||||
|
assert normalized_capability_matrix(required, required) == {
|
||||||
|
"worker_effective": required,
|
||||||
|
"task_required": required,
|
||||||
|
"eligible": True,
|
||||||
|
}
|
||||||
|
assert has_capabilities(["agent.runtime.openclaw"], required) is False
|
||||||
|
assert has_capabilities(["engineering.autonomous"], required) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_autonomous_capability_near_matches_remain_fail_closed() -> None:
|
||||||
|
worker = ["agent.runtime.openclaw", "engineering.autonomous"]
|
||||||
|
assert has_capabilities(worker, ["engineering.autonomous.extra"]) is False
|
||||||
|
assert has_capabilities(worker, ["engineering.*"]) is False
|
||||||
@@ -0,0 +1,293 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0,str(ROOT/"active-runtime/app"))
|
||||||
|
|
||||||
|
from aios_agent_task_execution_rebind import (
|
||||||
|
ExecutionRebindError,
|
||||||
|
ExecutionRebindRequest,
|
||||||
|
ExecutionRebindStore,
|
||||||
|
)
|
||||||
|
from aios_online_agent_execution import (
|
||||||
|
GovernedExecutionBinding,
|
||||||
|
project_execution_envelope,
|
||||||
|
)
|
||||||
|
|
||||||
|
AUTHORIZATION="owner-execution-rebind-20260818"
|
||||||
|
TASK_ID="task-rebind-a"
|
||||||
|
JOB_ID="job-rebind-a"
|
||||||
|
BASE="a"*40
|
||||||
|
BRANCH="agent/task-rebind-a"
|
||||||
|
IDEMPOTENCY="task-rebind-a-binding-v1"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _auth(monkeypatch):
|
||||||
|
monkeypatch.setenv(
|
||||||
|
"AIOS_AGENT_TASK_EXECUTION_REBIND_AUTHORIZATION_ID",
|
||||||
|
AUTHORIZATION,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def canon(value):
|
||||||
|
return hashlib.sha256(
|
||||||
|
json.dumps(value,sort_keys=True,separators=(",",":")).encode()
|
||||||
|
).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def binding():
|
||||||
|
return GovernedExecutionBinding.from_mapping({
|
||||||
|
"task_id":TASK_ID,
|
||||||
|
"job_id":JOB_ID,
|
||||||
|
"repository":"aios/aios",
|
||||||
|
"base_oid":BASE,
|
||||||
|
"expected_branch_oid":BASE,
|
||||||
|
"target_branch":BRANCH,
|
||||||
|
"branch_mode":"CREATE_NEW",
|
||||||
|
"generation":1,
|
||||||
|
"idempotency_key":IDEMPOTENCY,
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def old_envelope():
|
||||||
|
return project_execution_envelope(
|
||||||
|
binding(),
|
||||||
|
{
|
||||||
|
"action":"source_capture",
|
||||||
|
"adapter":"compact_interface_map",
|
||||||
|
"capability_scope":[
|
||||||
|
"git.read","git.test","governed.write.branch","governed.write.file"
|
||||||
|
],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def new_envelope():
|
||||||
|
return project_execution_envelope(
|
||||||
|
binding(),
|
||||||
|
{
|
||||||
|
"action":"deterministic",
|
||||||
|
"adapter":"deterministic",
|
||||||
|
"capability_scope":[
|
||||||
|
"git.read","git.test","governed.write.branch","governed.write.file"
|
||||||
|
],
|
||||||
|
"change_set":[
|
||||||
|
{
|
||||||
|
"path":"active-runtime/app/example.py",
|
||||||
|
"operation":"write_file",
|
||||||
|
"content":"VALUE=1\\n",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"verification":[
|
||||||
|
{
|
||||||
|
"label":"unit",
|
||||||
|
"argv":["/usr/bin/python3","-m","pytest","-q"],
|
||||||
|
"cwd":".",
|
||||||
|
"timeout_seconds":120,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"governance":{
|
||||||
|
"commit_idempotency_key":"task-rebind-a-commit-v1",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def make_db(tmp_path, *, status="QUEUED", revision=5, attempt=2, lease=False):
|
||||||
|
path=tmp_path/"state.sqlite3"
|
||||||
|
con=sqlite3.connect(path)
|
||||||
|
con.executescript("""
|
||||||
|
CREATE TABLE tasks(
|
||||||
|
task_id TEXT PRIMARY KEY,
|
||||||
|
status TEXT NOT NULL,
|
||||||
|
attempt INTEGER NOT NULL,
|
||||||
|
max_attempts INTEGER NOT NULL,
|
||||||
|
revision INTEGER NOT NULL,
|
||||||
|
claimed_by TEXT,
|
||||||
|
lease_token_hash TEXT,
|
||||||
|
lease_expires_at TEXT,
|
||||||
|
verification_json TEXT,
|
||||||
|
result_json TEXT,
|
||||||
|
metadata_json TEXT,
|
||||||
|
required_capabilities_json TEXT
|
||||||
|
);
|
||||||
|
CREATE TABLE task_events(
|
||||||
|
event_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
task_id TEXT,
|
||||||
|
event_type TEXT,
|
||||||
|
agent_id TEXT,
|
||||||
|
payload_json TEXT,
|
||||||
|
created_at TEXT
|
||||||
|
);
|
||||||
|
""")
|
||||||
|
verification={"decision":"NEEDS_REWORK","reason":"WRONG_ENVELOPE_BEFORE_PLAN"}
|
||||||
|
old=old_envelope()
|
||||||
|
metadata={
|
||||||
|
"execution":old,
|
||||||
|
"execution_projection":{
|
||||||
|
"schema":"aios.online-agent.task-projection.v1",
|
||||||
|
"task_id":TASK_ID,
|
||||||
|
"job_id":JOB_ID,
|
||||||
|
"execution_digest":old["execution_digest"],
|
||||||
|
},
|
||||||
|
"preserved":{"value":"unchanged"},
|
||||||
|
}
|
||||||
|
con.execute(
|
||||||
|
"INSERT INTO tasks VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||||
|
(
|
||||||
|
TASK_ID,status,attempt,3,revision,
|
||||||
|
"previous-worker" if not lease else "active-worker",
|
||||||
|
"lease-hash" if lease else None,
|
||||||
|
"2099-01-01T00:00:00Z" if lease else None,
|
||||||
|
json.dumps(verification),
|
||||||
|
json.dumps({"status":"SUCCEEDED"}),
|
||||||
|
json.dumps(metadata),
|
||||||
|
json.dumps([
|
||||||
|
"git.read","git.test","governed.write.branch","governed.write.file"
|
||||||
|
]),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
con.commit();con.close()
|
||||||
|
return path,verification
|
||||||
|
|
||||||
|
|
||||||
|
def request(verification, *, revision=5, old=None, new=None, key="rebind-task-a-v1"):
|
||||||
|
old=old or old_envelope()
|
||||||
|
new=new or new_envelope()
|
||||||
|
return ExecutionRebindRequest.from_mapping({
|
||||||
|
"expected_status":"QUEUED",
|
||||||
|
"expected_revision":revision,
|
||||||
|
"expected_attempt":2,
|
||||||
|
"previous_verification_sha256":canon(verification),
|
||||||
|
"expected_old_execution_digest":old["execution_digest"],
|
||||||
|
"expected_new_execution_digest":new["execution_digest"],
|
||||||
|
"authorization_id":AUTHORIZATION,
|
||||||
|
"idempotency_key":key,
|
||||||
|
"reason":"projection-before-plan race recovery",
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def resolver(_task_id,_caps):
|
||||||
|
return new_envelope()
|
||||||
|
|
||||||
|
|
||||||
|
def test_rebind_cas_updates_only_execution_projection_and_audits(tmp_path):
|
||||||
|
path,verification=make_db(tmp_path)
|
||||||
|
result=ExecutionRebindStore(path,resolver).rebind(
|
||||||
|
TASK_ID,request(verification)
|
||||||
|
)
|
||||||
|
assert result["status"]=="QUEUED"
|
||||||
|
assert result["revision"]==6
|
||||||
|
assert result["idempotent"] is False
|
||||||
|
|
||||||
|
con=sqlite3.connect(path)
|
||||||
|
row=con.execute(
|
||||||
|
"SELECT status,attempt,revision,claimed_by,metadata_json FROM tasks"
|
||||||
|
).fetchone()
|
||||||
|
assert row[:4]==("QUEUED",2,6,None)
|
||||||
|
metadata=json.loads(row[4])
|
||||||
|
assert metadata["preserved"]=={"value":"unchanged"}
|
||||||
|
assert metadata["execution"]["action"]=="deterministic"
|
||||||
|
assert metadata["execution"]["execution_digest"]==new_envelope()["execution_digest"]
|
||||||
|
assert metadata["execution_projection"]["execution_digest"]==new_envelope()["execution_digest"]
|
||||||
|
|
||||||
|
event=con.execute(
|
||||||
|
"SELECT event_type,payload_json FROM task_events"
|
||||||
|
).fetchone()
|
||||||
|
assert event[0]=="TASK_EXECUTION_REBIND_AUTHORIZED"
|
||||||
|
payload=json.loads(event[1])
|
||||||
|
assert payload["old_action"]=="source_capture"
|
||||||
|
assert payload["new_action"]=="deterministic"
|
||||||
|
assert payload["stale_claim_normalized"] is True
|
||||||
|
assert con.execute("SELECT COUNT(*) FROM task_execution_rebinds").fetchone()[0]==1
|
||||||
|
|
||||||
|
|
||||||
|
def test_rebind_idempotent_replay(tmp_path):
|
||||||
|
path,verification=make_db(tmp_path)
|
||||||
|
store=ExecutionRebindStore(path,resolver)
|
||||||
|
req=request(verification)
|
||||||
|
first=store.rebind(TASK_ID,req)
|
||||||
|
second=store.rebind(TASK_ID,req)
|
||||||
|
assert first["idempotent"] is False
|
||||||
|
assert second["idempotent"] is True
|
||||||
|
assert second["new_execution_digest"]==first["new_execution_digest"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_rebind_rejects_active_lease(tmp_path):
|
||||||
|
path,verification=make_db(tmp_path,lease=True)
|
||||||
|
with pytest.raises(ExecutionRebindError,match="EXECUTION_REBIND_ACTIVE_LEASE_DENY"):
|
||||||
|
ExecutionRebindStore(path,resolver).rebind(TASK_ID,request(verification))
|
||||||
|
|
||||||
|
|
||||||
|
def test_rebind_rejects_verification_drift(tmp_path):
|
||||||
|
path,_=make_db(tmp_path)
|
||||||
|
with pytest.raises(ExecutionRebindError,match="EXECUTION_REBIND_VERIFICATION_DRIFT"):
|
||||||
|
ExecutionRebindStore(path,resolver).rebind(
|
||||||
|
TASK_ID,request({"decision":"NEEDS_REWORK","reason":"other"})
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_rebind_rejects_nonqueued_task(tmp_path):
|
||||||
|
path,verification=make_db(tmp_path,status="VERIFYING")
|
||||||
|
with pytest.raises(ExecutionRebindError,match="EXECUTION_REBIND_CAS_MISMATCH"):
|
||||||
|
ExecutionRebindStore(path,resolver).rebind(TASK_ID,request(verification))
|
||||||
|
|
||||||
|
|
||||||
|
def test_rebind_rejects_non_deterministic_canonical_resolution(tmp_path):
|
||||||
|
path,verification=make_db(tmp_path)
|
||||||
|
def wrong(_task,_caps):
|
||||||
|
return old_envelope()
|
||||||
|
with pytest.raises(ExecutionRebindError,match="EXECUTION_REBIND_DETERMINISTIC_PLAN_REQUIRED"):
|
||||||
|
ExecutionRebindStore(path,wrong).rebind(TASK_ID,request(verification))
|
||||||
|
|
||||||
|
|
||||||
|
def test_rebind_rejects_binding_drift(tmp_path):
|
||||||
|
path,verification=make_db(tmp_path)
|
||||||
|
different=project_execution_envelope(
|
||||||
|
GovernedExecutionBinding.from_mapping({
|
||||||
|
"task_id":TASK_ID,
|
||||||
|
"job_id":"job-other",
|
||||||
|
"repository":"aios/aios",
|
||||||
|
"base_oid":BASE,
|
||||||
|
"expected_branch_oid":BASE,
|
||||||
|
"target_branch":BRANCH,
|
||||||
|
"branch_mode":"CREATE_NEW",
|
||||||
|
"generation":1,
|
||||||
|
"idempotency_key":IDEMPOTENCY,
|
||||||
|
}),
|
||||||
|
{
|
||||||
|
"action":"deterministic",
|
||||||
|
"capability_scope":["git.read"],
|
||||||
|
"change_set":[{"path":"x.py","operation":"write_file","content":"x=1\\n"}],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
def wrong(_task,_caps):
|
||||||
|
return different
|
||||||
|
with pytest.raises(Exception):
|
||||||
|
ExecutionRebindStore(path,wrong).rebind(TASK_ID,request(verification))
|
||||||
|
|
||||||
|
|
||||||
|
def test_rebind_request_is_narrow_and_digest_bound():
|
||||||
|
verification={"decision":"NEEDS_REWORK"}
|
||||||
|
data=request(verification).public()
|
||||||
|
data["arbitrary_metadata"]={"x":1}
|
||||||
|
with pytest.raises(ExecutionRebindError,match="EXECUTION_REBIND_FIELDS_INVALID"):
|
||||||
|
ExecutionRebindRequest.from_mapping(data)
|
||||||
|
|
||||||
|
|
||||||
|
def test_execution_rebind_unit_is_unprivileged_and_owner_credential_bound():
|
||||||
|
unit=(ROOT/"deploy/systemd/aios-agent-task-execution-rebind.service").read_text()
|
||||||
|
assert "User=aios-agent-task" in unit
|
||||||
|
assert "Group=aios-agent-task" in unit
|
||||||
|
assert "LoadCredential=agent_task_control_token:" in unit
|
||||||
|
assert "Environment=AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892" in unit
|
||||||
|
assert "RestrictAddressFamilies=AF_UNIX AF_INET" in unit
|
||||||
|
assert "User=root" not in unit
|
||||||
@@ -0,0 +1,452 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0,str(ROOT/"active-runtime/app"))
|
||||||
|
|
||||||
|
from aios_agent_task_initial_execution_projection import (
|
||||||
|
InitialExecutionProjectionError,
|
||||||
|
InitialExecutionProjectionRequest,
|
||||||
|
InitialExecutionProjectionStore,
|
||||||
|
SourceCaptureProjectionRecoveryRequest,
|
||||||
|
)
|
||||||
|
from aios_online_agent_execution import (
|
||||||
|
GovernedExecutionBinding,
|
||||||
|
project_execution_envelope,
|
||||||
|
)
|
||||||
|
|
||||||
|
AUTHORIZATION="owner-initial-execution-projection-recovery-20260820"
|
||||||
|
TASK_ID="task-initial-projection-a"
|
||||||
|
JOB_ID="job-initial-projection-a"
|
||||||
|
BASE="a"*40
|
||||||
|
BRANCH="agent/task-initial-projection-a"
|
||||||
|
IDEMPOTENCY="task-initial-projection-binding-v1"
|
||||||
|
CAPABILITY="agent.runtime.openclaw"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _auth(monkeypatch):
|
||||||
|
monkeypatch.setenv(
|
||||||
|
"AIOS_AGENT_TASK_INITIAL_EXECUTION_PROJECTION_AUTHORIZATION_ID",
|
||||||
|
AUTHORIZATION,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def canon(value):
|
||||||
|
return hashlib.sha256(
|
||||||
|
json.dumps(value,sort_keys=True,separators=(",",":")).encode()
|
||||||
|
).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def binding():
|
||||||
|
return GovernedExecutionBinding.from_mapping({
|
||||||
|
"task_id":TASK_ID,
|
||||||
|
"job_id":JOB_ID,
|
||||||
|
"repository":"aios/aios",
|
||||||
|
"base_oid":BASE,
|
||||||
|
"expected_branch_oid":BASE,
|
||||||
|
"target_branch":BRANCH,
|
||||||
|
"branch_mode":"CREATE_NEW",
|
||||||
|
"generation":1,
|
||||||
|
"idempotency_key":IDEMPOTENCY,
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def runtime_input():
|
||||||
|
return {
|
||||||
|
"goal":"Supervise release preparation",
|
||||||
|
"instructions":"Observe and resume only the same lineage.",
|
||||||
|
"success_criteria":[{"id":"heartbeat","description":"Heartbeat observed"}],
|
||||||
|
"constraints":{"production_write":False},
|
||||||
|
"scope":{"scope_type":"legacy","tenant_id":None,"project_id":None},
|
||||||
|
"trace_id":TASK_ID,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def envelope(*, job_id=JOB_ID, action="agent_runtime", capability=CAPABILITY):
|
||||||
|
bound=binding()
|
||||||
|
if job_id != JOB_ID:
|
||||||
|
bound=GovernedExecutionBinding.from_mapping({**bound.public(),"job_id":job_id})
|
||||||
|
return project_execution_envelope(
|
||||||
|
bound,
|
||||||
|
{
|
||||||
|
"action":action,
|
||||||
|
"adapter":"openclaw" if action=="agent_runtime" else "source_capture",
|
||||||
|
"capability_scope":[capability],
|
||||||
|
"runtime_input":runtime_input() if action=="agent_runtime" else None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def make_db(tmp_path, *, claimed_by=None, lease=False, metadata=None, caps=None, event="TASK_CREATED"):
|
||||||
|
path=tmp_path/"state.sqlite3"
|
||||||
|
con=sqlite3.connect(path)
|
||||||
|
con.executescript("""
|
||||||
|
CREATE TABLE tasks(
|
||||||
|
task_id TEXT PRIMARY KEY,
|
||||||
|
goal TEXT NOT NULL,
|
||||||
|
instructions TEXT NOT NULL,
|
||||||
|
success_criteria_json TEXT NOT NULL,
|
||||||
|
required_capabilities_json TEXT NOT NULL,
|
||||||
|
constraints_json TEXT NOT NULL,
|
||||||
|
metadata_json TEXT NOT NULL,
|
||||||
|
deadline TEXT,
|
||||||
|
priority INTEGER NOT NULL,
|
||||||
|
max_attempts INTEGER NOT NULL,
|
||||||
|
attempt INTEGER NOT NULL DEFAULT 0,
|
||||||
|
status TEXT NOT NULL,
|
||||||
|
claimed_by TEXT,
|
||||||
|
lease_token_hash TEXT,
|
||||||
|
lease_expires_at TEXT,
|
||||||
|
result_json TEXT,
|
||||||
|
result_digest TEXT,
|
||||||
|
result_idempotency_key TEXT,
|
||||||
|
verification_json TEXT,
|
||||||
|
revision INTEGER NOT NULL DEFAULT 1,
|
||||||
|
created_at TEXT NOT NULL,
|
||||||
|
updated_at TEXT NOT NULL
|
||||||
|
);
|
||||||
|
CREATE TABLE events(
|
||||||
|
event_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
task_id TEXT,
|
||||||
|
agent_id TEXT,
|
||||||
|
event_type TEXT NOT NULL,
|
||||||
|
payload_json TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL
|
||||||
|
);
|
||||||
|
""")
|
||||||
|
metadata={} if metadata is None else metadata
|
||||||
|
caps=[] if caps is None else caps
|
||||||
|
con.execute(
|
||||||
|
"INSERT INTO tasks VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||||
|
(
|
||||||
|
TASK_ID,"Supervise release preparation",
|
||||||
|
"Observe and resume only the same lineage.",
|
||||||
|
json.dumps([{"id":"heartbeat","description":"Heartbeat observed"}]),
|
||||||
|
json.dumps(caps),json.dumps({"production_write":False}),json.dumps(metadata),
|
||||||
|
None,100,20,0,"QUEUED",claimed_by,
|
||||||
|
"lease-hash" if lease else None,
|
||||||
|
"2099-01-01T00:00:00Z" if lease else None,
|
||||||
|
None,None,None,None,1,"2026-08-20T00:00:00Z","2026-08-20T00:00:00Z",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
con.execute(
|
||||||
|
"INSERT INTO events(task_id,agent_id,event_type,payload_json,created_at) VALUES (?,?,?,?,?)",
|
||||||
|
(TASK_ID,None,event,"{}","2026-08-20T00:00:00Z"),
|
||||||
|
)
|
||||||
|
con.commit();con.close()
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def request(*, digest=None, key="initial-projection-a-v1"):
|
||||||
|
return InitialExecutionProjectionRequest.from_mapping({
|
||||||
|
"expected_status":"QUEUED",
|
||||||
|
"expected_revision":1,
|
||||||
|
"expected_attempt":0,
|
||||||
|
"expected_metadata_sha256":canon({}),
|
||||||
|
"expected_required_capabilities_sha256":canon([]),
|
||||||
|
"expected_job_id":JOB_ID,
|
||||||
|
"expected_execution_digest":digest or envelope()["execution_digest"],
|
||||||
|
"required_capability":CAPABILITY,
|
||||||
|
"adapter":"openclaw",
|
||||||
|
"authorization_id":AUTHORIZATION,
|
||||||
|
"idempotency_key":key,
|
||||||
|
"reason":"initial governed supervisor runtime projection",
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def resolver(_task_id,_job_id,_capability,_adapter,received_runtime_input):
|
||||||
|
assert received_runtime_input==runtime_input()
|
||||||
|
return envelope()
|
||||||
|
|
||||||
|
|
||||||
|
def source_capture_result():
|
||||||
|
old=envelope(action="source_capture",capability="git.read")
|
||||||
|
return {
|
||||||
|
"task_id":TASK_ID,
|
||||||
|
"job_id":JOB_ID,
|
||||||
|
"repository":"aios/aios",
|
||||||
|
"base_oid":BASE,
|
||||||
|
"expected_branch_oid":BASE,
|
||||||
|
"target_branch":BRANCH,
|
||||||
|
"generation":1,
|
||||||
|
"status":"SUCCEEDED",
|
||||||
|
"classification":"NEEDS_REWORK_SOURCE_CAPTURE",
|
||||||
|
"execution_mode":"source_capture",
|
||||||
|
"execution_digest":old["execution_digest"],
|
||||||
|
"changed_files":[],
|
||||||
|
"writer_lease":"NONE",
|
||||||
|
"verification_state":"NEEDS_REWORK",
|
||||||
|
"direct_main_write":False,
|
||||||
|
"production_write":False,
|
||||||
|
"force_push":False,
|
||||||
|
"protected_ref_bypass":False,
|
||||||
|
"secret_material_exposed":False,
|
||||||
|
"source_snapshot":{"schema":"test.snapshot.v1","payload_sha256":"d"*64},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def make_source_capture_rework_db(tmp_path, *, mutate=None, events=None):
|
||||||
|
path=make_db(tmp_path)
|
||||||
|
result=source_capture_result()
|
||||||
|
verification={
|
||||||
|
"decision":"NEEDS_REWORK",
|
||||||
|
"reason":"SOURCE_CAPTURE_COMPLETE_RUNTIME_PROJECTION_REQUIRED",
|
||||||
|
}
|
||||||
|
if mutate:
|
||||||
|
mutate(result,verification)
|
||||||
|
event_types=events or [
|
||||||
|
"TASK_CLAIMED","TASK_HEARTBEAT","RESULT_SUBMITTED","TASK_VERIFIED"
|
||||||
|
]
|
||||||
|
con=sqlite3.connect(path)
|
||||||
|
con.execute(
|
||||||
|
"UPDATE tasks SET attempt=1,status='QUEUED',claimed_by=NULL,revision=5,"
|
||||||
|
"result_json=?,result_digest=?,result_idempotency_key=?,verification_json=?",
|
||||||
|
(
|
||||||
|
json.dumps(result,sort_keys=True,separators=(",",":")),
|
||||||
|
canon(result),"source-capture-result-v1",
|
||||||
|
json.dumps(verification,sort_keys=True,separators=(",",":")),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for event_type in event_types:
|
||||||
|
payload={}
|
||||||
|
if event_type=="TASK_VERIFIED":
|
||||||
|
payload={"decision":"NEEDS_REWORK","next_state":"QUEUED"}
|
||||||
|
con.execute(
|
||||||
|
"INSERT INTO events(task_id,agent_id,event_type,payload_json,created_at) "
|
||||||
|
"VALUES (?,?,?,?,?)",
|
||||||
|
(TASK_ID,"generic-worker",event_type,json.dumps(payload),"2026-08-20T00:01:00Z"),
|
||||||
|
)
|
||||||
|
con.commit();con.close()
|
||||||
|
return path,result,verification
|
||||||
|
|
||||||
|
|
||||||
|
def recovery_request(result,verification,*,digest=None,key="source-capture-recovery-a-v1"):
|
||||||
|
return SourceCaptureProjectionRecoveryRequest.from_mapping({
|
||||||
|
"expected_status":"QUEUED",
|
||||||
|
"expected_revision":5,
|
||||||
|
"expected_attempt":1,
|
||||||
|
"expected_metadata_sha256":canon({}),
|
||||||
|
"expected_required_capabilities_sha256":canon([]),
|
||||||
|
"previous_verification_sha256":canon(verification),
|
||||||
|
"expected_source_capture_result_sha256":canon(result),
|
||||||
|
"expected_source_capture_execution_digest":result["execution_digest"],
|
||||||
|
"expected_job_id":JOB_ID,
|
||||||
|
"expected_execution_digest":digest or envelope()["execution_digest"],
|
||||||
|
"required_capability":CAPABILITY,
|
||||||
|
"adapter":"openclaw",
|
||||||
|
"authorization_id":AUTHORIZATION,
|
||||||
|
"idempotency_key":key,
|
||||||
|
"reason":"first durable runtime projection after governed source capture",
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def test_initial_projection_cas_persists_envelope_capability_and_event(tmp_path):
|
||||||
|
path=make_db(tmp_path)
|
||||||
|
result=InitialExecutionProjectionStore(path,resolver).project(TASK_ID,request())
|
||||||
|
assert result["revision"]==2
|
||||||
|
assert result["attempt"]==0
|
||||||
|
assert result["idempotent"] is False
|
||||||
|
con=sqlite3.connect(path)
|
||||||
|
row=con.execute(
|
||||||
|
"SELECT status,attempt,revision,claimed_by,metadata_json,required_capabilities_json FROM tasks"
|
||||||
|
).fetchone()
|
||||||
|
assert row[:4]==("QUEUED",0,2,None)
|
||||||
|
metadata=json.loads(row[4])
|
||||||
|
assert metadata["execution"]["action"]=="agent_runtime"
|
||||||
|
assert metadata["execution"]["adapter"]=="openclaw"
|
||||||
|
assert metadata["execution"]["governance"]["writer_lease"] is False
|
||||||
|
assert json.loads(row[5])==[CAPABILITY]
|
||||||
|
events=con.execute("SELECT event_type,payload_json FROM events ORDER BY event_id").fetchall()
|
||||||
|
assert [event[0] for event in events]==[
|
||||||
|
"TASK_CREATED","TASK_INITIAL_EXECUTION_PROJECTION_AUTHORIZED"
|
||||||
|
]
|
||||||
|
payload=json.loads(events[-1][1])
|
||||||
|
assert payload["job_id"]==JOB_ID
|
||||||
|
assert payload["main_write"] is False
|
||||||
|
assert payload["production_write"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_initial_projection_idempotent_exact_replay(tmp_path):
|
||||||
|
path=make_db(tmp_path)
|
||||||
|
store=InitialExecutionProjectionStore(path,resolver)
|
||||||
|
first=store.project(TASK_ID,request())
|
||||||
|
second=store.project(TASK_ID,request())
|
||||||
|
assert first["idempotent"] is False
|
||||||
|
assert second["idempotent"] is True
|
||||||
|
assert second["execution_digest"]==first["execution_digest"]
|
||||||
|
con=sqlite3.connect(path)
|
||||||
|
assert con.execute("SELECT COUNT(*) FROM task_initial_execution_projections").fetchone()[0]==1
|
||||||
|
assert con.execute("SELECT COUNT(*) FROM events").fetchone()[0]==2
|
||||||
|
|
||||||
|
|
||||||
|
def test_source_capture_recovery_projects_first_runtime_envelope(tmp_path):
|
||||||
|
path,result,verification=make_source_capture_rework_db(tmp_path)
|
||||||
|
response=InitialExecutionProjectionStore(path,resolver).project_from_source_capture(
|
||||||
|
TASK_ID,recovery_request(result,verification)
|
||||||
|
)
|
||||||
|
assert response["revision"]==6
|
||||||
|
assert response["attempt"]==1
|
||||||
|
assert response["recovery_from_source_capture"] is True
|
||||||
|
assert response["idempotent"] is False
|
||||||
|
con=sqlite3.connect(path)
|
||||||
|
row=con.execute(
|
||||||
|
"SELECT status,attempt,revision,claimed_by,metadata_json,"
|
||||||
|
"required_capabilities_json,result_json,verification_json FROM tasks"
|
||||||
|
).fetchone()
|
||||||
|
assert row[:4]==("QUEUED",1,6,None)
|
||||||
|
metadata=json.loads(row[4])
|
||||||
|
assert metadata["execution"]["action"]=="agent_runtime"
|
||||||
|
assert metadata["execution"]["adapter"]=="openclaw"
|
||||||
|
assert metadata["execution"]["governance"]["writer_lease"] is False
|
||||||
|
assert json.loads(row[5])==[CAPABILITY]
|
||||||
|
assert json.loads(row[6])==result
|
||||||
|
assert json.loads(row[7])==verification
|
||||||
|
events=con.execute("SELECT event_type,payload_json FROM events ORDER BY event_id").fetchall()
|
||||||
|
assert [event[0] for event in events]==[
|
||||||
|
"TASK_CREATED","TASK_CLAIMED","TASK_HEARTBEAT","RESULT_SUBMITTED",
|
||||||
|
"TASK_VERIFIED","TASK_INITIAL_EXECUTION_PROJECTION_AUTHORIZED",
|
||||||
|
]
|
||||||
|
payload=json.loads(events[-1][1])
|
||||||
|
assert payload["recovery_from_source_capture"] is True
|
||||||
|
assert payload["source_capture_execution_digest"]==result["execution_digest"]
|
||||||
|
assert payload["main_write"] is False
|
||||||
|
assert payload["production_write"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_source_capture_recovery_idempotent_exact_replay(tmp_path):
|
||||||
|
path,result,verification=make_source_capture_rework_db(tmp_path)
|
||||||
|
store=InitialExecutionProjectionStore(path,resolver)
|
||||||
|
first=store.project_from_source_capture(TASK_ID,recovery_request(result,verification))
|
||||||
|
second=store.project_from_source_capture(TASK_ID,recovery_request(result,verification))
|
||||||
|
assert first["idempotent"] is False
|
||||||
|
assert second["idempotent"] is True
|
||||||
|
assert second["execution_digest"]==first["execution_digest"]
|
||||||
|
con=sqlite3.connect(path)
|
||||||
|
assert con.execute(
|
||||||
|
"SELECT COUNT(*) FROM task_source_capture_projection_recoveries"
|
||||||
|
).fetchone()[0]==1
|
||||||
|
assert con.execute("SELECT COUNT(*) FROM events").fetchone()[0]==6
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"mutate,error",
|
||||||
|
[
|
||||||
|
(lambda result,verification: result.__setitem__("changed_files",["x"]),
|
||||||
|
"SOURCE_CAPTURE_RESULT_CONTRACT_INVALID"),
|
||||||
|
(lambda result,verification: result.__setitem__("job_id","job-other"),
|
||||||
|
"SOURCE_CAPTURE_RESULT_CONTRACT_INVALID"),
|
||||||
|
(lambda result,verification: verification.__setitem__("decision","SUCCEEDED"),
|
||||||
|
"SOURCE_CAPTURE_NEEDS_REWORK_VERIFICATION_REQUIRED"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_source_capture_recovery_rejects_unsafe_result_or_verification(
|
||||||
|
tmp_path,mutate,error
|
||||||
|
):
|
||||||
|
path,result,verification=make_source_capture_rework_db(tmp_path,mutate=mutate)
|
||||||
|
with pytest.raises(InitialExecutionProjectionError,match=error):
|
||||||
|
InitialExecutionProjectionStore(path,resolver).project_from_source_capture(
|
||||||
|
TASK_ID,recovery_request(result,verification)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"events",
|
||||||
|
[
|
||||||
|
["TASK_CLAIMED","RESULT_SUBMITTED","TASK_VERIFIED"],
|
||||||
|
["TASK_CLAIMED","TASK_HEARTBEAT","RESULT_SUBMITTED","TASK_VERIFIED","TASK_CLAIMED"],
|
||||||
|
["TASK_CLAIMED","TASK_HEARTBEAT","TASK_VERIFIED","RESULT_SUBMITTED"],
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_source_capture_recovery_rejects_invalid_event_chain(tmp_path,events):
|
||||||
|
path,result,verification=make_source_capture_rework_db(tmp_path,events=events)
|
||||||
|
with pytest.raises(
|
||||||
|
InitialExecutionProjectionError,match="SOURCE_CAPTURE_RECOVERY_EVENT_CHAIN_INVALID"
|
||||||
|
):
|
||||||
|
InitialExecutionProjectionStore(path,resolver).project_from_source_capture(
|
||||||
|
TASK_ID,recovery_request(result,verification)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_source_capture_recovery_rejects_execution_digest_drift(tmp_path):
|
||||||
|
path,result,verification=make_source_capture_rework_db(tmp_path)
|
||||||
|
with pytest.raises(InitialExecutionProjectionError,match="EXECUTION_DIGEST_DRIFT"):
|
||||||
|
InitialExecutionProjectionStore(path,resolver).project_from_source_capture(
|
||||||
|
TASK_ID,recovery_request(result,verification,digest="f"*64)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_source_capture_recovery_request_is_strict(tmp_path):
|
||||||
|
_path,result,verification=make_source_capture_rework_db(tmp_path)
|
||||||
|
data=recovery_request(result,verification).public()
|
||||||
|
data["metadata_patch"]={"execution":"forbidden"}
|
||||||
|
with pytest.raises(InitialExecutionProjectionError,match="FIELDS_INVALID"):
|
||||||
|
SourceCaptureProjectionRecoveryRequest.from_mapping(data)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"kwargs,error",
|
||||||
|
[
|
||||||
|
({"claimed_by":"worker-a"},"TASK_ALREADY_CLAIMED"),
|
||||||
|
({"lease":True},"ACTIVE_LEASE_DENY"),
|
||||||
|
({"metadata":{"execution":{}}},"TASK_EXECUTION_ALREADY_PROJECTED"),
|
||||||
|
({"caps":["git.read"]},"TASK_INITIAL_CAPABILITIES_MUST_BE_EMPTY"),
|
||||||
|
({"event":"TASK_CLAIMED"},"TASK_INITIAL_EVENT_CHAIN_REQUIRED"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_initial_projection_rejects_noninitial_state(tmp_path,kwargs,error):
|
||||||
|
path=make_db(tmp_path,**kwargs)
|
||||||
|
with pytest.raises(InitialExecutionProjectionError,match=error):
|
||||||
|
InitialExecutionProjectionStore(path,resolver).project(TASK_ID,request())
|
||||||
|
|
||||||
|
|
||||||
|
def test_initial_projection_rejects_binding_drift(tmp_path):
|
||||||
|
path=make_db(tmp_path)
|
||||||
|
def wrong(*_args):
|
||||||
|
return envelope(job_id="job-other")
|
||||||
|
with pytest.raises(InitialExecutionProjectionError,match="GOVERNED_BINDING_CONFLICT"):
|
||||||
|
InitialExecutionProjectionStore(path,wrong).project(TASK_ID,request())
|
||||||
|
|
||||||
|
|
||||||
|
def test_initial_projection_rejects_nonruntime_envelope(tmp_path):
|
||||||
|
path=make_db(tmp_path)
|
||||||
|
def wrong(*_args):
|
||||||
|
return envelope(action="source_capture",capability="git.read")
|
||||||
|
with pytest.raises(InitialExecutionProjectionError):
|
||||||
|
InitialExecutionProjectionStore(path,wrong).project(TASK_ID,request())
|
||||||
|
|
||||||
|
|
||||||
|
def test_initial_projection_rejects_execution_digest_drift(tmp_path):
|
||||||
|
path=make_db(tmp_path)
|
||||||
|
with pytest.raises(InitialExecutionProjectionError,match="EXECUTION_DIGEST_DRIFT"):
|
||||||
|
InitialExecutionProjectionStore(path,resolver).project(
|
||||||
|
TASK_ID,request(digest="f"*64)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_request_is_narrow_and_initial_state_bound():
|
||||||
|
data=request().public()
|
||||||
|
data["arbitrary_metadata"]={"x":1}
|
||||||
|
with pytest.raises(InitialExecutionProjectionError,match="FIELDS_INVALID"):
|
||||||
|
InitialExecutionProjectionRequest.from_mapping(data)
|
||||||
|
data=request().public();data["expected_attempt"]=1
|
||||||
|
with pytest.raises(InitialExecutionProjectionError,match="INITIAL_STATE_REQUIRED"):
|
||||||
|
InitialExecutionProjectionRequest.from_mapping(data)
|
||||||
|
|
||||||
|
|
||||||
|
def test_initial_projection_unit_is_unprivileged_and_owner_credential_bound():
|
||||||
|
unit=(ROOT/"deploy/systemd/aios-agent-task-initial-execution-projection.service").read_text()
|
||||||
|
assert "User=aios-agent-task" in unit
|
||||||
|
assert "Group=aios-agent-task" in unit
|
||||||
|
assert "LoadCredential=agent_task_control_token:" in unit
|
||||||
|
assert "Environment=AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892" in unit
|
||||||
|
assert "RestrictAddressFamilies=AF_UNIX AF_INET" in unit
|
||||||
|
assert "ReadWritePaths=/var/lib/aios-agent-task" in unit
|
||||||
|
assert "User=root" not in unit
|
||||||
@@ -0,0 +1,117 @@
|
|||||||
|
from pathlib import Path
|
||||||
|
import hashlib, json, sqlite3, sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0,str(ROOT/"active-runtime/app"))
|
||||||
|
from aios_agent_task_retry_extension import RetryExtensionRequest, RetryExtensionStore, RetryExtensionError
|
||||||
|
|
||||||
|
AUTHORIZATION_ID="owner-scheme-a-20260817"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _authorization_id(monkeypatch):
|
||||||
|
monkeypatch.setenv("AIOS_AGENT_TASK_AUTHORIZATION_ID", AUTHORIZATION_ID)
|
||||||
|
|
||||||
|
|
||||||
|
def canon(v):
|
||||||
|
return hashlib.sha256(json.dumps(v,sort_keys=True,separators=(",",":")).encode()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def db(tmp_path):
|
||||||
|
p=tmp_path/"state.sqlite3"
|
||||||
|
c=sqlite3.connect(p)
|
||||||
|
c.executescript("""
|
||||||
|
CREATE TABLE tasks(
|
||||||
|
task_id TEXT PRIMARY KEY,status TEXT,attempt INTEGER,max_attempts INTEGER,revision INTEGER,
|
||||||
|
claimed_by TEXT,lease_token_hash TEXT,lease_expires_at TEXT,verification_json TEXT,result_json TEXT
|
||||||
|
);
|
||||||
|
CREATE TABLE task_events(
|
||||||
|
event_id INTEGER PRIMARY KEY AUTOINCREMENT,task_id TEXT,event_type TEXT,agent_id TEXT,
|
||||||
|
payload_json TEXT,created_at TEXT
|
||||||
|
);
|
||||||
|
""")
|
||||||
|
verification={"decision":"NEEDS_REWORK","reason":"BASE_OID_BINDING_INVALID"}
|
||||||
|
c.execute("INSERT INTO tasks VALUES (?,?,?,?,?,?,?,?,?,?)",
|
||||||
|
("task-a","FAILED",10,10,41,None,None,None,json.dumps(verification),json.dumps({"status":"FAILED"})))
|
||||||
|
c.commit(); c.close()
|
||||||
|
return p,verification
|
||||||
|
|
||||||
|
|
||||||
|
def request(v):
|
||||||
|
return RetryExtensionRequest.from_mapping({
|
||||||
|
"expected_status":"FAILED","expected_revision":41,"expected_attempt":10,"expected_max_attempts":10,
|
||||||
|
"next_max_attempts":11,"previous_verification_sha256":canon(v),
|
||||||
|
"authorization_id":AUTHORIZATION_ID,"idempotency_key":"retry-task-a-attempt11",
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def test_retry_extension_cas_and_audit(tmp_path):
|
||||||
|
p,v=db(tmp_path)
|
||||||
|
out=RetryExtensionStore(p).extend("task-a",request(v))
|
||||||
|
assert out["status"]=="QUEUED" and out["max_attempts"]==11 and out["revision"]==42
|
||||||
|
c=sqlite3.connect(p)
|
||||||
|
assert c.execute("select status,attempt,max_attempts,revision from tasks").fetchone()==("QUEUED",10,11,42)
|
||||||
|
event=c.execute("select event_type,payload_json from task_events").fetchone()
|
||||||
|
assert event[0]=="TASK_RETRY_EXTENSION_AUTHORIZED"
|
||||||
|
assert json.loads(event[1])["authorization_id"]=="owner-scheme-a-20260817"
|
||||||
|
|
||||||
|
|
||||||
|
def test_retry_extension_idempotent_replay(tmp_path):
|
||||||
|
p,v=db(tmp_path); store=RetryExtensionStore(p); req=request(v)
|
||||||
|
first=store.extend("task-a",req); second=store.extend("task-a",req)
|
||||||
|
assert first["idempotent"] is False and second["idempotent"] is True
|
||||||
|
c=sqlite3.connect(p)
|
||||||
|
assert c.execute("select count(*) from task_retry_extensions").fetchone()[0]==1
|
||||||
|
|
||||||
|
|
||||||
|
def test_retry_extension_normalizes_stale_terminal_claim_without_active_lease(tmp_path):
|
||||||
|
p,v=db(tmp_path)
|
||||||
|
c=sqlite3.connect(p)
|
||||||
|
c.execute("update tasks set claimed_by='previous-worker'")
|
||||||
|
c.commit(); c.close()
|
||||||
|
out=RetryExtensionStore(p).extend("task-a",request(v))
|
||||||
|
assert out["status"]=="QUEUED"
|
||||||
|
c=sqlite3.connect(p)
|
||||||
|
assert c.execute(
|
||||||
|
"select claimed_by,lease_token_hash,lease_expires_at from tasks"
|
||||||
|
).fetchone()==(None,None,None)
|
||||||
|
payload=json.loads(c.execute("select payload_json from task_events").fetchone()[0])
|
||||||
|
assert payload["previous_claimed_by"]=="previous-worker"
|
||||||
|
assert payload["stale_terminal_claim_normalized"] is True
|
||||||
|
assert c.execute("select count(*) from task_events").fetchone()[0]==1
|
||||||
|
|
||||||
|
|
||||||
|
def test_retry_extension_verification_drift_denied(tmp_path):
|
||||||
|
p,v=db(tmp_path)
|
||||||
|
bad=request({"decision":"NEEDS_REWORK","reason":"other"})
|
||||||
|
with pytest.raises(RetryExtensionError,match="RETRY_EXTENSION_VERIFICATION_DRIFT"):
|
||||||
|
RetryExtensionStore(p).extend("task-a",bad)
|
||||||
|
|
||||||
|
|
||||||
|
def test_retry_extension_authorization_id_mismatch_rejected(tmp_path, monkeypatch):
|
||||||
|
p,v=db(tmp_path)
|
||||||
|
monkeypatch.setenv("AIOS_AGENT_TASK_AUTHORIZATION_ID", "other-owner-id")
|
||||||
|
with pytest.raises(RetryExtensionError,match="RETRY_EXTENSION_AUTHORIZATION_ID_MISMATCH"):
|
||||||
|
RetryExtensionStore(p).extend("task-a",request(v))
|
||||||
|
|
||||||
|
|
||||||
|
def test_retry_extension_bounded_one_step():
|
||||||
|
v={"decision":"NEEDS_REWORK"}
|
||||||
|
data={
|
||||||
|
"expected_status":"FAILED","expected_revision":41,"expected_attempt":10,"expected_max_attempts":10,
|
||||||
|
"next_max_attempts":12,"previous_verification_sha256":canon(v),
|
||||||
|
"authorization_id":"owner-scheme-a-20260817","idempotency_key":"retry-task-a-attempt11",
|
||||||
|
}
|
||||||
|
with pytest.raises(RetryExtensionError,match="RETRY_EXTENSION_MUST_BE_BOUNDED_ONE_STEP"):
|
||||||
|
RetryExtensionRequest.from_mapping(data)
|
||||||
|
|
||||||
|
|
||||||
|
def test_retry_extension_unit_is_unprivileged_and_requires_owner_binding():
|
||||||
|
unit=(ROOT/"deploy/systemd/aios-agent-task-retry-extension.service").read_text()
|
||||||
|
assert "User=aios-agent-task" in unit
|
||||||
|
assert "Group=aios-agent-task" in unit
|
||||||
|
assert "LoadCredential=agent_task_control_token:" in unit
|
||||||
|
assert "EnvironmentFile=/etc/aios-gitops/agent-task-retry-extension.env" in unit
|
||||||
|
assert "User=root" not in unit
|
||||||
@@ -0,0 +1,188 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0,str(ROOT/"active-runtime/app"))
|
||||||
|
|
||||||
|
from aios_agent_task_initial_execution_projection import InitialExecutionProjectionError
|
||||||
|
from aios_agent_task_supervisor_coordinator_projection import (
|
||||||
|
COORDINATOR_CAPABILITIES,
|
||||||
|
RUNTIME_CAPABILITY,
|
||||||
|
SupervisorCoordinatorProjectionRequest,
|
||||||
|
SupervisorCoordinatorProjectionStore,
|
||||||
|
)
|
||||||
|
from aios_online_agent_execution import (
|
||||||
|
GovernedExecutionBinding,
|
||||||
|
project_execution_envelope,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
TASK="task-supervisor-coordinator-a"
|
||||||
|
JOB="job-supervisor-coordinator-a"
|
||||||
|
BASE="a"*40
|
||||||
|
BRANCH="agent/task-supervisor-coordinator-a"
|
||||||
|
BINDING_KEY="task-supervisor-coordinator-binding-v1"
|
||||||
|
AUTH="owner-supervisor-coordinator-projection-recovery-20260820"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def authorization(monkeypatch):
|
||||||
|
monkeypatch.setenv(
|
||||||
|
"AIOS_AGENT_TASK_INITIAL_EXECUTION_PROJECTION_AUTHORIZATION_ID",AUTH
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def sha(value):
|
||||||
|
return hashlib.sha256(
|
||||||
|
json.dumps(value,sort_keys=True,separators=(",",":")).encode()
|
||||||
|
).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def binding():
|
||||||
|
return GovernedExecutionBinding.from_mapping({
|
||||||
|
"task_id":TASK,"job_id":JOB,"repository":"aios/aios",
|
||||||
|
"base_oid":BASE,"expected_branch_oid":BASE,"target_branch":BRANCH,
|
||||||
|
"branch_mode":"CREATE_NEW","generation":1,"idempotency_key":BINDING_KEY,
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def runtime_input():
|
||||||
|
return {
|
||||||
|
"goal":"Durably supervise release preparation",
|
||||||
|
"instructions":"Wait for exact connector ACK and preserve the same lineage.",
|
||||||
|
"success_criteria":[{"id":"heartbeat","description":"Coordinator heartbeat"}],
|
||||||
|
"constraints":{},
|
||||||
|
"scope":{"scope_type":"legacy","tenant_id":None,"project_id":None},
|
||||||
|
"trace_id":TASK,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def envelope(capabilities):
|
||||||
|
return project_execution_envelope(binding(),{
|
||||||
|
"action":"agent_runtime","adapter":"openclaw",
|
||||||
|
"capability_scope":capabilities,"runtime_input":runtime_input(),
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def planner_result(old_digest,*,access=False):
|
||||||
|
return {
|
||||||
|
"task_id":TASK,"job_id":JOB,"repository":"aios/aios",
|
||||||
|
"base_oid":BASE,"expected_branch_oid":BASE,"target_branch":BRANCH,
|
||||||
|
"generation":1,"status":"SUCCEEDED","execution_mode":"agent_runtime",
|
||||||
|
"agent_runtime":"openclaw","execution_digest":old_digest,
|
||||||
|
"control_plane_access_granted_to_runtime":access,
|
||||||
|
"credential_material_exposed":False,"secret_material_exposed":False,
|
||||||
|
"changed_files":[],"writer_lease":"NONE","direct_main_write":False,
|
||||||
|
"production_write":False,"force_push":False,"protected_ref_bypass":False,
|
||||||
|
"classification":"EXTERNAL_AGENT_RUNTIME_SUCCEEDED",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def make_db(tmp_path,*,access=False,heartbeat=True):
|
||||||
|
tmp_path.mkdir(parents=True,exist_ok=True)
|
||||||
|
path=tmp_path/"state.sqlite3";con=sqlite3.connect(path)
|
||||||
|
con.executescript("""
|
||||||
|
CREATE TABLE tasks(
|
||||||
|
task_id TEXT PRIMARY KEY,goal TEXT NOT NULL,instructions TEXT NOT NULL,
|
||||||
|
success_criteria_json TEXT NOT NULL,required_capabilities_json TEXT NOT NULL,
|
||||||
|
constraints_json TEXT NOT NULL,metadata_json TEXT NOT NULL,deadline TEXT,
|
||||||
|
priority INTEGER NOT NULL,max_attempts INTEGER NOT NULL,attempt INTEGER NOT NULL,
|
||||||
|
status TEXT NOT NULL,claimed_by TEXT,lease_token_hash TEXT,lease_expires_at TEXT,
|
||||||
|
result_json TEXT,result_digest TEXT,result_idempotency_key TEXT,
|
||||||
|
verification_json TEXT,revision INTEGER NOT NULL,created_at TEXT,updated_at TEXT
|
||||||
|
);
|
||||||
|
CREATE TABLE events(
|
||||||
|
event_id INTEGER PRIMARY KEY AUTOINCREMENT,task_id TEXT,agent_id TEXT,
|
||||||
|
event_type TEXT NOT NULL,payload_json TEXT NOT NULL,created_at TEXT NOT NULL
|
||||||
|
);
|
||||||
|
""")
|
||||||
|
old=envelope([RUNTIME_CAPABILITY]);result=planner_result(old["execution_digest"],access=access)
|
||||||
|
verification={"decision":"NEEDS_REWORK","reason":"HOST_COORDINATOR_REQUIRED"}
|
||||||
|
metadata={"execution":old,"execution_projection":{
|
||||||
|
"schema":"aios.online-agent.task-projection.v1","task_id":TASK,
|
||||||
|
"job_id":JOB,"execution_digest":old["execution_digest"],
|
||||||
|
}}
|
||||||
|
con.execute("INSERT INTO tasks VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",(
|
||||||
|
TASK,runtime_input()["goal"],runtime_input()["instructions"],
|
||||||
|
json.dumps(runtime_input()["success_criteria"]),json.dumps([RUNTIME_CAPABILITY]),
|
||||||
|
"{}",json.dumps(metadata,sort_keys=True,separators=(",",":")),None,100,20,2,
|
||||||
|
"QUEUED",None,None,None,json.dumps(result,sort_keys=True,separators=(",",":")),
|
||||||
|
sha(result),"planner-result-v1",json.dumps(verification,sort_keys=True,separators=(",",":")),
|
||||||
|
11,"2026-08-20T00:00:00Z","2026-08-20T00:00:00Z",
|
||||||
|
))
|
||||||
|
events=["TASK_CREATED","TASK_INITIAL_EXECUTION_PROJECTION_AUTHORIZED","TASK_CLAIMED"]
|
||||||
|
if heartbeat: events.append("TASK_HEARTBEAT")
|
||||||
|
events += ["RESULT_SUBMITTED","TASK_VERIFIED"]
|
||||||
|
for event in events:
|
||||||
|
payload={"decision":"NEEDS_REWORK","next_state":"QUEUED"} if event=="TASK_VERIFIED" else {}
|
||||||
|
con.execute("INSERT INTO events(task_id,agent_id,event_type,payload_json,created_at) VALUES (?,?,?,?,?)",(TASK,"worker",event,json.dumps(payload),"2026-08-20T00:00:00Z"))
|
||||||
|
con.commit();con.close()
|
||||||
|
return path,metadata,result,verification,old
|
||||||
|
|
||||||
|
|
||||||
|
def request(metadata,result,verification,old,new):
|
||||||
|
return SupervisorCoordinatorProjectionRequest.from_mapping({
|
||||||
|
"expected_status":"QUEUED","expected_revision":11,"expected_attempt":2,
|
||||||
|
"expected_metadata_sha256":sha(metadata),
|
||||||
|
"expected_required_capabilities_sha256":sha([RUNTIME_CAPABILITY]),
|
||||||
|
"previous_verification_sha256":sha(verification),
|
||||||
|
"expected_planner_result_sha256":sha(result),
|
||||||
|
"expected_old_execution_digest":old["execution_digest"],
|
||||||
|
"expected_new_execution_digest":new["execution_digest"],
|
||||||
|
"expected_job_id":JOB,"authorization_id":AUTH,
|
||||||
|
"idempotency_key":"task-supervisor-coordinator-projection-v1",
|
||||||
|
"reason":"credential-isolated host coordinator required",
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def test_projects_exact_coordinator_capabilities_and_preserves_planner_evidence(tmp_path):
|
||||||
|
path,metadata,result,verification,old=make_db(tmp_path)
|
||||||
|
new=envelope(COORDINATOR_CAPABILITIES)
|
||||||
|
store=SupervisorCoordinatorProjectionStore(path,lambda task,job,runtime:new)
|
||||||
|
response=store.project(TASK,request(metadata,result,verification,old,new))
|
||||||
|
assert response["revision"]==12 and response["attempt"]==2
|
||||||
|
assert response["required_capabilities"]==COORDINATOR_CAPABILITIES
|
||||||
|
assert response["credential_forwarded_to_planner"] is False
|
||||||
|
con=sqlite3.connect(path)
|
||||||
|
row=con.execute("select status,revision,metadata_json,required_capabilities_json,result_json from tasks").fetchone()
|
||||||
|
assert row[:2]==("QUEUED",12)
|
||||||
|
assert json.loads(row[2])["execution"]["execution_digest"]==new["execution_digest"]
|
||||||
|
assert json.loads(row[3])==COORDINATOR_CAPABILITIES
|
||||||
|
assert json.loads(row[4])==result
|
||||||
|
assert con.execute("select event_type from events order by event_id desc limit 1").fetchone()[0]=="TASK_SUPERVISOR_COORDINATOR_PROJECTION_AUTHORIZED"
|
||||||
|
|
||||||
|
|
||||||
|
def test_exact_replay_is_idempotent(tmp_path):
|
||||||
|
path,metadata,result,verification,old=make_db(tmp_path);new=envelope(COORDINATOR_CAPABILITIES)
|
||||||
|
store=SupervisorCoordinatorProjectionStore(path,lambda task,job,runtime:new)
|
||||||
|
req=request(metadata,result,verification,old,new)
|
||||||
|
assert store.project(TASK,req)["idempotent"] is False
|
||||||
|
assert store.project(TASK,req)["idempotent"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_rejects_planner_with_control_access_or_missing_heartbeat(tmp_path):
|
||||||
|
path,metadata,result,verification,old=make_db(tmp_path,access=True);new=envelope(COORDINATOR_CAPABILITIES)
|
||||||
|
with pytest.raises(InitialExecutionProjectionError,match="RESULT_CONTRACT_INVALID"):
|
||||||
|
SupervisorCoordinatorProjectionStore(path,lambda *args:new).project(
|
||||||
|
TASK,request(metadata,result,verification,old,new)
|
||||||
|
)
|
||||||
|
path,metadata,result,verification,old=make_db(tmp_path/"second",heartbeat=False)
|
||||||
|
with pytest.raises(InitialExecutionProjectionError,match="EVENT_CHAIN_INVALID"):
|
||||||
|
SupervisorCoordinatorProjectionStore(path,lambda *args:new).project(
|
||||||
|
TASK,request(metadata,result,verification,old,new)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_request_rejects_arbitrary_metadata_patch(tmp_path):
|
||||||
|
path,metadata,result,verification,old=make_db(tmp_path);new=envelope(COORDINATOR_CAPABILITIES)
|
||||||
|
value=request(metadata,result,verification,old,new).public();value["metadata_patch"]={}
|
||||||
|
with pytest.raises(InitialExecutionProjectionError,match="FIELDS_INVALID"):
|
||||||
|
SupervisorCoordinatorProjectionRequest.from_mapping(value)
|
||||||
@@ -0,0 +1,252 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
APP=ROOT/"active-runtime/app"
|
||||||
|
sys.path.insert(0,str(APP))
|
||||||
|
|
||||||
|
import aios_all_modified_files_packager as packager
|
||||||
|
|
||||||
|
|
||||||
|
def init_repo(tmp_path: Path) -> tuple[Path, Path, str]:
|
||||||
|
bare=tmp_path/"repo.git"
|
||||||
|
work=tmp_path/"work"
|
||||||
|
subprocess.run(["git","init","--bare",str(bare)],check=True,capture_output=True)
|
||||||
|
subprocess.run(["git","clone",str(bare),str(work)],check=True,capture_output=True)
|
||||||
|
subprocess.run(["git","-C",str(work),"config","user.email","test@example.invalid"],check=True)
|
||||||
|
subprocess.run(["git","-C",str(work),"config","user.name","AIOS Test"],check=True)
|
||||||
|
(work/"README.md").write_text("base\n",encoding="utf-8")
|
||||||
|
subprocess.run(["git","-C",str(work),"add","README.md"],check=True)
|
||||||
|
subprocess.run(["git","-C",str(work),"commit","-m","base"],check=True,capture_output=True)
|
||||||
|
subprocess.run(["git","-C",str(work),"push","origin","HEAD:refs/heads/main"],check=True,capture_output=True)
|
||||||
|
base=subprocess.check_output(["git","-C",str(work),"rev-parse","HEAD"],text=True).strip()
|
||||||
|
return bare,work,base
|
||||||
|
|
||||||
|
|
||||||
|
def commit(work: Path, message: str) -> str:
|
||||||
|
subprocess.run(["git","-C",str(work),"add","-A"],check=True)
|
||||||
|
subprocess.run(["git","-C",str(work),"commit","-m",message],check=True,capture_output=True)
|
||||||
|
candidate=subprocess.check_output(["git","-C",str(work),"rev-parse","HEAD"],text=True).strip()
|
||||||
|
subprocess.run(["git","-C",str(work),"push","origin","HEAD:refs/heads/main"],check=True,capture_output=True)
|
||||||
|
return candidate
|
||||||
|
|
||||||
|
|
||||||
|
def real_pem_header() -> bytes:
|
||||||
|
return b"-----" + b"BEGIN " + b"PRIVATE KEY" + b"-----"
|
||||||
|
|
||||||
|
|
||||||
|
def bearer_value() -> bytes:
|
||||||
|
return b"Authorization" + b": Bearer " + (b"a"*40)
|
||||||
|
|
||||||
|
|
||||||
|
def key_value_secret() -> bytes:
|
||||||
|
return b"api_key" + b"=" + (b"x"*24)
|
||||||
|
|
||||||
|
|
||||||
|
def test_plain_private_key_words_are_not_secret_findings():
|
||||||
|
for value in (
|
||||||
|
b"BEGIN PRIVATE KEY",
|
||||||
|
b"PRIVATE KEY",
|
||||||
|
b"PEM private-key header",
|
||||||
|
b"assert 'BEGIN PRIVATE KEY' in message",
|
||||||
|
):
|
||||||
|
assert packager.secret_finding(value) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_complete_pem_private_key_header_is_rejected():
|
||||||
|
assert packager.secret_finding(real_pem_header())=="PEM_PRIVATE_KEY"
|
||||||
|
assert packager.secret_finding(
|
||||||
|
b"-----BEGIN OPENSSH " + b"PRIVATE KEY-----"
|
||||||
|
)=="PEM_PRIVATE_KEY"
|
||||||
|
|
||||||
|
|
||||||
|
def test_bearer_and_key_value_secret_detection_remains_fail_closed():
|
||||||
|
assert packager.secret_finding(bearer_value())=="AUTHORIZATION_BEARER"
|
||||||
|
assert packager.secret_finding(key_value_secret())=="KEY_VALUE_SECRET"
|
||||||
|
|
||||||
|
|
||||||
|
def test_short_or_descriptive_secret_words_do_not_false_positive():
|
||||||
|
assert packager.secret_finding(b"secret=short") is None
|
||||||
|
assert packager.secret_finding(b"password field is validated") is None
|
||||||
|
assert packager.secret_finding(b"api_key example placeholder") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_end_to_end_bundle_accepts_literal_begin_private_key_text(tmp_path):
|
||||||
|
bare,work,base=init_repo(tmp_path)
|
||||||
|
(work/"tests").mkdir()
|
||||||
|
(work/"tests"/"test_text.py").write_text(
|
||||||
|
'ERROR = "BEGIN PRIVATE KEY is not a complete PEM header"\n',
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
(work/"docs").mkdir()
|
||||||
|
(work/"docs"/"note.md").write_text("Scanner regression.\n",encoding="utf-8")
|
||||||
|
candidate=commit(work,"candidate")
|
||||||
|
|
||||||
|
extra=tmp_path/"execution"
|
||||||
|
extra.mkdir()
|
||||||
|
(extra/"evidence.json").write_text('{"status":"PASS"}\n',encoding="utf-8")
|
||||||
|
out=tmp_path/"out"
|
||||||
|
|
||||||
|
summary=packager.build_bundle(
|
||||||
|
git_dir=bare,
|
||||||
|
base=base,
|
||||||
|
candidate=candidate,
|
||||||
|
task_id="r42-packager-test",
|
||||||
|
extra_dir=extra,
|
||||||
|
output_root=out,
|
||||||
|
now=datetime(2026,8,19,tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert summary["changed_file_count"]==2
|
||||||
|
assert summary["execution_file_count"]==1
|
||||||
|
assert summary["repository_document_count"]==1
|
||||||
|
assert summary["execution_document_count"]==0
|
||||||
|
bundle_path=Path(summary["bundle_path"])
|
||||||
|
assert bundle_path.is_file()
|
||||||
|
assert packager.sha256(bundle_path.read_bytes())==summary["bundle_sha256"]
|
||||||
|
|
||||||
|
with zipfile.ZipFile(bundle_path) as archive:
|
||||||
|
names=set(archive.namelist())
|
||||||
|
assert "repo/after/tests/test_text.py" in names
|
||||||
|
assert "repo/after/docs/note.md" in names
|
||||||
|
assert "execution/evidence.json" in names
|
||||||
|
assert "documents-index.json" in names
|
||||||
|
assert "all-modified-files-manifest.json" in names
|
||||||
|
assert "checksums.sha256" in names
|
||||||
|
manifest=json.loads(archive.read("all-modified-files-manifest.json"))
|
||||||
|
assert manifest["schema"]=="aios.all-modified-files.bundle.v1"
|
||||||
|
assert manifest["candidate_oid"]==candidate
|
||||||
|
|
||||||
|
packager.verify_checksums(out/"bundle")
|
||||||
|
|
||||||
|
|
||||||
|
def test_end_to_end_bundle_rejects_real_pem_header(tmp_path):
|
||||||
|
bare,work,base=init_repo(tmp_path)
|
||||||
|
(work/"credential.txt").write_bytes(
|
||||||
|
real_pem_header()+b"\nnot-a-real-key-body\n"
|
||||||
|
)
|
||||||
|
candidate=commit(work,"credential-like candidate")
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
packager.PackagerError,
|
||||||
|
match="PEM_PRIVATE_KEY:after:credential.txt",
|
||||||
|
):
|
||||||
|
packager.build_bundle(
|
||||||
|
git_dir=bare,
|
||||||
|
base=base,
|
||||||
|
candidate=candidate,
|
||||||
|
task_id="r42-packager-pem-reject",
|
||||||
|
output_root=tmp_path/"out",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_execution_evidence_bearer_is_rejected(tmp_path):
|
||||||
|
bare,work,base=init_repo(tmp_path)
|
||||||
|
(work/"normal.txt").write_text("normal\n",encoding="utf-8")
|
||||||
|
candidate=commit(work,"normal")
|
||||||
|
extra=tmp_path/"execution"
|
||||||
|
extra.mkdir()
|
||||||
|
(extra/"unsafe.txt").write_bytes(bearer_value()+b"\n")
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
packager.PackagerError,
|
||||||
|
match="AUTHORIZATION_BEARER:execution:unsafe.txt",
|
||||||
|
):
|
||||||
|
packager.build_bundle(
|
||||||
|
git_dir=bare,
|
||||||
|
base=base,
|
||||||
|
candidate=candidate,
|
||||||
|
task_id="r42-packager-bearer-reject",
|
||||||
|
extra_dir=extra,
|
||||||
|
output_root=tmp_path/"out",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_execution_symlink_is_rejected(tmp_path):
|
||||||
|
bare,work,base=init_repo(tmp_path)
|
||||||
|
(work/"normal.txt").write_text("normal\n",encoding="utf-8")
|
||||||
|
candidate=commit(work,"normal")
|
||||||
|
extra=tmp_path/"execution"
|
||||||
|
extra.mkdir()
|
||||||
|
target=extra/"target.txt"
|
||||||
|
target.write_text("safe\n",encoding="utf-8")
|
||||||
|
(extra/"link.txt").symlink_to(target)
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
packager.PackagerError,
|
||||||
|
match="EXTRA_FILE_SYMLINK_REJECTED",
|
||||||
|
):
|
||||||
|
packager.build_bundle(
|
||||||
|
git_dir=bare,
|
||||||
|
base=base,
|
||||||
|
candidate=candidate,
|
||||||
|
task_id="r42-packager-symlink-reject",
|
||||||
|
extra_dir=extra,
|
||||||
|
output_root=tmp_path/"out",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_non_ancestor_lineage_is_rejected(tmp_path):
|
||||||
|
bare,work,base=init_repo(tmp_path)
|
||||||
|
(work/"one.txt").write_text("one\n",encoding="utf-8")
|
||||||
|
first=commit(work,"first")
|
||||||
|
|
||||||
|
subprocess.run(["git","-C",str(work),"checkout","--detach",base],check=True,capture_output=True)
|
||||||
|
(work/"other.txt").write_text("other\n",encoding="utf-8")
|
||||||
|
subprocess.run(["git","-C",str(work),"add","other.txt"],check=True)
|
||||||
|
subprocess.run(["git","-C",str(work),"commit","-m","other"],check=True,capture_output=True)
|
||||||
|
other=subprocess.check_output(["git","-C",str(work),"rev-parse","HEAD"],text=True).strip()
|
||||||
|
subprocess.run(
|
||||||
|
["git","-C",str(work),"push","origin","HEAD:refs/heads/other"],
|
||||||
|
check=True,
|
||||||
|
capture_output=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(packager.PackagerError,match="LINEAGE_INVALID"):
|
||||||
|
packager.validate_lineage(bare,first,other)
|
||||||
|
|
||||||
|
|
||||||
|
def test_bundle_checksum_closure_detects_tamper(tmp_path):
|
||||||
|
bare,work,base=init_repo(tmp_path)
|
||||||
|
(work/"normal.txt").write_text("normal\n",encoding="utf-8")
|
||||||
|
candidate=commit(work,"normal")
|
||||||
|
out=tmp_path/"out"
|
||||||
|
packager.build_bundle(
|
||||||
|
git_dir=bare,
|
||||||
|
base=base,
|
||||||
|
candidate=candidate,
|
||||||
|
task_id="r42-packager-checksum",
|
||||||
|
output_root=out,
|
||||||
|
)
|
||||||
|
(out/"bundle"/"repo"/"after"/"normal.txt").write_text(
|
||||||
|
"tampered\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
with pytest.raises(
|
||||||
|
packager.PackagerError,
|
||||||
|
match="CHECKSUM_CLOSURE_INVALID",
|
||||||
|
):
|
||||||
|
packager.verify_checksums(out/"bundle")
|
||||||
|
|
||||||
|
|
||||||
|
def test_source_does_not_contain_complete_pem_header_literal():
|
||||||
|
source=(ROOT/"active-runtime/app/aios_all_modified_files_packager.py").read_bytes()
|
||||||
|
tests=(ROOT/"tests/test_all_modified_files_packager.py").read_bytes()
|
||||||
|
complete=real_pem_header()
|
||||||
|
assert complete not in source
|
||||||
|
assert complete not in tests
|
||||||
|
|
||||||
|
|
||||||
|
def test_document_path_contract():
|
||||||
|
assert packager.is_document_path("docs/spec.md")
|
||||||
|
assert packager.is_document_path("README")
|
||||||
|
assert packager.is_document_path("CHANGELOG.md")
|
||||||
|
assert not packager.is_document_path("active-runtime/app/main.py")
|
||||||
@@ -0,0 +1,157 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
APP=ROOT/"active-runtime/app"
|
||||||
|
sys.path.insert(0,str(APP))
|
||||||
|
|
||||||
|
import aios_autonomous_control_plane as acp
|
||||||
|
import aios_capability_contract as contract
|
||||||
|
import aios_release_online_agent_worker as worker
|
||||||
|
|
||||||
|
|
||||||
|
def _ready_probes():
|
||||||
|
return {
|
||||||
|
"credential_projection": lambda: True,
|
||||||
|
"trusted_builder": lambda: True,
|
||||||
|
"authenticated_probe": lambda: True,
|
||||||
|
"release_api_health": lambda: True,
|
||||||
|
"worker_liveness": lambda: True,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _task():
|
||||||
|
return {
|
||||||
|
"task_id":"r42-autonomous-release-admission-test",
|
||||||
|
"required_capabilities":["release:candidate"],
|
||||||
|
"constraints":{
|
||||||
|
"release_candidate":{
|
||||||
|
"candidate_id":"r42-release-candidate-v2r2c9-test",
|
||||||
|
"source_commit":"1"*40,
|
||||||
|
"version":"V2R2C9",
|
||||||
|
"projection_id":"projection-"+"a"*32,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_capability_contract_registry_is_closed_and_release_is_versioned():
|
||||||
|
status=contract.validate_capability_contract_registry()
|
||||||
|
assert status["valid"] is True
|
||||||
|
release=contract.capability_contract("release:candidate")
|
||||||
|
assert release is not None
|
||||||
|
assert release.min_version==1
|
||||||
|
assert release.max_version==1
|
||||||
|
assert release.permissions==("prepare_release",)
|
||||||
|
assert "credential_projection" in release.dependencies
|
||||||
|
assert "trusted_builder" in release.dependencies
|
||||||
|
assert "authenticated_probe" in release.health_requirements
|
||||||
|
assert "production_deploy" in release.forbidden
|
||||||
|
assert "runtime_switch" in release.forbidden
|
||||||
|
|
||||||
|
|
||||||
|
def test_unknown_agent_capability_cannot_register_or_match():
|
||||||
|
assert contract.registered_capability("release:future",worker=True) is False
|
||||||
|
assert contract.registered_capability("release:*",worker=True) is False
|
||||||
|
assert contract.has_capabilities(["release:future"],["release:future"]) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_admission_requires_all_release_execution_readiness():
|
||||||
|
decision=acp.runtime_admission(
|
||||||
|
worker_capabilities=["release:candidate"],
|
||||||
|
required_capabilities=["release:candidate"],
|
||||||
|
requested_operations=["prepare_release"],
|
||||||
|
probes=_ready_probes(),
|
||||||
|
)
|
||||||
|
assert decision.admitted is True
|
||||||
|
assert decision.state=="EXECUTION_READY"
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_admission_blocks_credential_failure_before_execution():
|
||||||
|
probes=_ready_probes()
|
||||||
|
probes["credential_projection"]=lambda:False
|
||||||
|
probes["authenticated_probe"]=lambda:False
|
||||||
|
decision=acp.runtime_admission(
|
||||||
|
worker_capabilities=["release:candidate"],
|
||||||
|
required_capabilities=["release:candidate"],
|
||||||
|
requested_operations=["prepare_release"],
|
||||||
|
probes=probes,
|
||||||
|
)
|
||||||
|
assert decision.admitted is False
|
||||||
|
assert decision.state=="EXECUTION_READY_FAILED"
|
||||||
|
assert decision.failure_class==acp.FailureClass.CREDENTIAL_PROJECTION_ERROR
|
||||||
|
assert decision.repair_level==acp.RepairLevel.AUTO_REPAIR_TASK
|
||||||
|
|
||||||
|
|
||||||
|
def test_forbidden_release_operation_requires_owner_boundary():
|
||||||
|
decision=acp.runtime_admission(
|
||||||
|
worker_capabilities=["release:candidate"],
|
||||||
|
required_capabilities=["release:candidate"],
|
||||||
|
requested_operations=["production_deploy"],
|
||||||
|
probes=_ready_probes(),
|
||||||
|
)
|
||||||
|
assert decision.admitted is False
|
||||||
|
assert decision.failure_class==acp.FailureClass.POLICY_DENIED
|
||||||
|
assert decision.repair_level==acp.RepairLevel.OWNER_APPROVAL
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"error,phase,expected,level,consumes",
|
||||||
|
[
|
||||||
|
("Permission denied: gitops.tokens","release",acp.FailureClass.CREDENTIAL_PROJECTION_ERROR,acp.RepairLevel.AUTO_REPAIR_TASK,False),
|
||||||
|
("HTTP_TRANSPORT_UNAVAILABLE","network",acp.FailureClass.DEPENDENCY_UNAVAILABLE,acp.RepairLevel.AUTO_LOCAL,False),
|
||||||
|
("TASK_CAPABILITY_SCOPE_INVALID","validation",acp.FailureClass.CAPABILITY_CONTRACT_ERROR,acp.RepairLevel.AUTO_REPAIR_TASK,False),
|
||||||
|
("lease lost","heartbeat",acp.FailureClass.LEASE_LOST,acp.RepairLevel.AUTO_LOCAL,False),
|
||||||
|
("business permanent invalid release","release",acp.FailureClass.BUSINESS_PERMANENT,acp.RepairLevel.MANUAL,True),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_failure_classification_drives_repair_policy(error,phase,expected,level,consumes):
|
||||||
|
failure=acp.classify_failure(error,phase=phase)
|
||||||
|
assert failure==expected
|
||||||
|
repair=acp.repair_for_failure(failure)
|
||||||
|
assert repair.level==level
|
||||||
|
assert repair.consumes_business_attempt is consumes
|
||||||
|
|
||||||
|
|
||||||
|
def test_release_worker_does_not_claim_when_admission_is_not_ready(monkeypatch):
|
||||||
|
calls=[]
|
||||||
|
def fake_http(base,method,route,token,body=None,timeout=30):
|
||||||
|
calls.append((base,method,route,body))
|
||||||
|
if route=="/health":
|
||||||
|
return 200,{"status":"ok"}
|
||||||
|
if route=="/createReleaseCandidate" and body=={}:
|
||||||
|
return 500,{"error":"Permission denied: gitops.tokens"}
|
||||||
|
if route.endswith("/claim"):
|
||||||
|
raise AssertionError("claim must not happen before EXECUTION_READY")
|
||||||
|
raise AssertionError(route)
|
||||||
|
|
||||||
|
events=[]
|
||||||
|
monkeypatch.setattr(worker,"http_json",fake_http)
|
||||||
|
monkeypatch.setattr(worker,"audit",lambda event,**fields:events.append((event,fields)))
|
||||||
|
monkeypatch.setattr(worker,"write_liveness",lambda *args,**kwargs:None)
|
||||||
|
|
||||||
|
assert worker.execute_task(
|
||||||
|
_task(),
|
||||||
|
agent_token="a"*32,
|
||||||
|
release_token="r"*32,
|
||||||
|
) is False
|
||||||
|
assert not any(route.endswith("/claim") for _,_,route,_ in calls)
|
||||||
|
blocked=[item for item in events if item[0]=="TASK_ADMISSION_BLOCKED"]
|
||||||
|
assert len(blocked)==1
|
||||||
|
assert blocked[0][1]["admission"]["state"]=="EXECUTION_READY_FAILED"
|
||||||
|
|
||||||
|
|
||||||
|
def test_system_invariant_snapshot_is_degraded_when_dependency_is_missing():
|
||||||
|
probes=_ready_probes()
|
||||||
|
probes["trusted_builder"]=lambda:False
|
||||||
|
snapshot=acp.system_invariant_snapshot(
|
||||||
|
worker_capabilities=["release:candidate"],
|
||||||
|
required_capabilities=["release:candidate"],
|
||||||
|
probes=probes,
|
||||||
|
)
|
||||||
|
assert snapshot["status"]=="DEGRADED"
|
||||||
|
assert snapshot["execution_ready"] is False
|
||||||
@@ -0,0 +1,627 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
APP=ROOT/"active-runtime/app"
|
||||||
|
sys.path.insert(0,str(APP))
|
||||||
|
|
||||||
|
import aios_durable_owner_authorization as auth
|
||||||
|
|
||||||
|
|
||||||
|
NOW=datetime(2026,8,19,6,0,0,tzinfo=timezone.utc)
|
||||||
|
ACCOUNT="chatgpt-governed-implementer-singlehost"
|
||||||
|
|
||||||
|
|
||||||
|
def write_root_file(path: Path, value: object) -> Path:
|
||||||
|
path.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
path.write_text(json.dumps(value,sort_keys=True)+"\n",encoding="utf-8")
|
||||||
|
path.chmod(0o600)
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def policy_value(**overrides):
|
||||||
|
value={
|
||||||
|
"schema":auth.POLICY_SCHEMA,
|
||||||
|
"grant_id":"owner-permanent-preparation-20260819",
|
||||||
|
"owner_identity":"owner-singlehost",
|
||||||
|
"account":ACCOUNT,
|
||||||
|
"capabilities":[
|
||||||
|
"governed-task:project",
|
||||||
|
"release:candidate",
|
||||||
|
],
|
||||||
|
"issued_at":"2026-08-19T05:00:00Z",
|
||||||
|
"not_before":"2026-08-19T05:00:00Z",
|
||||||
|
"persistent_until_revoked":True,
|
||||||
|
"constraints":{
|
||||||
|
"repository":"aios/aios",
|
||||||
|
"allowed_target_prefix":"agent/",
|
||||||
|
"release_version_prefix":"V2R2C",
|
||||||
|
"max_task_risk":"engineering",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
value.update(overrides)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def load_policy(tmp_path,monkeypatch,**overrides):
|
||||||
|
path=write_root_file(tmp_path/"policy.json",policy_value(**overrides))
|
||||||
|
revocations=write_root_file(
|
||||||
|
tmp_path/"revocations.json",
|
||||||
|
{"revoked_grant_ids":[]},
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(auth,"validate_root_file",lambda path,**kwargs:path)
|
||||||
|
return auth.DurablePolicy.load(
|
||||||
|
path,
|
||||||
|
revocations,
|
||||||
|
now=NOW,
|
||||||
|
),path,revocations
|
||||||
|
|
||||||
|
|
||||||
|
def governed_request(**overrides):
|
||||||
|
value={
|
||||||
|
"task_id":"r42-owner-auth-projection-test-20260819",
|
||||||
|
"repository":"aios/aios",
|
||||||
|
"base_oid":"8"*40,
|
||||||
|
"expected_branch_oid":"8"*40,
|
||||||
|
"target_branch":"agent/r42-owner-auth-projection-test-20260819",
|
||||||
|
"generation":1,
|
||||||
|
"risk":"engineering",
|
||||||
|
"description":"project existing governed task",
|
||||||
|
"idempotency_key":"binding-owner-auth-projection-v1",
|
||||||
|
"branch_mode":"CREATE_NEW",
|
||||||
|
"priority":100,
|
||||||
|
"max_attempts":3,
|
||||||
|
"require_online_task_projection":True,
|
||||||
|
"online_task":{
|
||||||
|
"goal":"Analyze source read-only.",
|
||||||
|
"instructions":"Use source capture first.",
|
||||||
|
"success_criteria":[
|
||||||
|
{"id":"source","description":"Return source snapshot."}
|
||||||
|
],
|
||||||
|
"constraints":{"repository_mutation":"forbidden"},
|
||||||
|
"required_capabilities":["git.read","git.test"],
|
||||||
|
"priority":100,
|
||||||
|
"max_attempts":3,
|
||||||
|
},
|
||||||
|
"execution":{
|
||||||
|
"action":"source_capture",
|
||||||
|
"adapter":"source_capture",
|
||||||
|
"capability_scope":["git.read","git.test"],
|
||||||
|
"source_signals":["durable authorization projection"],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
value.update(overrides)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def release_request(**overrides):
|
||||||
|
value={
|
||||||
|
"candidate_id":"r42-pf1-release-candidate-20260819",
|
||||||
|
"source_commit":"8"*40,
|
||||||
|
"version":"V2R2C2",
|
||||||
|
}
|
||||||
|
value.update(overrides)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class FakeUpstream:
|
||||||
|
def __init__(self):
|
||||||
|
self.project_calls=[]
|
||||||
|
|
||||||
|
def project_governed_task(self,body):
|
||||||
|
self.project_calls.append(dict(body))
|
||||||
|
return 200,{
|
||||||
|
"task_id":body["task_id"],
|
||||||
|
"job_id":"job-"+"a"*32,
|
||||||
|
"state":"BOUND",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def broker(tmp_path,monkeypatch):
|
||||||
|
policy,path,revocations=load_policy(tmp_path,monkeypatch)
|
||||||
|
store=auth.ReceiptStore(tmp_path/"receipts.sqlite3")
|
||||||
|
upstream=FakeUpstream()
|
||||||
|
instance=auth.Broker(
|
||||||
|
policy_path=path,
|
||||||
|
revocation_path=revocations,
|
||||||
|
store=store,
|
||||||
|
upstream=upstream,
|
||||||
|
expected_account=ACCOUNT,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
auth.DurablePolicy,
|
||||||
|
"load",
|
||||||
|
classmethod(lambda cls,*args,**kwargs:policy),
|
||||||
|
)
|
||||||
|
return instance,upstream,store,policy
|
||||||
|
|
||||||
|
|
||||||
|
def test_allowed_persistent_capabilities_are_only_preparation_scopes():
|
||||||
|
assert auth.ALLOWED_PERSISTENT_CAPABILITIES=={
|
||||||
|
"governed-task:project",
|
||||||
|
"release:candidate",
|
||||||
|
}
|
||||||
|
assert auth.ALLOWED_PERSISTENT_CAPABILITIES.isdisjoint(
|
||||||
|
auth.FORBIDDEN_PERSISTENT_CAPABILITIES
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"capability",
|
||||||
|
sorted(auth.FORBIDDEN_PERSISTENT_CAPABILITIES),
|
||||||
|
)
|
||||||
|
def test_policy_rejects_high_risk_persistent_capability(
|
||||||
|
tmp_path,monkeypatch,capability
|
||||||
|
):
|
||||||
|
bad=policy_value(
|
||||||
|
capabilities=["governed-task:project",capability]
|
||||||
|
)
|
||||||
|
path=write_root_file(tmp_path/"policy.json",bad)
|
||||||
|
revocations=write_root_file(
|
||||||
|
tmp_path/"revocations.json",
|
||||||
|
{"revoked_grant_ids":[]},
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(auth,"validate_root_file",lambda path,**kwargs:path)
|
||||||
|
with pytest.raises(
|
||||||
|
auth.AuthorizationError,
|
||||||
|
match="POLICY_CAPABILITY_NOT_PERSISTABLE",
|
||||||
|
):
|
||||||
|
auth.DurablePolicy.load(path,revocations,now=NOW)
|
||||||
|
|
||||||
|
|
||||||
|
def test_policy_is_persistent_until_revoked(tmp_path,monkeypatch):
|
||||||
|
policy,_,_=load_policy(tmp_path,monkeypatch)
|
||||||
|
assert policy.persistent_until_revoked is True
|
||||||
|
assert policy.account==ACCOUNT
|
||||||
|
assert policy.capabilities=={
|
||||||
|
"governed-task:project",
|
||||||
|
"release:candidate",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_revocation_fails_closed(tmp_path,monkeypatch):
|
||||||
|
path=write_root_file(tmp_path/"policy.json",policy_value())
|
||||||
|
revocations=write_root_file(
|
||||||
|
tmp_path/"revocations.json",
|
||||||
|
{"revoked_grant_ids":["owner-permanent-preparation-20260819"]},
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(auth,"validate_root_file",lambda path,**kwargs:path)
|
||||||
|
with pytest.raises(auth.AuthorizationError,match="OWNER_GRANT_REVOKED"):
|
||||||
|
auth.DurablePolicy.load(path,revocations,now=NOW)
|
||||||
|
|
||||||
|
|
||||||
|
def test_future_policy_is_not_active(tmp_path,monkeypatch):
|
||||||
|
with pytest.raises(auth.AuthorizationError,match="POLICY_NOT_ACTIVE"):
|
||||||
|
load_policy(
|
||||||
|
tmp_path,
|
||||||
|
monkeypatch,
|
||||||
|
issued_at="2026-08-20T05:00:00Z",
|
||||||
|
not_before="2026-08-20T05:00:00Z",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_governed_projection_request_is_tightly_constrained(
|
||||||
|
tmp_path,monkeypatch
|
||||||
|
):
|
||||||
|
policy,_,_=load_policy(tmp_path,monkeypatch)
|
||||||
|
decision=policy.authorize(
|
||||||
|
"governed-task:project",
|
||||||
|
governed_request(),
|
||||||
|
)
|
||||||
|
assert decision["decision"]=="ALLOW"
|
||||||
|
assert decision["persistent_until_revoked"] is True
|
||||||
|
|
||||||
|
bad_caps=governed_request()
|
||||||
|
bad_caps["online_task"]={
|
||||||
|
**bad_caps["online_task"],
|
||||||
|
"required_capabilities":["release:candidate"],
|
||||||
|
}
|
||||||
|
bad_action=governed_request()
|
||||||
|
bad_action["execution"]={
|
||||||
|
**bad_action["execution"],
|
||||||
|
"action":"deterministic",
|
||||||
|
"adapter":"deterministic",
|
||||||
|
}
|
||||||
|
bad_scope=governed_request()
|
||||||
|
bad_scope["execution"]={
|
||||||
|
**bad_scope["execution"],
|
||||||
|
"capability_scope":["git.read"],
|
||||||
|
}
|
||||||
|
for bad in (
|
||||||
|
governed_request(repository="other/repo"),
|
||||||
|
governed_request(target_branch="main"),
|
||||||
|
governed_request(generation=2),
|
||||||
|
governed_request(risk="high"),
|
||||||
|
governed_request(expected_branch_oid="9"*40),
|
||||||
|
governed_request(branch_mode="CONTINUE_EXISTING"),
|
||||||
|
governed_request(priority=101),
|
||||||
|
governed_request(max_attempts=4),
|
||||||
|
governed_request(require_online_task_projection=False),
|
||||||
|
bad_caps,
|
||||||
|
bad_action,
|
||||||
|
bad_scope,
|
||||||
|
):
|
||||||
|
with pytest.raises(auth.AuthorizationError):
|
||||||
|
policy.authorize("governed-task:project",bad)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def test_write_capable_projection_is_allowed_only_as_source_capture_placeholder(
|
||||||
|
tmp_path,monkeypatch
|
||||||
|
):
|
||||||
|
policy,_,_=load_policy(tmp_path,monkeypatch)
|
||||||
|
request=governed_request()
|
||||||
|
request["online_task"]={
|
||||||
|
**request["online_task"],
|
||||||
|
"required_capabilities":[
|
||||||
|
"git.read",
|
||||||
|
"git.test",
|
||||||
|
"governed.write.branch",
|
||||||
|
"governed.write.file",
|
||||||
|
],
|
||||||
|
}
|
||||||
|
request["execution"]={
|
||||||
|
**request["execution"],
|
||||||
|
"capability_scope":[
|
||||||
|
"git.read",
|
||||||
|
"git.test",
|
||||||
|
"governed.write.branch",
|
||||||
|
"governed.write.file",
|
||||||
|
],
|
||||||
|
}
|
||||||
|
decision=policy.authorize("governed-task:project",request)
|
||||||
|
assert decision["decision"]=="ALLOW"
|
||||||
|
assert request["execution"]["action"]=="source_capture"
|
||||||
|
|
||||||
|
def test_release_candidate_request_is_tightly_constrained(
|
||||||
|
tmp_path,monkeypatch
|
||||||
|
):
|
||||||
|
policy,_,_=load_policy(tmp_path,monkeypatch)
|
||||||
|
decision=policy.authorize("release:candidate",release_request())
|
||||||
|
assert decision["decision"]=="ALLOW"
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
auth.AuthorizationError,
|
||||||
|
match="RELEASE_VERSION_NOT_AUTHORIZED",
|
||||||
|
):
|
||||||
|
policy.authorize(
|
||||||
|
"release:candidate",
|
||||||
|
release_request(version="V3R1C1"),
|
||||||
|
)
|
||||||
|
with pytest.raises(
|
||||||
|
auth.AuthorizationError,
|
||||||
|
match="RELEASE_CANDIDATE_FIELDS_INVALID",
|
||||||
|
):
|
||||||
|
policy.authorize(
|
||||||
|
"release:candidate",
|
||||||
|
{**release_request(),"execute":True},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_governed_task_projection_executes_canonical_upstream(
|
||||||
|
tmp_path,monkeypatch
|
||||||
|
):
|
||||||
|
instance,upstream,store,_=broker(tmp_path,monkeypatch)
|
||||||
|
request=governed_request()
|
||||||
|
response=instance.execute(
|
||||||
|
"governed-task:project",
|
||||||
|
request,
|
||||||
|
idempotency_key="owner-auth-projection-operation-0001",
|
||||||
|
)
|
||||||
|
assert len(upstream.project_calls)==1
|
||||||
|
assert upstream.project_calls[0]==request
|
||||||
|
assert response["operation_state"]=="EXECUTED"
|
||||||
|
assert response["external_capability_projection_required"] is False
|
||||||
|
assert response["separate_owner_release_approval_still_required"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_release_candidate_authorization_does_not_invent_builder_upstream(
|
||||||
|
tmp_path,monkeypatch
|
||||||
|
):
|
||||||
|
instance,upstream,store,_=broker(tmp_path,monkeypatch)
|
||||||
|
response=instance.execute(
|
||||||
|
"release:candidate",
|
||||||
|
release_request(),
|
||||||
|
idempotency_key="owner-auth-release-candidate-operation-0001",
|
||||||
|
)
|
||||||
|
assert upstream.project_calls==[]
|
||||||
|
assert response["operation_state"]==(
|
||||||
|
"AUTHORIZED_PENDING_EXTERNAL_CAPABILITY_PROJECTION"
|
||||||
|
)
|
||||||
|
assert response["upstream_status"] is None
|
||||||
|
assert response["upstream_response"] is None
|
||||||
|
assert response["external_capability_projection_required"] is True
|
||||||
|
assert response["projection_id"].startswith("projection-")
|
||||||
|
assert response["projection_state"]=="PENDING_EXTERNAL_CONNECTOR"
|
||||||
|
assert response["separate_owner_release_approval_still_required"] is True
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def test_release_authorization_creates_durable_projection_outbox(
|
||||||
|
tmp_path,monkeypatch
|
||||||
|
):
|
||||||
|
instance,_,store,policy=broker(tmp_path,monkeypatch)
|
||||||
|
request=release_request()
|
||||||
|
response=instance.execute(
|
||||||
|
"release:candidate",
|
||||||
|
request,
|
||||||
|
idempotency_key="owner-auth-release-outbox-0001",
|
||||||
|
)
|
||||||
|
projection=store.get_projection(response["projection_id"])
|
||||||
|
assert projection is not None
|
||||||
|
assert projection["schema"]==auth.PROJECTION_SCHEMA
|
||||||
|
assert projection["state"]=="PENDING_EXTERNAL_CONNECTOR"
|
||||||
|
assert projection["grant_id"]==policy.grant_id
|
||||||
|
assert projection["account"]==ACCOUNT
|
||||||
|
assert projection["capability"]=="release:candidate"
|
||||||
|
assert projection["request"]==request
|
||||||
|
assert projection["separate_owner_release_approval_still_required"] is True
|
||||||
|
|
||||||
|
listed=instance.projections()
|
||||||
|
assert listed["schema"]==auth.PROJECTION_LIST_SCHEMA
|
||||||
|
assert listed["count"]==1
|
||||||
|
assert listed["items"][0]["projection_id"]==response["projection_id"]
|
||||||
|
assert listed["external_connector_ack_supported"] is False
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def test_existing_pre_outbox_release_receipt_is_backfilled_on_exact_retry(
|
||||||
|
tmp_path,monkeypatch
|
||||||
|
):
|
||||||
|
instance,_,store,policy=broker(tmp_path,monkeypatch)
|
||||||
|
request=release_request()
|
||||||
|
decision=policy.authorize("release:candidate",request)
|
||||||
|
key="owner-auth-release-pre-outbox-0001"
|
||||||
|
legacy_response={
|
||||||
|
"schema":auth.RECEIPT_SCHEMA,
|
||||||
|
"grant_id":policy.grant_id,
|
||||||
|
"owner_identity":policy.owner_identity,
|
||||||
|
"account":policy.account,
|
||||||
|
"capability":"release:candidate",
|
||||||
|
"policy_sha256":policy.policy_sha256,
|
||||||
|
"request_sha256":decision["request_sha256"],
|
||||||
|
"idempotency_key":key,
|
||||||
|
"operation_state":"AUTHORIZED_PENDING_EXTERNAL_CAPABILITY_PROJECTION",
|
||||||
|
"upstream_status":None,
|
||||||
|
"upstream_response":None,
|
||||||
|
"external_capability_projection_required":True,
|
||||||
|
"separate_owner_release_approval_still_required":True,
|
||||||
|
}
|
||||||
|
store.put(
|
||||||
|
key,
|
||||||
|
"release:candidate",
|
||||||
|
decision["request_sha256"],
|
||||||
|
legacy_response,
|
||||||
|
)
|
||||||
|
response=instance.execute(
|
||||||
|
"release:candidate",
|
||||||
|
request,
|
||||||
|
idempotency_key=key,
|
||||||
|
)
|
||||||
|
assert response["projection_id"].startswith("projection-")
|
||||||
|
assert response["projection_state"]=="PENDING_EXTERNAL_CONNECTOR"
|
||||||
|
assert store.get_projection(response["projection_id"]) is not None
|
||||||
|
persisted=store.get(key)["response"]
|
||||||
|
assert persisted["projection_id"]==response["projection_id"]
|
||||||
|
|
||||||
|
def test_projection_id_is_deterministic_and_exact_retry_does_not_duplicate(
|
||||||
|
tmp_path,monkeypatch
|
||||||
|
):
|
||||||
|
instance,_,store,_=broker(tmp_path,monkeypatch)
|
||||||
|
key="owner-auth-release-outbox-0002"
|
||||||
|
first=instance.execute("release:candidate",release_request(),idempotency_key=key)
|
||||||
|
second=instance.execute("release:candidate",release_request(),idempotency_key=key)
|
||||||
|
assert second==first
|
||||||
|
con=sqlite3.connect(store.path)
|
||||||
|
rows=con.execute("select projection_id from projection_outbox").fetchall()
|
||||||
|
assert rows==[(first["projection_id"],)]
|
||||||
|
|
||||||
|
|
||||||
|
def test_projection_enqueue_recovers_crash_between_outbox_and_receipt(
|
||||||
|
tmp_path,monkeypatch
|
||||||
|
):
|
||||||
|
instance,_,store,policy=broker(tmp_path,monkeypatch)
|
||||||
|
request=release_request()
|
||||||
|
decision=policy.authorize("release:candidate",request)
|
||||||
|
key="owner-auth-release-outbox-crash-0001"
|
||||||
|
first=store.enqueue_projection(
|
||||||
|
idempotency_key=key,
|
||||||
|
grant_id=policy.grant_id,
|
||||||
|
account=policy.account,
|
||||||
|
capability="release:candidate",
|
||||||
|
policy_sha256=policy.policy_sha256,
|
||||||
|
request_sha256=decision["request_sha256"],
|
||||||
|
request=request,
|
||||||
|
)
|
||||||
|
# Simulate a later retry after the process crashed before receipt creation.
|
||||||
|
second=store.enqueue_projection(
|
||||||
|
idempotency_key=key,
|
||||||
|
grant_id=policy.grant_id,
|
||||||
|
account=policy.account,
|
||||||
|
capability="release:candidate",
|
||||||
|
policy_sha256=policy.policy_sha256,
|
||||||
|
request_sha256=decision["request_sha256"],
|
||||||
|
request=request,
|
||||||
|
)
|
||||||
|
assert second==first
|
||||||
|
|
||||||
|
|
||||||
|
def test_projection_idempotency_conflict_is_fail_closed(
|
||||||
|
tmp_path,monkeypatch
|
||||||
|
):
|
||||||
|
instance,_,store,policy=broker(tmp_path,monkeypatch)
|
||||||
|
first=release_request()
|
||||||
|
first_decision=policy.authorize("release:candidate",first)
|
||||||
|
key="owner-auth-release-outbox-conflict-0001"
|
||||||
|
store.enqueue_projection(
|
||||||
|
idempotency_key=key,
|
||||||
|
grant_id=policy.grant_id,
|
||||||
|
account=policy.account,
|
||||||
|
capability="release:candidate",
|
||||||
|
policy_sha256=policy.policy_sha256,
|
||||||
|
request_sha256=first_decision["request_sha256"],
|
||||||
|
request=first,
|
||||||
|
)
|
||||||
|
second=release_request(version="V2R2C3")
|
||||||
|
second_decision=policy.authorize("release:candidate",second)
|
||||||
|
with pytest.raises(auth.AuthorizationError,match="PROJECTION_IDEMPOTENCY_CONFLICT"):
|
||||||
|
store.enqueue_projection(
|
||||||
|
idempotency_key=key,
|
||||||
|
grant_id=policy.grant_id,
|
||||||
|
account=policy.account,
|
||||||
|
capability="release:candidate",
|
||||||
|
policy_sha256=policy.policy_sha256,
|
||||||
|
request_sha256=second_decision["request_sha256"],
|
||||||
|
request=second,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_revocation_blocks_projection_enumeration(tmp_path,monkeypatch):
|
||||||
|
instance,_,store,policy=broker(tmp_path,monkeypatch)
|
||||||
|
response=instance.execute(
|
||||||
|
"release:candidate",
|
||||||
|
release_request(),
|
||||||
|
idempotency_key="owner-auth-release-outbox-revoke-0001",
|
||||||
|
)
|
||||||
|
# Restore real policy loading and write a revocation for this grant.
|
||||||
|
monkeypatch.undo()
|
||||||
|
revocations=instance.revocation_path
|
||||||
|
revocations.write_text(
|
||||||
|
json.dumps({"revoked_grant_ids":[policy.grant_id]})+"\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
revocations.chmod(0o600)
|
||||||
|
monkeypatch.setattr(auth,"validate_root_file",lambda path,**kwargs:path)
|
||||||
|
with pytest.raises(auth.AuthorizationError,match="OWNER_GRANT_REVOKED"):
|
||||||
|
instance.projections()
|
||||||
|
# Evidence is retained; revocation does not erase history.
|
||||||
|
assert store.get_projection(response["projection_id"]) is not None
|
||||||
|
|
||||||
|
|
||||||
|
def test_projection_outbox_contains_no_credentials(tmp_path,monkeypatch):
|
||||||
|
instance,_,store,_=broker(tmp_path,monkeypatch)
|
||||||
|
instance.execute(
|
||||||
|
"release:candidate",
|
||||||
|
release_request(),
|
||||||
|
idempotency_key="owner-auth-release-outbox-schema-0001",
|
||||||
|
)
|
||||||
|
con=sqlite3.connect(store.path)
|
||||||
|
columns=[row[1] for row in con.execute("pragma table_info(projection_outbox)")]
|
||||||
|
joined=" ".join(columns).lower()
|
||||||
|
for forbidden in ("token","credential","private_key","secret"):
|
||||||
|
assert forbidden not in joined
|
||||||
|
payload=con.execute("select payload_json from projection_outbox").fetchone()[0]
|
||||||
|
assert "Bearer " not in payload
|
||||||
|
assert "submitOwnerApproval" not in payload
|
||||||
|
assert "startDeployment" not in payload
|
||||||
|
|
||||||
|
def test_exact_retry_returns_same_receipt_without_second_projection(
|
||||||
|
tmp_path,monkeypatch
|
||||||
|
):
|
||||||
|
instance,upstream,_,_=broker(tmp_path,monkeypatch)
|
||||||
|
request=governed_request()
|
||||||
|
first=instance.execute(
|
||||||
|
"governed-task:project",
|
||||||
|
request,
|
||||||
|
idempotency_key="owner-auth-projection-operation-0002",
|
||||||
|
)
|
||||||
|
second=instance.execute(
|
||||||
|
"governed-task:project",
|
||||||
|
request,
|
||||||
|
idempotency_key="owner-auth-projection-operation-0002",
|
||||||
|
)
|
||||||
|
assert second==first
|
||||||
|
assert len(upstream.project_calls)==1
|
||||||
|
|
||||||
|
|
||||||
|
def test_idempotency_conflict_fails_closed(tmp_path,monkeypatch):
|
||||||
|
instance,_,_,_=broker(tmp_path,monkeypatch)
|
||||||
|
instance.execute(
|
||||||
|
"release:candidate",
|
||||||
|
release_request(),
|
||||||
|
idempotency_key="owner-auth-release-operation-0002",
|
||||||
|
)
|
||||||
|
with pytest.raises(auth.AuthorizationError,match="IDEMPOTENCY_CONFLICT"):
|
||||||
|
instance.execute(
|
||||||
|
"release:candidate",
|
||||||
|
release_request(version="V2R2C3"),
|
||||||
|
idempotency_key="owner-auth-release-operation-0002",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_receipt_store_contains_no_owner_policy_or_credentials(
|
||||||
|
tmp_path,monkeypatch
|
||||||
|
):
|
||||||
|
instance,_,store,_=broker(tmp_path,monkeypatch)
|
||||||
|
instance.execute(
|
||||||
|
"release:candidate",
|
||||||
|
release_request(),
|
||||||
|
idempotency_key="owner-auth-release-operation-0003",
|
||||||
|
)
|
||||||
|
con=sqlite3.connect(store.path)
|
||||||
|
columns=[row[1] for row in con.execute("pragma table_info(receipts)")]
|
||||||
|
joined=" ".join(columns).lower()
|
||||||
|
assert "token" not in joined
|
||||||
|
assert "credential" not in joined
|
||||||
|
assert "policy_json" not in columns
|
||||||
|
|
||||||
|
|
||||||
|
def test_systemd_broker_uses_unix_socket_and_only_host_control_credential():
|
||||||
|
unit=(
|
||||||
|
ROOT/"deploy/systemd/aios-owner-durable-authorization-broker.service"
|
||||||
|
).read_text()
|
||||||
|
assert "LoadCredential=agent_task_control_token:" in unit
|
||||||
|
assert "release_candidate_token" not in unit
|
||||||
|
assert "18947" not in unit
|
||||||
|
assert "AIOS_RELEASE_CANDIDATE_API_URL" not in unit
|
||||||
|
assert "AIOS_OWNER_AUTH_BROKER_SOCKET=%t/aios-owner-authorization-broker/broker.sock" in unit
|
||||||
|
assert "Group=aios-executor-exchange" in unit
|
||||||
|
assert "RestrictAddressFamilies=AF_UNIX AF_INET" in unit
|
||||||
|
assert "IPAddressDeny=any" in unit
|
||||||
|
assert "IPAddressAllow=127.0.0.0/8" in unit
|
||||||
|
assert "InaccessiblePaths=/data/aios-gitops /home" in unit
|
||||||
|
|
||||||
|
|
||||||
|
def test_source_contains_no_release_builder_endpoint_or_high_risk_execution():
|
||||||
|
source=(
|
||||||
|
ROOT/"active-runtime/app/aios_durable_owner_authorization.py"
|
||||||
|
).read_text()
|
||||||
|
assert "/createReleaseCandidate" not in source
|
||||||
|
assert "18947" not in source
|
||||||
|
assert "ThreadingHTTPServer" not in source
|
||||||
|
assert "ThreadingUnixStreamServer" in source
|
||||||
|
assert "startDeployment" in source
|
||||||
|
assert "tag_publish" in source
|
||||||
|
assert "baseline_registry_finalize" in source
|
||||||
|
assert "runtime_switch" in source
|
||||||
|
assert "AUTHORIZED_PENDING_EXTERNAL_CAPABILITY_PROJECTION" in source
|
||||||
|
assert "PENDING_EXTERNAL_CONNECTOR" in source
|
||||||
|
assert "projection_outbox" in source
|
||||||
|
assert "external_connector_ack_supported" in source
|
||||||
|
assert "SAFE_PROJECTED_TASK_CAPABILITIES" in source
|
||||||
|
assert "INITIAL_EXECUTION_ACTION_NOT_AUTHORIZED" in source
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def test_docs_avoid_new_broker_bearer_secret():
|
||||||
|
doc=(
|
||||||
|
ROOT/"docs/superpowers/specs/2026-08-19-durable-owner-authorization.md"
|
||||||
|
).read_text()
|
||||||
|
assert "AF_UNIX" in doc
|
||||||
|
assert "avoids introducing another long-lived bearer secret" in doc
|
||||||
|
|
||||||
|
def test_docs_preserve_separate_owner_release_approval():
|
||||||
|
doc=(
|
||||||
|
ROOT/"docs/superpowers/specs/2026-08-19-durable-owner-authorization.md"
|
||||||
|
).read_text()
|
||||||
|
assert "persistent until explicitly revoked" in doc.lower()
|
||||||
|
assert "Owner signature" in doc
|
||||||
|
assert "AUTHORIZED_PENDING_EXTERNAL_CAPABILITY_PROJECTION" in doc
|
||||||
|
assert "PENDING_EXTERNAL_CONNECTOR" in doc
|
||||||
@@ -0,0 +1,232 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
APP=ROOT/"active-runtime/app"
|
||||||
|
sys.path.insert(0,str(APP))
|
||||||
|
|
||||||
|
import aios_online_agent_worker as worker
|
||||||
|
from aios_external_agent_runtime import (
|
||||||
|
ExternalAgentExchange,
|
||||||
|
ExternalAgentRuntimeError,
|
||||||
|
validate_request,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def envelope():
|
||||||
|
value=object.__new__(worker.ExecutionEnvelope)
|
||||||
|
object.__setattr__(value,"task_id","task-attempt-identity")
|
||||||
|
object.__setattr__(value,"job_id","job-attempt-identity")
|
||||||
|
object.__setattr__(value,"digest","d"*64)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def test_external_request_id_is_stable_within_attempt_and_changes_across_attempts():
|
||||||
|
env=envelope()
|
||||||
|
a1=worker.external_agent_request_id(env,"hermes",6)
|
||||||
|
a1_replay=worker.external_agent_request_id(env,"hermes",6)
|
||||||
|
a2=worker.external_agent_request_id(env,"hermes",7)
|
||||||
|
assert a1==a1_replay
|
||||||
|
assert a1!=a2
|
||||||
|
assert a1.startswith("external-hermes-")
|
||||||
|
assert a2.startswith("external-hermes-")
|
||||||
|
|
||||||
|
|
||||||
|
def test_external_request_id_rejects_invalid_attempt():
|
||||||
|
env=envelope()
|
||||||
|
with pytest.raises(worker.WorkerError,match="TASK_ATTEMPT_INVALID"):
|
||||||
|
worker.external_agent_request_id(env,"hermes",0)
|
||||||
|
|
||||||
|
|
||||||
|
def test_external_request_schema_binds_attempt():
|
||||||
|
value=validate_request({
|
||||||
|
"schema":"aios.external-agent-runtime.request.v1",
|
||||||
|
"request_id":"external-hermes-0123456789abcdef",
|
||||||
|
"runtime":"hermes",
|
||||||
|
"task_id":"task-attempt-identity",
|
||||||
|
"attempt":7,
|
||||||
|
"execution_digest":"d"*64,
|
||||||
|
"payload":{
|
||||||
|
"goal":"Return ok.",
|
||||||
|
"instructions":"",
|
||||||
|
"success_criteria":[],
|
||||||
|
"constraints":{},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
assert value["attempt"]==7
|
||||||
|
|
||||||
|
|
||||||
|
def test_external_request_schema_rejects_missing_attempt():
|
||||||
|
with pytest.raises(ExternalAgentRuntimeError,match="EXTERNAL_AGENT_ATTEMPT_INVALID"):
|
||||||
|
validate_request({
|
||||||
|
"schema":"aios.external-agent-runtime.request.v1",
|
||||||
|
"request_id":"external-hermes-0123456789abcdef",
|
||||||
|
"runtime":"hermes",
|
||||||
|
"task_id":"task-attempt-identity",
|
||||||
|
"execution_digest":"d"*64,
|
||||||
|
"payload":{
|
||||||
|
"goal":"Return ok.",
|
||||||
|
"instructions":"",
|
||||||
|
"success_criteria":[],
|
||||||
|
"constraints":{},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def _write_bound_result(path, request):
|
||||||
|
import hashlib
|
||||||
|
from aios_external_agent_runtime import canonical, sha256
|
||||||
|
material={
|
||||||
|
"schema":"aios.external-agent-runtime.result.v1",
|
||||||
|
"request_id":request["request_id"],
|
||||||
|
"request_sha256":sha256(canonical(request)),
|
||||||
|
"runtime":request["runtime"],
|
||||||
|
"task_id":request["task_id"],
|
||||||
|
"attempt":request["attempt"],
|
||||||
|
"execution_digest":request["execution_digest"],
|
||||||
|
"exit_code":0,
|
||||||
|
"duration_ms":1,
|
||||||
|
"output_text":"ok",
|
||||||
|
"output_sha256":hashlib.sha256(b"ok").hexdigest(),
|
||||||
|
"stderr_sha256":hashlib.sha256(b"").hexdigest(),
|
||||||
|
"credential_material_exposed":False,
|
||||||
|
"control_plane_access_granted":False,
|
||||||
|
}
|
||||||
|
material["result_sha256"]=sha256(canonical(material))
|
||||||
|
path.write_text(json.dumps(material,sort_keys=True),encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def test_exchange_result_binding_rejects_wrong_attempt(tmp_path):
|
||||||
|
exchange=ExternalAgentExchange(tmp_path)
|
||||||
|
request={
|
||||||
|
"schema":"aios.external-agent-runtime.request.v1",
|
||||||
|
"request_id":"external-hermes-0123456789abcdef",
|
||||||
|
"runtime":"hermes",
|
||||||
|
"task_id":"task-attempt-identity",
|
||||||
|
"attempt":7,
|
||||||
|
"execution_digest":"d"*64,
|
||||||
|
"payload":{
|
||||||
|
"goal":"Return ok.",
|
||||||
|
"instructions":"",
|
||||||
|
"success_criteria":[],
|
||||||
|
"constraints":{},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
request=validate_request(request)
|
||||||
|
result_path=tmp_path/"result.json"
|
||||||
|
_write_bound_result(result_path,request)
|
||||||
|
value=json.loads(result_path.read_text())
|
||||||
|
value["attempt"]=6
|
||||||
|
# Recompute digest so the binding check, not the digest check, is what fails.
|
||||||
|
from aios_external_agent_runtime import canonical,sha256
|
||||||
|
material=dict(value); material.pop("result_sha256")
|
||||||
|
value["result_sha256"]=sha256(canonical(material))
|
||||||
|
result_path.write_text(json.dumps(value),encoding="utf-8")
|
||||||
|
with pytest.raises(ExternalAgentRuntimeError,match="EXTERNAL_AGENT_RESULT_BINDING_INVALID"):
|
||||||
|
exchange._validated_result(
|
||||||
|
result_path,
|
||||||
|
request,
|
||||||
|
sha256(canonical(request)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_model_governance_request_id_is_attempt_scoped():
|
||||||
|
env=envelope()
|
||||||
|
attempt6=worker.external_agent_request_id(env,"hermes",6)+":model"
|
||||||
|
attempt7=worker.external_agent_request_id(env,"hermes",7)+":model"
|
||||||
|
assert attempt6!=attempt7
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_previous_attempt_does_not_poison_next_model_governance_reservation(tmp_path):
|
||||||
|
from aios_model_governance import (
|
||||||
|
ModelGovernanceStore,
|
||||||
|
ModelRecord,
|
||||||
|
ProviderRecord,
|
||||||
|
RouteRequest,
|
||||||
|
ScopePolicy,
|
||||||
|
)
|
||||||
|
|
||||||
|
store=ModelGovernanceStore(tmp_path/"model-governance.sqlite3",now_fn=lambda:1_800_000_000.0)
|
||||||
|
store.register_provider(
|
||||||
|
ProviderRecord(
|
||||||
|
"oneapi",
|
||||||
|
"runtime-profile:hermes",
|
||||||
|
max_failures=5,
|
||||||
|
cooldown_seconds=60,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
store.register_model(
|
||||||
|
ModelRecord(
|
||||||
|
"deepseek-v4-flash",
|
||||||
|
"oneapi",
|
||||||
|
"v1",
|
||||||
|
("agent.runtime.hermes","text"),
|
||||||
|
input_cost_micros_per_1k=1,
|
||||||
|
output_cost_micros_per_1k=1,
|
||||||
|
max_input_tokens=10000,
|
||||||
|
max_output_tokens=1000,
|
||||||
|
evaluation_sha256="a"*64,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
store.set_scope_policy(
|
||||||
|
ScopePolicy(
|
||||||
|
"tenant",
|
||||||
|
"model-governance-live",
|
||||||
|
"staging",
|
||||||
|
allowed_models=("deepseek-v4-flash",),
|
||||||
|
required_capabilities=("text",),
|
||||||
|
daily_token_budget=100000,
|
||||||
|
daily_cost_budget_micros=100000,
|
||||||
|
fallback_limit=1,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def route(request_id):
|
||||||
|
return store.route(
|
||||||
|
RouteRequest(
|
||||||
|
request_id=request_id,
|
||||||
|
task_id="r42-model-governance-hermes-live-20260818",
|
||||||
|
trace_id="trace-hermes-model-governance-live",
|
||||||
|
scope_type="tenant",
|
||||||
|
tenant_id="model-governance-live",
|
||||||
|
project_id="staging",
|
||||||
|
required_capabilities=("agent.runtime.hermes",),
|
||||||
|
estimated_input_tokens=100,
|
||||||
|
max_output_tokens=50,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
attempt6=route("external-hermes-attempt6:model")
|
||||||
|
store.record_failure(attempt6.request_id,"provider_process_failed")
|
||||||
|
row6=store.conn.execute(
|
||||||
|
"select state from model_governance_reservations where request_id=?",
|
||||||
|
(attempt6.request_id,),
|
||||||
|
).fetchone()
|
||||||
|
assert row6["state"]=="FAILED"
|
||||||
|
|
||||||
|
attempt7=route("external-hermes-attempt7:model")
|
||||||
|
assert attempt7.request_id!=attempt6.request_id
|
||||||
|
row7=store.conn.execute(
|
||||||
|
"select state from model_governance_reservations where request_id=?",
|
||||||
|
(attempt7.request_id,),
|
||||||
|
).fetchone()
|
||||||
|
assert row7["state"]=="RESERVED"
|
||||||
|
|
||||||
|
settled=store.record_success(
|
||||||
|
attempt7.request_id,
|
||||||
|
input_tokens=90,
|
||||||
|
output_tokens=40,
|
||||||
|
usage_source="runtime",
|
||||||
|
)
|
||||||
|
assert settled["state"]=="SETTLED"
|
||||||
|
assert store.conn.execute(
|
||||||
|
"select state from model_governance_reservations where request_id=?",
|
||||||
|
(attempt6.request_id,),
|
||||||
|
).fetchone()["state"]=="FAILED"
|
||||||
|
assert store.verify_audit_chain() is True
|
||||||
|
store.close()
|
||||||
@@ -0,0 +1,421 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
APP=ROOT/"active-runtime/app"
|
||||||
|
sys.path.insert(0,str(APP))
|
||||||
|
|
||||||
|
from aios_external_agent_runtime import (
|
||||||
|
ExternalAgentRuntimeError,
|
||||||
|
RuntimeProfile,
|
||||||
|
execute_request,
|
||||||
|
)
|
||||||
|
from aios_online_agent_execution import (
|
||||||
|
ExecutionProjectionError,
|
||||||
|
GovernedExecutionBinding,
|
||||||
|
project_online_task,
|
||||||
|
validate_execution_envelope,
|
||||||
|
)
|
||||||
|
import aios_online_agent_worker as worker
|
||||||
|
|
||||||
|
|
||||||
|
BASE="a"*40
|
||||||
|
|
||||||
|
|
||||||
|
def binding(task_id="task-hermes-runtime"):
|
||||||
|
return GovernedExecutionBinding.from_mapping({
|
||||||
|
"task_id":task_id,
|
||||||
|
"job_id":"job-"+task_id,
|
||||||
|
"repository":"aios/aios",
|
||||||
|
"base_oid":BASE,
|
||||||
|
"expected_branch_oid":BASE,
|
||||||
|
"target_branch":"agent/"+task_id,
|
||||||
|
"branch_mode":"CREATE_NEW",
|
||||||
|
"generation":1,
|
||||||
|
"idempotency_key":task_id+"-binding-v1",
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def task_request(task_id, runtime):
|
||||||
|
return {
|
||||||
|
"task_id":task_id,
|
||||||
|
"goal":"Summarize the requested operation.",
|
||||||
|
"instructions":"Return a concise result.",
|
||||||
|
"success_criteria":[{"id":"done","description":"A result is returned."}],
|
||||||
|
"constraints":{"network":"runtime-managed"},
|
||||||
|
"required_capabilities":[f"agent.runtime.{runtime}"],
|
||||||
|
"max_attempts":3,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def projected(task_id="task-hermes-runtime", runtime="hermes"):
|
||||||
|
return project_online_task(
|
||||||
|
binding(task_id),
|
||||||
|
task_request(task_id,runtime),
|
||||||
|
{
|
||||||
|
"action":"agent_runtime",
|
||||||
|
"adapter":runtime,
|
||||||
|
"capability_scope":[f"agent.runtime.{runtime}"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def clear_secret_env(monkeypatch):
|
||||||
|
for key in list(os.environ):
|
||||||
|
upper=key.upper()
|
||||||
|
if any(marker in upper for marker in (
|
||||||
|
"TOKEN","SECRET","PASSWORD","CREDENTIAL","AUTHORIZATION","PRIVATE_KEY"
|
||||||
|
)):
|
||||||
|
monkeypatch.delenv(key,raising=False)
|
||||||
|
monkeypatch.setenv("CREDENTIALS_DIRECTORY","/definitely-not-present-aios-test")
|
||||||
|
|
||||||
|
|
||||||
|
def make_helper(tmp_path):
|
||||||
|
helper=tmp_path/"runtime_helper.py"
|
||||||
|
helper.write_text(
|
||||||
|
"import json,sys\n"
|
||||||
|
"value=json.load(sys.stdin)\n"
|
||||||
|
"print(json.dumps({'answer':'ok','goal':value['goal']},sort_keys=True))\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
return helper
|
||||||
|
|
||||||
|
|
||||||
|
def make_profile(tmp_path,runtime,helper):
|
||||||
|
root=tmp_path/"profiles";root.mkdir(exist_ok=True)
|
||||||
|
(root/f"{runtime}.json").write_text(
|
||||||
|
json.dumps({
|
||||||
|
"schema":"aios.external-agent-runtime.profile.v1",
|
||||||
|
"runtime":runtime,
|
||||||
|
"argv":["/usr/bin/python3",str(helper)],
|
||||||
|
"input_mode":"stdin_json",
|
||||||
|
"output_mode":"stdout_json",
|
||||||
|
"timeout_seconds":30,
|
||||||
|
"max_output_bytes":65536,
|
||||||
|
}),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
return root
|
||||||
|
|
||||||
|
|
||||||
|
def make_text_request(tmp_path, runtime, payload=None):
|
||||||
|
request={
|
||||||
|
"schema":"aios.external-agent-runtime.request.v1",
|
||||||
|
"request_id":f"external-{runtime}-0123456789abcdef",
|
||||||
|
"runtime":runtime,
|
||||||
|
"task_id":f"task-{runtime}-runtime",
|
||||||
|
"attempt":1,
|
||||||
|
"execution_digest":"d"*64,
|
||||||
|
"payload":payload or {
|
||||||
|
"goal":"Return ok.",
|
||||||
|
"instructions":"No side effects.",
|
||||||
|
"success_criteria":[],
|
||||||
|
"constraints":{},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
request_path=tmp_path/"request.json"
|
||||||
|
result_path=tmp_path/"result.json"
|
||||||
|
request_path.write_text(json.dumps(request),encoding="utf-8")
|
||||||
|
return request_path, result_path
|
||||||
|
|
||||||
|
|
||||||
|
def test_agent_runtime_projection_is_read_only_and_task_bound():
|
||||||
|
value=projected()
|
||||||
|
execution=value["metadata"]["execution"]
|
||||||
|
assert execution["action"]=="agent_runtime"
|
||||||
|
assert execution["adapter"]=="hermes"
|
||||||
|
assert execution["runtime_input"]["goal"]==value["goal"]
|
||||||
|
assert execution["runtime_input"]["instructions"]==value["instructions"]
|
||||||
|
assert execution["governance"]["writer_lease"] is False
|
||||||
|
assert execution["change_set"]==[]
|
||||||
|
assert execution["verification"]==[]
|
||||||
|
validate_execution_envelope(
|
||||||
|
execution,
|
||||||
|
binding=binding(),
|
||||||
|
worker_capabilities=["agent.runtime.hermes"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_agent_runtime_projection_requires_runtime_capability():
|
||||||
|
with pytest.raises(ExecutionProjectionError,match="AGENT_RUNTIME_CAPABILITY_REQUIRED"):
|
||||||
|
project_online_task(
|
||||||
|
binding(),
|
||||||
|
task_request("task-hermes-runtime","hermes"),
|
||||||
|
{
|
||||||
|
"action":"agent_runtime",
|
||||||
|
"adapter":"hermes",
|
||||||
|
"capability_scope":["git.read"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_agent_runtime_projection_rejects_mutation():
|
||||||
|
with pytest.raises(ExecutionProjectionError,match="AGENT_RUNTIME_MUTATION_DENY"):
|
||||||
|
project_online_task(
|
||||||
|
binding(),
|
||||||
|
task_request("task-hermes-runtime","hermes"),
|
||||||
|
{
|
||||||
|
"action":"agent_runtime",
|
||||||
|
"adapter":"hermes",
|
||||||
|
"capability_scope":["agent.runtime.hermes"],
|
||||||
|
"change_set":[{"path":"x.py","operation":"write_file","content":"x=1\n"}],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_agent_runtime_projection_rejects_secret_constraint_keys():
|
||||||
|
request=task_request("task-hermes-runtime","hermes")
|
||||||
|
request["constraints"]={"api_token":"do-not-project"}
|
||||||
|
with pytest.raises(ExecutionProjectionError,match="AGENT_RUNTIME_SECRET_FIELD_DENY"):
|
||||||
|
project_online_task(
|
||||||
|
binding(),
|
||||||
|
request,
|
||||||
|
{
|
||||||
|
"action":"agent_runtime",
|
||||||
|
"adapter":"hermes",
|
||||||
|
"capability_scope":["agent.runtime.hermes"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("runtime",["hermes","openclaw"])
|
||||||
|
def test_external_executor_runs_root_owned_profile_without_shell(tmp_path,monkeypatch,runtime):
|
||||||
|
clear_secret_env(monkeypatch)
|
||||||
|
helper=make_helper(tmp_path)
|
||||||
|
profile_root=make_profile(tmp_path,runtime,helper)
|
||||||
|
request={
|
||||||
|
"schema":"aios.external-agent-runtime.request.v1",
|
||||||
|
"request_id":f"external-{runtime}-0123456789abcdef",
|
||||||
|
"runtime":runtime,
|
||||||
|
"task_id":f"task-{runtime}-runtime",
|
||||||
|
"attempt":1,
|
||||||
|
"execution_digest":"d"*64,
|
||||||
|
"payload":{
|
||||||
|
"goal":"Return ok.",
|
||||||
|
"instructions":"No side effects.",
|
||||||
|
"success_criteria":[],
|
||||||
|
"constraints":{},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
request_path=tmp_path/"request.json"
|
||||||
|
result_path=tmp_path/"result.json"
|
||||||
|
request_path.write_text(json.dumps(request),encoding="utf-8")
|
||||||
|
execute_request(request_path,result_path,profile_root=profile_root)
|
||||||
|
result=json.loads(result_path.read_text())
|
||||||
|
assert result["runtime"]==runtime
|
||||||
|
assert result["exit_code"]==0
|
||||||
|
assert result["output_json"]["answer"]=="ok"
|
||||||
|
assert result["credential_material_exposed"] is False
|
||||||
|
assert result["control_plane_access_granted"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_profile_rejects_shell_mode(tmp_path):
|
||||||
|
root=tmp_path/"profiles";root.mkdir()
|
||||||
|
(root/"hermes.json").write_text(json.dumps({
|
||||||
|
"schema":"aios.external-agent-runtime.profile.v1",
|
||||||
|
"runtime":"hermes",
|
||||||
|
"argv":["/usr/bin/python3","-c","print('x')"],
|
||||||
|
"input_mode":"stdin_text",
|
||||||
|
"output_mode":"stdout_text",
|
||||||
|
}))
|
||||||
|
with pytest.raises(ExternalAgentRuntimeError,match="EXTERNAL_AGENT_SHELL_MODE_DENY"):
|
||||||
|
RuntimeProfile.load("hermes",root)
|
||||||
|
|
||||||
|
|
||||||
|
def test_external_executor_projects_only_selected_provider_credential(tmp_path,monkeypatch):
|
||||||
|
clear_secret_env(monkeypatch)
|
||||||
|
helper=tmp_path/"credential_helper.py"
|
||||||
|
helper.write_text(
|
||||||
|
"import os\n"
|
||||||
|
"value=os.environ.get('OPENAI_API_KEY','')\n"
|
||||||
|
"print('AIOS_PROVIDER_PROJECTED_OK' if value == 'test-provider-key' else 'AIOS_PROVIDER_PROJECTED_FAIL')\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
profile_root=tmp_path/"profiles";profile_root.mkdir()
|
||||||
|
(profile_root/"hermes.json").write_text(json.dumps({
|
||||||
|
"schema":"aios.external-agent-runtime.profile.v1",
|
||||||
|
"runtime":"hermes",
|
||||||
|
"argv":["/usr/bin/python3",str(helper)],
|
||||||
|
"input_mode":"stdin_text",
|
||||||
|
"output_mode":"stdout_text",
|
||||||
|
"environment":{"RUNTIME_PROFILE_MARKER":"staging"},
|
||||||
|
"credential_env":{
|
||||||
|
"OPENAI_API_KEY":{
|
||||||
|
"credential":"provider.env",
|
||||||
|
"format":"dotenv",
|
||||||
|
"key":"ONEAPI_REGULAR_TOKEN",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}),encoding="utf-8")
|
||||||
|
credential_dir=tmp_path/"credentials";credential_dir.mkdir()
|
||||||
|
(credential_dir/"provider.env").write_text(
|
||||||
|
"ONEAPI_REGULAR_TOKEN=test-provider-key\nOTHER_PROVIDER_TOKEN=not-projected\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
monkeypatch.setenv("CREDENTIALS_DIRECTORY",str(credential_dir))
|
||||||
|
request_path,result_path=make_text_request(tmp_path,"hermes")
|
||||||
|
execute_request(request_path,result_path,profile_root=profile_root)
|
||||||
|
result=json.loads(result_path.read_text())
|
||||||
|
assert "AIOS_PROVIDER_PROJECTED_OK" in result["output_text"]
|
||||||
|
assert "test-provider-key" not in result["output_text"]
|
||||||
|
assert result["credential_material_exposed"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_profile_rejects_credential_path_traversal(tmp_path):
|
||||||
|
root=tmp_path/"profiles";root.mkdir()
|
||||||
|
(root/"hermes.json").write_text(json.dumps({
|
||||||
|
"schema":"aios.external-agent-runtime.profile.v1",
|
||||||
|
"runtime":"hermes",
|
||||||
|
"argv":["/usr/bin/python3","-V"],
|
||||||
|
"input_mode":"stdin_text",
|
||||||
|
"output_mode":"stdout_text",
|
||||||
|
"credential_env":{
|
||||||
|
"OPENAI_API_KEY":{
|
||||||
|
"credential":"../provider.env",
|
||||||
|
"format":"dotenv",
|
||||||
|
"key":"ONEAPI_REGULAR_TOKEN",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}),encoding="utf-8")
|
||||||
|
with pytest.raises(ExternalAgentRuntimeError,match="EXTERNAL_AGENT_CREDENTIAL_NAME_INVALID"):
|
||||||
|
RuntimeProfile.load("hermes",root)
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_profile_prompt_placeholder_is_root_owned(tmp_path,monkeypatch):
|
||||||
|
clear_secret_env(monkeypatch)
|
||||||
|
helper=tmp_path/"prompt_helper.py"
|
||||||
|
helper.write_text(
|
||||||
|
"import sys\n"
|
||||||
|
"print('AIOS_PROMPT_PLACEHOLDER_OK' if 'Goal:' in sys.argv[1] else 'AIOS_PROMPT_PLACEHOLDER_FAIL')\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
profile_root=tmp_path/"profiles";profile_root.mkdir()
|
||||||
|
(profile_root/"hermes.json").write_text(json.dumps({
|
||||||
|
"schema":"aios.external-agent-runtime.profile.v1",
|
||||||
|
"runtime":"hermes",
|
||||||
|
"argv":["/usr/bin/python3",str(helper),"{prompt}"],
|
||||||
|
"input_mode":"stdin_text",
|
||||||
|
"output_mode":"stdout_text",
|
||||||
|
}),encoding="utf-8")
|
||||||
|
request_path,result_path=make_text_request(tmp_path,"hermes")
|
||||||
|
execute_request(request_path,result_path,profile_root=profile_root)
|
||||||
|
result=json.loads(result_path.read_text())
|
||||||
|
assert "AIOS_PROMPT_PLACEHOLDER_OK" in result["output_text"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("runtime,capability",[
|
||||||
|
("hermes","agent.runtime.hermes"),
|
||||||
|
("openclaw","agent.runtime.openclaw"),
|
||||||
|
])
|
||||||
|
def test_worker_external_adapter_builds_readonly_result(monkeypatch,tmp_path,runtime,capability):
|
||||||
|
mapped=projected(f"task-{runtime}-runtime",runtime)
|
||||||
|
envelope=worker.ExecutionEnvelope.from_task(
|
||||||
|
mapped,
|
||||||
|
worker_capabilities=(capability,),
|
||||||
|
)
|
||||||
|
class FakeExchange:
|
||||||
|
def __init__(self,*args,**kwargs): pass
|
||||||
|
def execute(self,**kwargs):
|
||||||
|
return {
|
||||||
|
"schema":"aios.external-agent-runtime.result.v1",
|
||||||
|
"request_id":kwargs["request_id"],
|
||||||
|
"request_sha256":"e"*64,
|
||||||
|
"runtime":runtime,
|
||||||
|
"task_id":kwargs["task_id"],
|
||||||
|
"attempt":kwargs["attempt"],
|
||||||
|
"execution_digest":kwargs["execution_digest"],
|
||||||
|
"exit_code":0,
|
||||||
|
"duration_ms":10,
|
||||||
|
"output_text":"done",
|
||||||
|
"output_sha256":"f"*64,
|
||||||
|
"stderr_sha256":"0"*64,
|
||||||
|
"credential_material_exposed":False,
|
||||||
|
"control_plane_access_granted":False,
|
||||||
|
"result_sha256":"1"*64,
|
||||||
|
}
|
||||||
|
monkeypatch.setattr(worker,"ExternalAgentExchange",FakeExchange)
|
||||||
|
monkeypatch.setattr(worker,"audit",lambda *a,**k:None)
|
||||||
|
executor=worker.HermesExecutor() if runtime=="hermes" else worker.OpenClawExecutor()
|
||||||
|
execution=executor.execute(envelope,tmp_path,attempt=1)
|
||||||
|
result=worker.build_agent_runtime_result(envelope.task_id,envelope,execution)
|
||||||
|
assert result["status"]=="SUCCEEDED"
|
||||||
|
assert result["classification"]=="EXTERNAL_AGENT_RUNTIME_SUCCEEDED"
|
||||||
|
assert result["execution_mode"]=="agent_runtime"
|
||||||
|
assert result["agent_runtime"]==runtime
|
||||||
|
assert result["writer_lease"]=="NONE"
|
||||||
|
assert result["changed_files"]==[]
|
||||||
|
assert result["control_plane_access_granted_to_runtime"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_worker_adapter_filter_defaults_exclude_external_runtimes():
|
||||||
|
assert "hermes" not in worker.WORKER_ALLOWED_ADAPTERS
|
||||||
|
assert "openclaw" not in worker.WORKER_ALLOWED_ADAPTERS
|
||||||
|
|
||||||
|
|
||||||
|
def test_external_executor_systemd_has_no_control_credentials_or_repository_access():
|
||||||
|
unit=(ROOT/"deploy/systemd/aios-external-agent-executor.service").read_text()
|
||||||
|
assert "User=aios-engineering-executor" in unit
|
||||||
|
assert "LoadCredential=aios-external-agent-oneapi-env:/data/agent/secrets/oneapi-model-ready.env" in unit
|
||||||
|
assert "LoadCredentialEncrypted" not in unit
|
||||||
|
assert "InaccessiblePaths=/run/credentials" not in unit
|
||||||
|
assert "InaccessiblePaths=/etc/aios-gitops" in unit
|
||||||
|
assert "InaccessiblePaths=/data/aios-gitops" in unit
|
||||||
|
assert "InaccessiblePaths=/data/agent/secrets" in unit
|
||||||
|
assert "IPAddressDeny=127.0.0.0/8" in unit
|
||||||
|
assert "IPAddressDeny=::1/128" in unit
|
||||||
|
assert "IPAddressAllow=172.18.0.2/32" in unit
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"unit_name,capability,adapter,state_root",
|
||||||
|
[
|
||||||
|
(
|
||||||
|
"aios-hermes-online-agent-worker.service",
|
||||||
|
"agent.runtime.hermes",
|
||||||
|
"hermes",
|
||||||
|
"/var/lib/aios-online-agent-hermes",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"aios-openclaw-online-agent-worker.service",
|
||||||
|
"agent.runtime.openclaw",
|
||||||
|
"openclaw",
|
||||||
|
"/var/lib/aios-online-agent-openclaw",
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_dedicated_worker_units_are_capability_and_adapter_scoped(
|
||||||
|
unit_name,capability,adapter,state_root
|
||||||
|
):
|
||||||
|
unit=(ROOT/"deploy/systemd"/unit_name).read_text()
|
||||||
|
assert f"Environment=AIOS_WORKER_CAPABILITIES={capability}" in unit
|
||||||
|
assert f"Environment=AIOS_WORKER_ALLOWED_ADAPTERS={adapter}" in unit
|
||||||
|
assert f"Environment=AIOS_WORKER_STATE_ROOT={state_root}" in unit
|
||||||
|
assert "AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892" in unit
|
||||||
|
assert "LoadCredentialEncrypted=agent_token:" in unit
|
||||||
|
|
||||||
|
|
||||||
|
def test_example_profiles_are_non_authoritative_examples_only():
|
||||||
|
for runtime in ("hermes","openclaw"):
|
||||||
|
value=json.loads(
|
||||||
|
(ROOT/"config/external-agent-runtime.d"/f"{runtime}.example.json").read_text()
|
||||||
|
)
|
||||||
|
assert value["schema"]=="aios.external-agent-runtime.profile.v1"
|
||||||
|
assert value["runtime"]==runtime
|
||||||
|
assert value["input_mode"]=="stdin_text"
|
||||||
|
assert value["output_mode"]=="stdout_text"
|
||||||
|
|
||||||
|
|
||||||
|
def test_staging_profiles_are_provider_only_and_root_profile_bound():
|
||||||
|
hermes=json.loads((ROOT/"config/external-agent-runtime.d/hermes.profile.json").read_text())
|
||||||
|
openclaw=json.loads((ROOT/"config/external-agent-runtime.d/openclaw.profile.json").read_text())
|
||||||
|
for runtime,value in (("hermes",hermes),("openclaw",openclaw)):
|
||||||
|
assert value["runtime"]==runtime
|
||||||
|
assert set(value["credential_env"])=={"OPENAI_API_KEY"}
|
||||||
|
assert value["credential_env"]["OPENAI_API_KEY"]["credential"]=="aios-external-agent-oneapi-env"
|
||||||
|
assert "OPENAI_API_KEY" not in value.get("environment",{})
|
||||||
@@ -0,0 +1,205 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from email.message import Message
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import stat
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
APP=ROOT/"active-runtime/app"
|
||||||
|
sys.path.insert(0,str(APP))
|
||||||
|
|
||||||
|
import aios_gpt_https_edge as edge
|
||||||
|
|
||||||
|
|
||||||
|
class Headers(dict):
|
||||||
|
def get(self,key,default=None):
|
||||||
|
return super().get(key,default)
|
||||||
|
|
||||||
|
|
||||||
|
def test_default_and_public_bind_require_explicit_owner_approval():
|
||||||
|
assert edge.public_bind_allowed("127.0.0.1","0")=="127.0.0.1"
|
||||||
|
assert edge.public_bind_allowed("::1","0")=="::1"
|
||||||
|
with pytest.raises(RuntimeError,match="OWNER_APPROVAL_REQUIRED"):
|
||||||
|
edge.public_bind_allowed("0.0.0.0","0")
|
||||||
|
with pytest.raises(RuntimeError,match="OWNER_APPROVAL_REQUIRED"):
|
||||||
|
edge.public_bind_allowed("::","")
|
||||||
|
assert edge.public_bind_allowed("0.0.0.0","1")=="0.0.0.0"
|
||||||
|
|
||||||
|
|
||||||
|
def test_arbitrary_public_bind_address_is_denied_even_with_approval():
|
||||||
|
with pytest.raises(RuntimeError,match="BIND_ADDRESS_INVALID"):
|
||||||
|
edge.public_bind_allowed("192.0.2.10","1")
|
||||||
|
|
||||||
|
|
||||||
|
def test_public_domain_validation():
|
||||||
|
assert edge.validated_public_domain("tasks.example.com")=="tasks.example.com"
|
||||||
|
assert edge.validated_public_domain("")==""
|
||||||
|
with pytest.raises(RuntimeError,match="PUBLIC_DOMAIN_INVALID"):
|
||||||
|
edge.validated_public_domain("https://example.com")
|
||||||
|
with pytest.raises(RuntimeError,match="PUBLIC_DOMAIN_INVALID"):
|
||||||
|
edge.validated_public_domain("localhost")
|
||||||
|
|
||||||
|
|
||||||
|
def test_route_allowlist_accepts_only_readonly_api():
|
||||||
|
assert edge.allowed_route("POST","/v1/read-only-tasks")=="/v1/read-only-tasks"
|
||||||
|
assert edge.allowed_route("GET","/v1/read-only-tasks/external-request-0001")==(
|
||||||
|
"/v1/read-only-tasks/external-request-0001"
|
||||||
|
)
|
||||||
|
assert edge.allowed_route("GET","/health")=="/health"
|
||||||
|
assert edge.allowed_route("GET","/openapi.json")=="/openapi.json"
|
||||||
|
for method,path in [
|
||||||
|
("POST","/v2/governed-task"),
|
||||||
|
("GET","/v1/tasks/x"),
|
||||||
|
("POST","/v1/read-only-tasks/x"),
|
||||||
|
("DELETE","/v1/read-only-tasks/x"),
|
||||||
|
]:
|
||||||
|
with pytest.raises(edge.EdgeError,match="ROUTE_DENY"):
|
||||||
|
edge.allowed_route(method,path)
|
||||||
|
|
||||||
|
|
||||||
|
def test_query_and_fragment_are_denied():
|
||||||
|
with pytest.raises(edge.EdgeError,match="QUERY_OR_FRAGMENT_DENY"):
|
||||||
|
edge.allowed_route("GET","/health?x=1")
|
||||||
|
with pytest.raises(edge.EdgeError,match="QUERY_OR_FRAGMENT_DENY"):
|
||||||
|
edge.allowed_route("GET","/openapi.json#x")
|
||||||
|
|
||||||
|
|
||||||
|
def test_forwarded_headers_are_minimal_and_require_bearer():
|
||||||
|
incoming=Headers({
|
||||||
|
"Authorization":"Bearer "+"a"*40,
|
||||||
|
"Content-Type":"application/json; charset=utf-8",
|
||||||
|
"Cookie":"secret=1",
|
||||||
|
"X-Forwarded-For":"203.0.113.10",
|
||||||
|
"X-Api-Key":"do-not-forward",
|
||||||
|
})
|
||||||
|
forwarded=edge.forwarded_headers(incoming,body=True)
|
||||||
|
assert forwarded=={
|
||||||
|
"Accept":"application/json",
|
||||||
|
"Authorization":"Bearer "+"a"*40,
|
||||||
|
"Content-Type":"application/json",
|
||||||
|
}
|
||||||
|
with pytest.raises(edge.EdgeError,match="AUTHORIZATION_REQUIRED"):
|
||||||
|
edge.forwarded_headers(Headers({}),body=False)
|
||||||
|
|
||||||
|
|
||||||
|
def test_invalid_content_type_is_denied():
|
||||||
|
with pytest.raises(edge.EdgeError,match="CONTENT_TYPE_INVALID"):
|
||||||
|
edge.forwarded_headers(
|
||||||
|
Headers({
|
||||||
|
"Authorization":"Bearer "+"a"*40,
|
||||||
|
"Content-Type":"text/plain",
|
||||||
|
}),
|
||||||
|
body=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_rate_limiter_caps_window_and_allows_expiry():
|
||||||
|
now=[100.0]
|
||||||
|
limiter=edge.RateLimiter(window_seconds=60,limit=2,burst=1,now_fn=lambda:now[0])
|
||||||
|
assert limiter.allowed("client")
|
||||||
|
assert limiter.allowed("client")
|
||||||
|
assert limiter.allowed("client")
|
||||||
|
assert not limiter.allowed("client")
|
||||||
|
now[0]=161.0
|
||||||
|
assert limiter.allowed("client")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def test_rate_limiter_client_table_is_bounded():
|
||||||
|
now=[100.0]
|
||||||
|
limiter=edge.RateLimiter(
|
||||||
|
window_seconds=60,
|
||||||
|
limit=10,
|
||||||
|
burst=0,
|
||||||
|
max_clients=2,
|
||||||
|
now_fn=lambda:now[0],
|
||||||
|
)
|
||||||
|
assert limiter.allowed("a")
|
||||||
|
assert limiter.allowed("b")
|
||||||
|
assert not limiter.allowed("c")
|
||||||
|
now[0]=161.0
|
||||||
|
assert limiter.allowed("c")
|
||||||
|
assert len(limiter._events) <= 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_audit_source_does_not_log_sensitive_request_material():
|
||||||
|
source=(ROOT/"active-runtime/app/aios_gpt_https_edge.py").read_text()
|
||||||
|
assert '"authorization", "body", "token", "cookie", "client_ip"' in source
|
||||||
|
assert "_client_pseudonym" in source
|
||||||
|
|
||||||
|
def test_proxy_upstream_is_fixed():
|
||||||
|
proxy=edge.Proxy()
|
||||||
|
assert proxy.upstream=="http://127.0.0.1:18945"
|
||||||
|
bad=edge.Proxy(upstream="http://127.0.0.1:18892")
|
||||||
|
with pytest.raises(edge.EdgeError,match="UPSTREAM_DRIFT_DENY"):
|
||||||
|
bad.request("GET","/health",{"Authorization":"Bearer "+"a"*40},None)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def test_public_bind_requires_approved_domain():
|
||||||
|
assert edge.validated_edge_identity("127.0.0.1","0","")==("127.0.0.1","")
|
||||||
|
with pytest.raises(RuntimeError,match="PUBLIC_DOMAIN_REQUIRED"):
|
||||||
|
edge.validated_edge_identity("0.0.0.0","1","")
|
||||||
|
assert edge.validated_edge_identity(
|
||||||
|
"0.0.0.0","1","tasks.example.com"
|
||||||
|
)==("0.0.0.0","tasks.example.com")
|
||||||
|
|
||||||
|
def test_tls_key_requires_no_group_or_other_permissions(tmp_path):
|
||||||
|
key=tmp_path/"key.pem"
|
||||||
|
key.write_text("x"*64,encoding="utf-8")
|
||||||
|
key.chmod(0o600)
|
||||||
|
assert edge.validate_tls_file(key,key=True)==key
|
||||||
|
key.chmod(0o644)
|
||||||
|
with pytest.raises(RuntimeError,match="TLS_KEY_MODE_INVALID"):
|
||||||
|
edge.validate_tls_file(key,key=True)
|
||||||
|
|
||||||
|
|
||||||
|
def test_tls_symlink_is_denied(tmp_path):
|
||||||
|
real=tmp_path/"cert.pem"
|
||||||
|
real.write_text("x"*64,encoding="utf-8")
|
||||||
|
link=tmp_path/"link.pem"
|
||||||
|
link.symlink_to(real)
|
||||||
|
with pytest.raises(RuntimeError,match="TLS_FILE_INVALID"):
|
||||||
|
edge.validate_tls_file(link,key=False)
|
||||||
|
|
||||||
|
|
||||||
|
def test_systemd_unit_is_disabled_by_owner_approval_condition_and_loopback_default():
|
||||||
|
unit=(ROOT/"deploy/systemd/aios-gpt-https-edge.service").read_text()
|
||||||
|
assert "ConditionPathExists=/etc/aios-gitops/gpt-https-edge.approved" in unit
|
||||||
|
assert "AIOS_GPT_EDGE_HOST=127.0.0.1" in unit
|
||||||
|
assert "AIOS_GPT_EDGE_PUBLIC_BIND_APPROVED=0" in unit
|
||||||
|
assert "AIOS_GPT_EDGE_PORT=18443" in unit
|
||||||
|
assert "LoadCredential=tls_cert:" in unit
|
||||||
|
assert "LoadCredential=tls_key:" in unit
|
||||||
|
assert "InaccessiblePaths=/data/aios-gitops" in unit
|
||||||
|
assert "InaccessiblePaths=" in unit and "/var/lib/aios-agent-task" in unit
|
||||||
|
assert "NoNewPrivileges=yes" in unit
|
||||||
|
assert "ProtectSystem=strict" in unit
|
||||||
|
|
||||||
|
|
||||||
|
def test_systemd_unit_does_not_reference_internal_control_credentials():
|
||||||
|
unit=(ROOT/"deploy/systemd/aios-gpt-https-edge.service").read_text()
|
||||||
|
assert "agent_task_control" not in unit
|
||||||
|
assert "gitea" not in unit.lower()
|
||||||
|
assert "18892" not in unit
|
||||||
|
assert "18940" not in unit
|
||||||
|
|
||||||
|
|
||||||
|
def test_edge_source_does_not_proxy_internal_bridge_or_task_api():
|
||||||
|
source=(ROOT/"active-runtime/app/aios_gpt_https_edge.py").read_text()
|
||||||
|
assert 'UPSTREAM = "http://127.0.0.1:18945"' in source
|
||||||
|
assert "18892" not in source
|
||||||
|
assert "18940" not in source
|
||||||
|
assert "/v2/governed-task" not in source
|
||||||
|
assert "/v1/tasks/" not in source
|
||||||
|
|
||||||
|
|
||||||
|
def test_docs_keep_public_activation_separately_approved():
|
||||||
|
doc=(ROOT/"docs/superpowers/specs/2026-08-19-gpt-https-edge.md").read_text()
|
||||||
|
assert "explicit owner approval" in doc.lower()
|
||||||
|
assert "PUBLIC_NETWORK_EXPOSURE=NONE" in doc
|
||||||
|
assert "PUBLIC_CUSTOM_GPT_DIRECT_AI0S_ACCESS=NOT_VERIFIED" in doc
|
||||||
@@ -0,0 +1,362 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
APP=ROOT/"active-runtime/app"
|
||||||
|
sys.path.insert(0,str(APP))
|
||||||
|
|
||||||
|
import aios_gpt_https_edge_activation as activation
|
||||||
|
|
||||||
|
|
||||||
|
CANDIDATE="69a51d50c7f6cd9045346d41e338acebe4b6f1e8"
|
||||||
|
NOW=datetime(2026,8,19,2,0,0,tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def approval_value(cert_sha: str, **overrides):
|
||||||
|
value={
|
||||||
|
"schema":"aios.gpt-https-edge.public-approval.v1",
|
||||||
|
"approval_id":"owner-edge-approval-0001",
|
||||||
|
"candidate_oid":CANDIDATE,
|
||||||
|
"domain":"tasks.example.com",
|
||||||
|
"public_bind":True,
|
||||||
|
"bind_host":"0.0.0.0",
|
||||||
|
"port":18443,
|
||||||
|
"tls_cert_sha256":cert_sha,
|
||||||
|
"issued_at":"2026-08-19T01:00:00Z",
|
||||||
|
"not_before":"2026-08-19T01:00:00Z",
|
||||||
|
"expires_at":"2026-08-20T01:00:00Z",
|
||||||
|
"owner":"owner@example",
|
||||||
|
}
|
||||||
|
value.update(overrides)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def write_owner_file(path: Path, data: bytes, mode=0o600):
|
||||||
|
path.write_bytes(data)
|
||||||
|
path.chmod(mode)
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def load_approval(tmp_path, monkeypatch, **overrides):
|
||||||
|
tmp_path.mkdir(parents=True,exist_ok=True)
|
||||||
|
cert=write_owner_file(tmp_path/"cert.pem",b"certificate-material-"*4,0o600)
|
||||||
|
value=approval_value(hashlib.sha256(cert.read_bytes()).hexdigest(),**overrides)
|
||||||
|
approval=write_owner_file(
|
||||||
|
tmp_path/"approval.json",
|
||||||
|
(json.dumps(value,sort_keys=True)+"\n").encode(),
|
||||||
|
0o600,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
activation,
|
||||||
|
"validate_root_file",
|
||||||
|
lambda path, **kwargs: path,
|
||||||
|
)
|
||||||
|
return activation.Approval.load(
|
||||||
|
approval,
|
||||||
|
candidate_oid=CANDIDATE,
|
||||||
|
cert_path=cert,
|
||||||
|
now=NOW,
|
||||||
|
),approval,cert
|
||||||
|
|
||||||
|
|
||||||
|
class FakeSystemd:
|
||||||
|
def __init__(self):
|
||||||
|
self.active_units={activation.INGRESS_UNIT}
|
||||||
|
self.enabled_units=set()
|
||||||
|
self.calls=[]
|
||||||
|
self.exec_start=""
|
||||||
|
self.environment=""
|
||||||
|
self.fail_restart=False
|
||||||
|
|
||||||
|
def run(self,*args,check=True):
|
||||||
|
self.calls.append(args)
|
||||||
|
class Result:
|
||||||
|
returncode=0
|
||||||
|
stdout=""
|
||||||
|
if args[:2]==("restart",activation.EDGE_UNIT):
|
||||||
|
if self.fail_restart:
|
||||||
|
raise subprocess.CalledProcessError(1,args)
|
||||||
|
self.active_units.add(activation.EDGE_UNIT)
|
||||||
|
elif args[:2]==("stop",activation.EDGE_UNIT):
|
||||||
|
self.active_units.discard(activation.EDGE_UNIT)
|
||||||
|
elif args[:2]==("enable",activation.EDGE_UNIT):
|
||||||
|
self.enabled_units.add(activation.EDGE_UNIT)
|
||||||
|
elif args[:2]==("disable",activation.EDGE_UNIT):
|
||||||
|
self.enabled_units.discard(activation.EDGE_UNIT)
|
||||||
|
return Result()
|
||||||
|
|
||||||
|
def active(self,unit):
|
||||||
|
return unit in self.active_units
|
||||||
|
|
||||||
|
def enabled(self,unit):
|
||||||
|
return unit in self.enabled_units
|
||||||
|
|
||||||
|
def show(self,unit,prop):
|
||||||
|
if prop=="ExecStart":
|
||||||
|
return self.exec_start
|
||||||
|
if prop=="Environment":
|
||||||
|
return self.environment
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_approval_binds_candidate_domain_bind_port_and_cert(tmp_path,monkeypatch):
|
||||||
|
approval,_,cert=load_approval(tmp_path,monkeypatch)
|
||||||
|
assert approval.candidate_oid==CANDIDATE
|
||||||
|
assert approval.domain=="tasks.example.com"
|
||||||
|
assert approval.bind_host=="0.0.0.0"
|
||||||
|
assert approval.port==18443
|
||||||
|
assert approval.tls_cert_sha256==hashlib.sha256(cert.read_bytes()).hexdigest()
|
||||||
|
assert approval.expires_at>approval.not_before
|
||||||
|
|
||||||
|
|
||||||
|
def test_approval_rejects_candidate_mismatch(tmp_path,monkeypatch):
|
||||||
|
cert=write_owner_file(tmp_path/"cert.pem",b"certificate-material-"*4)
|
||||||
|
value=approval_value(hashlib.sha256(cert.read_bytes()).hexdigest())
|
||||||
|
approval=write_owner_file(tmp_path/"approval.json",json.dumps(value).encode())
|
||||||
|
monkeypatch.setattr(activation,"validate_root_file",lambda path,**kwargs:path)
|
||||||
|
with pytest.raises(activation.ActivationError,match="CANDIDATE_MISMATCH"):
|
||||||
|
activation.Approval.load(
|
||||||
|
approval,
|
||||||
|
candidate_oid="0"*40,
|
||||||
|
cert_path=cert,
|
||||||
|
now=NOW,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_approval_rejects_cert_hash_mismatch(tmp_path,monkeypatch):
|
||||||
|
cert=write_owner_file(tmp_path/"cert.pem",b"certificate-material-"*4)
|
||||||
|
value=approval_value("a"*64)
|
||||||
|
approval=write_owner_file(tmp_path/"approval.json",json.dumps(value).encode())
|
||||||
|
monkeypatch.setattr(activation,"validate_root_file",lambda path,**kwargs:path)
|
||||||
|
with pytest.raises(activation.ActivationError,match="CERT_SHA_MISMATCH"):
|
||||||
|
activation.Approval.load(
|
||||||
|
approval,
|
||||||
|
candidate_oid=CANDIDATE,
|
||||||
|
cert_path=cert,
|
||||||
|
now=NOW,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_approval_rejects_expired_not_yet_valid_and_long_lived(tmp_path,monkeypatch):
|
||||||
|
with pytest.raises(activation.ActivationError,match="APPROVAL_EXPIRED"):
|
||||||
|
load_approval(
|
||||||
|
tmp_path/"expired",
|
||||||
|
monkeypatch,
|
||||||
|
issued_at="2026-08-17T01:00:00Z",
|
||||||
|
not_before="2026-08-17T01:00:00Z",
|
||||||
|
expires_at="2026-08-18T01:00:00Z",
|
||||||
|
)
|
||||||
|
# Use distinct temp roots because helper writes fixed names.
|
||||||
|
|
||||||
|
with pytest.raises(activation.ActivationError,match="NOT_YET_VALID"):
|
||||||
|
load_approval(
|
||||||
|
tmp_path/"future",
|
||||||
|
monkeypatch,
|
||||||
|
issued_at="2026-08-19T03:00:00Z",
|
||||||
|
not_before="2026-08-19T03:00:00Z",
|
||||||
|
expires_at="2026-08-20T03:00:00Z",
|
||||||
|
)
|
||||||
|
with pytest.raises(activation.ActivationError,match="VALIDITY_TOO_LONG"):
|
||||||
|
load_approval(
|
||||||
|
tmp_path/"long",
|
||||||
|
monkeypatch,
|
||||||
|
issued_at="2026-08-18T01:00:00Z",
|
||||||
|
not_before="2026-08-18T01:00:00Z",
|
||||||
|
expires_at="2026-08-26T01:00:00Z",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def test_validate_release_requires_edge_and_controller_and_denies_symlink(tmp_path):
|
||||||
|
release=tmp_path/"release"
|
||||||
|
app=release/"active-runtime/app"
|
||||||
|
app.mkdir(parents=True)
|
||||||
|
edge_file=app/"aios_gpt_https_edge.py"
|
||||||
|
controller=app/"aios_gpt_https_edge_activation.py"
|
||||||
|
edge_file.write_text("edge\n")
|
||||||
|
controller.write_text("controller\n")
|
||||||
|
value=activation.validate_release(release)
|
||||||
|
assert value["edge"]==edge_file
|
||||||
|
assert value["controller"]==controller
|
||||||
|
controller.unlink()
|
||||||
|
with pytest.raises(activation.ActivationError,match="IMMUTABLE_RELEASE_INVALID"):
|
||||||
|
activation.validate_release(release)
|
||||||
|
|
||||||
|
def test_render_dropin_is_exact_and_contains_no_secret(tmp_path,monkeypatch):
|
||||||
|
approval,_,_=load_approval(tmp_path,monkeypatch)
|
||||||
|
release=Path("/opt/aios/releases")/(CANDIDATE+"-gpt-https-edge-public")
|
||||||
|
text=activation.render_dropin(approval,release)
|
||||||
|
assert "AIOS_GPT_EDGE_HOST=0.0.0.0" in text
|
||||||
|
assert "AIOS_GPT_EDGE_PORT=18443" in text
|
||||||
|
assert "AIOS_GPT_EDGE_PUBLIC_BIND_APPROVED=1" in text
|
||||||
|
assert "AIOS_GPT_EDGE_PUBLIC_DOMAIN=tasks.example.com" in text
|
||||||
|
assert CANDIDATE in text
|
||||||
|
for forbidden in ("Bearer ","agent_task_control","tls_key","gpt_ingress_token"):
|
||||||
|
assert forbidden not in text
|
||||||
|
|
||||||
|
|
||||||
|
def test_receipt_contains_binding_but_no_owner_secret_material(tmp_path,monkeypatch):
|
||||||
|
approval,_,_=load_approval(tmp_path,monkeypatch)
|
||||||
|
receipt=activation.receipt_for(
|
||||||
|
approval,
|
||||||
|
Path("/opt/aios/releases/test"),
|
||||||
|
state="ACTIVE",
|
||||||
|
)
|
||||||
|
assert receipt["approval_id"]==approval.approval_id
|
||||||
|
assert receipt["candidate_oid"]==CANDIDATE
|
||||||
|
assert receipt["domain"]=="tasks.example.com"
|
||||||
|
serialized=json.dumps(receipt)
|
||||||
|
assert "Bearer " not in serialized
|
||||||
|
assert "private" not in serialized.lower()
|
||||||
|
|
||||||
|
|
||||||
|
def test_activate_is_idempotent_for_same_active_approval(tmp_path,monkeypatch):
|
||||||
|
approval,_,cert=load_approval(tmp_path,monkeypatch)
|
||||||
|
release=tmp_path/"release"
|
||||||
|
app=release/"active-runtime/app/aios_gpt_https_edge.py"
|
||||||
|
controller=release/"active-runtime/app/aios_gpt_https_edge_activation.py"
|
||||||
|
app.parent.mkdir(parents=True)
|
||||||
|
app.write_text("print('x')\n")
|
||||||
|
controller.write_text("print('controller')\n")
|
||||||
|
key=write_owner_file(tmp_path/"key.pem",b"private-key-material-"*4,0o600)
|
||||||
|
dropin=tmp_path/"edge.d"/activation.DROPIN_NAME
|
||||||
|
state=tmp_path/"state.json"
|
||||||
|
systemd=FakeSystemd()
|
||||||
|
|
||||||
|
monkeypatch.setattr(activation.os,"geteuid",lambda:0)
|
||||||
|
monkeypatch.setattr(activation,"require_ingress_ready",lambda _:None)
|
||||||
|
monkeypatch.setattr(activation,"validate_tls_pair",lambda *a,**k:{"cert_sha256":approval.tls_cert_sha256})
|
||||||
|
monkeypatch.setattr(activation,"tls_probe",lambda *a,**k:None)
|
||||||
|
|
||||||
|
systemd.exec_start=str(app)
|
||||||
|
systemd.environment=(
|
||||||
|
"AIOS_GPT_EDGE_HOST=0.0.0.0 "
|
||||||
|
"AIOS_GPT_EDGE_PORT=18443 "
|
||||||
|
"AIOS_GPT_EDGE_PUBLIC_BIND_APPROVED=1 "
|
||||||
|
"AIOS_GPT_EDGE_PUBLIC_DOMAIN=tasks.example.com"
|
||||||
|
)
|
||||||
|
first=activation.activate(
|
||||||
|
approval=approval,
|
||||||
|
release_root=release,
|
||||||
|
cert_path=cert,
|
||||||
|
key_path=key,
|
||||||
|
dropin_path=dropin,
|
||||||
|
state_path=state,
|
||||||
|
systemd=systemd,
|
||||||
|
)
|
||||||
|
calls_after_first=len(systemd.calls)
|
||||||
|
second=activation.activate(
|
||||||
|
approval=approval,
|
||||||
|
release_root=release,
|
||||||
|
cert_path=cert,
|
||||||
|
key_path=key,
|
||||||
|
dropin_path=dropin,
|
||||||
|
state_path=state,
|
||||||
|
systemd=systemd,
|
||||||
|
)
|
||||||
|
assert first["state"]=="ACTIVE"
|
||||||
|
assert second==json.loads(state.read_text())
|
||||||
|
assert len(systemd.calls)==calls_after_first
|
||||||
|
|
||||||
|
|
||||||
|
def test_activate_failure_after_mutation_rolls_back(tmp_path,monkeypatch):
|
||||||
|
approval,_,cert=load_approval(tmp_path,monkeypatch)
|
||||||
|
release=tmp_path/"release"
|
||||||
|
app=release/"active-runtime/app/aios_gpt_https_edge.py"
|
||||||
|
controller=release/"active-runtime/app/aios_gpt_https_edge_activation.py"
|
||||||
|
app.parent.mkdir(parents=True)
|
||||||
|
app.write_text("print('x')\n")
|
||||||
|
controller.write_text("print('controller')\n")
|
||||||
|
key=write_owner_file(tmp_path/"key.pem",b"private-key-material-"*4,0o600)
|
||||||
|
dropin=tmp_path/"edge.d"/activation.DROPIN_NAME
|
||||||
|
state=tmp_path/"state.json"
|
||||||
|
systemd=FakeSystemd()
|
||||||
|
systemd.fail_restart=True
|
||||||
|
|
||||||
|
monkeypatch.setattr(activation.os,"geteuid",lambda:0)
|
||||||
|
monkeypatch.setattr(activation,"require_ingress_ready",lambda _:None)
|
||||||
|
monkeypatch.setattr(activation,"validate_tls_pair",lambda *a,**k:{"cert_sha256":approval.tls_cert_sha256})
|
||||||
|
monkeypatch.setattr(activation,"tls_probe",lambda *a,**k:None)
|
||||||
|
|
||||||
|
with pytest.raises(subprocess.CalledProcessError):
|
||||||
|
activation.activate(
|
||||||
|
approval=approval,
|
||||||
|
release_root=release,
|
||||||
|
cert_path=cert,
|
||||||
|
key_path=key,
|
||||||
|
dropin_path=dropin,
|
||||||
|
state_path=state,
|
||||||
|
systemd=systemd,
|
||||||
|
)
|
||||||
|
assert not dropin.exists()
|
||||||
|
assert not systemd.active(activation.EDGE_UNIT)
|
||||||
|
assert json.loads(state.read_text())["state"]=="ROLLED_BACK"
|
||||||
|
|
||||||
|
|
||||||
|
def test_rollback_is_replay_safe_and_leaves_owner_assets_alone(tmp_path,monkeypatch):
|
||||||
|
approval_file=tmp_path/"approval.json"
|
||||||
|
cert=tmp_path/"cert.pem"
|
||||||
|
key=tmp_path/"key.pem"
|
||||||
|
for path in (approval_file,cert,key):
|
||||||
|
path.write_text("owner-asset"*4)
|
||||||
|
dropin=tmp_path/"edge.d"/activation.DROPIN_NAME
|
||||||
|
dropin.parent.mkdir(parents=True)
|
||||||
|
dropin.write_text("[Service]\n")
|
||||||
|
state=tmp_path/"state.json"
|
||||||
|
state.write_text(json.dumps({
|
||||||
|
"schema":activation.STATE_SCHEMA,
|
||||||
|
"state":"ACTIVE",
|
||||||
|
"candidate_oid":CANDIDATE,
|
||||||
|
"approval_id":"owner-edge-approval-0001",
|
||||||
|
"domain":"tasks.example.com",
|
||||||
|
}))
|
||||||
|
systemd=FakeSystemd()
|
||||||
|
systemd.active_units.add(activation.EDGE_UNIT)
|
||||||
|
systemd.enabled_units.add(activation.EDGE_UNIT)
|
||||||
|
monkeypatch.setattr(activation.os,"geteuid",lambda:0)
|
||||||
|
|
||||||
|
first=activation.rollback(
|
||||||
|
dropin_path=dropin,
|
||||||
|
state_path=state,
|
||||||
|
systemd=systemd,
|
||||||
|
reason="owner_requested",
|
||||||
|
)
|
||||||
|
second=activation.rollback(
|
||||||
|
dropin_path=dropin,
|
||||||
|
state_path=state,
|
||||||
|
systemd=systemd,
|
||||||
|
reason="owner_requested",
|
||||||
|
)
|
||||||
|
assert first["state"]=="ROLLED_BACK"
|
||||||
|
assert second["state"]=="ROLLED_BACK"
|
||||||
|
assert not dropin.exists()
|
||||||
|
assert not systemd.active(activation.EDGE_UNIT)
|
||||||
|
assert not systemd.enabled(activation.EDGE_UNIT)
|
||||||
|
for path in (approval_file,cert,key):
|
||||||
|
assert path.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_activation_source_never_creates_owner_approval_or_tls_secret():
|
||||||
|
source=(ROOT/"active-runtime/app/aios_gpt_https_edge_activation.py").read_text()
|
||||||
|
assert "secrets.token" not in source
|
||||||
|
assert "openssl" not in source.lower()
|
||||||
|
assert "BEGIN PRIVATE KEY" not in source
|
||||||
|
assert "gpt-https-edge.approved" in source
|
||||||
|
assert "gpt-https-edge-tls.key" in source
|
||||||
|
|
||||||
|
|
||||||
|
def test_docs_separate_implementation_activation_and_runtime_verification():
|
||||||
|
doc=(ROOT/"docs/superpowers/specs/2026-08-19-gpt-https-edge-activation.md").read_text()
|
||||||
|
assert "IMPLEMENTED" in doc
|
||||||
|
assert "OWNER_APPROVED" in doc
|
||||||
|
assert "ACTIVATED" in doc
|
||||||
|
assert "RUNTIME_VERIFIED" in doc
|
||||||
|
assert "does not own" in doc.lower()
|
||||||
@@ -0,0 +1,342 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
APP=ROOT/"active-runtime/app"
|
||||||
|
sys.path.insert(0,str(APP))
|
||||||
|
|
||||||
|
from aios_gpt_online_task_ingress import (
|
||||||
|
IngressError,
|
||||||
|
IngressService,
|
||||||
|
Store,
|
||||||
|
READONLY_CAPABILITIES,
|
||||||
|
branch_for,
|
||||||
|
normalized_request,
|
||||||
|
openapi_document,
|
||||||
|
task_id_for,
|
||||||
|
validated_bind_host,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
MAIN="3244f0f17f8845592ac94d585f5ebd65693c9f29"
|
||||||
|
JOB="job-0123456789abcdef0123456789abcdef"
|
||||||
|
|
||||||
|
|
||||||
|
def request(request_id="external-request-0001", **overrides):
|
||||||
|
value={
|
||||||
|
"schema":"aios.gpt-online-task-ingress.request.v1",
|
||||||
|
"request_id":request_id,
|
||||||
|
"goal":"Inspect the repository for durable worker state.",
|
||||||
|
"instructions":"Read only. Return a concise source analysis.",
|
||||||
|
"success_criteria":[
|
||||||
|
{"id":"analysis","description":"Return a source-backed analysis."}
|
||||||
|
],
|
||||||
|
"constraints":{
|
||||||
|
"max_files":50,
|
||||||
|
"focus_paths":["active-runtime/app","tests"],
|
||||||
|
"notes":"Do not mutate repository state.",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
value.update(overrides)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class FakeUpstream:
|
||||||
|
def __init__(self):
|
||||||
|
self.created=[]
|
||||||
|
self.projected=[]
|
||||||
|
self.tasks={}
|
||||||
|
self.snapshot={"authoritative_head_oid":MAIN}
|
||||||
|
|
||||||
|
def control_snapshot(self):
|
||||||
|
return dict(self.snapshot)
|
||||||
|
|
||||||
|
def create_governed_task(self, body):
|
||||||
|
self.created.append(dict(body))
|
||||||
|
task_id=body["task_id"]
|
||||||
|
job_id=JOB
|
||||||
|
self.tasks[task_id]={
|
||||||
|
"task_id":task_id,
|
||||||
|
"status":"PROJECTED",
|
||||||
|
"attempt":0,
|
||||||
|
"metadata":{
|
||||||
|
"execution":{
|
||||||
|
"task_id":task_id,
|
||||||
|
"job_id":job_id,
|
||||||
|
"base_oid":body["base_oid"],
|
||||||
|
"action":"source_capture",
|
||||||
|
"capability_scope":list(READONLY_CAPABILITIES),
|
||||||
|
"governance":{"writer_lease":False},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
return 201,{"task_id":task_id,"job_id":job_id,"state":"BOUND"}
|
||||||
|
|
||||||
|
def execution_envelope(self, body):
|
||||||
|
self.projected.append(dict(body))
|
||||||
|
task_id=body["task_id"]
|
||||||
|
return 200,{
|
||||||
|
"schema":"aios.online-agent.execution.v2",
|
||||||
|
"task_id":task_id,
|
||||||
|
"job_id":body["job_id"],
|
||||||
|
"repository":"aios/aios",
|
||||||
|
"base_oid":MAIN,
|
||||||
|
"action":"source_capture",
|
||||||
|
"capability_scope":list(READONLY_CAPABILITIES),
|
||||||
|
"governance":{"writer_lease":False},
|
||||||
|
}
|
||||||
|
|
||||||
|
def get_task(self, task_id):
|
||||||
|
value=self.tasks.get(task_id)
|
||||||
|
return (404,{}) if value is None else (200,dict(value))
|
||||||
|
|
||||||
|
|
||||||
|
def service(tmp_path):
|
||||||
|
upstream=FakeUpstream()
|
||||||
|
store=Store(tmp_path/"state.sqlite3")
|
||||||
|
return IngressService(upstream,store),upstream,store
|
||||||
|
|
||||||
|
|
||||||
|
def test_request_schema_is_narrow_and_rejects_capability_or_execution_control():
|
||||||
|
value=normalized_request(request())
|
||||||
|
assert value["goal"].startswith("Inspect")
|
||||||
|
for field in ("capability_scope","required_capabilities","action","adapter","repository","base_oid"):
|
||||||
|
bad=request()
|
||||||
|
bad[field]="evil"
|
||||||
|
with pytest.raises(IngressError,match="UNKNOWN_FIELDS"):
|
||||||
|
normalized_request(bad)
|
||||||
|
|
||||||
|
|
||||||
|
def test_constraints_are_bounded_and_path_traversal_is_denied():
|
||||||
|
bad=request(constraints={"focus_paths":["../etc/passwd"]})
|
||||||
|
with pytest.raises(IngressError,match="FOCUS_PATHS_INVALID"):
|
||||||
|
normalized_request(bad)
|
||||||
|
bad=request(constraints={"max_files":1000})
|
||||||
|
with pytest.raises(IngressError,match="MAX_FILES_INVALID"):
|
||||||
|
normalized_request(bad)
|
||||||
|
|
||||||
|
|
||||||
|
def test_non_loopback_bind_is_denied():
|
||||||
|
assert validated_bind_host("127.0.0.1")=="127.0.0.1"
|
||||||
|
with pytest.raises(RuntimeError,match="GPT_INGRESS_NON_LOOPBACK_BIND_DENY"):
|
||||||
|
validated_bind_host("0.0.0.0")
|
||||||
|
with pytest.raises(RuntimeError,match="GPT_INGRESS_NON_LOOPBACK_BIND_DENY"):
|
||||||
|
validated_bind_host("192.0.2.10")
|
||||||
|
|
||||||
|
|
||||||
|
def test_task_and_branch_ids_are_deterministic_and_bounded():
|
||||||
|
a=task_id_for("external-request-0001")
|
||||||
|
b=task_id_for("external-request-0001")
|
||||||
|
c=task_id_for("external-request-0002")
|
||||||
|
assert a==b
|
||||||
|
assert a!=c
|
||||||
|
assert a.startswith("gpt-readonly-")
|
||||||
|
branch=branch_for(a)
|
||||||
|
assert branch.startswith("agent/external-gpt-readonly/")
|
||||||
|
assert len(a)<180
|
||||||
|
|
||||||
|
|
||||||
|
def test_create_translates_to_canonical_readonly_governed_task(tmp_path):
|
||||||
|
svc,upstream,store=service(tmp_path)
|
||||||
|
response=svc.create(request())
|
||||||
|
assert response["state"]=="BOUND"
|
||||||
|
assert response["write_access"] is False
|
||||||
|
assert response["official_verification_required"] is True
|
||||||
|
assert response["capabilities"]==list(READONLY_CAPABILITIES)
|
||||||
|
|
||||||
|
assert len(upstream.created)==1
|
||||||
|
body=upstream.created[0]
|
||||||
|
assert body["repository"]=="aios/aios"
|
||||||
|
assert body["base_oid"]==MAIN
|
||||||
|
assert body["expected_branch_oid"]==MAIN
|
||||||
|
assert body["branch_mode"]=="CREATE_NEW"
|
||||||
|
assert body["risk"]=="low"
|
||||||
|
assert body["priority"]==0
|
||||||
|
assert body["max_attempts"]==3
|
||||||
|
assert body["require_online_task_projection"] is True
|
||||||
|
assert body["online_task"]["required_capabilities"]==list(READONLY_CAPABILITIES)
|
||||||
|
assert body["online_task"]["constraints"]["repository_mutation"]=="forbidden"
|
||||||
|
assert body["online_task"]["constraints"]["credential_access"]=="forbidden"
|
||||||
|
assert body["execution"]["action"]=="source_capture"
|
||||||
|
assert body["execution"]["adapter"]=="source_capture"
|
||||||
|
assert body["execution"]["capability_scope"]==list(READONLY_CAPABILITIES)
|
||||||
|
assert "writer_lease" not in body["execution"]
|
||||||
|
|
||||||
|
assert len(upstream.projected)==1
|
||||||
|
projection=upstream.projected[0]
|
||||||
|
assert projection["task_id"]==response["task_id"]
|
||||||
|
assert projection["job_id"]==JOB
|
||||||
|
assert projection["required_capabilities"]==list(READONLY_CAPABILITIES)
|
||||||
|
|
||||||
|
row=store.get("external-request-0001")
|
||||||
|
assert row is not None
|
||||||
|
assert row["state"]=="BOUND"
|
||||||
|
assert row["job_id"]==JOB
|
||||||
|
assert row["base_oid"]==MAIN
|
||||||
|
|
||||||
|
|
||||||
|
def test_exact_retry_is_idempotent_without_duplicate_bridge_call(tmp_path):
|
||||||
|
svc,upstream,_=service(tmp_path)
|
||||||
|
first=svc.create(request())
|
||||||
|
second=svc.create(request())
|
||||||
|
assert second==first
|
||||||
|
assert len(upstream.created)==1
|
||||||
|
assert len(upstream.projected)==1
|
||||||
|
|
||||||
|
|
||||||
|
def test_same_request_id_with_different_content_is_conflict(tmp_path):
|
||||||
|
svc,upstream,_=service(tmp_path)
|
||||||
|
svc.create(request())
|
||||||
|
with pytest.raises(IngressError,match="IDEMPOTENCY_CONFLICT"):
|
||||||
|
svc.create(request(goal="Different goal."))
|
||||||
|
assert len(upstream.created)==1
|
||||||
|
|
||||||
|
|
||||||
|
def test_pending_request_recovers_existing_canonical_task_after_crash(tmp_path):
|
||||||
|
svc,upstream,store=service(tmp_path)
|
||||||
|
raw=normalized_request(request())
|
||||||
|
task_id=task_id_for(raw["request_id"])
|
||||||
|
store.reserve(raw,task_id)
|
||||||
|
|
||||||
|
upstream.tasks[task_id]={
|
||||||
|
"task_id":task_id,
|
||||||
|
"status":"PROJECTED",
|
||||||
|
"attempt":0,
|
||||||
|
"metadata":{
|
||||||
|
"execution":{
|
||||||
|
"task_id":task_id,
|
||||||
|
"job_id":JOB,
|
||||||
|
"base_oid":MAIN,
|
||||||
|
"action":"source_capture",
|
||||||
|
"capability_scope":list(READONLY_CAPABILITIES),
|
||||||
|
"governance":{"writer_lease":False},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
result=svc.create(request())
|
||||||
|
assert result["job_id"]==JOB
|
||||||
|
assert result["task_id"]==task_id
|
||||||
|
assert len(upstream.created)==0
|
||||||
|
row=store.get(raw["request_id"])
|
||||||
|
assert row["state"]=="BOUND"
|
||||||
|
|
||||||
|
|
||||||
|
def test_status_is_sanitized_and_never_proxies_raw_execution_or_result_text(tmp_path):
|
||||||
|
svc,upstream,store=service(tmp_path)
|
||||||
|
created=svc.create(request())
|
||||||
|
task_id=created["task_id"]
|
||||||
|
upstream.tasks[task_id]={
|
||||||
|
"task_id":task_id,
|
||||||
|
"status":"SUCCEEDED",
|
||||||
|
"attempt":1,
|
||||||
|
"claimed_by":"online-engineering-worker-cn-network-mkt-generic-20260815",
|
||||||
|
"metadata":{
|
||||||
|
"execution":{
|
||||||
|
"job_id":JOB,
|
||||||
|
"base_oid":MAIN,
|
||||||
|
"action":"source_capture",
|
||||||
|
"capability_scope":list(READONLY_CAPABILITIES),
|
||||||
|
"governance":{"writer_lease":False},
|
||||||
|
"secret_marker":"do-not-return",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"result":{
|
||||||
|
"status":"SUCCEEDED",
|
||||||
|
"classification":"NEEDS_REWORK_SOURCE_CAPTURE",
|
||||||
|
"execution_mode":"source_capture",
|
||||||
|
"output_text":"raw source text that must not be proxied",
|
||||||
|
"source_snapshot":{
|
||||||
|
"payload_sha256":"a"*64,
|
||||||
|
"file_count":42,
|
||||||
|
"files":["secret/path"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"verification":{
|
||||||
|
"decision":"SUCCEEDED",
|
||||||
|
"classification":"GENERIC_ONLINE_TASK_AUTO_CONSUMPTION_VERIFIED",
|
||||||
|
"private":"do-not-return",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
status=svc.status("external-request-0001")
|
||||||
|
assert status["state"]=="SUCCEEDED"
|
||||||
|
assert status["attempt"]==1
|
||||||
|
assert status["claimed"] is True
|
||||||
|
assert status["official_verification"]=="SUCCEEDED"
|
||||||
|
assert status["source_snapshot_sha256"]=="a"*64
|
||||||
|
assert status["source_snapshot_file_count"]==42
|
||||||
|
serialized=json.dumps(status)
|
||||||
|
assert "raw source text" not in serialized
|
||||||
|
assert "secret_marker" not in serialized
|
||||||
|
assert "secret/path" not in serialized
|
||||||
|
assert "do-not-return" not in serialized
|
||||||
|
|
||||||
|
|
||||||
|
def test_pending_status_can_recover_job_binding(tmp_path):
|
||||||
|
svc,upstream,store=service(tmp_path)
|
||||||
|
raw=normalized_request(request())
|
||||||
|
task_id=task_id_for(raw["request_id"])
|
||||||
|
store.reserve(raw,task_id)
|
||||||
|
upstream.tasks[task_id]={
|
||||||
|
"task_id":task_id,
|
||||||
|
"status":"PROJECTED",
|
||||||
|
"attempt":0,
|
||||||
|
"metadata":{
|
||||||
|
"execution":{
|
||||||
|
"job_id":JOB,
|
||||||
|
"base_oid":MAIN,
|
||||||
|
"action":"source_capture",
|
||||||
|
"capability_scope":list(READONLY_CAPABILITIES),
|
||||||
|
"governance":{"writer_lease":False},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
value=svc.status(raw["request_id"])
|
||||||
|
assert value["job_id"]==JOB
|
||||||
|
assert store.get(raw["request_id"])["state"]=="BOUND"
|
||||||
|
|
||||||
|
|
||||||
|
def test_openapi_exposes_only_create_readonly_and_status_operations():
|
||||||
|
doc=openapi_document()
|
||||||
|
assert doc["openapi"]=="3.1.0"
|
||||||
|
paths=doc["paths"]
|
||||||
|
assert set(paths)=={
|
||||||
|
"/v1/read-only-tasks",
|
||||||
|
"/v1/read-only-tasks/{request_id}",
|
||||||
|
}
|
||||||
|
rendered=json.dumps(doc)
|
||||||
|
for forbidden in ("writer-lease","commit","verify","production","tag","baseline"):
|
||||||
|
assert forbidden not in rendered.lower()
|
||||||
|
|
||||||
|
|
||||||
|
def test_systemd_unit_is_loopback_credential_isolated_and_has_no_repository_access():
|
||||||
|
unit=(ROOT/"deploy/systemd/aios-gpt-online-task-ingress.service").read_text()
|
||||||
|
assert "DynamicUser=yes" in unit
|
||||||
|
assert "LoadCredential=gpt_ingress_token:" in unit
|
||||||
|
assert "LoadCredential=agent_task_control_token:" in unit
|
||||||
|
assert "AIOS_GPT_INGRESS_HOST=127.0.0.1" in unit
|
||||||
|
assert "AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892" in unit
|
||||||
|
assert "AIOS_AGENT_TASK_API_URL=http://127.0.0.1:18940" in unit
|
||||||
|
assert "IPAddressDeny=any" in unit
|
||||||
|
assert "IPAddressAllow=127.0.0.0/8" in unit
|
||||||
|
assert "InaccessiblePaths=/data/aios-gitops /etc/aios-gitops /home" in unit
|
||||||
|
assert "ProtectSystem=strict" in unit
|
||||||
|
assert "NoNewPrivileges=yes" in unit
|
||||||
|
assert "MemoryDenyWriteExecute=yes" in unit
|
||||||
|
assert "0.0.0.0" not in unit
|
||||||
|
|
||||||
|
|
||||||
|
def test_ingress_sqlite_contains_no_request_body_or_credentials(tmp_path):
|
||||||
|
svc,upstream,store=service(tmp_path)
|
||||||
|
svc.create(request(instructions="sensitive-looking but non-secret user text"))
|
||||||
|
con=sqlite3.connect(store.path)
|
||||||
|
columns=[row[1] for row in con.execute("pragma table_info(external_task_requests)")]
|
||||||
|
assert "request_json" not in columns
|
||||||
|
assert "goal" not in columns
|
||||||
|
assert "instructions" not in columns
|
||||||
|
assert "token" not in " ".join(columns).lower()
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
from pathlib import Path
|
||||||
|
import hashlib,json,os,tarfile,sys,tempfile
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0,str(ROOT/"active-runtime/app"))
|
||||||
|
import aios_isolated_executor as mod
|
||||||
|
|
||||||
|
def make_archive(tmp_path):
|
||||||
|
root=tmp_path/"src"; root.mkdir()
|
||||||
|
(root/"hello.py").write_text("VALUE = 1\n")
|
||||||
|
archive=tmp_path/"source.tgz"
|
||||||
|
with tarfile.open(archive,"w:gz") as tf: tf.add(root/"hello.py",arcname="hello.py")
|
||||||
|
return archive
|
||||||
|
|
||||||
|
def test_isolated_executor_applies_hash_bound_change(tmp_path,monkeypatch):
|
||||||
|
archive=make_archive(tmp_path)
|
||||||
|
monkeypatch.delenv("CREDENTIALS_DIRECTORY",raising=False)
|
||||||
|
monkeypatch.setattr(mod,"assert_uncredentialed",lambda:None)
|
||||||
|
request={
|
||||||
|
"schema":"aios.isolated-execution.request.v1",
|
||||||
|
"request_id":"task-req-0000000000000000",
|
||||||
|
"source_archive":str(archive),
|
||||||
|
"source_archive_sha256":mod.sha256(archive.read_bytes()),
|
||||||
|
"change_set":[{
|
||||||
|
"path":"hello.py","operation":"replace_text","old":"VALUE = 1","new":"VALUE = 2","count":1,
|
||||||
|
"expected_sha256":mod.sha256(b"VALUE = 1\n")
|
||||||
|
}],
|
||||||
|
"verification":[{"label":"compile","argv":["/usr/bin/python3","-m","py_compile","hello.py"],"cwd":".","timeout_seconds":30}]
|
||||||
|
}
|
||||||
|
req=tmp_path/"req.json"; res=tmp_path/"res.json"
|
||||||
|
req.write_text(json.dumps(request))
|
||||||
|
mod.execute_request(req,res)
|
||||||
|
out=json.loads(res.read_text())
|
||||||
|
assert out["request_id"]=="task-req-0000000000000000"
|
||||||
|
assert out["credential_isolation"]["credential_env_absent"] is True
|
||||||
|
changed=out["changed_files"][0]
|
||||||
|
assert changed["path"]=="hello.py" and changed["deleted"] is False
|
||||||
|
|
||||||
|
def test_isolated_executor_source_drift_denied(tmp_path,monkeypatch):
|
||||||
|
archive=make_archive(tmp_path); monkeypatch.setattr(mod,"assert_uncredentialed",lambda:None)
|
||||||
|
request={
|
||||||
|
"schema":"aios.isolated-execution.request.v1","request_id":"task-req-0000000000000001",
|
||||||
|
"source_archive":str(archive),
|
||||||
|
"source_archive_sha256":mod.sha256(archive.read_bytes()),
|
||||||
|
"change_set":[{"path":"hello.py","operation":"write_file","content":"x\n","expected_sha256":"0"*64}],
|
||||||
|
"verification":[]
|
||||||
|
}
|
||||||
|
req=tmp_path/"req.json"; res=tmp_path/"res.json"; req.write_text(json.dumps(request))
|
||||||
|
with pytest.raises(mod.IsolatedExecutorError,match="SOURCE_DRIFT"):
|
||||||
|
mod.execute_request(req,res)
|
||||||
|
|
||||||
|
def test_credential_boundary_eacces_is_accepted(monkeypatch):
|
||||||
|
class DeniedDirectory:
|
||||||
|
def exists(self): return True
|
||||||
|
def is_symlink(self): return False
|
||||||
|
def is_dir(self): return True
|
||||||
|
def iterdir(self): raise PermissionError(13,"permission denied")
|
||||||
|
monkeypatch.setattr(mod.os,"access",lambda *_args,**_kwargs:False)
|
||||||
|
assert mod._credential_exposure(DeniedDirectory()) is None
|
||||||
|
|
||||||
|
def test_systemd_unit_has_no_credentials_and_private_network():
|
||||||
|
unit=(ROOT/"deploy/systemd/aios-online-agent-isolated-executor.service").read_text()
|
||||||
|
assert "PrivateNetwork=true" in unit
|
||||||
|
assert "InaccessiblePaths=/run/credentials" in unit
|
||||||
|
assert "LoadCredential" not in unit
|
||||||
|
assert "SupplementaryGroups=gitops" not in unit
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user