Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
dd416fa7ac | ||
|
|
d5dc3c0f31 | ||
|
|
a3df11b693 |
@@ -13,7 +13,6 @@ from hashlib import sha256
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import json
|
import json
|
||||||
from typing import Any
|
from typing import Any
|
||||||
from message_handoff import stable_event_id
|
|
||||||
|
|
||||||
|
|
||||||
def utc_now() -> str:
|
def utc_now() -> str:
|
||||||
@@ -56,19 +55,13 @@ class AgentEventBus:
|
|||||||
payload: dict[str, Any],
|
payload: dict[str, Any],
|
||||||
correlation_id: str | None = None,
|
correlation_id: str | None = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
event_id = stable_event_id(
|
event_id = "evt_" + sha256(
|
||||||
source=source, target=target, message_type=kind, payload=payload,
|
json.dumps(
|
||||||
tenant_id=(payload.get('tenant_id') if isinstance(payload, dict) else None),
|
{"kind": kind, "source": source, "summary": summary, "payload": payload},
|
||||||
user_id=(payload.get('user_id') if isinstance(payload, dict) else None),
|
sort_keys=True,
|
||||||
project_id=(payload.get('project_id') if isinstance(payload, dict) else None),
|
ensure_ascii=False,
|
||||||
conversation_id=(payload.get('conversation_id') if isinstance(payload, dict) else None),
|
).encode("utf-8")
|
||||||
task_id=(payload.get('task_id') if isinstance(payload, dict) else None),
|
).hexdigest()[:16]
|
||||||
run_id=(payload.get('run_id') if isinstance(payload, dict) else None),
|
|
||||||
correlation_id=(payload.get('correlation_id') if isinstance(payload, dict) else None),
|
|
||||||
causation_id=(payload.get('causation_id') if isinstance(payload, dict) else None),
|
|
||||||
sequence=(payload.get('sequence') if isinstance(payload, dict) else None),
|
|
||||||
idempotency_key=(payload.get('idempotency_key') if isinstance(payload, dict) else None),
|
|
||||||
)
|
|
||||||
event = AgentEvent(event_id, kind, source, target, summary, payload, correlation_id=correlation_id)
|
event = AgentEvent(event_id, kind, source, target, summary, payload, correlation_id=correlation_id)
|
||||||
self._write("pending", event.to_dict())
|
self._write("pending", event.to_dict())
|
||||||
self._audit("created", event.to_dict())
|
self._audit("created", event.to_dict())
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ class AgentEventObservabilityBridge:
|
|||||||
def __init__(self, hooks: AgentObservabilityHooks | None = None):
|
def __init__(self, hooks: AgentObservabilityHooks | None = None):
|
||||||
self.hooks = hooks or AgentObservabilityHooks()
|
self.hooks = hooks or AgentObservabilityHooks()
|
||||||
|
|
||||||
def consume(self, event: dict, *, actor=None, trace=None):
|
def consume(self, event: dict):
|
||||||
kind = event.get("kind")
|
kind = event.get("kind")
|
||||||
payload = event.get("payload", {})
|
payload = event.get("payload", {})
|
||||||
if kind == "agent_started":
|
if kind == "agent_started":
|
||||||
@@ -16,8 +16,6 @@ class AgentEventObservabilityBridge:
|
|||||||
visible_scope=payload.get("scope", "self"),
|
visible_scope=payload.get("scope", "self"),
|
||||||
entry_agent=payload.get("agent", "unknown"),
|
entry_agent=payload.get("agent", "unknown"),
|
||||||
execution_id=event.get("id"),
|
execution_id=event.get("id"),
|
||||||
actor=actor,
|
|
||||||
trace=trace,
|
|
||||||
)
|
)
|
||||||
if kind == "agent_delegated":
|
if kind == "agent_delegated":
|
||||||
return self.hooks.delegate(event["execution_id"], payload.get("agent", "unknown"))
|
return self.hooks.delegate(event["execution_id"], payload.get("agent", "unknown"))
|
||||||
|
|||||||
@@ -31,18 +31,6 @@ class AgentExecutionRecord:
|
|||||||
status: str = "completed"
|
status: str = "completed"
|
||||||
started_at: str = field(default_factory=utc_now)
|
started_at: str = field(default_factory=utc_now)
|
||||||
ended_at: str = field(default_factory=utc_now)
|
ended_at: str = field(default_factory=utc_now)
|
||||||
trace_id: str | None = None
|
|
||||||
span_id: str | None = None
|
|
||||||
parent_span_id: str | None = None
|
|
||||||
correlation_id: str | None = None
|
|
||||||
causation_id: str | None = None
|
|
||||||
tenant_id: str | None = None
|
|
||||||
project_id: str | None = None
|
|
||||||
conversation_id: str | None = None
|
|
||||||
task_id: str | None = None
|
|
||||||
claim_id: str | None = None
|
|
||||||
run_id: str | None = None
|
|
||||||
message_id: str | None = None
|
|
||||||
|
|
||||||
def to_dict(self):
|
def to_dict(self):
|
||||||
return asdict(self)
|
return asdict(self)
|
||||||
|
|||||||
@@ -9,49 +9,25 @@ try:
|
|||||||
except Exception:
|
except Exception:
|
||||||
APIRouter = None
|
APIRouter = None
|
||||||
|
|
||||||
from .agent_observability import AgentExecutionRecord, AgentObservability
|
from .agent_observability import AgentExecutionRecord
|
||||||
from .agent_observability_dashboard import AgentObservabilityDashboard
|
from .agent_observability_dashboard import AgentObservabilityDashboard
|
||||||
from .agent_observability_store import AgentObservabilityStore
|
from .agent_observability_store import AgentObservabilityStore
|
||||||
from .trace_tool_credential_governance import GovernanceActor, TraceContext, authorize_scope
|
|
||||||
|
|
||||||
store = AgentObservabilityStore()
|
store = AgentObservabilityStore()
|
||||||
dashboard = AgentObservabilityDashboard(store.load())
|
dashboard = AgentObservabilityDashboard(store.load())
|
||||||
|
|
||||||
|
|
||||||
def record_agent_execution(payload: dict[str, Any], *, actor: GovernanceActor | dict[str, Any] | None = None, trace: TraceContext | None = None):
|
def record_agent_execution(payload: dict[str, Any]):
|
||||||
scoped = any(payload.get(name) is not None for name in (
|
record = AgentExecutionRecord(**payload)
|
||||||
"trace_id", "span_id", "tenant_id", "project_id", "conversation_id", "task_id", "run_id"
|
|
||||||
))
|
|
||||||
values = dict(payload)
|
|
||||||
if scoped:
|
|
||||||
if actor is None or trace is None:
|
|
||||||
raise RuntimeError("TRACE_ACTOR_CONTEXT_REQUIRED")
|
|
||||||
trusted = GovernanceActor.from_value(actor)
|
|
||||||
authorize_scope(trusted, trace.scope)
|
|
||||||
bound = {
|
|
||||||
"trace_id": trace.trace_id, "span_id": trace.span_id,
|
|
||||||
"parent_span_id": trace.parent_span_id, "correlation_id": trace.correlation_id,
|
|
||||||
"causation_id": trace.causation_id, "tenant_id": trace.scope.tenant_id,
|
|
||||||
"user_id": trace.scope.user_id, "project_id": trace.scope.project_id,
|
|
||||||
"conversation_id": trace.scope.conversation_id, "task_id": trace.task_id,
|
|
||||||
"claim_id": trace.claim_id, "run_id": trace.scope.run_id, "message_id": trace.message_id,
|
|
||||||
}
|
|
||||||
for name, authoritative in bound.items():
|
|
||||||
if values.get(name) not in (None, authoritative):
|
|
||||||
raise RuntimeError("TRACE_SCOPE_MISMATCH_DENY:" + name)
|
|
||||||
values[name] = authoritative
|
|
||||||
record = AgentExecutionRecord(**values)
|
|
||||||
return {"status": "ok", "record": store.append(record)}
|
return {"status": "ok", "record": store.append(record)}
|
||||||
|
|
||||||
|
|
||||||
def get_agent_observability(user_id: str | None = None, group_id: str | None = None, role: str = "user", *, actor: GovernanceActor | dict[str, Any] | None = None):
|
def get_agent_observability(user_id: str | None = None, group_id: str | None = None, role: str = "user"):
|
||||||
records = store.query(user_id, group_id, role, actor=actor)
|
records = store.query(user_id, group_id, role)
|
||||||
visible = AgentObservability()
|
|
||||||
visible.records = [AgentExecutionRecord(**record) for record in records]
|
|
||||||
return {
|
return {
|
||||||
"status": "ok",
|
"status": "ok",
|
||||||
"records": records,
|
"records": records,
|
||||||
"summary": visible.summary(None),
|
"summary": store.load().summary(user_id if role != "owner" else None),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -14,11 +14,6 @@ from uuid import uuid4
|
|||||||
|
|
||||||
from .agent_observability import AgentExecutionRecord
|
from .agent_observability import AgentExecutionRecord
|
||||||
from .agent_observability_store import AgentObservabilityStore
|
from .agent_observability_store import AgentObservabilityStore
|
||||||
from .trace_tool_credential_governance import (
|
|
||||||
GovernanceActor,
|
|
||||||
TraceContext,
|
|
||||||
authorize_scope,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def utc_now() -> str:
|
def utc_now() -> str:
|
||||||
@@ -41,7 +36,6 @@ class AgentRunContext:
|
|||||||
input_tokens: int = 0
|
input_tokens: int = 0
|
||||||
output_tokens: int = 0
|
output_tokens: int = 0
|
||||||
status: str = "running"
|
status: str = "running"
|
||||||
trace: TraceContext | None = None
|
|
||||||
|
|
||||||
def delegate(self, agent_name: str) -> None:
|
def delegate(self, agent_name: str) -> None:
|
||||||
if agent_name not in self.path:
|
if agent_name not in self.path:
|
||||||
@@ -65,19 +59,7 @@ class AgentObservabilityHooks:
|
|||||||
visible_scope: str,
|
visible_scope: str,
|
||||||
entry_agent: str,
|
entry_agent: str,
|
||||||
execution_id: str | None = None,
|
execution_id: str | None = None,
|
||||||
*,
|
|
||||||
actor: GovernanceActor | dict[str, Any] | None = None,
|
|
||||||
trace: TraceContext | None = None,
|
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
if execution_id is not None and execution_id in self.active:
|
|
||||||
raise RuntimeError("OBSERVABILITY_EXECUTION_CONFLICT")
|
|
||||||
if trace is not None:
|
|
||||||
if actor is None:
|
|
||||||
raise RuntimeError("TRACE_ACTOR_CONTEXT_REQUIRED")
|
|
||||||
trusted_actor = GovernanceActor.from_value(actor)
|
|
||||||
authorize_scope(trusted_actor, trace.scope)
|
|
||||||
if trace.scope.user_id is not None and trace.scope.user_id != user_id:
|
|
||||||
raise RuntimeError("TRACE_SCOPE_MISMATCH_DENY:user_id")
|
|
||||||
run = AgentRunContext(
|
run = AgentRunContext(
|
||||||
execution_id=execution_id or f"exec_{uuid4().hex[:12]}",
|
execution_id=execution_id or f"exec_{uuid4().hex[:12]}",
|
||||||
user_id=user_id,
|
user_id=user_id,
|
||||||
@@ -85,7 +67,6 @@ class AgentObservabilityHooks:
|
|||||||
entry_agent=entry_agent,
|
entry_agent=entry_agent,
|
||||||
path=[entry_agent],
|
path=[entry_agent],
|
||||||
used_hermes=entry_agent.lower() == "hermes",
|
used_hermes=entry_agent.lower() == "hermes",
|
||||||
trace=trace,
|
|
||||||
)
|
)
|
||||||
self.active[run.execution_id] = run
|
self.active[run.execution_id] = run
|
||||||
return {"status": "started", "execution_id": run.execution_id, "run": run.__dict__}
|
return {"status": "started", "execution_id": run.execution_id, "run": run.__dict__}
|
||||||
@@ -125,18 +106,6 @@ class AgentObservabilityHooks:
|
|||||||
status=status,
|
status=status,
|
||||||
started_at=run.started_at,
|
started_at=run.started_at,
|
||||||
ended_at=ended_at,
|
ended_at=ended_at,
|
||||||
trace_id=run.trace.trace_id if run.trace else None,
|
|
||||||
span_id=run.trace.span_id if run.trace else None,
|
|
||||||
parent_span_id=run.trace.parent_span_id if run.trace else None,
|
|
||||||
correlation_id=run.trace.correlation_id if run.trace else None,
|
|
||||||
causation_id=run.trace.causation_id if run.trace else None,
|
|
||||||
tenant_id=run.trace.scope.tenant_id if run.trace else None,
|
|
||||||
project_id=run.trace.scope.project_id if run.trace else None,
|
|
||||||
conversation_id=run.trace.scope.conversation_id if run.trace else None,
|
|
||||||
task_id=run.trace.task_id if run.trace else None,
|
|
||||||
claim_id=run.trace.claim_id if run.trace else None,
|
|
||||||
run_id=run.trace.scope.run_id if run.trace else None,
|
|
||||||
message_id=run.trace.message_id if run.trace else None,
|
|
||||||
)
|
)
|
||||||
return {"status": "stored", "record": self.store.append(record)}
|
return {"status": "stored", "record": self.store.append(record)}
|
||||||
|
|
||||||
|
|||||||
@@ -12,12 +12,6 @@ from pathlib import Path
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from .agent_observability import AgentExecutionRecord, AgentObservability
|
from .agent_observability import AgentExecutionRecord, AgentObservability
|
||||||
from .trace_tool_credential_governance import (
|
|
||||||
GovernanceActor,
|
|
||||||
GovernanceError,
|
|
||||||
GovernanceScope,
|
|
||||||
authorize_scope,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class AgentObservabilityStore:
|
class AgentObservabilityStore:
|
||||||
@@ -29,11 +23,6 @@ class AgentObservabilityStore:
|
|||||||
def append(self, record: AgentExecutionRecord) -> dict[str, Any]:
|
def append(self, record: AgentExecutionRecord) -> dict[str, Any]:
|
||||||
path = self.records_dir / f"{record.execution_id}.json"
|
path = self.records_dir / f"{record.execution_id}.json"
|
||||||
data = record.to_dict()
|
data = record.to_dict()
|
||||||
if path.exists():
|
|
||||||
existing = json.loads(path.read_text(encoding="utf-8"))
|
|
||||||
if existing == data:
|
|
||||||
return existing
|
|
||||||
raise RuntimeError("OBSERVABILITY_EXECUTION_CONFLICT")
|
|
||||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||||
return data
|
return data
|
||||||
|
|
||||||
@@ -44,36 +33,8 @@ class AgentObservabilityStore:
|
|||||||
observability.records.append(AgentExecutionRecord(**raw))
|
observability.records.append(AgentExecutionRecord(**raw))
|
||||||
return observability
|
return observability
|
||||||
|
|
||||||
def query(self, user_id: str | None = None, group_id: str | None = None, role: str = "user", *, actor: GovernanceActor | dict[str, Any] | None = None) -> list[dict[str, Any]]:
|
def query(self, user_id: str | None = None, group_id: str | None = None, role: str = "user") -> list[dict[str, Any]]:
|
||||||
records = [record.to_dict() for record in self.load().records]
|
records = [record.to_dict() for record in self.load().records]
|
||||||
scoped = [record for record in records if record.get("tenant_id") is not None]
|
|
||||||
legacy = [record for record in records if record.get("tenant_id") is None]
|
|
||||||
if actor is not None:
|
|
||||||
trusted = GovernanceActor.from_value(actor)
|
|
||||||
visible: list[dict[str, Any]] = []
|
|
||||||
for record in scoped:
|
|
||||||
try:
|
|
||||||
authorize_scope(
|
|
||||||
trusted,
|
|
||||||
GovernanceScope(
|
|
||||||
"tenant",
|
|
||||||
tenant_id=record.get("tenant_id"),
|
|
||||||
user_id=record.get("user_id"),
|
|
||||||
project_id=record.get("project_id"),
|
|
||||||
conversation_id=record.get("conversation_id"),
|
|
||||||
run_id=record.get("run_id"),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
except GovernanceError:
|
|
||||||
continue
|
|
||||||
visible.append(record)
|
|
||||||
if trusted.privileged:
|
|
||||||
visible.extend(legacy)
|
|
||||||
else:
|
|
||||||
visible.extend(record for record in legacy if record.get("user_id") == trusted.scope.user_id)
|
|
||||||
return visible
|
|
||||||
# Caller-supplied role remains a legacy compatibility path only.
|
|
||||||
records = legacy
|
|
||||||
if role == "owner":
|
if role == "owner":
|
||||||
return records
|
return records
|
||||||
if role == "group_admin" and group_id:
|
if role == "group_admin" and group_id:
|
||||||
|
|||||||
@@ -1,722 +0,0 @@
|
|||||||
"""Owner-controlled CAS-bound execution-envelope rebind for Online Agent Tasks.
|
|
||||||
|
|
||||||
This service exists for a narrow recovery case: a Task was projected/claimed with a read-only
|
|
||||||
source_capture envelope before its already-registered deterministic plan became authoritative.
|
|
||||||
|
|
||||||
It never creates Tasks/Jobs/branches and never accepts arbitrary metadata patches. It only replaces
|
|
||||||
metadata.execution + the derived execution_projection for the same governed binding, after an
|
|
||||||
official NEEDS_REWORK transition has returned the Task to QUEUED with no active lease.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
import hashlib
|
|
||||||
import hmac
|
|
||||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
from pathlib import Path
|
|
||||||
import re
|
|
||||||
import sqlite3
|
|
||||||
from typing import Any, Callable, Mapping, Sequence
|
|
||||||
import urllib.error
|
|
||||||
import urllib.request
|
|
||||||
|
|
||||||
from aios_online_agent_execution import (
|
|
||||||
GovernedExecutionBinding,
|
|
||||||
validate_execution_envelope,
|
|
||||||
)
|
|
||||||
|
|
||||||
TASK_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{2,179}$")
|
|
||||||
KEY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{7,159}$")
|
|
||||||
SHA_RE = re.compile(r"^[0-9a-f]{64}$")
|
|
||||||
MAX_BODY = 64 * 1024
|
|
||||||
AUTHORIZATION_ID_ENV = "AIOS_AGENT_TASK_EXECUTION_REBIND_AUTHORIZATION_ID"
|
|
||||||
|
|
||||||
|
|
||||||
class ExecutionRebindError(RuntimeError):
|
|
||||||
def __init__(self, code: str, status: int = 409) -> None:
|
|
||||||
super().__init__(code)
|
|
||||||
self.code = code
|
|
||||||
self.status = status
|
|
||||||
|
|
||||||
|
|
||||||
def _canonical(value: object) -> bytes:
|
|
||||||
return json.dumps(
|
|
||||||
value, sort_keys=True, separators=(",", ":"), ensure_ascii=False
|
|
||||||
).encode("utf-8")
|
|
||||||
|
|
||||||
|
|
||||||
def _sha(value: object) -> str:
|
|
||||||
return hashlib.sha256(_canonical(value)).hexdigest()
|
|
||||||
|
|
||||||
|
|
||||||
def _now() -> str:
|
|
||||||
return datetime.now(timezone.utc).isoformat()
|
|
||||||
|
|
||||||
|
|
||||||
def _require_string(
|
|
||||||
value: object,
|
|
||||||
code: str,
|
|
||||||
pattern: re.Pattern[str] | None = None,
|
|
||||||
) -> str:
|
|
||||||
if not isinstance(value, str) or not value.strip():
|
|
||||||
raise ExecutionRebindError(code, 400)
|
|
||||||
value = value.strip()
|
|
||||||
if pattern is not None and pattern.fullmatch(value) is None:
|
|
||||||
raise ExecutionRebindError(code, 400)
|
|
||||||
return value
|
|
||||||
|
|
||||||
|
|
||||||
def _required_authorization_id() -> str:
|
|
||||||
value = os.environ.get(AUTHORIZATION_ID_ENV, "").strip()
|
|
||||||
if not value:
|
|
||||||
raise RuntimeError("TASK_EXECUTION_REBIND_AUTHORIZATION_ID_MISSING")
|
|
||||||
if KEY_RE.fullmatch(value) is None:
|
|
||||||
raise RuntimeError("TASK_EXECUTION_REBIND_AUTHORIZATION_ID_INVALID")
|
|
||||||
return value
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class ExecutionRebindRequest:
|
|
||||||
expected_status: str
|
|
||||||
expected_revision: int
|
|
||||||
expected_attempt: int
|
|
||||||
previous_verification_sha256: str
|
|
||||||
expected_old_execution_digest: str
|
|
||||||
expected_new_execution_digest: str
|
|
||||||
authorization_id: str
|
|
||||||
idempotency_key: str
|
|
||||||
reason: str
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_mapping(cls, value: Mapping[str, Any]) -> "ExecutionRebindRequest":
|
|
||||||
allowed = {
|
|
||||||
"expected_status",
|
|
||||||
"expected_revision",
|
|
||||||
"expected_attempt",
|
|
||||||
"previous_verification_sha256",
|
|
||||||
"expected_old_execution_digest",
|
|
||||||
"expected_new_execution_digest",
|
|
||||||
"authorization_id",
|
|
||||||
"idempotency_key",
|
|
||||||
"reason",
|
|
||||||
}
|
|
||||||
if set(value) != allowed:
|
|
||||||
raise ExecutionRebindError("EXECUTION_REBIND_FIELDS_INVALID", 400)
|
|
||||||
|
|
||||||
status = _require_string(value["expected_status"], "EXPECTED_STATUS_INVALID")
|
|
||||||
if status != "QUEUED":
|
|
||||||
raise ExecutionRebindError("EXPECTED_STATUS_INVALID", 400)
|
|
||||||
|
|
||||||
revision = value["expected_revision"]
|
|
||||||
attempt = value["expected_attempt"]
|
|
||||||
if (
|
|
||||||
not isinstance(revision, int)
|
|
||||||
or isinstance(revision, bool)
|
|
||||||
or revision < 0
|
|
||||||
or not isinstance(attempt, int)
|
|
||||||
or isinstance(attempt, bool)
|
|
||||||
or attempt < 1
|
|
||||||
):
|
|
||||||
raise ExecutionRebindError("EXECUTION_REBIND_INTEGER_INVALID", 400)
|
|
||||||
|
|
||||||
verification = _require_string(
|
|
||||||
value["previous_verification_sha256"], "VERIFICATION_SHA256_INVALID"
|
|
||||||
)
|
|
||||||
old_digest = _require_string(
|
|
||||||
value["expected_old_execution_digest"], "OLD_EXECUTION_DIGEST_INVALID"
|
|
||||||
)
|
|
||||||
new_digest = _require_string(
|
|
||||||
value["expected_new_execution_digest"], "NEW_EXECUTION_DIGEST_INVALID"
|
|
||||||
)
|
|
||||||
for digest, code in (
|
|
||||||
(verification, "VERIFICATION_SHA256_INVALID"),
|
|
||||||
(old_digest, "OLD_EXECUTION_DIGEST_INVALID"),
|
|
||||||
(new_digest, "NEW_EXECUTION_DIGEST_INVALID"),
|
|
||||||
):
|
|
||||||
if SHA_RE.fullmatch(digest) is None:
|
|
||||||
raise ExecutionRebindError(code, 400)
|
|
||||||
if old_digest == new_digest:
|
|
||||||
raise ExecutionRebindError("EXECUTION_REBIND_DIGEST_UNCHANGED", 400)
|
|
||||||
|
|
||||||
authorization = _require_string(
|
|
||||||
value["authorization_id"], "AUTHORIZATION_ID_INVALID", KEY_RE
|
|
||||||
)
|
|
||||||
key = _require_string(value["idempotency_key"], "IDEMPOTENCY_KEY_INVALID", KEY_RE)
|
|
||||||
reason = _require_string(value["reason"], "EXECUTION_REBIND_REASON_INVALID")
|
|
||||||
if len(reason) > 512:
|
|
||||||
raise ExecutionRebindError("EXECUTION_REBIND_REASON_INVALID", 400)
|
|
||||||
|
|
||||||
return cls(
|
|
||||||
status,
|
|
||||||
revision,
|
|
||||||
attempt,
|
|
||||||
verification,
|
|
||||||
old_digest,
|
|
||||||
new_digest,
|
|
||||||
authorization,
|
|
||||||
key,
|
|
||||||
reason,
|
|
||||||
)
|
|
||||||
|
|
||||||
def public(self) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"expected_status": self.expected_status,
|
|
||||||
"expected_revision": self.expected_revision,
|
|
||||||
"expected_attempt": self.expected_attempt,
|
|
||||||
"previous_verification_sha256": self.previous_verification_sha256,
|
|
||||||
"expected_old_execution_digest": self.expected_old_execution_digest,
|
|
||||||
"expected_new_execution_digest": self.expected_new_execution_digest,
|
|
||||||
"authorization_id": self.authorization_id,
|
|
||||||
"idempotency_key": self.idempotency_key,
|
|
||||||
"reason": self.reason,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class CanonicalBridgeExecutionResolver:
|
|
||||||
"""Resolve the already-registered plan from the canonical governed Bridge."""
|
|
||||||
|
|
||||||
def __init__(self, base_url: str, control_token: str) -> None:
|
|
||||||
self.base_url = base_url.rstrip("/")
|
|
||||||
self.control_token = control_token
|
|
||||||
|
|
||||||
def __call__(
|
|
||||||
self,
|
|
||||||
task_id: str,
|
|
||||||
required_capabilities: Sequence[str],
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
body = json.dumps(
|
|
||||||
{
|
|
||||||
"task_id": task_id,
|
|
||||||
"required_capabilities": list(required_capabilities),
|
|
||||||
},
|
|
||||||
sort_keys=True,
|
|
||||||
separators=(",", ":"),
|
|
||||||
).encode("utf-8")
|
|
||||||
request = urllib.request.Request(
|
|
||||||
self.base_url + "/v2/execution-envelope",
|
|
||||||
method="POST",
|
|
||||||
data=body,
|
|
||||||
headers={
|
|
||||||
"Authorization": "Bearer " + self.control_token,
|
|
||||||
"Content-Type": "application/json",
|
|
||||||
"Accept": "application/json",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
with urllib.request.urlopen(request, timeout=15) as response:
|
|
||||||
raw = response.read()
|
|
||||||
status = response.status
|
|
||||||
except urllib.error.HTTPError as exc:
|
|
||||||
raw = exc.read()
|
|
||||||
status = exc.code
|
|
||||||
try:
|
|
||||||
value = json.loads(raw.decode("utf-8"))
|
|
||||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
||||||
raise ExecutionRebindError("CANONICAL_EXECUTION_RESPONSE_INVALID", 502) from exc
|
|
||||||
if not isinstance(value, Mapping):
|
|
||||||
raise ExecutionRebindError("CANONICAL_EXECUTION_RESPONSE_INVALID", 502)
|
|
||||||
if status != 200:
|
|
||||||
raise ExecutionRebindError(
|
|
||||||
str(value.get("error") or "CANONICAL_EXECUTION_RESOLUTION_FAILED"),
|
|
||||||
502 if status >= 500 else 409,
|
|
||||||
)
|
|
||||||
return dict(value)
|
|
||||||
|
|
||||||
|
|
||||||
class ExecutionRebindStore:
|
|
||||||
"""Official owner-controlled mutation with CAS, plan resolution and durable audit."""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
db_path: Path,
|
|
||||||
resolver: Callable[[str, Sequence[str]], Mapping[str, Any]],
|
|
||||||
) -> None:
|
|
||||||
self.db_path = Path(db_path)
|
|
||||||
self.resolver = resolver
|
|
||||||
|
|
||||||
def _connect(self) -> sqlite3.Connection:
|
|
||||||
connection = sqlite3.connect(self.db_path, timeout=30.0)
|
|
||||||
connection.row_factory = sqlite3.Row
|
|
||||||
connection.execute("PRAGMA foreign_keys=ON")
|
|
||||||
connection.execute("PRAGMA busy_timeout=30000")
|
|
||||||
return connection
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _columns(
|
|
||||||
connection: sqlite3.Connection,
|
|
||||||
table: str,
|
|
||||||
) -> dict[str, sqlite3.Row]:
|
|
||||||
return {
|
|
||||||
str(row["name"]): row
|
|
||||||
for row in connection.execute(f'PRAGMA table_info("{table}")')
|
|
||||||
}
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _json_value(raw: object, code: str) -> dict[str, Any]:
|
|
||||||
if isinstance(raw, Mapping):
|
|
||||||
return dict(raw)
|
|
||||||
if not isinstance(raw, str):
|
|
||||||
raise ExecutionRebindError(code, 409)
|
|
||||||
try:
|
|
||||||
value = json.loads(raw)
|
|
||||||
except json.JSONDecodeError as exc:
|
|
||||||
raise ExecutionRebindError(code, 409) from exc
|
|
||||||
if not isinstance(value, Mapping):
|
|
||||||
raise ExecutionRebindError(code, 409)
|
|
||||||
return dict(value)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _verification_hash(raw: object) -> str:
|
|
||||||
if raw in (None, ""):
|
|
||||||
value: object = {}
|
|
||||||
elif isinstance(raw, str):
|
|
||||||
try:
|
|
||||||
value = json.loads(raw)
|
|
||||||
except json.JSONDecodeError as exc:
|
|
||||||
raise ExecutionRebindError("TASK_VERIFICATION_INVALID", 409) from exc
|
|
||||||
else:
|
|
||||||
value = raw
|
|
||||||
return _sha(value)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _metadata_column(columns: Mapping[str, sqlite3.Row]) -> str:
|
|
||||||
for name in ("metadata_json", "metadata"):
|
|
||||||
if name in columns:
|
|
||||||
return name
|
|
||||||
raise ExecutionRebindError("TASK_METADATA_SCHEMA_UNSUPPORTED", 503)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _required_capabilities(
|
|
||||||
task: sqlite3.Row,
|
|
||||||
columns: Mapping[str, sqlite3.Row],
|
|
||||||
old_execution: Mapping[str, Any],
|
|
||||||
) -> list[str]:
|
|
||||||
for field in ("required_capabilities_json", "required_capabilities"):
|
|
||||||
if field not in columns:
|
|
||||||
continue
|
|
||||||
raw = task[field]
|
|
||||||
if raw in (None, ""):
|
|
||||||
continue
|
|
||||||
if isinstance(raw, str):
|
|
||||||
try:
|
|
||||||
value = json.loads(raw)
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
value = [item for item in raw.split(",") if item.strip()]
|
|
||||||
else:
|
|
||||||
value = raw
|
|
||||||
if isinstance(value, list) and all(isinstance(item, str) for item in value):
|
|
||||||
return [item.strip() for item in value if item.strip()]
|
|
||||||
value = old_execution.get("capability_scope")
|
|
||||||
if isinstance(value, list) and all(isinstance(item, str) for item in value):
|
|
||||||
return [item for item in value if item]
|
|
||||||
raise ExecutionRebindError("TASK_CAPABILITIES_UNAVAILABLE", 503)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _event_table(
|
|
||||||
connection: sqlite3.Connection,
|
|
||||||
) -> tuple[str, dict[str, sqlite3.Row]]:
|
|
||||||
tables = {
|
|
||||||
row["name"]
|
|
||||||
for row in connection.execute(
|
|
||||||
"SELECT name FROM sqlite_master WHERE type='table'"
|
|
||||||
)
|
|
||||||
}
|
|
||||||
for name in ("task_events", "events"):
|
|
||||||
if name not in tables:
|
|
||||||
continue
|
|
||||||
columns = ExecutionRebindStore._columns(connection, name)
|
|
||||||
if "task_id" in columns and ("event_type" in columns or "type" in columns):
|
|
||||||
return name, columns
|
|
||||||
raise ExecutionRebindError("TASK_EVENT_TABLE_UNSUPPORTED", 503)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _insert_event(
|
|
||||||
connection: sqlite3.Connection,
|
|
||||||
task_id: str,
|
|
||||||
payload: Mapping[str, Any],
|
|
||||||
) -> None:
|
|
||||||
table, columns = ExecutionRebindStore._event_table(connection)
|
|
||||||
values: dict[str, Any] = {"task_id": task_id}
|
|
||||||
if "event_type" in columns:
|
|
||||||
values["event_type"] = "TASK_EXECUTION_REBIND_AUTHORIZED"
|
|
||||||
else:
|
|
||||||
values["type"] = "TASK_EXECUTION_REBIND_AUTHORIZED"
|
|
||||||
payload_text = json.dumps(payload, sort_keys=True, separators=(",", ":"))
|
|
||||||
if "payload_json" in columns:
|
|
||||||
values["payload_json"] = payload_text
|
|
||||||
elif "payload" in columns:
|
|
||||||
values["payload"] = payload_text
|
|
||||||
else:
|
|
||||||
raise ExecutionRebindError("TASK_EVENT_PAYLOAD_UNSUPPORTED", 503)
|
|
||||||
if "created_at" in columns:
|
|
||||||
values["created_at"] = _now()
|
|
||||||
if "agent_id" in columns:
|
|
||||||
values["agent_id"] = "owner-control"
|
|
||||||
if "actor_id" in columns:
|
|
||||||
values["actor_id"] = "owner-control"
|
|
||||||
names = list(values)
|
|
||||||
quoted = ",".join('"' + name + '"' for name in names)
|
|
||||||
placeholders = ",".join("?" for _ in names)
|
|
||||||
connection.execute(
|
|
||||||
f'INSERT INTO "{table}" ({quoted}) VALUES ({placeholders})',
|
|
||||||
tuple(values[name] for name in names),
|
|
||||||
)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _ensure_rebind_table(connection: sqlite3.Connection) -> None:
|
|
||||||
connection.execute(
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS task_execution_rebinds (
|
|
||||||
task_id TEXT NOT NULL,
|
|
||||||
idempotency_key TEXT NOT NULL,
|
|
||||||
request_sha256 TEXT NOT NULL,
|
|
||||||
authorization_id TEXT NOT NULL,
|
|
||||||
previous_verification_sha256 TEXT NOT NULL,
|
|
||||||
old_execution_digest TEXT NOT NULL,
|
|
||||||
new_execution_digest TEXT NOT NULL,
|
|
||||||
expected_revision INTEGER NOT NULL,
|
|
||||||
applied_revision INTEGER NOT NULL,
|
|
||||||
attempt INTEGER NOT NULL,
|
|
||||||
reason TEXT NOT NULL,
|
|
||||||
created_at TEXT NOT NULL,
|
|
||||||
PRIMARY KEY(task_id,idempotency_key)
|
|
||||||
)
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
|
|
||||||
def _load_and_validate(
|
|
||||||
self,
|
|
||||||
connection: sqlite3.Connection,
|
|
||||||
task_id: str,
|
|
||||||
request: ExecutionRebindRequest,
|
|
||||||
) -> tuple[sqlite3.Row, dict[str, sqlite3.Row], str, dict[str, Any], dict[str, Any]]:
|
|
||||||
columns = self._columns(connection, "tasks")
|
|
||||||
required = {
|
|
||||||
"task_id",
|
|
||||||
"status",
|
|
||||||
"attempt",
|
|
||||||
"revision",
|
|
||||||
"claimed_by",
|
|
||||||
"lease_token_hash",
|
|
||||||
"lease_expires_at",
|
|
||||||
"verification_json",
|
|
||||||
}
|
|
||||||
if not required.issubset(columns):
|
|
||||||
raise ExecutionRebindError("TASK_SCHEMA_EXECUTION_REBIND_UNSUPPORTED", 503)
|
|
||||||
metadata_column = self._metadata_column(columns)
|
|
||||||
task = connection.execute(
|
|
||||||
"SELECT * FROM tasks WHERE task_id=?",
|
|
||||||
(task_id,),
|
|
||||||
).fetchone()
|
|
||||||
if task is None:
|
|
||||||
raise ExecutionRebindError("TASK_NOT_FOUND", 404)
|
|
||||||
|
|
||||||
if (
|
|
||||||
str(task["status"]) != request.expected_status
|
|
||||||
or int(task["revision"]) != request.expected_revision
|
|
||||||
or int(task["attempt"]) != request.expected_attempt
|
|
||||||
):
|
|
||||||
raise ExecutionRebindError("EXECUTION_REBIND_CAS_MISMATCH", 409)
|
|
||||||
if task["lease_token_hash"] not in (None, "") or task["lease_expires_at"] not in (None, ""):
|
|
||||||
raise ExecutionRebindError("EXECUTION_REBIND_ACTIVE_LEASE_DENY", 409)
|
|
||||||
|
|
||||||
verification_sha = self._verification_hash(task["verification_json"])
|
|
||||||
if verification_sha != request.previous_verification_sha256:
|
|
||||||
raise ExecutionRebindError("EXECUTION_REBIND_VERIFICATION_DRIFT", 409)
|
|
||||||
|
|
||||||
metadata = self._json_value(
|
|
||||||
task[metadata_column],
|
|
||||||
"TASK_METADATA_INVALID",
|
|
||||||
)
|
|
||||||
execution = metadata.get("execution")
|
|
||||||
if not isinstance(execution, Mapping):
|
|
||||||
raise ExecutionRebindError("TASK_EXECUTION_METADATA_MISSING", 409)
|
|
||||||
old_execution = validate_execution_envelope(dict(execution))
|
|
||||||
if old_execution["execution_digest"] != request.expected_old_execution_digest:
|
|
||||||
raise ExecutionRebindError("EXECUTION_REBIND_OLD_DIGEST_DRIFT", 409)
|
|
||||||
if old_execution.get("action") != "source_capture":
|
|
||||||
raise ExecutionRebindError("EXECUTION_REBIND_SOURCE_CAPTURE_REQUIRED", 409)
|
|
||||||
|
|
||||||
projection = metadata.get("execution_projection")
|
|
||||||
if isinstance(projection, Mapping):
|
|
||||||
projected_digest = str(projection.get("execution_digest") or "")
|
|
||||||
if projected_digest and projected_digest != request.expected_old_execution_digest:
|
|
||||||
raise ExecutionRebindError("EXECUTION_REBIND_PROJECTION_DRIFT", 409)
|
|
||||||
return task, columns, metadata_column, metadata, old_execution
|
|
||||||
|
|
||||||
def rebind(
|
|
||||||
self,
|
|
||||||
task_id: str,
|
|
||||||
request: ExecutionRebindRequest,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
task_id = _require_string(task_id, "TASK_ID_INVALID", TASK_RE)
|
|
||||||
required_authorization = _required_authorization_id()
|
|
||||||
if request.authorization_id != required_authorization:
|
|
||||||
raise ExecutionRebindError("EXECUTION_REBIND_AUTHORIZATION_ID_MISMATCH", 403)
|
|
||||||
|
|
||||||
request_sha = _sha(request.public())
|
|
||||||
|
|
||||||
# Idempotent replay must win before CAS validation because a successful
|
|
||||||
# rebind necessarily increments the Task revision and changes the persisted envelope.
|
|
||||||
with self._connect() as connection:
|
|
||||||
self._ensure_rebind_table(connection)
|
|
||||||
replay = connection.execute(
|
|
||||||
"SELECT * FROM task_execution_rebinds WHERE task_id=? AND idempotency_key=?",
|
|
||||||
(task_id, request.idempotency_key),
|
|
||||||
).fetchone()
|
|
||||||
if replay is not None:
|
|
||||||
if replay["request_sha256"] != request_sha:
|
|
||||||
raise ExecutionRebindError("EXECUTION_REBIND_IDEMPOTENCY_CONFLICT", 409)
|
|
||||||
connection.commit()
|
|
||||||
return {
|
|
||||||
"task_id": task_id,
|
|
||||||
"status": "QUEUED",
|
|
||||||
"attempt": int(replay["attempt"]),
|
|
||||||
"revision": int(replay["applied_revision"]),
|
|
||||||
"old_execution_digest": replay["old_execution_digest"],
|
|
||||||
"new_execution_digest": replay["new_execution_digest"],
|
|
||||||
"authorization_id": replay["authorization_id"],
|
|
||||||
"idempotent": True,
|
|
||||||
}
|
|
||||||
|
|
||||||
# Read/CAS snapshot first. Canonical Bridge resolution happens outside the DB write lock.
|
|
||||||
with self._connect() as connection:
|
|
||||||
task, columns, _, _, old_execution = self._load_and_validate(
|
|
||||||
connection, task_id, request
|
|
||||||
)
|
|
||||||
capabilities = self._required_capabilities(task, columns, old_execution)
|
|
||||||
|
|
||||||
binding = GovernedExecutionBinding.from_mapping(old_execution)
|
|
||||||
resolved = dict(self.resolver(task_id, capabilities))
|
|
||||||
new_execution = validate_execution_envelope(resolved, binding=binding)
|
|
||||||
if new_execution.get("action") != "deterministic":
|
|
||||||
raise ExecutionRebindError("EXECUTION_REBIND_DETERMINISTIC_PLAN_REQUIRED", 409)
|
|
||||||
if new_execution["execution_digest"] != request.expected_new_execution_digest:
|
|
||||||
raise ExecutionRebindError("EXECUTION_REBIND_NEW_DIGEST_DRIFT", 409)
|
|
||||||
if new_execution["execution_digest"] == old_execution["execution_digest"]:
|
|
||||||
raise ExecutionRebindError("EXECUTION_REBIND_DIGEST_UNCHANGED", 409)
|
|
||||||
|
|
||||||
# Re-read under BEGIN IMMEDIATE and apply only if the exact snapshot is unchanged.
|
|
||||||
with self._connect() as connection:
|
|
||||||
connection.execute("BEGIN IMMEDIATE")
|
|
||||||
self._ensure_rebind_table(connection)
|
|
||||||
|
|
||||||
replay = connection.execute(
|
|
||||||
"SELECT * FROM task_execution_rebinds WHERE task_id=? AND idempotency_key=?",
|
|
||||||
(task_id, request.idempotency_key),
|
|
||||||
).fetchone()
|
|
||||||
if replay is not None:
|
|
||||||
if replay["request_sha256"] != request_sha:
|
|
||||||
raise ExecutionRebindError("EXECUTION_REBIND_IDEMPOTENCY_CONFLICT", 409)
|
|
||||||
connection.commit()
|
|
||||||
return {
|
|
||||||
"task_id": task_id,
|
|
||||||
"status": "QUEUED",
|
|
||||||
"attempt": int(replay["attempt"]),
|
|
||||||
"revision": int(replay["applied_revision"]),
|
|
||||||
"old_execution_digest": replay["old_execution_digest"],
|
|
||||||
"new_execution_digest": replay["new_execution_digest"],
|
|
||||||
"authorization_id": replay["authorization_id"],
|
|
||||||
"idempotent": True,
|
|
||||||
}
|
|
||||||
|
|
||||||
task, columns, metadata_column, metadata, current_old = self._load_and_validate(
|
|
||||||
connection, task_id, request
|
|
||||||
)
|
|
||||||
if current_old["execution_digest"] != old_execution["execution_digest"]:
|
|
||||||
raise ExecutionRebindError("EXECUTION_REBIND_OLD_DIGEST_DRIFT", 409)
|
|
||||||
|
|
||||||
new_metadata = dict(metadata)
|
|
||||||
new_metadata["execution"] = new_execution
|
|
||||||
new_metadata["execution_projection"] = {
|
|
||||||
"schema": "aios.online-agent.task-projection.v1",
|
|
||||||
"task_id": binding.task_id,
|
|
||||||
"job_id": binding.job_id,
|
|
||||||
"execution_digest": new_execution["execution_digest"],
|
|
||||||
}
|
|
||||||
applied_revision = request.expected_revision + 1
|
|
||||||
previous_claimed_by = task["claimed_by"]
|
|
||||||
|
|
||||||
cursor = connection.execute(
|
|
||||||
f"""
|
|
||||||
UPDATE tasks
|
|
||||||
SET "{metadata_column}"=?, revision=?, claimed_by=NULL,
|
|
||||||
lease_token_hash=NULL, lease_expires_at=NULL
|
|
||||||
WHERE task_id=? AND status=? AND revision=? AND attempt=?
|
|
||||||
AND claimed_by IS ? AND lease_token_hash IS NULL AND lease_expires_at IS NULL
|
|
||||||
""",
|
|
||||||
(
|
|
||||||
json.dumps(new_metadata, sort_keys=True, separators=(",", ":")),
|
|
||||||
applied_revision,
|
|
||||||
task_id,
|
|
||||||
request.expected_status,
|
|
||||||
request.expected_revision,
|
|
||||||
request.expected_attempt,
|
|
||||||
previous_claimed_by,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
if cursor.rowcount != 1:
|
|
||||||
raise ExecutionRebindError("EXECUTION_REBIND_CAS_MISMATCH", 409)
|
|
||||||
|
|
||||||
event_payload = {
|
|
||||||
"schema": "aios.task-execution-rebind.event.v1",
|
|
||||||
"task_id": task_id,
|
|
||||||
"job_id": binding.job_id,
|
|
||||||
"authorization_id": request.authorization_id,
|
|
||||||
"idempotency_key": request.idempotency_key,
|
|
||||||
"reason": request.reason,
|
|
||||||
"previous_verification_sha256": request.previous_verification_sha256,
|
|
||||||
"old_execution_digest": old_execution["execution_digest"],
|
|
||||||
"new_execution_digest": new_execution["execution_digest"],
|
|
||||||
"old_action": old_execution.get("action"),
|
|
||||||
"new_action": new_execution.get("action"),
|
|
||||||
"previous_revision": request.expected_revision,
|
|
||||||
"applied_revision": applied_revision,
|
|
||||||
"attempt": request.expected_attempt,
|
|
||||||
"request_sha256": request_sha,
|
|
||||||
"previous_claimed_by": previous_claimed_by,
|
|
||||||
"stale_claim_normalized": previous_claimed_by not in (None, ""),
|
|
||||||
}
|
|
||||||
self._insert_event(connection, task_id, event_payload)
|
|
||||||
connection.execute(
|
|
||||||
"""
|
|
||||||
INSERT INTO task_execution_rebinds(
|
|
||||||
task_id,idempotency_key,request_sha256,authorization_id,
|
|
||||||
previous_verification_sha256,old_execution_digest,new_execution_digest,
|
|
||||||
expected_revision,applied_revision,attempt,reason,created_at
|
|
||||||
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)
|
|
||||||
""",
|
|
||||||
(
|
|
||||||
task_id,
|
|
||||||
request.idempotency_key,
|
|
||||||
request_sha,
|
|
||||||
request.authorization_id,
|
|
||||||
request.previous_verification_sha256,
|
|
||||||
old_execution["execution_digest"],
|
|
||||||
new_execution["execution_digest"],
|
|
||||||
request.expected_revision,
|
|
||||||
applied_revision,
|
|
||||||
request.expected_attempt,
|
|
||||||
request.reason,
|
|
||||||
_now(),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
connection.commit()
|
|
||||||
return {
|
|
||||||
"task_id": task_id,
|
|
||||||
"status": "QUEUED",
|
|
||||||
"attempt": request.expected_attempt,
|
|
||||||
"revision": applied_revision,
|
|
||||||
"old_execution_digest": old_execution["execution_digest"],
|
|
||||||
"new_execution_digest": new_execution["execution_digest"],
|
|
||||||
"authorization_id": request.authorization_id,
|
|
||||||
"idempotent": False,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class ExecutionRebindHandler(BaseHTTPRequestHandler):
|
|
||||||
store: ExecutionRebindStore | None = None
|
|
||||||
control_token: str = ""
|
|
||||||
|
|
||||||
def log_message(self, *_args: Any) -> None:
|
|
||||||
return
|
|
||||||
|
|
||||||
def _json(self, value: Mapping[str, Any], status: int) -> None:
|
|
||||||
data = json.dumps(
|
|
||||||
dict(value), sort_keys=True, separators=(",", ":")
|
|
||||||
).encode("utf-8")
|
|
||||||
self.send_response(status)
|
|
||||||
self.send_header("Content-Type", "application/json")
|
|
||||||
self.send_header("Content-Length", str(len(data)))
|
|
||||||
self.end_headers()
|
|
||||||
self.wfile.write(data)
|
|
||||||
|
|
||||||
def _authorized(self) -> bool:
|
|
||||||
header = self.headers.get("Authorization", "")
|
|
||||||
return (
|
|
||||||
header.startswith("Bearer ")
|
|
||||||
and bool(self.control_token)
|
|
||||||
and hmac.compare_digest(header[7:], self.control_token)
|
|
||||||
)
|
|
||||||
|
|
||||||
def do_GET(self) -> None:
|
|
||||||
if self.path == "/health":
|
|
||||||
self._json(
|
|
||||||
{"status": "ok", "service": "aios-agent-task-execution-rebind"},
|
|
||||||
200,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
self._json({"error": "NOT_FOUND"}, 404)
|
|
||||||
|
|
||||||
def do_POST(self) -> None:
|
|
||||||
match = re.fullmatch(r"/v1/tasks/([^/]+)/execution-rebind", self.path)
|
|
||||||
if match is None:
|
|
||||||
self._json({"error": "NOT_FOUND"}, 404)
|
|
||||||
return
|
|
||||||
if not self._authorized():
|
|
||||||
self._json({"error": "UNAUTHORIZED"}, 401)
|
|
||||||
return
|
|
||||||
try:
|
|
||||||
length = int(self.headers.get("Content-Length", "0"))
|
|
||||||
except ValueError:
|
|
||||||
self._json({"error": "CONTENT_LENGTH_INVALID"}, 400)
|
|
||||||
return
|
|
||||||
if length < 1 or length > MAX_BODY:
|
|
||||||
self._json({"error": "REQUEST_SIZE_INVALID"}, 413)
|
|
||||||
return
|
|
||||||
try:
|
|
||||||
value = json.loads(self.rfile.read(length).decode("utf-8"))
|
|
||||||
if not isinstance(value, Mapping):
|
|
||||||
raise ValueError
|
|
||||||
request = ExecutionRebindRequest.from_mapping(value)
|
|
||||||
if self.store is None:
|
|
||||||
raise ExecutionRebindError("RUNTIME_UNAVAILABLE", 503)
|
|
||||||
result = self.store.rebind(match.group(1), request)
|
|
||||||
self._json(result, 200 if result["idempotent"] else 201)
|
|
||||||
except ExecutionRebindError as exc:
|
|
||||||
self._json({"error": exc.code}, exc.status)
|
|
||||||
except (UnicodeDecodeError, json.JSONDecodeError, ValueError):
|
|
||||||
self._json({"error": "REQUEST_INVALID"}, 400)
|
|
||||||
|
|
||||||
|
|
||||||
def load_control_token(path: Path) -> str:
|
|
||||||
if path.is_symlink() or not path.is_file():
|
|
||||||
raise RuntimeError("TASK_EXECUTION_REBIND_CONTROL_CREDENTIAL_UNAVAILABLE")
|
|
||||||
value = path.read_text(encoding="utf-8").strip()
|
|
||||||
if len(value) < 32:
|
|
||||||
raise RuntimeError("TASK_EXECUTION_REBIND_CONTROL_CREDENTIAL_INVALID")
|
|
||||||
return value
|
|
||||||
|
|
||||||
|
|
||||||
def serve() -> None:
|
|
||||||
db = Path(os.environ.get("AIOS_AGENT_TASK_DB", "/var/lib/aios-agent-task/state.sqlite3"))
|
|
||||||
_required_authorization_id()
|
|
||||||
token_path = Path(
|
|
||||||
os.environ.get(
|
|
||||||
"AIOS_AGENT_TASK_CONTROL_TOKEN_FILE",
|
|
||||||
str(
|
|
||||||
Path(os.environ.get("CREDENTIALS_DIRECTORY", "/run/credentials"))
|
|
||||||
/ "agent_task_control_token"
|
|
||||||
),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
token = load_control_token(token_path)
|
|
||||||
bridge = os.environ.get(
|
|
||||||
"AIOS_GOVERNED_WRITE_API_URL",
|
|
||||||
"http://127.0.0.1:18892",
|
|
||||||
)
|
|
||||||
host = os.environ.get("AIOS_TASK_EXECUTION_REBIND_HOST", "127.0.0.1")
|
|
||||||
port = int(os.environ.get("AIOS_TASK_EXECUTION_REBIND_PORT", "18942"))
|
|
||||||
ExecutionRebindHandler.store = ExecutionRebindStore(
|
|
||||||
db,
|
|
||||||
CanonicalBridgeExecutionResolver(bridge, token),
|
|
||||||
)
|
|
||||||
ExecutionRebindHandler.control_token = token
|
|
||||||
ThreadingHTTPServer((host, port), ExecutionRebindHandler).serve_forever()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
serve()
|
|
||||||
@@ -1,427 +0,0 @@
|
|||||||
"""Owner-controlled, CAS-bound retry extension for exhausted Online Agent Tasks."""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
import hashlib
|
|
||||||
import hmac
|
|
||||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
from pathlib import Path
|
|
||||||
import re
|
|
||||||
import sqlite3
|
|
||||||
from typing import Any, Mapping
|
|
||||||
|
|
||||||
TASK_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{2,179}$")
|
|
||||||
KEY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{7,159}$")
|
|
||||||
MAX_BODY = 64 * 1024
|
|
||||||
AUTHORIZE_EXTENSION_ID_ENV = "AIOS_AGENT_TASK_AUTHORIZATION_ID"
|
|
||||||
|
|
||||||
|
|
||||||
class RetryExtensionError(RuntimeError):
|
|
||||||
def __init__(self, code: str, status: int = 409) -> None:
|
|
||||||
super().__init__(code)
|
|
||||||
self.code = code
|
|
||||||
self.status = status
|
|
||||||
|
|
||||||
|
|
||||||
def _canonical(value: object) -> bytes:
|
|
||||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
|
||||||
|
|
||||||
|
|
||||||
def _sha(value: object) -> str:
|
|
||||||
return hashlib.sha256(_canonical(value)).hexdigest()
|
|
||||||
|
|
||||||
|
|
||||||
def _required_authorization_id() -> str:
|
|
||||||
value = os.environ.get(AUTHORIZE_EXTENSION_ID_ENV, "").strip()
|
|
||||||
if not value:
|
|
||||||
raise RuntimeError("TASK_RETRY_AUTHORIZATION_ID_MISSING")
|
|
||||||
if KEY_RE.fullmatch(value) is None:
|
|
||||||
raise RuntimeError("TASK_RETRY_AUTHORIZATION_ID_INVALID")
|
|
||||||
return value
|
|
||||||
|
|
||||||
|
|
||||||
def _now() -> str:
|
|
||||||
return datetime.now(timezone.utc).isoformat()
|
|
||||||
|
|
||||||
|
|
||||||
def _require_string(value: object, code: str, pattern: re.Pattern[str] | None = None) -> str:
|
|
||||||
if not isinstance(value, str) or not value.strip():
|
|
||||||
raise RetryExtensionError(code, 400)
|
|
||||||
value = value.strip()
|
|
||||||
if pattern is not None and pattern.fullmatch(value) is None:
|
|
||||||
raise RetryExtensionError(code, 400)
|
|
||||||
return value
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class RetryExtensionRequest:
|
|
||||||
expected_status: str
|
|
||||||
expected_revision: int
|
|
||||||
expected_attempt: int
|
|
||||||
expected_max_attempts: int
|
|
||||||
next_max_attempts: int
|
|
||||||
previous_verification_sha256: str
|
|
||||||
authorization_id: str
|
|
||||||
idempotency_key: str
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_mapping(cls, value: Mapping[str, Any]) -> "RetryExtensionRequest":
|
|
||||||
allowed = {
|
|
||||||
"expected_status", "expected_revision", "expected_attempt", "expected_max_attempts",
|
|
||||||
"next_max_attempts", "previous_verification_sha256", "authorization_id", "idempotency_key",
|
|
||||||
}
|
|
||||||
if set(value) != allowed:
|
|
||||||
raise RetryExtensionError("RETRY_EXTENSION_FIELDS_INVALID", 400)
|
|
||||||
status = _require_string(value["expected_status"], "EXPECTED_STATUS_INVALID")
|
|
||||||
if status != "FAILED":
|
|
||||||
raise RetryExtensionError("EXPECTED_STATUS_INVALID", 400)
|
|
||||||
ints: dict[str, int] = {}
|
|
||||||
for field in ("expected_revision", "expected_attempt", "expected_max_attempts", "next_max_attempts"):
|
|
||||||
item = value[field]
|
|
||||||
if not isinstance(item, int) or isinstance(item, bool) or item < 0:
|
|
||||||
raise RetryExtensionError("RETRY_EXTENSION_INTEGER_INVALID", 400)
|
|
||||||
ints[field] = item
|
|
||||||
if ints["expected_attempt"] != ints["expected_max_attempts"]:
|
|
||||||
raise RetryExtensionError("RETRY_EXTENSION_NOT_EXHAUSTED", 409)
|
|
||||||
if ints["next_max_attempts"] != ints["expected_max_attempts"] + 1:
|
|
||||||
raise RetryExtensionError("RETRY_EXTENSION_MUST_BE_BOUNDED_ONE_STEP", 400)
|
|
||||||
if ints["next_max_attempts"] > 50:
|
|
||||||
raise RetryExtensionError("RETRY_EXTENSION_LIMIT_EXCEEDED", 400)
|
|
||||||
verification = _require_string(value["previous_verification_sha256"], "VERIFICATION_SHA256_INVALID")
|
|
||||||
if re.fullmatch(r"[0-9a-f]{64}", verification) is None:
|
|
||||||
raise RetryExtensionError("VERIFICATION_SHA256_INVALID", 400)
|
|
||||||
authorization = _require_string(value["authorization_id"], "AUTHORIZATION_ID_INVALID", KEY_RE)
|
|
||||||
key = _require_string(value["idempotency_key"], "IDEMPOTENCY_KEY_INVALID", KEY_RE)
|
|
||||||
return cls(
|
|
||||||
status,
|
|
||||||
ints["expected_revision"],
|
|
||||||
ints["expected_attempt"],
|
|
||||||
ints["expected_max_attempts"],
|
|
||||||
ints["next_max_attempts"],
|
|
||||||
verification,
|
|
||||||
authorization,
|
|
||||||
key,
|
|
||||||
)
|
|
||||||
|
|
||||||
def public(self) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"expected_status": self.expected_status,
|
|
||||||
"expected_revision": self.expected_revision,
|
|
||||||
"expected_attempt": self.expected_attempt,
|
|
||||||
"expected_max_attempts": self.expected_max_attempts,
|
|
||||||
"next_max_attempts": self.next_max_attempts,
|
|
||||||
"previous_verification_sha256": self.previous_verification_sha256,
|
|
||||||
"authorization_id": self.authorization_id,
|
|
||||||
"idempotency_key": self.idempotency_key,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class RetryExtensionStore:
|
|
||||||
"""Owns the official mutation. Direct/manual DB edits remain forbidden."""
|
|
||||||
|
|
||||||
def __init__(self, db_path: Path) -> None:
|
|
||||||
self.db_path = Path(db_path)
|
|
||||||
|
|
||||||
def _connect(self) -> sqlite3.Connection:
|
|
||||||
connection = sqlite3.connect(self.db_path, timeout=30.0)
|
|
||||||
connection.row_factory = sqlite3.Row
|
|
||||||
connection.execute("PRAGMA foreign_keys=ON")
|
|
||||||
connection.execute("PRAGMA busy_timeout=30000")
|
|
||||||
return connection
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _columns(connection: sqlite3.Connection, table: str) -> dict[str, sqlite3.Row]:
|
|
||||||
return {str(row["name"]): row for row in connection.execute(f'PRAGMA table_info("{table}")')}
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _event_table(connection: sqlite3.Connection) -> tuple[str, dict[str, sqlite3.Row]]:
|
|
||||||
tables = {
|
|
||||||
row["name"]
|
|
||||||
for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'")
|
|
||||||
}
|
|
||||||
for name in ("task_events", "events"):
|
|
||||||
if name not in tables:
|
|
||||||
continue
|
|
||||||
columns = RetryExtensionStore._columns(connection, name)
|
|
||||||
if "task_id" in columns and ("event_type" in columns or "type" in columns):
|
|
||||||
return name, columns
|
|
||||||
raise RetryExtensionError("TASK_EVENT_TABLE_UNSUPPORTED", 503)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _verification_hash(raw: object) -> str:
|
|
||||||
if raw in (None, ""):
|
|
||||||
value: object = {}
|
|
||||||
elif isinstance(raw, str):
|
|
||||||
try:
|
|
||||||
value = json.loads(raw)
|
|
||||||
except json.JSONDecodeError as exc:
|
|
||||||
raise RetryExtensionError("TASK_VERIFICATION_INVALID", 409) from exc
|
|
||||||
else:
|
|
||||||
value = raw
|
|
||||||
return _sha(value)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _ensure_extension_table(connection: sqlite3.Connection) -> None:
|
|
||||||
connection.execute(
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS task_retry_extensions (
|
|
||||||
task_id TEXT NOT NULL,
|
|
||||||
idempotency_key TEXT NOT NULL,
|
|
||||||
request_sha256 TEXT NOT NULL,
|
|
||||||
authorization_id TEXT NOT NULL,
|
|
||||||
previous_verification_sha256 TEXT NOT NULL,
|
|
||||||
previous_max_attempts INTEGER NOT NULL,
|
|
||||||
next_max_attempts INTEGER NOT NULL,
|
|
||||||
expected_revision INTEGER NOT NULL,
|
|
||||||
applied_revision INTEGER NOT NULL,
|
|
||||||
created_at TEXT NOT NULL,
|
|
||||||
PRIMARY KEY(task_id, idempotency_key)
|
|
||||||
)
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _insert_event(
|
|
||||||
connection: sqlite3.Connection,
|
|
||||||
task_id: str,
|
|
||||||
payload: Mapping[str, Any],
|
|
||||||
) -> None:
|
|
||||||
table, columns = RetryExtensionStore._event_table(connection)
|
|
||||||
values: dict[str, Any] = {"task_id": task_id}
|
|
||||||
if "event_type" in columns:
|
|
||||||
values["event_type"] = "TASK_RETRY_EXTENSION_AUTHORIZED"
|
|
||||||
else:
|
|
||||||
values["type"] = "TASK_RETRY_EXTENSION_AUTHORIZED"
|
|
||||||
payload_text = json.dumps(payload, sort_keys=True, separators=(",", ":"))
|
|
||||||
if "payload_json" in columns:
|
|
||||||
values["payload_json"] = payload_text
|
|
||||||
elif "payload" in columns:
|
|
||||||
values["payload"] = payload_text
|
|
||||||
else:
|
|
||||||
raise RetryExtensionError("TASK_EVENT_PAYLOAD_UNSUPPORTED", 503)
|
|
||||||
if "created_at" in columns:
|
|
||||||
values["created_at"] = _now()
|
|
||||||
if "agent_id" in columns:
|
|
||||||
values["agent_id"] = "owner-control"
|
|
||||||
if "actor_id" in columns:
|
|
||||||
values["actor_id"] = "owner-control"
|
|
||||||
names = list(values)
|
|
||||||
sql = f'INSERT INTO "{table}" ({",".join(f"""\"{name}\"""" for name in names)}) VALUES ({",".join("?" for _ in names)})'
|
|
||||||
connection.execute(sql, tuple(values[name] for name in names))
|
|
||||||
|
|
||||||
def extend(self, task_id: str, request: RetryExtensionRequest) -> dict[str, Any]:
|
|
||||||
task_id = _require_string(task_id, "TASK_ID_INVALID", TASK_RE)
|
|
||||||
required_authorization_id = _required_authorization_id()
|
|
||||||
if request.authorization_id != required_authorization_id:
|
|
||||||
raise RetryExtensionError("RETRY_EXTENSION_AUTHORIZATION_ID_MISMATCH", 403)
|
|
||||||
request_payload = request.public()
|
|
||||||
request_sha = _sha(request_payload)
|
|
||||||
with self._connect() as connection:
|
|
||||||
connection.execute("BEGIN IMMEDIATE")
|
|
||||||
self._ensure_extension_table(connection)
|
|
||||||
replay = connection.execute(
|
|
||||||
"SELECT * FROM task_retry_extensions WHERE task_id=? AND idempotency_key=?",
|
|
||||||
(task_id, request.idempotency_key),
|
|
||||||
).fetchone()
|
|
||||||
if replay is not None:
|
|
||||||
if replay["request_sha256"] != request_sha:
|
|
||||||
raise RetryExtensionError("RETRY_EXTENSION_IDEMPOTENCY_CONFLICT", 409)
|
|
||||||
connection.commit()
|
|
||||||
return {
|
|
||||||
"task_id": task_id,
|
|
||||||
"status": "QUEUED",
|
|
||||||
"attempt": request.expected_attempt,
|
|
||||||
"max_attempts": int(replay["next_max_attempts"]),
|
|
||||||
"revision": int(replay["applied_revision"]),
|
|
||||||
"idempotent": True,
|
|
||||||
"authorization_id": replay["authorization_id"],
|
|
||||||
"previous_verification_sha256": replay["previous_verification_sha256"],
|
|
||||||
}
|
|
||||||
|
|
||||||
columns = self._columns(connection, "tasks")
|
|
||||||
required = {
|
|
||||||
"task_id", "status", "attempt", "max_attempts", "revision", "claimed_by",
|
|
||||||
"lease_token_hash", "lease_expires_at", "verification_json",
|
|
||||||
}
|
|
||||||
if not required.issubset(columns):
|
|
||||||
raise RetryExtensionError("TASK_SCHEMA_RETRY_EXTENSION_UNSUPPORTED", 503)
|
|
||||||
task = connection.execute("SELECT * FROM tasks WHERE task_id=?", (task_id,)).fetchone()
|
|
||||||
if task is None:
|
|
||||||
raise RetryExtensionError("TASK_NOT_FOUND", 404)
|
|
||||||
actual_verification_sha = self._verification_hash(task["verification_json"])
|
|
||||||
if actual_verification_sha != request.previous_verification_sha256:
|
|
||||||
raise RetryExtensionError("RETRY_EXTENSION_VERIFICATION_DRIFT", 409)
|
|
||||||
if (
|
|
||||||
task["status"] != request.expected_status
|
|
||||||
or int(task["revision"]) != request.expected_revision
|
|
||||||
or int(task["attempt"]) != request.expected_attempt
|
|
||||||
or int(task["max_attempts"]) != request.expected_max_attempts
|
|
||||||
):
|
|
||||||
raise RetryExtensionError("RETRY_EXTENSION_CAS_MISMATCH", 409)
|
|
||||||
for lease_field in ("lease_token_hash", "lease_expires_at"):
|
|
||||||
if lease_field in columns and task[lease_field] not in (None, ""):
|
|
||||||
raise RetryExtensionError("RETRY_EXTENSION_ACTIVE_LEASE_DENY", 409)
|
|
||||||
|
|
||||||
previous_claimed_by = task["claimed_by"]
|
|
||||||
applied_revision = request.expected_revision + 1
|
|
||||||
cursor = connection.execute(
|
|
||||||
"""
|
|
||||||
UPDATE tasks
|
|
||||||
SET status='QUEUED', max_attempts=?, revision=?, claimed_by=NULL,
|
|
||||||
lease_token_hash=NULL, lease_expires_at=NULL
|
|
||||||
WHERE task_id=? AND status=? AND revision=? AND attempt=? AND max_attempts=?
|
|
||||||
AND claimed_by IS ?
|
|
||||||
AND lease_token_hash IS NULL AND lease_expires_at IS NULL
|
|
||||||
""",
|
|
||||||
(
|
|
||||||
request.next_max_attempts,
|
|
||||||
applied_revision,
|
|
||||||
task_id,
|
|
||||||
request.expected_status,
|
|
||||||
request.expected_revision,
|
|
||||||
request.expected_attempt,
|
|
||||||
request.expected_max_attempts,
|
|
||||||
previous_claimed_by,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
if cursor.rowcount != 1:
|
|
||||||
raise RetryExtensionError("RETRY_EXTENSION_CAS_MISMATCH", 409)
|
|
||||||
|
|
||||||
event_payload = {
|
|
||||||
"schema": "aios.task-retry-extension.event.v1",
|
|
||||||
"task_id": task_id,
|
|
||||||
"authorization_id": request.authorization_id,
|
|
||||||
"idempotency_key": request.idempotency_key,
|
|
||||||
"previous_verification_sha256": request.previous_verification_sha256,
|
|
||||||
"previous_max_attempts": request.expected_max_attempts,
|
|
||||||
"next_max_attempts": request.next_max_attempts,
|
|
||||||
"previous_revision": request.expected_revision,
|
|
||||||
"applied_revision": applied_revision,
|
|
||||||
"request_sha256": request_sha,
|
|
||||||
"previous_claimed_by": previous_claimed_by,
|
|
||||||
"stale_terminal_claim_normalized": previous_claimed_by not in (None, ""),
|
|
||||||
}
|
|
||||||
self._insert_event(connection, task_id, event_payload)
|
|
||||||
connection.execute(
|
|
||||||
"""
|
|
||||||
INSERT INTO task_retry_extensions(
|
|
||||||
task_id,idempotency_key,request_sha256,authorization_id,previous_verification_sha256,
|
|
||||||
previous_max_attempts,next_max_attempts,expected_revision,applied_revision,created_at
|
|
||||||
) VALUES (?,?,?,?,?,?,?,?,?,?)
|
|
||||||
""",
|
|
||||||
(
|
|
||||||
task_id,
|
|
||||||
request.idempotency_key,
|
|
||||||
request_sha,
|
|
||||||
request.authorization_id,
|
|
||||||
request.previous_verification_sha256,
|
|
||||||
request.expected_max_attempts,
|
|
||||||
request.next_max_attempts,
|
|
||||||
request.expected_revision,
|
|
||||||
applied_revision,
|
|
||||||
_now(),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
connection.commit()
|
|
||||||
return {
|
|
||||||
"task_id": task_id,
|
|
||||||
"status": "QUEUED",
|
|
||||||
"attempt": request.expected_attempt,
|
|
||||||
"max_attempts": request.next_max_attempts,
|
|
||||||
"revision": applied_revision,
|
|
||||||
"idempotent": False,
|
|
||||||
"authorization_id": request.authorization_id,
|
|
||||||
"previous_verification_sha256": request.previous_verification_sha256,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class RetryExtensionHandler(BaseHTTPRequestHandler):
|
|
||||||
store: RetryExtensionStore | None = None
|
|
||||||
control_token: str = ""
|
|
||||||
|
|
||||||
def log_message(self, *_args: Any) -> None:
|
|
||||||
return
|
|
||||||
|
|
||||||
def _json(self, value: Mapping[str, Any], status: int) -> None:
|
|
||||||
data = json.dumps(dict(value), sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
||||||
self.send_response(status)
|
|
||||||
self.send_header("Content-Type", "application/json")
|
|
||||||
self.send_header("Content-Length", str(len(data)))
|
|
||||||
self.end_headers()
|
|
||||||
self.wfile.write(data)
|
|
||||||
|
|
||||||
def _authorized(self) -> bool:
|
|
||||||
header = self.headers.get("Authorization", "")
|
|
||||||
return (
|
|
||||||
header.startswith("Bearer ")
|
|
||||||
and bool(self.control_token)
|
|
||||||
and hmac.compare_digest(header[7:], self.control_token)
|
|
||||||
)
|
|
||||||
|
|
||||||
def do_GET(self) -> None:
|
|
||||||
if self.path == "/health":
|
|
||||||
self._json({"status": "ok", "service": "aios-agent-task-retry-extension"}, 200)
|
|
||||||
else:
|
|
||||||
self._json({"error": "NOT_FOUND"}, 404)
|
|
||||||
|
|
||||||
def do_POST(self) -> None:
|
|
||||||
match = re.fullmatch(r"/v1/tasks/([^/]+)/retry-extension", self.path)
|
|
||||||
if match is None:
|
|
||||||
self._json({"error": "NOT_FOUND"}, 404)
|
|
||||||
return
|
|
||||||
if not self._authorized():
|
|
||||||
self._json({"error": "UNAUTHORIZED"}, 401)
|
|
||||||
return
|
|
||||||
try:
|
|
||||||
length = int(self.headers.get("Content-Length", "0"))
|
|
||||||
except ValueError:
|
|
||||||
self._json({"error": "CONTENT_LENGTH_INVALID"}, 400)
|
|
||||||
return
|
|
||||||
if length < 1 or length > MAX_BODY:
|
|
||||||
self._json({"error": "REQUEST_SIZE_INVALID"}, 413)
|
|
||||||
return
|
|
||||||
try:
|
|
||||||
value = json.loads(self.rfile.read(length).decode("utf-8"))
|
|
||||||
if not isinstance(value, Mapping):
|
|
||||||
raise ValueError
|
|
||||||
request = RetryExtensionRequest.from_mapping(value)
|
|
||||||
if self.store is None:
|
|
||||||
raise RetryExtensionError("RUNTIME_UNAVAILABLE", 503)
|
|
||||||
result = self.store.extend(match.group(1), request)
|
|
||||||
self._json(result, 200 if result["idempotent"] else 201)
|
|
||||||
except RetryExtensionError as exc:
|
|
||||||
self._json({"error": exc.code}, exc.status)
|
|
||||||
except (UnicodeDecodeError, json.JSONDecodeError, ValueError):
|
|
||||||
self._json({"error": "REQUEST_INVALID"}, 400)
|
|
||||||
|
|
||||||
|
|
||||||
def load_control_token(path: Path) -> str:
|
|
||||||
if path.is_symlink() or not path.is_file():
|
|
||||||
raise RuntimeError("TASK_RETRY_CONTROL_CREDENTIAL_UNAVAILABLE")
|
|
||||||
value = path.read_text(encoding="utf-8").strip()
|
|
||||||
if len(value) < 32:
|
|
||||||
raise RuntimeError("TASK_RETRY_CONTROL_CREDENTIAL_INVALID")
|
|
||||||
return value
|
|
||||||
|
|
||||||
|
|
||||||
def serve() -> None:
|
|
||||||
db = Path(os.environ.get("AIOS_AGENT_TASK_DB", "/var/lib/aios-agent-task/state.sqlite3"))
|
|
||||||
_required_authorization_id()
|
|
||||||
token_path = Path(
|
|
||||||
os.environ.get(
|
|
||||||
"AIOS_AGENT_TASK_CONTROL_TOKEN_FILE",
|
|
||||||
str(Path(os.environ.get("CREDENTIALS_DIRECTORY", "/run/credentials")) / "agent_task_control_token"),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
host = os.environ.get("AIOS_TASK_RETRY_HOST", "127.0.0.1")
|
|
||||||
port = int(os.environ.get("AIOS_TASK_RETRY_PORT", "18941"))
|
|
||||||
RetryExtensionHandler.store = RetryExtensionStore(db)
|
|
||||||
RetryExtensionHandler.control_token = load_control_token(token_path)
|
|
||||||
ThreadingHTTPServer((host, port), RetryExtensionHandler).serve_forever()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
serve()
|
|
||||||
@@ -94,91 +94,6 @@ class AuthorityGitExecutor:
|
|||||||
self._git(["worktree", "add", "--detach", str(path), base_oid])
|
self._git(["worktree", "add", "--detach", str(path), base_oid])
|
||||||
return path
|
return path
|
||||||
|
|
||||||
def recover_failed_detached_worktree_exact(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
path: Path,
|
|
||||||
expected_oid: str,
|
|
||||||
stale_oid: str,
|
|
||||||
target_ref: str,
|
|
||||||
) -> str:
|
|
||||||
"""Recreate one exact clean, detached, single-attempt worktree at its durable head."""
|
|
||||||
_oid(expected_oid, "expected_oid")
|
|
||||||
_oid(stale_oid, "stale_oid")
|
|
||||||
if not TASK_BRANCH_RE.fullmatch(target_ref):
|
|
||||||
raise AuthorityGitError("TASK_BRANCH_SCOPE_DENIED")
|
|
||||||
if self.read_ref(target_ref) is not None:
|
|
||||||
raise AuthorityGitError("AUTHORITY_TARGET_BRANCH_ALREADY_EXISTS")
|
|
||||||
if (
|
|
||||||
not path.is_absolute()
|
|
||||||
or path.parent != self.worktree_root
|
|
||||||
or path.name in {"", ".", ".."}
|
|
||||||
or path.is_symlink()
|
|
||||||
or not path.is_dir()
|
|
||||||
):
|
|
||||||
raise AuthorityGitError("WORKTREE_RECOVERY_PATH_INVALID")
|
|
||||||
head = subprocess.run(
|
|
||||||
["git", "-C", str(path), "rev-parse", "HEAD"],
|
|
||||||
text=True, capture_output=True, check=False,
|
|
||||||
)
|
|
||||||
if head.returncode != 0 or head.stdout.strip() != stale_oid or stale_oid == expected_oid:
|
|
||||||
raise AuthorityGitError("WORKTREE_RECOVERY_HEAD_INVALID")
|
|
||||||
symbolic = subprocess.run(
|
|
||||||
["git", "-C", str(path), "symbolic-ref", "-q", "HEAD"],
|
|
||||||
text=True, capture_output=True, check=False,
|
|
||||||
)
|
|
||||||
if symbolic.returncode != 1:
|
|
||||||
raise AuthorityGitError("WORKTREE_RECOVERY_NOT_DETACHED")
|
|
||||||
status = subprocess.run(
|
|
||||||
["git", "-C", str(path), "status", "--porcelain=v1", "--untracked-files=all"],
|
|
||||||
text=True, capture_output=True, check=False,
|
|
||||||
)
|
|
||||||
if status.returncode != 0 or status.stdout:
|
|
||||||
raise AuthorityGitError("WORKTREE_RECOVERY_NOT_CLEAN")
|
|
||||||
parents = self._git(["rev-list", "--parents", "-n", "1", stale_oid]).split()
|
|
||||||
if parents != [stale_oid, expected_oid]:
|
|
||||||
raise AuthorityGitError("WORKTREE_RECOVERY_LINEAGE_INVALID")
|
|
||||||
registered = [
|
|
||||||
line.removeprefix("worktree ")
|
|
||||||
for line in self._git(["worktree", "list", "--porcelain"]).splitlines()
|
|
||||||
if line.startswith("worktree ")
|
|
||||||
]
|
|
||||||
if registered.count(str(path)) != 1:
|
|
||||||
raise AuthorityGitError("WORKTREE_RECOVERY_REGISTRATION_INVALID")
|
|
||||||
self._git(["worktree", "remove", "--force", str(path)])
|
|
||||||
if path.exists() or path.is_symlink():
|
|
||||||
raise AuthorityGitError("WORKTREE_RECOVERY_REMOVE_FAILED")
|
|
||||||
recreated = self.create_detached_worktree(expected_oid, path.name, path.parent)
|
|
||||||
readback = subprocess.run(
|
|
||||||
["git", "-C", str(recreated), "rev-parse", "HEAD"],
|
|
||||||
text=True, capture_output=True, check=False,
|
|
||||||
)
|
|
||||||
detached = subprocess.run(
|
|
||||||
["git", "-C", str(recreated), "symbolic-ref", "-q", "HEAD"],
|
|
||||||
text=True, capture_output=True, check=False,
|
|
||||||
)
|
|
||||||
if readback.returncode != 0 or readback.stdout.strip() != expected_oid or detached.returncode != 1:
|
|
||||||
raise AuthorityGitError("WORKTREE_RECOVERY_READBACK_MISMATCH")
|
|
||||||
return stale_oid
|
|
||||||
|
|
||||||
def create_task_branch_CAS(self, branch: str, expected_base_oid: str, new_oid: str) -> str:
|
|
||||||
if not TASK_BRANCH_RE.fullmatch(branch):
|
|
||||||
raise AuthorityGitError("TASK_BRANCH_SCOPE_DENIED")
|
|
||||||
_oid(expected_base_oid, "expected_base_oid")
|
|
||||||
_oid(new_oid, "new_oid")
|
|
||||||
current = self.read_ref(branch)
|
|
||||||
if current is not None:
|
|
||||||
if current == new_oid:
|
|
||||||
return new_oid
|
|
||||||
raise AuthorityGitError("AUTHORITY_TARGET_BRANCH_ALREADY_EXISTS")
|
|
||||||
if not self.ancestry_check(expected_base_oid, new_oid):
|
|
||||||
raise AuthorityGitError("NEW_OID_NOT_DESCENDANT")
|
|
||||||
self._git(["update-ref", branch, new_oid, "0" * 40])
|
|
||||||
readback = self.read_ref(branch)
|
|
||||||
if readback != new_oid:
|
|
||||||
raise AuthorityGitError("AUTHORITY_CREATE_READBACK_MISMATCH")
|
|
||||||
return readback
|
|
||||||
|
|
||||||
def advance_task_branch_CAS(self, branch: str, expected_oid: str, new_oid: str) -> str:
|
def advance_task_branch_CAS(self, branch: str, expected_oid: str, new_oid: str) -> str:
|
||||||
if not TASK_BRANCH_RE.fullmatch(branch):
|
if not TASK_BRANCH_RE.fullmatch(branch):
|
||||||
raise AuthorityGitError("TASK_BRANCH_SCOPE_DENIED")
|
raise AuthorityGitError("TASK_BRANCH_SCOPE_DENIED")
|
||||||
|
|||||||
@@ -1,163 +0,0 @@
|
|||||||
"""Canonical fail-closed capability semantics for task eligibility."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from collections.abc import Iterable
|
|
||||||
|
|
||||||
|
|
||||||
_TASK_ALIASES: dict[str, frozenset[str]] = {
|
|
||||||
"repository_read": frozenset({"repo.read"}),
|
|
||||||
"git.read": frozenset({"repo.read"}),
|
|
||||||
"repo.read": frozenset({"repo.read"}),
|
|
||||||
"test_execution": frozenset({"test.execute"}),
|
|
||||||
"git.test": frozenset({"test.execute"}),
|
|
||||||
"test.execute": frozenset({"test.execute"}),
|
|
||||||
"governed_git_write": frozenset({"governed.git.write.branch", "governed.git.write.file"}),
|
|
||||||
"governed.git.write": frozenset({"governed.git.write.branch", "governed.git.write.file"}),
|
|
||||||
"governed.write.branch": frozenset({"governed.git.write.branch"}),
|
|
||||||
"governed.write.file": frozenset({"governed.git.write.file"}),
|
|
||||||
"shell": frozenset({"runtime.shell"}),
|
|
||||||
"git": frozenset({"runtime.git"}),
|
|
||||||
"python": frozenset({"runtime.python"}),
|
|
||||||
"code.read": frozenset({"legacy.code.read"}),
|
|
||||||
"code.review": frozenset({"legacy.code.review"}),
|
|
||||||
"code.write": frozenset({"legacy.code.write"}),
|
|
||||||
"code.write.patch": frozenset({"legacy.code.write.patch"}),
|
|
||||||
"test.python": frozenset({"legacy.test.python"}),
|
|
||||||
"test.shell.allowlisted": frozenset({"legacy.test.shell.allowlisted"}),
|
|
||||||
"git.commit": frozenset({"legacy.git.commit"}),
|
|
||||||
"git.diff.read": frozenset({"legacy.git.diff.read"}),
|
|
||||||
"git.force_push": frozenset({"legacy.git.force_push"}),
|
|
||||||
"deploy.production": frozenset({"legacy.deploy.production"}),
|
|
||||||
"secret.transform": frozenset({"legacy.secret.transform"}),
|
|
||||||
"secure_file_transformer": frozenset({"legacy.secure_file_transformer"}),
|
|
||||||
"runtime.shadow": frozenset({"legacy.runtime.shadow"}),
|
|
||||||
"review.security": frozenset({"legacy.review.security"}),
|
|
||||||
"review.api-contract": frozenset({"legacy.review.api-contract"}),
|
|
||||||
"gpu.train": frozenset({"legacy.gpu.train"}),
|
|
||||||
"agent.runtime.hermes": frozenset({"agent.runtime.hermes"}),
|
|
||||||
"agent.runtime.openclaw": frozenset({"agent.runtime.openclaw"}),
|
|
||||||
}
|
|
||||||
|
|
||||||
_WORKER_ALIASES: dict[str, frozenset[str]] = {
|
|
||||||
"git.read": frozenset({"repo.read"}),
|
|
||||||
"repo.read": frozenset({"repo.read"}),
|
|
||||||
"repository_read": frozenset({"repo.read"}),
|
|
||||||
"git.test": frozenset({"test.execute"}),
|
|
||||||
"test.execute": frozenset({"test.execute"}),
|
|
||||||
"test_execution": frozenset({"test.execute"}),
|
|
||||||
"governed.write.branch": frozenset({"governed.git.write.branch"}),
|
|
||||||
"governed.git.write.branch": frozenset({"governed.git.write.branch"}),
|
|
||||||
"governed.write.file": frozenset({"governed.git.write.file"}),
|
|
||||||
"governed.git.write.file": frozenset({"governed.git.write.file"}),
|
|
||||||
"shell": frozenset({"runtime.shell"}),
|
|
||||||
"git": frozenset({"runtime.git"}),
|
|
||||||
"python": frozenset({"runtime.python"}),
|
|
||||||
"code.read": frozenset({"legacy.code.read"}),
|
|
||||||
"code.review": frozenset({"legacy.code.review"}),
|
|
||||||
"code.write": frozenset({"legacy.code.write"}),
|
|
||||||
"code.write.patch": frozenset({"legacy.code.write.patch"}),
|
|
||||||
"test.python": frozenset({"legacy.test.python"}),
|
|
||||||
"test.shell.allowlisted": frozenset({"legacy.test.shell.allowlisted"}),
|
|
||||||
"git.commit": frozenset({"legacy.git.commit"}),
|
|
||||||
"git.diff.read": frozenset({"legacy.git.diff.read"}),
|
|
||||||
"git.force_push": frozenset({"legacy.git.force_push"}),
|
|
||||||
"deploy.production": frozenset({"legacy.deploy.production"}),
|
|
||||||
"secret.transform": frozenset({"legacy.secret.transform"}),
|
|
||||||
"secure_file_transformer": frozenset({"legacy.secure_file_transformer"}),
|
|
||||||
"runtime.shadow": frozenset({"legacy.runtime.shadow"}),
|
|
||||||
"review.security": frozenset({"legacy.review.security"}),
|
|
||||||
"review.api-contract": frozenset({"legacy.review.api-contract"}),
|
|
||||||
"gpu.train": frozenset({"legacy.gpu.train"}),
|
|
||||||
"agent.runtime.hermes": frozenset({"agent.runtime.hermes"}),
|
|
||||||
"agent.runtime.openclaw": frozenset({"agent.runtime.openclaw"}),
|
|
||||||
}
|
|
||||||
|
|
||||||
_UNKNOWN_PREFIX = "exact:"
|
|
||||||
|
|
||||||
|
|
||||||
def _clean(value: object) -> str:
|
|
||||||
if not isinstance(value, str):
|
|
||||||
raise ValueError("capability must be a string")
|
|
||||||
text = value.strip().lower()
|
|
||||||
if not text:
|
|
||||||
raise ValueError("capability must not be empty")
|
|
||||||
return text
|
|
||||||
|
|
||||||
|
|
||||||
def normalize_required_capabilities(values: Iterable[object]) -> frozenset[str]:
|
|
||||||
"""Map task-facing names to one semantic vocabulary.
|
|
||||||
|
|
||||||
Unknown names remain explicit exact requirements. They are never treated
|
|
||||||
as wildcards or granted by a failed normalization.
|
|
||||||
"""
|
|
||||||
|
|
||||||
result: set[str] = set()
|
|
||||||
for value in values:
|
|
||||||
text = _clean(value)
|
|
||||||
if "*" in text:
|
|
||||||
result.add(_UNKNOWN_PREFIX + text)
|
|
||||||
else:
|
|
||||||
result.update(_TASK_ALIASES.get(text, frozenset({_UNKNOWN_PREFIX + text})))
|
|
||||||
return frozenset(result)
|
|
||||||
|
|
||||||
|
|
||||||
def normalize_worker_capabilities(values: Iterable[object]) -> frozenset[str]:
|
|
||||||
"""Map registered worker names to effective semantic capabilities."""
|
|
||||||
|
|
||||||
result: set[str] = set()
|
|
||||||
for value in values:
|
|
||||||
text = _clean(value)
|
|
||||||
if "*" in text:
|
|
||||||
continue
|
|
||||||
mapped = _WORKER_ALIASES.get(text)
|
|
||||||
if mapped is not None:
|
|
||||||
result.update(mapped)
|
|
||||||
if {
|
|
||||||
"governed.git.write.branch",
|
|
||||||
"governed.git.write.file",
|
|
||||||
}.issubset(result):
|
|
||||||
result.add("governed.git.write")
|
|
||||||
return frozenset(result)
|
|
||||||
|
|
||||||
|
|
||||||
def has_capabilities(
|
|
||||||
worker_capabilities: Iterable[object],
|
|
||||||
required_capabilities: Iterable[object],
|
|
||||||
) -> bool:
|
|
||||||
"""Return the single eligibility decision used by poll and claim."""
|
|
||||||
|
|
||||||
required = normalize_required_capabilities(required_capabilities)
|
|
||||||
effective = normalize_worker_capabilities(worker_capabilities)
|
|
||||||
if any(value.startswith(_UNKNOWN_PREFIX) for value in required):
|
|
||||||
return False
|
|
||||||
return required.issubset(effective)
|
|
||||||
|
|
||||||
|
|
||||||
def required_capabilities_subset(
|
|
||||||
granted_requirements: Iterable[object],
|
|
||||||
required_requirements: Iterable[object],
|
|
||||||
) -> bool:
|
|
||||||
"""Check semantic capability containment for delegation constraints."""
|
|
||||||
granted = normalize_required_capabilities(granted_requirements)
|
|
||||||
required = normalize_required_capabilities(required_requirements)
|
|
||||||
if any(value.startswith(_UNKNOWN_PREFIX) for value in granted | required):
|
|
||||||
return False
|
|
||||||
return required.issubset(granted)
|
|
||||||
|
|
||||||
|
|
||||||
def normalized_capability_matrix(
|
|
||||||
worker_capabilities: Iterable[object],
|
|
||||||
required_capabilities: Iterable[object],
|
|
||||||
) -> dict[str, object]:
|
|
||||||
"""Expose non-secret decision evidence for tests and diagnostics."""
|
|
||||||
|
|
||||||
worker_values = tuple(worker_capabilities)
|
|
||||||
required_values = tuple(required_capabilities)
|
|
||||||
worker = normalize_worker_capabilities(worker_values)
|
|
||||||
required = normalize_required_capabilities(required_values)
|
|
||||||
return {
|
|
||||||
"worker_effective": sorted(worker),
|
|
||||||
"task_required": sorted(required),
|
|
||||||
"eligible": has_capabilities(worker_values, required_values),
|
|
||||||
}
|
|
||||||
@@ -13,8 +13,6 @@ BRIDGE_OPERATIONS = (
|
|||||||
"getGovernedWriteCapabilities",
|
"getGovernedWriteCapabilities",
|
||||||
"createGovernedTask",
|
"createGovernedTask",
|
||||||
"acquireWriterLease",
|
"acquireWriterLease",
|
||||||
"bootstrapWriterLease",
|
|
||||||
"renewWriterLease",
|
|
||||||
"commitFileGoverned",
|
"commitFileGoverned",
|
||||||
"commitFilesGoverned",
|
"commitFilesGoverned",
|
||||||
"getGitOpsJob",
|
"getGitOpsJob",
|
||||||
@@ -33,8 +31,6 @@ BRIDGE_OPERATIONS = (
|
|||||||
OPERATION_REQUIRED_FIELDS = {
|
OPERATION_REQUIRED_FIELDS = {
|
||||||
"createGovernedTask": ("task_id", "repository", "base_oid", "target_branch", "risk", "description", "idempotency_key"),
|
"createGovernedTask": ("task_id", "repository", "base_oid", "target_branch", "risk", "description", "idempotency_key"),
|
||||||
"acquireWriterLease": ("schema", "task_id", "repository", "branch_mode", "base_oid", "expected_branch_oid", "target_branch", "generation", "lease_seconds"),
|
"acquireWriterLease": ("schema", "task_id", "repository", "branch_mode", "base_oid", "expected_branch_oid", "target_branch", "generation", "lease_seconds"),
|
||||||
"bootstrapWriterLease": ("schema", "task_id", "repository", "branch_mode", "base_oid", "expected_branch_oid", "target_branch", "generation", "lease_seconds"),
|
|
||||||
"renewWriterLease": ("schema", "task_id", "repository", "branch_mode", "base_oid", "expected_branch_oid", "target_branch", "generation", "lease_seconds"),
|
|
||||||
"commitFileGoverned": ("task_id", "repository", "expected_branch_oid", "target_branch", "generation", "idempotency_key", "path", "content_base64", "message"),
|
"commitFileGoverned": ("task_id", "repository", "expected_branch_oid", "target_branch", "generation", "idempotency_key", "path", "content_base64", "message"),
|
||||||
"commitFilesGoverned": ("task_id", "repository", "expected_branch_oid", "target_branch", "generation", "idempotency_key", "files", "message"),
|
"commitFilesGoverned": ("task_id", "repository", "expected_branch_oid", "target_branch", "generation", "idempotency_key", "files", "message"),
|
||||||
"getGitOpsJob": ("job_id",),
|
"getGitOpsJob": ("job_id",),
|
||||||
@@ -50,49 +46,6 @@ OPERATION_REQUIRED_FIELDS = {
|
|||||||
"getControlPlaneSnapshot": (),
|
"getControlPlaneSnapshot": (),
|
||||||
}
|
}
|
||||||
|
|
||||||
BRIDGE_OPERATION_CONTRACTS = {
|
|
||||||
"acquireWriterLease": {
|
|
||||||
"method": "POST",
|
|
||||||
"path": "/v2/writer-lease",
|
|
||||||
"required_fields": OPERATION_REQUIRED_FIELDS["acquireWriterLease"],
|
|
||||||
"response_fields": (
|
|
||||||
"task_id", "job_id", "generation", "lease_seconds", "lease_expires_at",
|
|
||||||
"create_target_branch", "worktree_mode", "cas_ref_update_required",
|
|
||||||
),
|
|
||||||
},
|
|
||||||
"bootstrapWriterLease": {
|
|
||||||
"method": "POST",
|
|
||||||
"path": "/v2/writer-lease/bootstrap",
|
|
||||||
"required_fields": OPERATION_REQUIRED_FIELDS["bootstrapWriterLease"],
|
|
||||||
"response_fields": (
|
|
||||||
"task_id", "job_id", "generation", "lease_seconds", "lease_expires_at",
|
|
||||||
"create_target_branch", "worktree_mode", "cas_ref_update_required",
|
|
||||||
"recovered", "bootstrapped", "runtime_rehydrated",
|
|
||||||
"durable_branch_lease_present", "idempotent",
|
|
||||||
),
|
|
||||||
},
|
|
||||||
"renewWriterLease": {
|
|
||||||
"method": "POST",
|
|
||||||
"path": "/v2/writer-lease/renew",
|
|
||||||
"required_fields": OPERATION_REQUIRED_FIELDS["renewWriterLease"],
|
|
||||||
"response_fields": (
|
|
||||||
"task_id", "job_id", "generation", "lease_seconds", "lease_expires_at",
|
|
||||||
"recovered", "bootstrapped", "runtime_rehydrated",
|
|
||||||
"durable_branch_lease_present", "idempotent",
|
|
||||||
),
|
|
||||||
},
|
|
||||||
"commitFileGoverned": {
|
|
||||||
"method": "POST", "path": "/v2/commit-file",
|
|
||||||
"required_fields": OPERATION_REQUIRED_FIELDS["commitFileGoverned"],
|
|
||||||
"response_fields": ("job_id", "commit_oid", "commit_visible", "branch_mode", "target_branch", "generation"),
|
|
||||||
},
|
|
||||||
"commitFilesGoverned": {
|
|
||||||
"method": "POST", "path": "/v2/commit-files",
|
|
||||||
"required_fields": OPERATION_REQUIRED_FIELDS["commitFilesGoverned"],
|
|
||||||
"response_fields": ("job_id", "commit_oid", "commit_visible", "branch_mode", "target_branch", "generation"),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class RuntimeDriftError(RuntimeError):
|
class RuntimeDriftError(RuntimeError):
|
||||||
pass
|
pass
|
||||||
@@ -169,14 +122,6 @@ class BridgeV2Adapter:
|
|||||||
"legacy_writes_enabled": False,
|
"legacy_writes_enabled": False,
|
||||||
"governed_writes_enabled": True,
|
"governed_writes_enabled": True,
|
||||||
"operations": list(BRIDGE_OPERATIONS),
|
"operations": list(BRIDGE_OPERATIONS),
|
||||||
"operation_contracts": {
|
|
||||||
operation: {
|
|
||||||
**contract,
|
|
||||||
"required_fields": list(contract["required_fields"]),
|
|
||||||
"response_fields": list(contract["response_fields"]),
|
|
||||||
}
|
|
||||||
for operation, contract in BRIDGE_OPERATION_CONTRACTS.items()
|
|
||||||
},
|
|
||||||
"force_push": False,
|
"force_push": False,
|
||||||
"main_write": False,
|
"main_write": False,
|
||||||
"tag_write": False,
|
"tag_write": False,
|
||||||
@@ -206,8 +151,6 @@ class BridgeV2Adapter:
|
|||||||
|
|
||||||
def createGovernedTask(self, request: Mapping[str, Any]) -> Any: return self._call("createGovernedTask", request)
|
def createGovernedTask(self, request: Mapping[str, Any]) -> Any: return self._call("createGovernedTask", request)
|
||||||
def acquireWriterLease(self, request: Mapping[str, Any]) -> Any: return self._call("acquireWriterLease", request)
|
def acquireWriterLease(self, request: Mapping[str, Any]) -> Any: return self._call("acquireWriterLease", request)
|
||||||
def bootstrapWriterLease(self, request: Mapping[str, Any]) -> Any: return self._call("bootstrapWriterLease", request)
|
|
||||||
def renewWriterLease(self, request: Mapping[str, Any]) -> Any: return self._call("renewWriterLease", request)
|
|
||||||
def commitFileGoverned(self, request: Mapping[str, Any]) -> Any: return self._call("commitFileGoverned", request)
|
def commitFileGoverned(self, request: Mapping[str, Any]) -> Any: return self._call("commitFileGoverned", request)
|
||||||
def commitFilesGoverned(self, request: Mapping[str, Any]) -> Any: return self._call("commitFilesGoverned", request)
|
def commitFilesGoverned(self, request: Mapping[str, Any]) -> Any: return self._call("commitFilesGoverned", request)
|
||||||
def getGitOpsJob(self, request: Mapping[str, Any]) -> Any: return self._call("getGitOpsJob", request)
|
def getGitOpsJob(self, request: Mapping[str, Any]) -> Any: return self._call("getGitOpsJob", request)
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -51,13 +51,11 @@ class AuthorityPort(Protocol):
|
|||||||
def branch_exists(self, branch: str) -> bool: ...
|
def branch_exists(self, branch: str) -> bool: ...
|
||||||
def branch_head(self, branch: str) -> str: ...
|
def branch_head(self, branch: str) -> str: ...
|
||||||
def is_ancestor(self, older: str, newer: str) -> bool: ...
|
def is_ancestor(self, older: str, newer: str) -> bool: ...
|
||||||
def create_task_branch_cas(self, branch: str, expected_base: str, new: str) -> str: ...
|
|
||||||
def compare_and_swap_branch(self, branch: str, expected: str, new: str) -> str: ...
|
def compare_and_swap_branch(self, branch: str, expected: str, new: str) -> str: ...
|
||||||
|
|
||||||
|
|
||||||
class RemotePort(Protocol):
|
class RemotePort(Protocol):
|
||||||
def repository_identity(self) -> str: ...
|
def repository_identity(self) -> str: ...
|
||||||
def push_new_exact(self, branch: str, new: str) -> None: ...
|
|
||||||
def push_exact(self, branch: str, expected: str, new: str) -> None: ...
|
def push_exact(self, branch: str, expected: str, new: str) -> None: ...
|
||||||
def read_git_ref(self, branch: str) -> str | None: ...
|
def read_git_ref(self, branch: str) -> str | None: ...
|
||||||
def read_api_branch(self, branch: str) -> str | None: ...
|
def read_api_branch(self, branch: str) -> str | None: ...
|
||||||
@@ -105,9 +103,6 @@ class AuthorityGitAdapter:
|
|||||||
def is_ancestor(self, older: str, newer: str) -> bool:
|
def is_ancestor(self, older: str, newer: str) -> bool:
|
||||||
return self.executor.ancestry_check(older, newer)
|
return self.executor.ancestry_check(older, newer)
|
||||||
|
|
||||||
def create_task_branch_cas(self, branch: str, expected_base: str, new: str) -> str:
|
|
||||||
return self.executor.create_task_branch_CAS(self._ref(branch), expected_base, new)
|
|
||||||
|
|
||||||
def compare_and_swap_branch(self, branch: str, expected: str, new: str) -> str:
|
def compare_and_swap_branch(self, branch: str, expected: str, new: str) -> str:
|
||||||
return self.executor.advance_task_branch_CAS(self._ref(branch), expected, new)
|
return self.executor.advance_task_branch_CAS(self._ref(branch), expected, new)
|
||||||
|
|
||||||
@@ -263,15 +258,6 @@ class GiteaContinuationRemote:
|
|||||||
raise ContinuationExecutionError("GITEA_GIT_READBACK_INVALID")
|
raise ContinuationExecutionError("GITEA_GIT_READBACK_INVALID")
|
||||||
return oid
|
return oid
|
||||||
|
|
||||||
def push_new_exact(self, branch: str, new: str) -> None:
|
|
||||||
self._require_projected()
|
|
||||||
self.repository_identity()
|
|
||||||
if self.read_git_ref(branch) is not None:
|
|
||||||
raise ContinuationExecutionError("GITEA_TARGET_BRANCH_ALREADY_EXISTS")
|
|
||||||
_run(["git", "push", self.remote_name, f"{new}:refs/heads/{branch}"], cwd=self.workspace)
|
|
||||||
if self.read_git_ref(branch) != new:
|
|
||||||
raise ContinuationExecutionError("GITEA_CREATE_READBACK_MISMATCH")
|
|
||||||
|
|
||||||
def push_exact(self, branch: str, expected: str, new: str) -> None:
|
def push_exact(self, branch: str, expected: str, new: str) -> None:
|
||||||
self._require_projected()
|
self._require_projected()
|
||||||
self.repository_identity()
|
self.repository_identity()
|
||||||
@@ -343,7 +329,6 @@ class ContinuationPermissions:
|
|||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
class ContinuationRuntimeBinding:
|
class ContinuationRuntimeBinding:
|
||||||
request: WriterLeaseRequestV2
|
request: WriterLeaseRequestV2
|
||||||
job_id: str
|
|
||||||
workspace_path: Path
|
workspace_path: Path
|
||||||
lease_not_before: datetime
|
lease_not_before: datetime
|
||||||
lease_expires_at: datetime
|
lease_expires_at: datetime
|
||||||
@@ -404,20 +389,13 @@ class ContinuationExecutor:
|
|||||||
def observe_generation(self, branch: str, generation: int) -> None:
|
def observe_generation(self, branch: str, generation: int) -> None:
|
||||||
self._observed_generations[branch] = max(generation, self._observed_generations.get(branch, 0))
|
self._observed_generations[branch] = max(generation, self._observed_generations.get(branch, 0))
|
||||||
|
|
||||||
def _max_generation(self, branch: str, *, exclude_job_id: str | None = None) -> int:
|
def _max_generation(self, branch: str) -> int:
|
||||||
observed = self._observed_generations.get(branch, 0)
|
observed = self._observed_generations.get(branch, 0)
|
||||||
with self.controller.store._connect() as connection:
|
with self.controller.store._connect() as connection:
|
||||||
if exclude_job_id is None:
|
row = connection.execute(
|
||||||
row = connection.execute(
|
"SELECT MAX(generation) AS generation FROM branch_leases WHERE branch=?",
|
||||||
"SELECT MAX(generation) AS generation FROM branch_leases WHERE branch=?",
|
(branch,),
|
||||||
(branch,),
|
).fetchone()
|
||||||
).fetchone()
|
|
||||||
else:
|
|
||||||
row = connection.execute(
|
|
||||||
"SELECT MAX(generation) AS generation FROM branch_leases "
|
|
||||||
"WHERE branch=? AND job_id!=?",
|
|
||||||
(branch, exclude_job_id),
|
|
||||||
).fetchone()
|
|
||||||
durable = int(row["generation"] or 0)
|
durable = int(row["generation"] or 0)
|
||||||
return max(observed, durable)
|
return max(observed, durable)
|
||||||
|
|
||||||
@@ -426,7 +404,6 @@ class ContinuationExecutor:
|
|||||||
return {
|
return {
|
||||||
"schema": WriterLeaseRequestV2.schema,
|
"schema": WriterLeaseRequestV2.schema,
|
||||||
"task_id": request.task_id,
|
"task_id": request.task_id,
|
||||||
"job_id": request.job_id,
|
|
||||||
"repository": request.repository,
|
"repository": request.repository,
|
||||||
"branch_mode": request.branch_mode,
|
"branch_mode": request.branch_mode,
|
||||||
"base_oid": request.base_oid,
|
"base_oid": request.base_oid,
|
||||||
@@ -436,21 +413,10 @@ class ContinuationExecutor:
|
|||||||
"lease_seconds": request.lease_seconds,
|
"lease_seconds": request.lease_seconds,
|
||||||
}
|
}
|
||||||
|
|
||||||
def _validate_preflight(self, binding: ContinuationRuntimeBinding, *, job_id: str | None = None) -> None:
|
def _validate_preflight(self, binding: ContinuationRuntimeBinding) -> None:
|
||||||
request = binding.request
|
request = binding.request
|
||||||
request_mapping = self._request_mapping(request)
|
normalized = validate_writer_lease_request_v2(self._request_mapping(request), self.authority)
|
||||||
normalized = validate_writer_lease_request_v2(request_mapping, self.authority)
|
if normalized != {**self._request_mapping(request), "create_target_branch": False, "worktree_mode": "detached_at_expected_head", "cas_ref_update_required": True}:
|
||||||
expected_normalized = {
|
|
||||||
**request_mapping,
|
|
||||||
"create_target_branch": request.branch_mode == "CREATE_NEW",
|
|
||||||
"worktree_mode": (
|
|
||||||
"detached_at_base_oid"
|
|
||||||
if request.branch_mode == "CREATE_NEW"
|
|
||||||
else "detached_at_expected_head"
|
|
||||||
),
|
|
||||||
"cas_ref_update_required": True,
|
|
||||||
}
|
|
||||||
if normalized != expected_normalized:
|
|
||||||
raise ContinuationExecutionError("CONTINUATION_NORMALIZATION_INVALID")
|
raise ContinuationExecutionError("CONTINUATION_NORMALIZATION_INVALID")
|
||||||
if not binding.original_receipt_valid or not binding.original_grant_consumed:
|
if not binding.original_receipt_valid or not binding.original_grant_consumed:
|
||||||
raise ContinuationExecutionError("ORIGINAL_GRANT_LINEAGE_INVALID")
|
raise ContinuationExecutionError("ORIGINAL_GRANT_LINEAGE_INVALID")
|
||||||
@@ -460,7 +426,7 @@ class ContinuationExecutor:
|
|||||||
raise ContinuationExecutionError("LEASE_NOT_ACTIVE")
|
raise ContinuationExecutionError("LEASE_NOT_ACTIVE")
|
||||||
if now >= binding.lease_expires_at:
|
if now >= binding.lease_expires_at:
|
||||||
raise ContinuationExecutionError("LEASE_EXPIRED")
|
raise ContinuationExecutionError("LEASE_EXPIRED")
|
||||||
if request.generation <= self._max_generation(request.target_branch, exclude_job_id=job_id):
|
if request.generation <= self._max_generation(request.target_branch):
|
||||||
raise ContinuationExecutionError("GENERATION_NOT_MONOTONIC")
|
raise ContinuationExecutionError("GENERATION_NOT_MONOTONIC")
|
||||||
if not self.workspace.is_detached():
|
if not self.workspace.is_detached():
|
||||||
raise ContinuationExecutionError("WORKTREE_NOT_DETACHED")
|
raise ContinuationExecutionError("WORKTREE_NOT_DETACHED")
|
||||||
@@ -486,102 +452,57 @@ class ContinuationExecutor:
|
|||||||
"generation": request.generation,
|
"generation": request.generation,
|
||||||
}
|
}
|
||||||
|
|
||||||
def execute(
|
def execute(self, binding: ContinuationRuntimeBinding, commit: ContinuationCommit) -> dict[str, object]:
|
||||||
self,
|
payload = commit.payload()
|
||||||
binding: ContinuationRuntimeBinding,
|
submitted = self.controller.submit(self._job_request(binding, commit, payload), payload=payload)
|
||||||
commit: ContinuationCommit,
|
current = self.controller.store.read(submitted.job_id)
|
||||||
*,
|
if JobState(current["state"]) == JobState.COMPLETED:
|
||||||
before_complete: Callable[[str], None] | None = None,
|
|
||||||
) -> dict[str, object]:
|
|
||||||
commit.payload()
|
|
||||||
if not binding.job_id or binding.request.job_id != binding.job_id:
|
|
||||||
raise ContinuationExecutionError("RUNTIME_LINEAGE_MISSING_JOB_ID")
|
|
||||||
current = self.controller.store.read(binding.job_id)
|
|
||||||
current_state = JobState(current["state"])
|
|
||||||
if current_state == JobState.COMPLETED:
|
|
||||||
return self.controller.store._receipt(current, duplicate=True).as_dict()
|
return self.controller.store._receipt(current, duplicate=True).as_dict()
|
||||||
if current_state == JobState.REMOTE_VERIFIED:
|
self._validate_preflight(binding)
|
||||||
receipt = json.loads(str(current["receipt_json"] or "{}"))
|
|
||||||
commit_oid = str(receipt.get("authority_oid") or "")
|
|
||||||
if not OID_RE.fullmatch(commit_oid):
|
|
||||||
raise ContinuationExecutionError("GOVERNED_COMMIT_OID_INVALID")
|
|
||||||
if before_complete is not None:
|
|
||||||
before_complete(commit_oid)
|
|
||||||
return self.controller.advance(
|
|
||||||
binding.job_id,
|
|
||||||
JobState.COMPLETED,
|
|
||||||
side_effect_state="REMOTE_VERIFIED",
|
|
||||||
receipt=receipt,
|
|
||||||
).as_dict()
|
|
||||||
self._validate_preflight(binding, job_id=binding.job_id)
|
|
||||||
request = binding.request
|
request = binding.request
|
||||||
projected = False
|
projected = False
|
||||||
try:
|
try:
|
||||||
self.credential.project()
|
self.credential.project()
|
||||||
projected = True
|
projected = True
|
||||||
row = self.controller.store.read(binding.job_id)
|
row = self.controller.store.read(submitted.job_id)
|
||||||
state = JobState(row["state"])
|
if JobState(row["state"]) == JobState.REQUESTED:
|
||||||
if state == JobState.REQUESTED:
|
|
||||||
self.controller.advance(
|
self.controller.advance(
|
||||||
binding.job_id,
|
submitted.job_id,
|
||||||
JobState.AUTHORIZED,
|
JobState.AUTHORIZED,
|
||||||
side_effect_state="AUTHORIZED",
|
side_effect_state="AUTHORIZED",
|
||||||
receipt={"task_id": request.task_id, "original_grant_consumed": True},
|
receipt={"task_id": request.task_id, "original_grant_consumed": True},
|
||||||
expected_revision=int(row["revision"]),
|
|
||||||
)
|
)
|
||||||
row = self.controller.store.read(binding.job_id)
|
row = self.controller.store.read(submitted.job_id)
|
||||||
state = JobState(row["state"])
|
if JobState(row["state"]) == JobState.AUTHORIZED:
|
||||||
if state == JobState.AUTHORIZED:
|
|
||||||
self.controller.lease_job(
|
self.controller.lease_job(
|
||||||
binding.job_id,
|
submitted.job_id,
|
||||||
lease_id=f"lease-{binding.job_id}",
|
lease_id=f"lease-{submitted.job_id}",
|
||||||
lineage_anchor_oid=request.base_oid,
|
lineage_anchor_oid=request.base_oid,
|
||||||
current_head_oid=request.expected_branch_oid,
|
current_head_oid=request.expected_branch_oid,
|
||||||
generation=request.generation,
|
generation=request.generation,
|
||||||
lease_expires_at=binding.lease_expires_at.astimezone(timezone.utc).isoformat(),
|
lease_expires_at=binding.lease_expires_at.astimezone(timezone.utc).isoformat(),
|
||||||
expected_revision=int(row["revision"]),
|
expected_revision=int(row["revision"]),
|
||||||
)
|
)
|
||||||
row = self.controller.store.read(binding.job_id)
|
self.controller.advance(submitted.job_id, JobState.WORKTREE_READY, side_effect_state="WORKTREE_READY")
|
||||||
state = JobState(row["state"])
|
self.controller.advance(submitted.job_id, JobState.EXECUTING, side_effect_state="IN_PROGRESS")
|
||||||
if state == JobState.LEASED:
|
|
||||||
self.controller.advance(binding.job_id, JobState.WORKTREE_READY,
|
|
||||||
side_effect_state="WORKTREE_READY", expected_revision=int(row["revision"]))
|
|
||||||
row = self.controller.store.read(binding.job_id)
|
|
||||||
state = JobState(row["state"])
|
|
||||||
if state == JobState.WORKTREE_READY:
|
|
||||||
self.controller.advance(binding.job_id, JobState.EXECUTING,
|
|
||||||
side_effect_state="IN_PROGRESS", expected_revision=int(row["revision"]))
|
|
||||||
elif state != JobState.EXECUTING:
|
|
||||||
raise ContinuationExecutionError("JOB_EXECUTION_STATE_INVALID")
|
|
||||||
new_oid = self.workspace.commit_files(commit.files, commit.message)
|
new_oid = self.workspace.commit_files(commit.files, commit.message)
|
||||||
if not OID_RE.fullmatch(new_oid) or not self.authority.is_ancestor(request.expected_branch_oid, new_oid):
|
if not OID_RE.fullmatch(new_oid) or not self.authority.is_ancestor(request.expected_branch_oid, new_oid):
|
||||||
raise ContinuationExecutionError("NON_FAST_FORWARD_COMMIT")
|
raise ContinuationExecutionError("NON_FAST_FORWARD_COMMIT")
|
||||||
if request.branch_mode == "CREATE_NEW":
|
authority_oid = self.authority.compare_and_swap_branch(request.target_branch, request.expected_branch_oid, new_oid)
|
||||||
authority_oid = self.authority.create_task_branch_cas(request.target_branch, request.base_oid, new_oid)
|
|
||||||
else:
|
|
||||||
authority_oid = self.authority.compare_and_swap_branch(request.target_branch, request.expected_branch_oid, new_oid)
|
|
||||||
if authority_oid != new_oid:
|
if authority_oid != new_oid:
|
||||||
raise ContinuationExecutionError("AUTHORITY_CAS_READBACK_MISMATCH")
|
raise ContinuationExecutionError("AUTHORITY_CAS_READBACK_MISMATCH")
|
||||||
self.controller.advance(
|
self.controller.advance(
|
||||||
binding.job_id,
|
submitted.job_id,
|
||||||
JobState.LOCAL_VERIFIED,
|
JobState.LOCAL_VERIFIED,
|
||||||
side_effect_state="LOCAL_VERIFIED",
|
side_effect_state="LOCAL_VERIFIED",
|
||||||
receipt={"authority_oid": authority_oid, "force_push": False},
|
receipt={"authority_oid": authority_oid, "force_push": False},
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
self.controller.advance(binding.job_id, JobState.REMOTE_SYNCING, side_effect_state="IN_PROGRESS")
|
self.controller.advance(submitted.job_id, JobState.REMOTE_SYNCING, side_effect_state="IN_PROGRESS")
|
||||||
if request.branch_mode == "CREATE_NEW":
|
self.remote.push_exact(request.target_branch, request.expected_branch_oid, new_oid)
|
||||||
self.remote.push_new_exact(request.target_branch, new_oid)
|
|
||||||
else:
|
|
||||||
self.remote.push_exact(request.target_branch, request.expected_branch_oid, new_oid)
|
|
||||||
except Exception:
|
except Exception:
|
||||||
return self._remote_pending(binding.job_id, authority_oid, new_oid)
|
return self._remote_pending(submitted.job_id, authority_oid, new_oid)
|
||||||
return self._finish_remote_verification(
|
return self._finish_remote_verification(submitted.job_id, authority_oid, new_oid)
|
||||||
binding.job_id,
|
|
||||||
authority_oid,
|
|
||||||
new_oid,
|
|
||||||
before_complete=before_complete,
|
|
||||||
)
|
|
||||||
finally:
|
finally:
|
||||||
if projected:
|
if projected:
|
||||||
self.credential.revoke()
|
self.credential.revoke()
|
||||||
@@ -604,14 +525,7 @@ class ContinuationExecutor:
|
|||||||
)
|
)
|
||||||
return result.as_dict()
|
return result.as_dict()
|
||||||
|
|
||||||
def _finish_remote_verification(
|
def _finish_remote_verification(self, job_id: str, authority_oid: str, new_oid: str) -> dict[str, object]:
|
||||||
self,
|
|
||||||
job_id: str,
|
|
||||||
authority_oid: str,
|
|
||||||
new_oid: str,
|
|
||||||
*,
|
|
||||||
before_complete: Callable[[str], None] | None = None,
|
|
||||||
) -> dict[str, object]:
|
|
||||||
row = self.controller.store.read(job_id)
|
row = self.controller.store.read(job_id)
|
||||||
branch = row["branch"]
|
branch = row["branch"]
|
||||||
request = json.loads(row["request_json"])["binding"]
|
request = json.loads(row["request_json"])["binding"]
|
||||||
@@ -628,20 +542,8 @@ class ContinuationExecutor:
|
|||||||
"reconciliation_decision": decision.action.value,
|
"reconciliation_decision": decision.action.value,
|
||||||
"force_push": False,
|
"force_push": False,
|
||||||
}
|
}
|
||||||
self.controller.advance(
|
self.controller.advance(job_id, JobState.REMOTE_VERIFIED, side_effect_state="REMOTE_VERIFIED", receipt=receipt)
|
||||||
job_id,
|
return self.controller.advance(job_id, JobState.COMPLETED, side_effect_state="REMOTE_VERIFIED", receipt=receipt).as_dict()
|
||||||
JobState.REMOTE_VERIFIED,
|
|
||||||
side_effect_state="REMOTE_VERIFIED",
|
|
||||||
receipt=receipt,
|
|
||||||
)
|
|
||||||
if before_complete is not None:
|
|
||||||
before_complete(new_oid)
|
|
||||||
return self.controller.advance(
|
|
||||||
job_id,
|
|
||||||
JobState.COMPLETED,
|
|
||||||
side_effect_state="REMOTE_VERIFIED",
|
|
||||||
receipt=receipt,
|
|
||||||
).as_dict()
|
|
||||||
|
|
||||||
def reconcile(self, job_id: str) -> dict[str, object]:
|
def reconcile(self, job_id: str) -> dict[str, object]:
|
||||||
row = self.controller.store.read(job_id)
|
row = self.controller.store.read(job_id)
|
||||||
|
|||||||
@@ -49,7 +49,6 @@ class WriterLeaseRequestV2:
|
|||||||
target_branch: str
|
target_branch: str
|
||||||
generation: int
|
generation: int
|
||||||
lease_seconds: int = 900
|
lease_seconds: int = 900
|
||||||
job_id: str = ""
|
|
||||||
|
|
||||||
schema = "aios.task-writer-lease.v2"
|
schema = "aios.task-writer-lease.v2"
|
||||||
accepted_schemas = frozenset({"aios.task-writer-lease.v2", "aios.task-writer-lease-request.v2"})
|
accepted_schemas = frozenset({"aios.task-writer-lease.v2", "aios.task-writer-lease-request.v2"})
|
||||||
@@ -57,8 +56,6 @@ class WriterLeaseRequestV2:
|
|||||||
def validate(self, probe: BranchProbe) -> None:
|
def validate(self, probe: BranchProbe) -> None:
|
||||||
if not self.task_id or self.repository != "aios/aios":
|
if not self.task_id or self.repository != "aios/aios":
|
||||||
raise ContinuationError("request_identity_invalid")
|
raise ContinuationError("request_identity_invalid")
|
||||||
if not self.job_id or not self.job_id.startswith("job-"):
|
|
||||||
raise ContinuationError("job_id_invalid")
|
|
||||||
if self.branch_mode not in {"CREATE_NEW", "CONTINUE_EXISTING"}:
|
if self.branch_mode not in {"CREATE_NEW", "CONTINUE_EXISTING"}:
|
||||||
raise ContinuationError("branch_mode_invalid")
|
raise ContinuationError("branch_mode_invalid")
|
||||||
_oid(self.base_oid, "base_oid")
|
_oid(self.base_oid, "base_oid")
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ from datetime import datetime, timedelta, timezone
|
|||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import secrets
|
import secrets
|
||||||
import re
|
|
||||||
from typing import Any, Callable, Mapping
|
from typing import Any, Callable, Mapping
|
||||||
|
|
||||||
|
|
||||||
@@ -22,14 +21,9 @@ class CredentialLease:
|
|||||||
scopes: tuple[str, ...]
|
scopes: tuple[str, ...]
|
||||||
provider: str
|
provider: str
|
||||||
state: str = "ACTIVE"
|
state: str = "ACTIVE"
|
||||||
scope_type: str = "system"
|
|
||||||
tenant_id: str | None = None
|
|
||||||
user_id: str | None = None
|
|
||||||
project_id: str | None = None
|
|
||||||
run_id: str | None = None
|
|
||||||
|
|
||||||
def as_dict(self) -> dict[str, Any]:
|
def as_dict(self) -> dict[str, Any]:
|
||||||
return {"lease_id": self.lease_id, "task_id": self.task_id, "repository": self.repository, "target_branch": self.target_branch, "generation": self.generation, "issued_at": self.issued_at, "expires_at": self.expires_at, "scopes": list(self.scopes), "provider": self.provider, "state": self.state, "scope_type": self.scope_type, "tenant_id": self.tenant_id, "user_id": self.user_id, "project_id": self.project_id, "run_id": self.run_id}
|
return {"lease_id": self.lease_id, "task_id": self.task_id, "repository": self.repository, "target_branch": self.target_branch, "generation": self.generation, "issued_at": self.issued_at, "expires_at": self.expires_at, "scopes": list(self.scopes), "provider": self.provider, "state": self.state}
|
||||||
|
|
||||||
|
|
||||||
class FileCredentialProvider:
|
class FileCredentialProvider:
|
||||||
@@ -49,19 +43,11 @@ class CredentialBroker:
|
|||||||
self.now = now or (lambda: datetime.now(timezone.utc))
|
self.now = now or (lambda: datetime.now(timezone.utc))
|
||||||
self._leases: dict[str, CredentialLease] = {}
|
self._leases: dict[str, CredentialLease] = {}
|
||||||
|
|
||||||
def issue(self, *, task_id: str, repository: str, target_branch: str, generation: int, scopes: tuple[str, ...] = ("git:push", "gitea:read"), lease_seconds: int = 900, scope_type: str = "system", tenant_id: str | None = None, user_id: str | None = None, project_id: str | None = None, run_id: str | None = None) -> dict[str, Any]:
|
def issue(self, *, task_id: str, repository: str, target_branch: str, generation: int, scopes: tuple[str, ...] = ("git:push", "gitea:read"), lease_seconds: int = 900) -> dict[str, Any]:
|
||||||
if repository != "aios/aios" or not target_branch.startswith("agent/") or generation < 1 or not 1 <= lease_seconds <= 86400:
|
if repository != "aios/aios" or not target_branch.startswith("agent/") or generation < 1 or not 1 <= lease_seconds <= 86400:
|
||||||
raise ValueError("CREDENTIAL_LEASE_BINDING_INVALID")
|
raise ValueError("CREDENTIAL_LEASE_BINDING_INVALID")
|
||||||
if scope_type == "tenant":
|
|
||||||
if not tenant_id:
|
|
||||||
raise ValueError("CREDENTIAL_SCOPE_DENY")
|
|
||||||
elif scope_type in {"system", "global"}:
|
|
||||||
if any((tenant_id, user_id, project_id, run_id)):
|
|
||||||
raise ValueError("CREDENTIAL_SCOPE_DENY")
|
|
||||||
else:
|
|
||||||
raise ValueError("CREDENTIAL_SCOPE_DENY")
|
|
||||||
now = self.now().astimezone(timezone.utc)
|
now = self.now().astimezone(timezone.utc)
|
||||||
lease = CredentialLease("lease-" + secrets.token_hex(16), task_id, repository, target_branch, generation, now.isoformat(), (now + timedelta(seconds=lease_seconds)).isoformat(), tuple(scopes), type(self.provider).__name__, "ACTIVE", scope_type, tenant_id, user_id, project_id, run_id)
|
lease = CredentialLease("lease-" + secrets.token_hex(16), task_id, repository, target_branch, generation, now.isoformat(), (now + timedelta(seconds=lease_seconds)).isoformat(), tuple(scopes), type(self.provider).__name__)
|
||||||
self._leases[lease.lease_id] = lease
|
self._leases[lease.lease_id] = lease
|
||||||
return lease.as_dict()
|
return lease.as_dict()
|
||||||
|
|
||||||
@@ -78,39 +64,7 @@ class CredentialBroker:
|
|||||||
lease = self._get(lease_id)
|
lease = self._get(lease_id)
|
||||||
if lease.state != "ACTIVE":
|
if lease.state != "ACTIVE":
|
||||||
raise RuntimeError("CREDENTIAL_LEASE_NOT_ACTIVE")
|
raise RuntimeError("CREDENTIAL_LEASE_NOT_ACTIVE")
|
||||||
environment = self.provider.environment(lease)
|
return self.provider.environment(lease)
|
||||||
for key, value in environment.items():
|
|
||||||
upper = str(key).upper()
|
|
||||||
if not isinstance(value, str):
|
|
||||||
raise RuntimeError("CREDENTIAL_SECRET_EXPOSURE_DENY")
|
|
||||||
sensitive = any(marker in upper for marker in ("CREDENTIAL", "TOKEN", "SECRET", "PASSWORD", "PRIVATE_KEY", "API_KEY"))
|
|
||||||
if re.search(r"(?i)bearer\s+\S+|(?:token|secret|password|api[_-]?key)\s*[:=]", value):
|
|
||||||
raise RuntimeError("CREDENTIAL_SECRET_EXPOSURE_DENY")
|
|
||||||
if sensitive and upper.endswith("_FILE"):
|
|
||||||
candidate = Path(value)
|
|
||||||
credential_root = Path("/run/credentials")
|
|
||||||
try:
|
|
||||||
resolved = candidate.resolve(strict=False)
|
|
||||||
except OSError as exc:
|
|
||||||
raise RuntimeError("CREDENTIAL_SECRET_EXPOSURE_DENY") from exc
|
|
||||||
if not candidate.is_absolute() or not (resolved == credential_root or credential_root in resolved.parents):
|
|
||||||
raise RuntimeError("CREDENTIAL_SECRET_EXPOSURE_DENY")
|
|
||||||
elif sensitive and upper.endswith("_REFERENCE"):
|
|
||||||
if re.fullmatch(r"(?:systemd-credential|vault|secret-manager):[A-Za-z0-9][A-Za-z0-9._:/#-]{0,255}", value) is None:
|
|
||||||
raise RuntimeError("CREDENTIAL_SECRET_EXPOSURE_DENY")
|
|
||||||
elif sensitive:
|
|
||||||
raise RuntimeError("CREDENTIAL_SECRET_EXPOSURE_DENY")
|
|
||||||
return environment
|
|
||||||
|
|
||||||
def authorize(self, lease_id: str, *, task_id: str, operation: str, scope_type: str = "system", tenant_id: str | None = None, user_id: str | None = None, project_id: str | None = None, run_id: str | None = None) -> Mapping[str, str]:
|
|
||||||
lease = self._get(lease_id)
|
|
||||||
if lease.state != "ACTIVE" or lease.task_id != task_id or operation not in lease.scopes:
|
|
||||||
raise RuntimeError("CREDENTIAL_SCOPE_DENY")
|
|
||||||
requested = (scope_type, tenant_id, user_id, project_id, run_id)
|
|
||||||
actual = (lease.scope_type, lease.tenant_id, lease.user_id, lease.project_id, lease.run_id)
|
|
||||||
if requested != actual:
|
|
||||||
raise RuntimeError("CREDENTIAL_SCOPE_DENY")
|
|
||||||
return self.environment(lease_id)
|
|
||||||
|
|
||||||
def status(self, lease_id: str) -> dict[str, Any]:
|
def status(self, lease_id: str) -> dict[str, Any]:
|
||||||
return self._get(lease_id).as_dict()
|
return self._get(lease_id).as_dict()
|
||||||
|
|||||||
@@ -1,710 +0,0 @@
|
|||||||
"""Credential-isolated external Agent runtime executor for Online Agent Tasks.
|
|
||||||
|
|
||||||
Task payloads never choose commands. A root-owned runtime profile chooses the executable and
|
|
||||||
I/O mode. The Online Agent worker writes a hash-bound request into an exchange directory; this
|
|
||||||
uncredentialed process executes Hermes/OpenClaw and returns a bounded, sanitized result.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from dataclasses import dataclass
|
|
||||||
import hashlib
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
from pathlib import Path
|
|
||||||
import re
|
|
||||||
import subprocess
|
|
||||||
import tempfile
|
|
||||||
import time
|
|
||||||
from typing import Any, Mapping, Sequence
|
|
||||||
|
|
||||||
REQUEST_SCHEMA = "aios.external-agent-runtime.request.v1"
|
|
||||||
RESULT_SCHEMA = "aios.external-agent-runtime.result.v1"
|
|
||||||
PROFILE_SCHEMA = "aios.external-agent-runtime.profile.v1"
|
|
||||||
SUPPORTED_RUNTIMES = frozenset({"hermes", "openclaw"})
|
|
||||||
REQUEST_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{15,254}$")
|
|
||||||
RUNTIME_RE = re.compile(r"^[a-z][a-z0-9_-]{2,31}$")
|
|
||||||
SENSITIVE_RE = re.compile(
|
|
||||||
r"(?i)(authorization\s*[:=]\s*bearer\s+\S+|"
|
|
||||||
r"(?:password|passwd|secret|token|api[_-]?key)\s*[:=]\s*\S+)"
|
|
||||||
)
|
|
||||||
SECRET_ENV_MARKERS = (
|
|
||||||
"TOKEN",
|
|
||||||
"SECRET",
|
|
||||||
"PASSWORD",
|
|
||||||
"CREDENTIAL",
|
|
||||||
"AUTHORIZATION",
|
|
||||||
"PRIVATE_KEY",
|
|
||||||
"API_KEY",
|
|
||||||
)
|
|
||||||
PROFILE_ENV_NAME_RE = re.compile(r"^[A-Z][A-Z0-9_]{1,63}$")
|
|
||||||
PROFILE_CREDENTIAL_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{2,127}$")
|
|
||||||
PROFILE_CREDENTIAL_KEY_RE = re.compile(r"^[A-Z][A-Z0-9_]{1,63}$")
|
|
||||||
PROVIDER_CREDENTIAL_ENV_NAMES = frozenset({"OPENAI_API_KEY"})
|
|
||||||
PROFILE_FIXED_ENV_DENY = frozenset(
|
|
||||||
{
|
|
||||||
"PATH",
|
|
||||||
"HOME",
|
|
||||||
"NO_COLOR",
|
|
||||||
"LC_ALL",
|
|
||||||
"LANG",
|
|
||||||
"LD_PRELOAD",
|
|
||||||
"LD_LIBRARY_PATH",
|
|
||||||
"PYTHONPATH",
|
|
||||||
"NODE_OPTIONS",
|
|
||||||
"BASH_ENV",
|
|
||||||
"SHELL",
|
|
||||||
"IFS",
|
|
||||||
}
|
|
||||||
)
|
|
||||||
MAX_REQUEST_BYTES = 256 * 1024
|
|
||||||
MAX_OUTPUT_BYTES = 128 * 1024
|
|
||||||
MAX_PROFILE_BYTES = 64 * 1024
|
|
||||||
MAX_CREDENTIAL_BYTES = 64 * 1024
|
|
||||||
DEFAULT_PROFILE_ROOT = Path("/etc/aios/external-agent-runtime.d")
|
|
||||||
DEFAULT_EXCHANGE_ROOT = Path("/var/lib/aios-external-agent-executor")
|
|
||||||
ALLOWED_EXECUTABLE_ROOTS = (
|
|
||||||
Path("/usr/bin"),
|
|
||||||
Path("/usr/local/bin"),
|
|
||||||
Path("/opt/aios/bin"),
|
|
||||||
Path("/opt/hermes"),
|
|
||||||
Path("/opt/openclaw"),
|
|
||||||
Path("/opt/aios-hermes-agent"),
|
|
||||||
Path("/opt/aios/openclaw-engineering"),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class ExternalAgentRuntimeError(RuntimeError):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def canonical(value: object) -> bytes:
|
|
||||||
return json.dumps(
|
|
||||||
value, ensure_ascii=False, sort_keys=True, separators=(",", ":")
|
|
||||||
).encode("utf-8")
|
|
||||||
|
|
||||||
|
|
||||||
def sha256(data: bytes) -> str:
|
|
||||||
return hashlib.sha256(data).hexdigest()
|
|
||||||
|
|
||||||
|
|
||||||
def sanitize_text(value: str) -> str:
|
|
||||||
return SENSITIVE_RE.sub("[REDACTED]", str(value))
|
|
||||||
|
|
||||||
|
|
||||||
def _bounded_text(value: str, limit: int = MAX_OUTPUT_BYTES) -> str:
|
|
||||||
data = sanitize_text(value).encode("utf-8", "replace")
|
|
||||||
if len(data) <= limit:
|
|
||||||
return data.decode("utf-8", "replace")
|
|
||||||
marker = b"\n...[TRUNCATED]...\n"
|
|
||||||
half = max(1, (limit - len(marker)) // 2)
|
|
||||||
return (data[:half] + marker + data[-half:]).decode("utf-8", "replace")
|
|
||||||
|
|
||||||
|
|
||||||
def _safe_runtime(value: object) -> str:
|
|
||||||
runtime = str(value or "").strip().lower()
|
|
||||||
if RUNTIME_RE.fullmatch(runtime) is None or runtime not in SUPPORTED_RUNTIMES:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_RUNTIME_UNSUPPORTED")
|
|
||||||
return runtime
|
|
||||||
|
|
||||||
|
|
||||||
def _assert_no_secret_env() -> None:
|
|
||||||
leaked = sorted(
|
|
||||||
key
|
|
||||||
for key, value in os.environ.items()
|
|
||||||
if value and any(marker in key.upper() for marker in SECRET_ENV_MARKERS)
|
|
||||||
)
|
|
||||||
# CREDENTIALS_DIRECTORY itself is systemd metadata; the directory is expected to be hidden.
|
|
||||||
leaked = [key for key in leaked if key != "CREDENTIALS_DIRECTORY"]
|
|
||||||
if leaked:
|
|
||||||
raise ExternalAgentRuntimeError(
|
|
||||||
"EXTERNAL_AGENT_CREDENTIAL_ENV_EXPOSED:" + ",".join(leaked)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _safe_executable(value: object) -> str:
|
|
||||||
if not isinstance(value, str) or not value.startswith("/"):
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_EXECUTABLE_ABSOLUTE_REQUIRED")
|
|
||||||
path = Path(value)
|
|
||||||
if path.is_symlink():
|
|
||||||
resolved = path.resolve(strict=True)
|
|
||||||
else:
|
|
||||||
resolved = path.resolve(strict=True)
|
|
||||||
if not resolved.is_file() or not os.access(resolved, os.X_OK):
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_EXECUTABLE_INVALID")
|
|
||||||
if not any(resolved == root or root in resolved.parents for root in ALLOWED_EXECUTABLE_ROOTS):
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_EXECUTABLE_ROOT_DENY")
|
|
||||||
return str(resolved)
|
|
||||||
|
|
||||||
|
|
||||||
def _safe_profile_script(value: object) -> str:
|
|
||||||
if not isinstance(value, str) or not value.startswith("/"):
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_SCRIPT_ABSOLUTE_REQUIRED")
|
|
||||||
path = Path(value)
|
|
||||||
if path.is_symlink():
|
|
||||||
resolved = path.resolve(strict=True)
|
|
||||||
else:
|
|
||||||
resolved = path.resolve(strict=True)
|
|
||||||
if not resolved.is_file():
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_SCRIPT_INVALID")
|
|
||||||
if not any(resolved == root or root in resolved.parents for root in ALLOWED_EXECUTABLE_ROOTS):
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_SCRIPT_ROOT_DENY")
|
|
||||||
return str(resolved)
|
|
||||||
|
|
||||||
|
|
||||||
def _profile_environment(value: object) -> tuple[tuple[str, str], ...]:
|
|
||||||
if value is None:
|
|
||||||
return ()
|
|
||||||
if not isinstance(value, Mapping):
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_PROFILE_ENVIRONMENT_INVALID")
|
|
||||||
result: list[tuple[str, str]] = []
|
|
||||||
for raw_name, raw_value in value.items():
|
|
||||||
if not isinstance(raw_name, str) or PROFILE_ENV_NAME_RE.fullmatch(raw_name) is None:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_PROFILE_ENVIRONMENT_NAME_INVALID")
|
|
||||||
if raw_name in PROFILE_FIXED_ENV_DENY:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_PROFILE_ENVIRONMENT_NAME_DENY")
|
|
||||||
if any(marker in raw_name.upper() for marker in SECRET_ENV_MARKERS):
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_PROFILE_SECRET_ENV_DENY")
|
|
||||||
if not isinstance(raw_value, str) or "\x00" in raw_value or "\n" in raw_value or "\r" in raw_value:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_PROFILE_ENVIRONMENT_VALUE_INVALID")
|
|
||||||
if len(raw_value) > 4096:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_PROFILE_ENVIRONMENT_VALUE_TOO_LARGE")
|
|
||||||
result.append((raw_name, raw_value))
|
|
||||||
return tuple(sorted(result))
|
|
||||||
|
|
||||||
|
|
||||||
def _profile_credential_environment(
|
|
||||||
value: object,
|
|
||||||
) -> tuple[tuple[str, str, str, str], ...]:
|
|
||||||
if value is None:
|
|
||||||
return ()
|
|
||||||
if not isinstance(value, Mapping):
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_PROFILE_CREDENTIAL_ENV_INVALID")
|
|
||||||
result: list[tuple[str, str, str, str]] = []
|
|
||||||
for raw_env_name, raw_spec in value.items():
|
|
||||||
if (
|
|
||||||
not isinstance(raw_env_name, str)
|
|
||||||
or raw_env_name not in PROVIDER_CREDENTIAL_ENV_NAMES
|
|
||||||
or PROFILE_ENV_NAME_RE.fullmatch(raw_env_name) is None
|
|
||||||
):
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_CREDENTIAL_ENV_NAME_DENY")
|
|
||||||
if not isinstance(raw_spec, Mapping):
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_CREDENTIAL_SPEC_INVALID")
|
|
||||||
credential = raw_spec.get("credential")
|
|
||||||
fmt = str(raw_spec.get("format") or "dotenv")
|
|
||||||
key = raw_spec.get("key")
|
|
||||||
if (
|
|
||||||
not isinstance(credential, str)
|
|
||||||
or PROFILE_CREDENTIAL_NAME_RE.fullmatch(credential) is None
|
|
||||||
):
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_CREDENTIAL_NAME_INVALID")
|
|
||||||
if fmt not in {"dotenv", "raw"}:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_CREDENTIAL_FORMAT_INVALID")
|
|
||||||
if fmt == "dotenv":
|
|
||||||
if not isinstance(key, str) or PROFILE_CREDENTIAL_KEY_RE.fullmatch(key) is None:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_CREDENTIAL_KEY_INVALID")
|
|
||||||
elif key is not None:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_CREDENTIAL_KEY_UNEXPECTED")
|
|
||||||
result.append((raw_env_name, credential, fmt, str(key or "")))
|
|
||||||
return tuple(sorted(result))
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class RuntimeProfile:
|
|
||||||
runtime: str
|
|
||||||
argv: tuple[str, ...]
|
|
||||||
input_mode: str
|
|
||||||
output_mode: str
|
|
||||||
timeout_seconds: int
|
|
||||||
max_output_bytes: int
|
|
||||||
environment: tuple[tuple[str, str], ...]
|
|
||||||
credential_env: tuple[tuple[str, str, str, str], ...]
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def load(
|
|
||||||
cls,
|
|
||||||
runtime: str,
|
|
||||||
profile_root: Path = DEFAULT_PROFILE_ROOT,
|
|
||||||
) -> "RuntimeProfile":
|
|
||||||
runtime = _safe_runtime(runtime)
|
|
||||||
root = Path(profile_root)
|
|
||||||
if root.is_symlink():
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_PROFILE_ROOT_SYMLINK_DENY")
|
|
||||||
path = root / f"{runtime}.json"
|
|
||||||
if path.is_symlink() or not path.is_file() or path.stat().st_size > MAX_PROFILE_BYTES:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_PROFILE_UNAVAILABLE")
|
|
||||||
try:
|
|
||||||
value = json.loads(path.read_text(encoding="utf-8"))
|
|
||||||
except json.JSONDecodeError as exc:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_PROFILE_INVALID") from exc
|
|
||||||
if not isinstance(value, Mapping) or value.get("schema") != PROFILE_SCHEMA:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_PROFILE_INVALID")
|
|
||||||
if _safe_runtime(value.get("runtime")) != runtime:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_PROFILE_RUNTIME_MISMATCH")
|
|
||||||
argv = value.get("argv")
|
|
||||||
if (
|
|
||||||
not isinstance(argv, Sequence)
|
|
||||||
or isinstance(argv, (str, bytes))
|
|
||||||
or not argv
|
|
||||||
or not all(isinstance(item, str) and item and "\x00" not in item for item in argv)
|
|
||||||
):
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_PROFILE_ARGV_INVALID")
|
|
||||||
executable = _safe_executable(argv[0])
|
|
||||||
normalized = [executable, *list(argv[1:])]
|
|
||||||
if any(item in {"sh", "bash", "-c", "--command"} for item in normalized[1:]):
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_SHELL_MODE_DENY")
|
|
||||||
joined_argv = " ".join(normalized)
|
|
||||||
if re.search(r"(?i)(token|secret|password|api[_-]?key|authorization)\s*[:=]", joined_argv):
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_PROFILE_SECRET_ARG_DENY")
|
|
||||||
if Path(executable).name in {"node", "nodejs"}:
|
|
||||||
if len(normalized) < 2 or normalized[1].startswith("-"):
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_NODE_SCRIPT_REQUIRED")
|
|
||||||
normalized[1] = _safe_profile_script(normalized[1])
|
|
||||||
if any(item in {"-e", "--eval", "-p", "--print", "-r", "--require"} for item in normalized[1:]):
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_NODE_CODE_MODE_DENY")
|
|
||||||
input_mode = str(value.get("input_mode") or "stdin_text")
|
|
||||||
output_mode = str(value.get("output_mode") or "stdout_text")
|
|
||||||
if input_mode not in {"stdin_text", "stdin_json", "file_json"}:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_INPUT_MODE_INVALID")
|
|
||||||
if output_mode not in {"stdout_text", "stdout_json", "file_json"}:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_OUTPUT_MODE_INVALID")
|
|
||||||
timeout = int(value.get("timeout_seconds") or 900)
|
|
||||||
max_output = int(value.get("max_output_bytes") or MAX_OUTPUT_BYTES)
|
|
||||||
if not 1 <= timeout <= 3600:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_TIMEOUT_INVALID")
|
|
||||||
if not 1024 <= max_output <= MAX_OUTPUT_BYTES:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_OUTPUT_LIMIT_INVALID")
|
|
||||||
environment = _profile_environment(value.get("environment"))
|
|
||||||
credential_env = _profile_credential_environment(value.get("credential_env"))
|
|
||||||
return cls(
|
|
||||||
runtime,
|
|
||||||
tuple(normalized),
|
|
||||||
input_mode,
|
|
||||||
output_mode,
|
|
||||||
timeout,
|
|
||||||
max_output,
|
|
||||||
environment,
|
|
||||||
credential_env,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def validate_request(value: Mapping[str, Any]) -> dict[str, Any]:
|
|
||||||
if value.get("schema") != REQUEST_SCHEMA:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_REQUEST_SCHEMA_INVALID")
|
|
||||||
request_id = str(value.get("request_id") or "")
|
|
||||||
if REQUEST_ID_RE.fullmatch(request_id) is None:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_REQUEST_ID_INVALID")
|
|
||||||
runtime = _safe_runtime(value.get("runtime"))
|
|
||||||
task_id = str(value.get("task_id") or "")
|
|
||||||
execution_digest = str(value.get("execution_digest") or "")
|
|
||||||
if not task_id or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:-]{2,179}", task_id):
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_TASK_ID_INVALID")
|
|
||||||
if re.fullmatch(r"[0-9a-f]{64}", execution_digest) is None:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_EXECUTION_DIGEST_INVALID")
|
|
||||||
payload = value.get("payload")
|
|
||||||
if not isinstance(payload, Mapping):
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_PAYLOAD_INVALID")
|
|
||||||
encoded = canonical(payload)
|
|
||||||
if len(encoded) > MAX_REQUEST_BYTES:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_PAYLOAD_TOO_LARGE")
|
|
||||||
goal = payload.get("goal")
|
|
||||||
instructions = payload.get("instructions")
|
|
||||||
if not isinstance(goal, str) or not goal.strip():
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_GOAL_REQUIRED")
|
|
||||||
if instructions is not None and not isinstance(instructions, str):
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_INSTRUCTIONS_INVALID")
|
|
||||||
return {
|
|
||||||
"schema": REQUEST_SCHEMA,
|
|
||||||
"request_id": request_id,
|
|
||||||
"runtime": runtime,
|
|
||||||
"task_id": task_id,
|
|
||||||
"execution_digest": execution_digest,
|
|
||||||
"payload": dict(payload),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def render_prompt(payload: Mapping[str, Any]) -> str:
|
|
||||||
goal = str(payload.get("goal") or "").strip()
|
|
||||||
instructions = str(payload.get("instructions") or "").strip()
|
|
||||||
criteria = payload.get("success_criteria") or []
|
|
||||||
constraints = payload.get("constraints") or {}
|
|
||||||
lines = [
|
|
||||||
"AIOS Online Agent Task",
|
|
||||||
"",
|
|
||||||
"Goal:",
|
|
||||||
goal,
|
|
||||||
]
|
|
||||||
if instructions:
|
|
||||||
lines.extend(["", "Instructions:", instructions])
|
|
||||||
if criteria:
|
|
||||||
lines.extend(["", "Success criteria:", json.dumps(criteria, ensure_ascii=False, sort_keys=True)])
|
|
||||||
if constraints:
|
|
||||||
lines.extend(["", "Constraints:", json.dumps(constraints, ensure_ascii=False, sort_keys=True)])
|
|
||||||
lines.extend([
|
|
||||||
"",
|
|
||||||
"Return the final task result only. Do not claim repository, production, credential, or control-plane side effects unless they are explicitly represented in the task payload.",
|
|
||||||
])
|
|
||||||
return "\n".join(lines)
|
|
||||||
|
|
||||||
|
|
||||||
def _argv_with_files(
|
|
||||||
profile: RuntimeProfile,
|
|
||||||
input_path: Path,
|
|
||||||
output_path: Path,
|
|
||||||
prompt: str,
|
|
||||||
) -> list[str]:
|
|
||||||
replacements = {
|
|
||||||
"{input_json}": str(input_path),
|
|
||||||
"{output_json}": str(output_path),
|
|
||||||
"{prompt}": prompt,
|
|
||||||
}
|
|
||||||
result = []
|
|
||||||
for item in profile.argv:
|
|
||||||
for marker, replacement in replacements.items():
|
|
||||||
item = item.replace(marker, replacement)
|
|
||||||
result.append(item)
|
|
||||||
if profile.input_mode == "file_json" and not any("{input_json}" in item for item in profile.argv):
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_INPUT_PLACEHOLDER_REQUIRED")
|
|
||||||
if profile.output_mode == "file_json" and not any("{output_json}" in item for item in profile.argv):
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_OUTPUT_PLACEHOLDER_REQUIRED")
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def _credential_environment(profile: RuntimeProfile) -> dict[str, str]:
|
|
||||||
if not profile.credential_env:
|
|
||||||
return {}
|
|
||||||
raw_root = os.environ.get("CREDENTIALS_DIRECTORY")
|
|
||||||
if not raw_root or not raw_root.startswith("/"):
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_CREDENTIAL_DIRECTORY_MISSING")
|
|
||||||
root = Path(raw_root)
|
|
||||||
if root.is_symlink() or not root.is_dir():
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_CREDENTIAL_DIRECTORY_INVALID")
|
|
||||||
result: dict[str, str] = {}
|
|
||||||
for env_name, credential, fmt, key in profile.credential_env:
|
|
||||||
path = root / credential
|
|
||||||
if path.is_symlink() or not path.is_file():
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_CREDENTIAL_FILE_INVALID")
|
|
||||||
try:
|
|
||||||
if path.stat().st_size > MAX_CREDENTIAL_BYTES:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_CREDENTIAL_TOO_LARGE")
|
|
||||||
raw = path.read_bytes()
|
|
||||||
except OSError as exc:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_CREDENTIAL_READ_FAILED") from exc
|
|
||||||
if b"\x00" in raw:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_CREDENTIAL_CONTENT_INVALID")
|
|
||||||
try:
|
|
||||||
text = raw.decode("utf-8")
|
|
||||||
except UnicodeDecodeError as exc:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_CREDENTIAL_CONTENT_INVALID") from exc
|
|
||||||
selected = ""
|
|
||||||
if fmt == "raw":
|
|
||||||
selected = text.strip()
|
|
||||||
else:
|
|
||||||
found = False
|
|
||||||
for line in text.splitlines():
|
|
||||||
candidate = line.strip()
|
|
||||||
if not candidate or candidate.startswith("#"):
|
|
||||||
continue
|
|
||||||
if candidate.startswith("export "):
|
|
||||||
candidate = candidate[7:].lstrip()
|
|
||||||
if "=" not in candidate:
|
|
||||||
continue
|
|
||||||
candidate_key, candidate_value = candidate.split("=", 1)
|
|
||||||
if candidate_key.strip() != key:
|
|
||||||
continue
|
|
||||||
if found:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_CREDENTIAL_KEY_DUPLICATE")
|
|
||||||
selected = candidate_value.strip()
|
|
||||||
if len(selected) >= 2 and selected[0] == selected[-1] and selected[0] in {"'", '"'}:
|
|
||||||
selected = selected[1:-1]
|
|
||||||
found = True
|
|
||||||
if not found:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_CREDENTIAL_KEY_MISSING")
|
|
||||||
if not selected or "\x00" in selected or "\n" in selected or "\r" in selected:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_CREDENTIAL_VALUE_INVALID")
|
|
||||||
if len(selected) > MAX_CREDENTIAL_BYTES:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_CREDENTIAL_VALUE_TOO_LARGE")
|
|
||||||
result[env_name] = selected
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_output(
|
|
||||||
profile: RuntimeProfile,
|
|
||||||
stdout: str,
|
|
||||||
output_path: Path,
|
|
||||||
) -> tuple[str, object]:
|
|
||||||
if profile.output_mode == "file_json":
|
|
||||||
if output_path.is_symlink() or not output_path.is_file():
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_OUTPUT_FILE_MISSING")
|
|
||||||
raw = output_path.read_bytes()
|
|
||||||
if len(raw) > profile.max_output_bytes:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_OUTPUT_TOO_LARGE")
|
|
||||||
text = raw.decode("utf-8", "replace")
|
|
||||||
try:
|
|
||||||
parsed = json.loads(text)
|
|
||||||
except json.JSONDecodeError as exc:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_OUTPUT_JSON_INVALID") from exc
|
|
||||||
return _bounded_text(text, profile.max_output_bytes), parsed
|
|
||||||
if len(stdout.encode("utf-8", "replace")) > profile.max_output_bytes:
|
|
||||||
stdout = _bounded_text(stdout, profile.max_output_bytes)
|
|
||||||
if profile.output_mode == "stdout_json":
|
|
||||||
try:
|
|
||||||
parsed = json.loads(stdout)
|
|
||||||
except json.JSONDecodeError as exc:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_OUTPUT_JSON_INVALID") from exc
|
|
||||||
return _bounded_text(stdout, profile.max_output_bytes), parsed
|
|
||||||
return _bounded_text(stdout, profile.max_output_bytes), None
|
|
||||||
|
|
||||||
|
|
||||||
def execute_request(
|
|
||||||
request_path: Path,
|
|
||||||
result_path: Path,
|
|
||||||
*,
|
|
||||||
profile_root: Path = DEFAULT_PROFILE_ROOT,
|
|
||||||
) -> None:
|
|
||||||
_assert_no_secret_env()
|
|
||||||
if request_path.is_symlink() or not request_path.is_file():
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_REQUEST_FILE_INVALID")
|
|
||||||
raw_request = request_path.read_bytes()
|
|
||||||
if len(raw_request) > MAX_REQUEST_BYTES:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_REQUEST_TOO_LARGE")
|
|
||||||
try:
|
|
||||||
request = validate_request(json.loads(raw_request.decode("utf-8")))
|
|
||||||
except json.JSONDecodeError as exc:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_REQUEST_JSON_INVALID") from exc
|
|
||||||
profile = RuntimeProfile.load(request["runtime"], profile_root)
|
|
||||||
|
|
||||||
with tempfile.TemporaryDirectory(prefix=f"aios-{request['runtime']}-") as temporary:
|
|
||||||
temp_root = Path(temporary)
|
|
||||||
(temp_root / "home").mkdir()
|
|
||||||
input_path = temp_root / "input.json"
|
|
||||||
output_path = temp_root / "output.json"
|
|
||||||
input_path.write_bytes(canonical(request["payload"]))
|
|
||||||
prompt = render_prompt(request["payload"])
|
|
||||||
argv = _argv_with_files(profile, input_path, output_path, prompt)
|
|
||||||
|
|
||||||
stdin_data = None
|
|
||||||
if profile.input_mode == "stdin_json":
|
|
||||||
stdin_data = canonical(request["payload"]).decode("utf-8")
|
|
||||||
elif profile.input_mode == "stdin_text":
|
|
||||||
stdin_data = prompt
|
|
||||||
|
|
||||||
child_env = {
|
|
||||||
"PATH": "/usr/bin:/usr/local/bin:/bin",
|
|
||||||
"LC_ALL": "C.UTF-8",
|
|
||||||
"LANG": "C.UTF-8",
|
|
||||||
"HOME": str(temp_root / "home"),
|
|
||||||
"NO_COLOR": "1",
|
|
||||||
}
|
|
||||||
child_env.update(dict(profile.environment))
|
|
||||||
child_env.update(_credential_environment(profile))
|
|
||||||
|
|
||||||
started = time.monotonic()
|
|
||||||
try:
|
|
||||||
completed = subprocess.run(
|
|
||||||
argv,
|
|
||||||
input=stdin_data,
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
check=False,
|
|
||||||
shell=False,
|
|
||||||
timeout=profile.timeout_seconds,
|
|
||||||
cwd=str(temp_root),
|
|
||||||
env=child_env,
|
|
||||||
)
|
|
||||||
except subprocess.TimeoutExpired as exc:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_TIMEOUT") from exc
|
|
||||||
|
|
||||||
if completed.returncode != 0:
|
|
||||||
raise ExternalAgentRuntimeError(
|
|
||||||
f"EXTERNAL_AGENT_PROCESS_FAILED:{completed.returncode}:"
|
|
||||||
+ _bounded_text(completed.stderr, 2048)
|
|
||||||
)
|
|
||||||
|
|
||||||
output_text, output_json = _parse_output(profile, completed.stdout, output_path)
|
|
||||||
result: dict[str, Any] = {
|
|
||||||
"schema": RESULT_SCHEMA,
|
|
||||||
"request_id": request["request_id"],
|
|
||||||
"request_sha256": sha256(canonical(request)),
|
|
||||||
"runtime": request["runtime"],
|
|
||||||
"task_id": request["task_id"],
|
|
||||||
"execution_digest": request["execution_digest"],
|
|
||||||
"exit_code": completed.returncode,
|
|
||||||
"duration_ms": int((time.monotonic() - started) * 1000),
|
|
||||||
"output_text": output_text,
|
|
||||||
"output_sha256": sha256(output_text.encode("utf-8", "replace")),
|
|
||||||
"stderr_sha256": sha256(completed.stderr.encode("utf-8", "replace")),
|
|
||||||
"credential_material_exposed": False,
|
|
||||||
"control_plane_access_granted": False,
|
|
||||||
}
|
|
||||||
if output_json is not None:
|
|
||||||
result["output_json"] = output_json
|
|
||||||
result["result_sha256"] = sha256(canonical(result))
|
|
||||||
result_path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
fd, name = tempfile.mkstemp(prefix=f".{result_path.name}.", dir=result_path.parent)
|
|
||||||
try:
|
|
||||||
os.fchmod(fd, 0o640)
|
|
||||||
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
|
||||||
json.dump(result, handle, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
|
||||||
handle.write("\n")
|
|
||||||
handle.flush()
|
|
||||||
os.fsync(handle.fileno())
|
|
||||||
os.replace(name, result_path)
|
|
||||||
finally:
|
|
||||||
if os.path.exists(name):
|
|
||||||
os.unlink(name)
|
|
||||||
|
|
||||||
|
|
||||||
class ExternalAgentExchange:
|
|
||||||
"""Worker-side idempotent request/result exchange."""
|
|
||||||
|
|
||||||
def __init__(self, root: Path = DEFAULT_EXCHANGE_ROOT) -> None:
|
|
||||||
self.root = Path(root)
|
|
||||||
|
|
||||||
def execute(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
request_id: str,
|
|
||||||
runtime: str,
|
|
||||||
task_id: str,
|
|
||||||
execution_digest: str,
|
|
||||||
payload: Mapping[str, Any],
|
|
||||||
timeout_seconds: int = 1800,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
request = validate_request(
|
|
||||||
{
|
|
||||||
"schema": REQUEST_SCHEMA,
|
|
||||||
"request_id": request_id,
|
|
||||||
"runtime": runtime,
|
|
||||||
"task_id": task_id,
|
|
||||||
"execution_digest": execution_digest,
|
|
||||||
"payload": dict(payload),
|
|
||||||
}
|
|
||||||
)
|
|
||||||
requests = self.root / "requests"
|
|
||||||
results = self.root / "results"
|
|
||||||
failed = self.root / "failed"
|
|
||||||
for directory in (requests, results, failed):
|
|
||||||
directory.mkdir(parents=True, exist_ok=True)
|
|
||||||
request_path = requests / f"{request_id}.json"
|
|
||||||
result_path = results / f"{request_id}.json"
|
|
||||||
failed_path = failed / f"{request_id}.json"
|
|
||||||
request_sha = sha256(canonical(request))
|
|
||||||
|
|
||||||
if result_path.is_file():
|
|
||||||
return self._validated_result(result_path, request, request_sha)
|
|
||||||
if failed_path.is_file():
|
|
||||||
value = json.loads(failed_path.read_text(encoding="utf-8"))
|
|
||||||
raise ExternalAgentRuntimeError(
|
|
||||||
str(value.get("error") or "EXTERNAL_AGENT_EXECUTOR_FAILED")
|
|
||||||
)
|
|
||||||
|
|
||||||
if request_path.exists():
|
|
||||||
try:
|
|
||||||
existing = json.loads(request_path.read_text(encoding="utf-8"))
|
|
||||||
except json.JSONDecodeError as exc:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_EXISTING_REQUEST_INVALID") from exc
|
|
||||||
if sha256(canonical(existing)) != request_sha:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_REQUEST_ID_CONFLICT")
|
|
||||||
else:
|
|
||||||
fd, name = tempfile.mkstemp(prefix=f".{request_path.name}.", dir=requests)
|
|
||||||
try:
|
|
||||||
os.fchmod(fd, 0o640)
|
|
||||||
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
|
||||||
json.dump(request, handle, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
|
||||||
handle.write("\n")
|
|
||||||
handle.flush()
|
|
||||||
os.fsync(handle.fileno())
|
|
||||||
os.replace(name, request_path)
|
|
||||||
finally:
|
|
||||||
if os.path.exists(name):
|
|
||||||
os.unlink(name)
|
|
||||||
|
|
||||||
deadline = time.monotonic() + timeout_seconds
|
|
||||||
while time.monotonic() < deadline:
|
|
||||||
if result_path.is_file():
|
|
||||||
return self._validated_result(result_path, request, request_sha)
|
|
||||||
if failed_path.is_file():
|
|
||||||
try:
|
|
||||||
value = json.loads(failed_path.read_text(encoding="utf-8"))
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
value = {"error": "EXTERNAL_AGENT_EXECUTOR_FAILED"}
|
|
||||||
raise ExternalAgentRuntimeError(
|
|
||||||
str(value.get("error") or "EXTERNAL_AGENT_EXECUTOR_FAILED")
|
|
||||||
)
|
|
||||||
time.sleep(0.25)
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_EXCHANGE_TIMEOUT")
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _validated_result(
|
|
||||||
result_path: Path,
|
|
||||||
request: Mapping[str, Any],
|
|
||||||
request_sha: str,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
try:
|
|
||||||
value = json.loads(result_path.read_text(encoding="utf-8"))
|
|
||||||
except json.JSONDecodeError as exc:
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_RESULT_INVALID") from exc
|
|
||||||
if (
|
|
||||||
not isinstance(value, Mapping)
|
|
||||||
or value.get("schema") != RESULT_SCHEMA
|
|
||||||
or value.get("request_id") != request["request_id"]
|
|
||||||
or value.get("request_sha256") != request_sha
|
|
||||||
or value.get("runtime") != request["runtime"]
|
|
||||||
or value.get("task_id") != request["task_id"]
|
|
||||||
or value.get("execution_digest") != request["execution_digest"]
|
|
||||||
):
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_RESULT_BINDING_INVALID")
|
|
||||||
material = dict(value)
|
|
||||||
declared = str(material.pop("result_sha256", ""))
|
|
||||||
if declared != sha256(canonical(material)):
|
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_RESULT_DIGEST_INVALID")
|
|
||||||
return dict(value)
|
|
||||||
|
|
||||||
|
|
||||||
def serve(
|
|
||||||
root: Path = DEFAULT_EXCHANGE_ROOT,
|
|
||||||
profile_root: Path = DEFAULT_PROFILE_ROOT,
|
|
||||||
) -> None:
|
|
||||||
_assert_no_secret_env()
|
|
||||||
root = Path(root)
|
|
||||||
for directory in (root / "requests", root / "results", root / "failed"):
|
|
||||||
directory.mkdir(parents=True, exist_ok=True)
|
|
||||||
while True:
|
|
||||||
for request_path in sorted((root / "requests").glob("*.json")):
|
|
||||||
if request_path.is_symlink():
|
|
||||||
continue
|
|
||||||
result_path = root / "results" / request_path.name
|
|
||||||
failed_path = root / "failed" / request_path.name
|
|
||||||
try:
|
|
||||||
execute_request(request_path, result_path, profile_root=profile_root)
|
|
||||||
request_path.unlink(missing_ok=True)
|
|
||||||
except Exception as exc:
|
|
||||||
failed_path.write_text(
|
|
||||||
json.dumps(
|
|
||||||
{"error": sanitize_text(str(exc) or type(exc).__name__)},
|
|
||||||
ensure_ascii=False,
|
|
||||||
sort_keys=True,
|
|
||||||
)
|
|
||||||
+ "\n",
|
|
||||||
encoding="utf-8",
|
|
||||||
)
|
|
||||||
request_path.unlink(missing_ok=True)
|
|
||||||
time.sleep(0.25)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
import argparse
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
parser.add_argument("--root", default=str(DEFAULT_EXCHANGE_ROOT))
|
|
||||||
parser.add_argument("--profile-root", default=str(DEFAULT_PROFILE_ROOT))
|
|
||||||
parser.add_argument("--request")
|
|
||||||
parser.add_argument("--result")
|
|
||||||
args = parser.parse_args()
|
|
||||||
if args.request and args.result:
|
|
||||||
execute_request(
|
|
||||||
Path(args.request),
|
|
||||||
Path(args.result),
|
|
||||||
profile_root=Path(args.profile_root),
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
serve(Path(args.root), Path(args.profile_root))
|
|
||||||
@@ -27,19 +27,6 @@ class ServiceUnavailable(JobError):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
|
||||||
class LeaseRenewalResult:
|
|
||||||
"""Durable outcome of one writer-lease recovery attempt."""
|
|
||||||
|
|
||||||
renewed: bool
|
|
||||||
idempotent: bool
|
|
||||||
durable: bool
|
|
||||||
lease_expires_at: str | None
|
|
||||||
|
|
||||||
def __bool__(self) -> bool:
|
|
||||||
return self.renewed or self.idempotent
|
|
||||||
|
|
||||||
|
|
||||||
class BindingConflict(JobError):
|
class BindingConflict(JobError):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@@ -79,14 +66,6 @@ def utc_now() -> str:
|
|||||||
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||||
|
|
||||||
|
|
||||||
def parse_utc(value: str) -> datetime:
|
|
||||||
normalized = value[:-1] + "+00:00" if value.endswith("Z") else value
|
|
||||||
parsed = datetime.fromisoformat(normalized)
|
|
||||||
if parsed.tzinfo is None:
|
|
||||||
raise ValueError("LEASE_EXPIRY_INVALID")
|
|
||||||
return parsed.astimezone(timezone.utc)
|
|
||||||
|
|
||||||
|
|
||||||
def canonical_json(value: Any) -> bytes:
|
def canonical_json(value: Any) -> bytes:
|
||||||
return (
|
return (
|
||||||
json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n"
|
json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n"
|
||||||
@@ -425,16 +404,13 @@ class GitOpsJobController:
|
|||||||
self.store._mirror(result)
|
self.store._mirror(result)
|
||||||
return self.store._receipt(result, duplicate=True)
|
return self.store._receipt(result, duplicate=True)
|
||||||
|
|
||||||
def update_lease(self, lease_id: str, *, expected_revision: int, current_head_oid: str, generation: int, state: str = "ACTIVE", lease_expires_at: str | None = None) -> None:
|
def update_lease(self, lease_id: str, *, expected_revision: int, current_head_oid: str, generation: int, state: str = "ACTIVE") -> None:
|
||||||
if len(current_head_oid) != 40 or generation < 1:
|
if len(current_head_oid) != 40 or generation < 1:
|
||||||
raise JobError("LEASE_BINDING_INVALID")
|
raise JobError("LEASE_BINDING_INVALID")
|
||||||
try:
|
try:
|
||||||
with self.store._connect() as connection:
|
with self.store._connect() as connection:
|
||||||
connection.execute("BEGIN IMMEDIATE")
|
connection.execute("BEGIN IMMEDIATE")
|
||||||
if lease_expires_at is None:
|
updated = connection.execute("UPDATE branch_leases SET current_head_oid=?,generation=?,state=?,revision=revision+1 WHERE lease_id=? AND revision=?", (current_head_oid, generation, state, lease_id, expected_revision))
|
||||||
updated = connection.execute("UPDATE branch_leases SET current_head_oid=?,generation=?,state=?,revision=revision+1 WHERE lease_id=? AND revision=?", (current_head_oid, generation, state, lease_id, expected_revision))
|
|
||||||
else:
|
|
||||||
updated = connection.execute("UPDATE branch_leases SET current_head_oid=?,generation=?,lease_expires_at=?,state=?,revision=revision+1 WHERE lease_id=? AND revision=?", (current_head_oid, generation, lease_expires_at, state, lease_id, expected_revision))
|
|
||||||
if updated.rowcount != 1:
|
if updated.rowcount != 1:
|
||||||
raise JobError("LEASE_REVISION_CAS_CONFLICT")
|
raise JobError("LEASE_REVISION_CAS_CONFLICT")
|
||||||
connection.commit()
|
connection.commit()
|
||||||
@@ -443,94 +419,6 @@ class GitOpsJobController:
|
|||||||
except sqlite3.Error as exc:
|
except sqlite3.Error as exc:
|
||||||
raise ServiceUnavailable("JOB_DB_UNAVAILABLE") from exc
|
raise ServiceUnavailable("JOB_DB_UNAVAILABLE") from exc
|
||||||
|
|
||||||
def renew_lease(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
branch: str,
|
|
||||||
lineage_anchor_oid: str,
|
|
||||||
current_head_oid: str,
|
|
||||||
generation: int,
|
|
||||||
lease_expires_at: str,
|
|
||||||
) -> LeaseRenewalResult:
|
|
||||||
if (
|
|
||||||
not branch
|
|
||||||
or len(lineage_anchor_oid) != 40
|
|
||||||
or len(current_head_oid) != 40
|
|
||||||
or generation < 1
|
|
||||||
or not lease_expires_at.strip()
|
|
||||||
):
|
|
||||||
raise JobError("LEASE_BINDING_INVALID")
|
|
||||||
try:
|
|
||||||
with self.store._connect() as connection:
|
|
||||||
connection.execute("BEGIN IMMEDIATE")
|
|
||||||
rows = connection.execute(
|
|
||||||
"SELECT lease_id, revision, generation, state, lease_expires_at "
|
|
||||||
"FROM branch_leases "
|
|
||||||
"WHERE branch=? AND lineage_anchor_oid=? AND current_head_oid=? "
|
|
||||||
"AND generation=? AND state='ACTIVE' ORDER BY revision DESC",
|
|
||||||
(branch, lineage_anchor_oid, current_head_oid, generation),
|
|
||||||
).fetchall()
|
|
||||||
if not rows:
|
|
||||||
connection.commit()
|
|
||||||
return LeaseRenewalResult(False, False, False, None)
|
|
||||||
if len(rows) != 1:
|
|
||||||
raise JobError("LEASE_BINDING_CONFLICT")
|
|
||||||
row = rows[0]
|
|
||||||
try:
|
|
||||||
current_expires_at = parse_utc(str(row["lease_expires_at"]))
|
|
||||||
requested_expires_at = parse_utc(lease_expires_at)
|
|
||||||
except (TypeError, ValueError) as exc:
|
|
||||||
raise JobError("LEASE_EXPIRY_INVALID") from exc
|
|
||||||
now = datetime.now(timezone.utc)
|
|
||||||
if requested_expires_at <= now:
|
|
||||||
raise JobError("LEASE_EXPIRY_INVALID")
|
|
||||||
if current_expires_at > now:
|
|
||||||
connection.commit()
|
|
||||||
return LeaseRenewalResult(
|
|
||||||
False,
|
|
||||||
True,
|
|
||||||
True,
|
|
||||||
str(row["lease_expires_at"]),
|
|
||||||
)
|
|
||||||
updated = connection.execute(
|
|
||||||
"UPDATE branch_leases SET lease_expires_at=?,revision=revision+1 "
|
|
||||||
"WHERE lease_id=? AND revision=? AND generation=? "
|
|
||||||
"AND state='ACTIVE' AND lease_expires_at=?",
|
|
||||||
(
|
|
||||||
lease_expires_at,
|
|
||||||
row["lease_id"],
|
|
||||||
int(row["revision"]),
|
|
||||||
generation,
|
|
||||||
row["lease_expires_at"],
|
|
||||||
),
|
|
||||||
)
|
|
||||||
if updated.rowcount != 1:
|
|
||||||
current = connection.execute(
|
|
||||||
"SELECT lease_expires_at, generation, state FROM branch_leases "
|
|
||||||
"WHERE lease_id=?",
|
|
||||||
(row["lease_id"],),
|
|
||||||
).fetchone()
|
|
||||||
if (
|
|
||||||
current is not None
|
|
||||||
and int(current["generation"]) == generation
|
|
||||||
and current["state"] == "ACTIVE"
|
|
||||||
and parse_utc(str(current["lease_expires_at"])) > now
|
|
||||||
):
|
|
||||||
connection.commit()
|
|
||||||
return LeaseRenewalResult(
|
|
||||||
False,
|
|
||||||
True,
|
|
||||||
True,
|
|
||||||
str(current["lease_expires_at"]),
|
|
||||||
)
|
|
||||||
raise JobError("LEASE_REVISION_CAS_CONFLICT")
|
|
||||||
connection.commit()
|
|
||||||
return LeaseRenewalResult(True, False, True, lease_expires_at)
|
|
||||||
except JobError:
|
|
||||||
raise
|
|
||||||
except sqlite3.Error as exc:
|
|
||||||
raise ServiceUnavailable("JOB_DB_UNAVAILABLE") from exc
|
|
||||||
|
|
||||||
def advance(self, job_id: str, state: JobState, *, side_effect_state: str, receipt: Mapping[str, Any] | None = None, expected_revision: int | None = None) -> JobReceipt:
|
def advance(self, job_id: str, state: JobState, *, side_effect_state: str, receipt: Mapping[str, Any] | None = None, expected_revision: int | None = None) -> JobReceipt:
|
||||||
return self._advance(
|
return self._advance(
|
||||||
job_id,
|
job_id,
|
||||||
|
|||||||
@@ -1,306 +0,0 @@
|
|||||||
"""Uncredentialed deterministic executor for governed Online Agent changes.
|
|
||||||
|
|
||||||
This process never receives Agent/Owner/Bridge credentials and never writes Git refs.
|
|
||||||
It operates only on an exchange bundle and returns hash-bound changed files + test evidence.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import base64
|
|
||||||
from dataclasses import dataclass
|
|
||||||
import errno
|
|
||||||
import hashlib
|
|
||||||
import io
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
from pathlib import Path
|
|
||||||
import re
|
|
||||||
import shutil
|
|
||||||
import socket
|
|
||||||
import subprocess
|
|
||||||
import tarfile
|
|
||||||
import tempfile
|
|
||||||
import time
|
|
||||||
from typing import Any, Mapping, Sequence
|
|
||||||
|
|
||||||
ALLOWED_PROGRAMS = {"/usr/bin/python3", "/usr/bin/git", "/bin/true", "/usr/bin/true"}
|
|
||||||
MAX_ARCHIVE_BYTES = 64 * 1024 * 1024
|
|
||||||
MAX_FILE_BYTES = 4 * 1024 * 1024
|
|
||||||
SECRET_ENV_TOKENS = ("TOKEN", "SECRET", "PASSWORD", "CREDENTIAL", "AUTHORIZATION", "PRIVATE_KEY")
|
|
||||||
REQUEST_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{15,254}$")
|
|
||||||
SENSITIVE_VALUE_RE = re.compile(
|
|
||||||
r"(?i)(authorization\s*[:=]\s*bearer\s+\S+|authorization\s+bearer\s+\S+|"
|
|
||||||
r"(?:password|passwd|secret|token|api[_-]?key)\s*[:=]\s*\S+)"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class IsolatedExecutorError(RuntimeError):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def sha256(data: bytes) -> str:
|
|
||||||
return hashlib.sha256(data).hexdigest()
|
|
||||||
|
|
||||||
|
|
||||||
def canonical(value: object) -> bytes:
|
|
||||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
|
||||||
|
|
||||||
|
|
||||||
def safe_relative(value: object) -> str:
|
|
||||||
if not isinstance(value, str) or not value or value.startswith("/"):
|
|
||||||
raise IsolatedExecutorError("PATH_INVALID")
|
|
||||||
path = Path(value)
|
|
||||||
if ".." in path.parts or "." in path.parts or any(part in {"", ".git"} for part in path.parts):
|
|
||||||
raise IsolatedExecutorError("PATH_INVALID")
|
|
||||||
return path.as_posix()
|
|
||||||
|
|
||||||
|
|
||||||
def resolve_under(root: Path, relative: str) -> Path:
|
|
||||||
root = root.resolve()
|
|
||||||
path = (root / safe_relative(relative)).resolve(strict=False)
|
|
||||||
if path == root or root not in path.parents:
|
|
||||||
raise IsolatedExecutorError("PATH_ESCAPE")
|
|
||||||
return path
|
|
||||||
|
|
||||||
|
|
||||||
def _credential_exposure(directory: Path) -> str | None:
|
|
||||||
"""Return an exposure code; EACCES is the intended credential isolation proof."""
|
|
||||||
try:
|
|
||||||
if not directory.exists():
|
|
||||||
return None
|
|
||||||
if directory.is_symlink():
|
|
||||||
return "CREDENTIAL_MOUNT_SYMLINK"
|
|
||||||
if os.access(directory, os.R_OK):
|
|
||||||
return "CREDENTIAL_MOUNT_ACCESSIBLE"
|
|
||||||
if directory.is_dir():
|
|
||||||
for child in directory.iterdir():
|
|
||||||
if child.is_file() and os.access(child, os.R_OK):
|
|
||||||
return "CREDENTIAL_FILE_ACCESSIBLE"
|
|
||||||
except PermissionError:
|
|
||||||
return None
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def assert_uncredentialed() -> None:
|
|
||||||
credentials = Path("/run/credentials")
|
|
||||||
candidates = {credentials}
|
|
||||||
credentials_directory = Path(os.environ.get("CREDENTIALS_DIRECTORY", "/run/credentials"))
|
|
||||||
candidates.add(credentials_directory)
|
|
||||||
leaked = sorted(
|
|
||||||
key for key, value in os.environ.items()
|
|
||||||
if value
|
|
||||||
and (any(token in key.upper() for token in SECRET_ENV_TOKENS) or SENSITIVE_VALUE_RE.search(str(value)))
|
|
||||||
)
|
|
||||||
if leaked:
|
|
||||||
raise IsolatedExecutorError("CREDENTIAL_ENV_EXPOSED:" + ",".join(leaked))
|
|
||||||
for directory in candidates:
|
|
||||||
exposure = _credential_exposure(directory)
|
|
||||||
if exposure is not None:
|
|
||||||
raise IsolatedExecutorError(exposure)
|
|
||||||
# In the intended PrivateNetwork namespace host loopback services are unreachable.
|
|
||||||
try:
|
|
||||||
probe = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
||||||
except OSError as exc:
|
|
||||||
if exc.errno in {errno.EACCES, errno.EPERM, errno.EAFNOSUPPORT}:
|
|
||||||
return
|
|
||||||
raise IsolatedExecutorError("NETWORK_ISOLATION_PROBE_FAILED") from exc
|
|
||||||
probe.settimeout(0.15)
|
|
||||||
try:
|
|
||||||
if probe.connect_ex(("127.0.0.1", 18892)) == 0:
|
|
||||||
raise IsolatedExecutorError("GOVERNED_BRIDGE_REACHABLE")
|
|
||||||
finally:
|
|
||||||
probe.close()
|
|
||||||
|
|
||||||
|
|
||||||
def extract_source(archive: Path, root: Path) -> None:
|
|
||||||
if archive.is_symlink() or not archive.is_file() or archive.stat().st_size > MAX_ARCHIVE_BYTES:
|
|
||||||
raise IsolatedExecutorError("SOURCE_ARCHIVE_INVALID")
|
|
||||||
with tarfile.open(archive, "r:gz") as tf:
|
|
||||||
for member in tf.getmembers():
|
|
||||||
path = Path(member.name)
|
|
||||||
if path.is_absolute() or ".." in path.parts or ".git" in path.parts:
|
|
||||||
raise IsolatedExecutorError("SOURCE_ARCHIVE_PATH_INVALID")
|
|
||||||
if member.issym() or member.islnk():
|
|
||||||
raise IsolatedExecutorError("SOURCE_ARCHIVE_LINK_DENY")
|
|
||||||
tf.extractall(root, filter="data")
|
|
||||||
|
|
||||||
|
|
||||||
def run_command(item: Mapping[str, Any], root: Path) -> dict[str, Any]:
|
|
||||||
argv = item.get("argv")
|
|
||||||
if not isinstance(argv, Sequence) or isinstance(argv, (str, bytes)) or not argv:
|
|
||||||
raise IsolatedExecutorError("COMMAND_INVALID")
|
|
||||||
argv = [str(part) for part in argv]
|
|
||||||
if argv[0] not in ALLOWED_PROGRAMS:
|
|
||||||
raise IsolatedExecutorError("COMMAND_PROGRAM_DENY")
|
|
||||||
if argv[0] == "/usr/bin/python3" and "-m" in argv:
|
|
||||||
index = argv.index("-m")
|
|
||||||
if index + 1 >= len(argv) or argv[index + 1] not in {"pytest", "py_compile"}:
|
|
||||||
raise IsolatedExecutorError("PYTHON_MODULE_DENY")
|
|
||||||
if argv[0] == "/usr/bin/git" and any(
|
|
||||||
token in {"push", "update-ref", "commit", "merge", "rebase", "reset", "clean", "branch"}
|
|
||||||
for token in argv[1:]
|
|
||||||
):
|
|
||||||
raise IsolatedExecutorError("GIT_MUTATION_DENY")
|
|
||||||
timeout = int(item.get("timeout_seconds") or 300)
|
|
||||||
if not 1 <= timeout <= 1800:
|
|
||||||
raise IsolatedExecutorError("COMMAND_TIMEOUT_INVALID")
|
|
||||||
cwd = root if item.get("cwd", ".") == "." else resolve_under(root, str(item.get("cwd")))
|
|
||||||
started = time.monotonic()
|
|
||||||
completed = subprocess.run(
|
|
||||||
argv,
|
|
||||||
cwd=cwd,
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
check=False,
|
|
||||||
timeout=timeout,
|
|
||||||
env={"PATH": "/usr/bin:/bin", "LC_ALL": "C.UTF-8"},
|
|
||||||
)
|
|
||||||
result = {
|
|
||||||
"label": str(item.get("label") or "command"),
|
|
||||||
"exit_code": completed.returncode,
|
|
||||||
"stdout_sha256": sha256(completed.stdout.encode("utf-8", "replace")),
|
|
||||||
"stderr_sha256": sha256(completed.stderr.encode("utf-8", "replace")),
|
|
||||||
"duration_ms": int((time.monotonic() - started) * 1000),
|
|
||||||
}
|
|
||||||
if completed.returncode != 0:
|
|
||||||
raise IsolatedExecutorError(
|
|
||||||
"COMMAND_FAILED:" + result["label"] + ":" + str(completed.returncode)
|
|
||||||
)
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def apply_changes(request: Mapping[str, Any], root: Path) -> list[str]:
|
|
||||||
changes = request.get("change_set")
|
|
||||||
if not isinstance(changes, list) or not changes:
|
|
||||||
raise IsolatedExecutorError("CHANGE_SET_INVALID")
|
|
||||||
changed: list[str] = []
|
|
||||||
for change in changes:
|
|
||||||
if not isinstance(change, Mapping):
|
|
||||||
raise IsolatedExecutorError("CHANGE_INVALID")
|
|
||||||
relative = safe_relative(change.get("path"))
|
|
||||||
path = resolve_under(root, relative)
|
|
||||||
current = path.read_bytes() if path.is_file() else None
|
|
||||||
expected = change.get("expected_sha256")
|
|
||||||
if expected is not None and (current is None or sha256(current) != expected):
|
|
||||||
raise IsolatedExecutorError("SOURCE_DRIFT:" + relative)
|
|
||||||
operation = change.get("operation")
|
|
||||||
if operation == "write_file":
|
|
||||||
content = change.get("content")
|
|
||||||
if not isinstance(content, str):
|
|
||||||
raise IsolatedExecutorError("WRITE_CONTENT_INVALID")
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
path.write_text(content, encoding="utf-8")
|
|
||||||
elif operation == "replace_text":
|
|
||||||
if current is None:
|
|
||||||
raise IsolatedExecutorError("REPLACE_SOURCE_MISSING")
|
|
||||||
old = str(change.get("old"))
|
|
||||||
new = str(change.get("new"))
|
|
||||||
count = int(change.get("count", 1))
|
|
||||||
text = current.decode("utf-8")
|
|
||||||
if text.count(old) != count:
|
|
||||||
raise IsolatedExecutorError("REPLACE_COUNT_INVALID:" + relative)
|
|
||||||
path.write_text(text.replace(old, new), encoding="utf-8")
|
|
||||||
elif operation == "delete_file":
|
|
||||||
if path.exists():
|
|
||||||
path.unlink()
|
|
||||||
else:
|
|
||||||
raise IsolatedExecutorError("CHANGE_OPERATION_INVALID")
|
|
||||||
changed.append(relative)
|
|
||||||
return sorted(set(changed))
|
|
||||||
|
|
||||||
|
|
||||||
def changed_bundle(root: Path, changed: list[str]) -> list[dict[str, Any]]:
|
|
||||||
files = []
|
|
||||||
for relative in changed:
|
|
||||||
path = resolve_under(root, relative)
|
|
||||||
if path.exists():
|
|
||||||
data = path.read_bytes()
|
|
||||||
if len(data) > MAX_FILE_BYTES:
|
|
||||||
raise IsolatedExecutorError("CHANGED_FILE_TOO_LARGE:" + relative)
|
|
||||||
files.append({
|
|
||||||
"path": relative,
|
|
||||||
"deleted": False,
|
|
||||||
"sha256": sha256(data),
|
|
||||||
"content_b64": base64.b64encode(data).decode("ascii"),
|
|
||||||
})
|
|
||||||
else:
|
|
||||||
files.append({"path": relative, "deleted": True, "sha256": None, "content_b64": None})
|
|
||||||
return files
|
|
||||||
|
|
||||||
|
|
||||||
def execute_request(request_path: Path, result_path: Path) -> None:
|
|
||||||
assert_uncredentialed()
|
|
||||||
request = json.loads(request_path.read_text(encoding="utf-8"))
|
|
||||||
if request.get("schema") != "aios.isolated-execution.request.v1":
|
|
||||||
raise IsolatedExecutorError("REQUEST_SCHEMA_INVALID")
|
|
||||||
request_id = request.get("request_id")
|
|
||||||
if not isinstance(request_id, str) or not REQUEST_ID_RE.fullmatch(request_id):
|
|
||||||
raise IsolatedExecutorError("REQUEST_ID_INVALID")
|
|
||||||
source_archive = Path(str(request.get("source_archive") or ""))
|
|
||||||
if not source_archive.is_absolute():
|
|
||||||
raise IsolatedExecutorError("SOURCE_ARCHIVE_PATH_INVALID")
|
|
||||||
expected_archive_sha = str(request.get("source_archive_sha256") or "")
|
|
||||||
if sha256(source_archive.read_bytes()) != expected_archive_sha:
|
|
||||||
raise IsolatedExecutorError("SOURCE_ARCHIVE_SHA_MISMATCH")
|
|
||||||
with tempfile.TemporaryDirectory(prefix="aios-isolated-executor-") as temporary:
|
|
||||||
root = Path(temporary) / "workspace"
|
|
||||||
root.mkdir()
|
|
||||||
extract_source(source_archive, root)
|
|
||||||
changed = apply_changes(request, root)
|
|
||||||
commands = [
|
|
||||||
run_command(item, root)
|
|
||||||
for item in request.get("verification", [])
|
|
||||||
if isinstance(item, Mapping)
|
|
||||||
]
|
|
||||||
result = {
|
|
||||||
"schema": "aios.isolated-execution.result.v1",
|
|
||||||
"request_id": request_id,
|
|
||||||
"request_sha256": sha256(canonical(request)),
|
|
||||||
"changed_files": changed_bundle(root, changed),
|
|
||||||
"verification": commands,
|
|
||||||
"credential_isolation": {
|
|
||||||
"credential_env_absent": True,
|
|
||||||
"credential_mount_inaccessible": True,
|
|
||||||
"governed_bridge_unreachable": True,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
result["result_sha256"] = sha256(canonical(result))
|
|
||||||
temporary_result = result_path.with_suffix(".tmp")
|
|
||||||
temporary_result.write_text(json.dumps(result, sort_keys=True, separators=(",", ":")), encoding="utf-8")
|
|
||||||
os.replace(temporary_result, result_path)
|
|
||||||
|
|
||||||
|
|
||||||
def serve(root: Path) -> None:
|
|
||||||
assert_uncredentialed()
|
|
||||||
requests = root / "requests"
|
|
||||||
results = root / "results"
|
|
||||||
failed = root / "failed"
|
|
||||||
for path in (requests, results, failed):
|
|
||||||
path.mkdir(parents=True, exist_ok=True)
|
|
||||||
while True:
|
|
||||||
for request in sorted(requests.glob("*.json")):
|
|
||||||
if request.is_symlink():
|
|
||||||
continue
|
|
||||||
result = results / request.name
|
|
||||||
try:
|
|
||||||
execute_request(request, result)
|
|
||||||
request.unlink()
|
|
||||||
except Exception as exc:
|
|
||||||
(failed / request.name).write_text(
|
|
||||||
json.dumps({"error": str(exc) or type(exc).__name__}, sort_keys=True),
|
|
||||||
encoding="utf-8",
|
|
||||||
)
|
|
||||||
request.unlink(missing_ok=True)
|
|
||||||
time.sleep(0.5)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
import argparse
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
parser.add_argument("--root", default="/var/lib/aios-online-agent-executor")
|
|
||||||
parser.add_argument("--request")
|
|
||||||
parser.add_argument("--result")
|
|
||||||
args = parser.parse_args()
|
|
||||||
if args.request and args.result:
|
|
||||||
execute_request(Path(args.request), Path(args.result))
|
|
||||||
else:
|
|
||||||
serve(Path(args.root))
|
|
||||||
@@ -1,302 +0,0 @@
|
|||||||
"""Durable checkpoint journal for the generic Online Agent worker.
|
|
||||||
|
|
||||||
The JSON task state remains a fast cache. This SQLite journal is the durable,
|
|
||||||
append-only evidence used for restart/resume/replay decisions.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
import hashlib
|
|
||||||
import json
|
|
||||||
from pathlib import Path
|
|
||||||
import sqlite3
|
|
||||||
from typing import Any, Mapping
|
|
||||||
|
|
||||||
PHASE_ORDER = {
|
|
||||||
"claimed": 10,
|
|
||||||
"workspace_prepared": 20,
|
|
||||||
"verification_passed": 30,
|
|
||||||
"agent_runtime_done": 40,
|
|
||||||
"source_capture_done": 40,
|
|
||||||
"commit_done": 40,
|
|
||||||
"reconciliation_required": 45,
|
|
||||||
"result_pending": 50,
|
|
||||||
"result_submitted": 60,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class CheckpointError(RuntimeError):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def canonical(value: object) -> bytes:
|
|
||||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, default=str).encode("utf-8")
|
|
||||||
|
|
||||||
|
|
||||||
def sha256_json(value: object) -> str:
|
|
||||||
return hashlib.sha256(canonical(value)).hexdigest()
|
|
||||||
|
|
||||||
|
|
||||||
def now() -> str:
|
|
||||||
return datetime.now(timezone.utc).isoformat()
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class CheckpointBinding:
|
|
||||||
task_id: str
|
|
||||||
attempt: int
|
|
||||||
execution_digest: str
|
|
||||||
job_id: str
|
|
||||||
repository: str
|
|
||||||
base_oid: str
|
|
||||||
expected_branch_oid: str
|
|
||||||
target_branch: str
|
|
||||||
generation: int
|
|
||||||
|
|
||||||
def material(self) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"task_id": self.task_id,
|
|
||||||
"attempt": self.attempt,
|
|
||||||
"execution_digest": self.execution_digest,
|
|
||||||
"job_id": self.job_id,
|
|
||||||
"repository": self.repository,
|
|
||||||
"base_oid": self.base_oid,
|
|
||||||
"expected_branch_oid": self.expected_branch_oid,
|
|
||||||
"target_branch": self.target_branch,
|
|
||||||
"generation": self.generation,
|
|
||||||
}
|
|
||||||
|
|
||||||
@property
|
|
||||||
def binding_sha256(self) -> str:
|
|
||||||
return sha256_json(self.material())
|
|
||||||
|
|
||||||
|
|
||||||
class CheckpointStore:
|
|
||||||
def __init__(self, path: Path) -> None:
|
|
||||||
self.path = Path(path)
|
|
||||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
with self._connect() as connection:
|
|
||||||
connection.executescript(
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS checkpoints(
|
|
||||||
task_id TEXT NOT NULL,
|
|
||||||
attempt INTEGER NOT NULL,
|
|
||||||
sequence INTEGER NOT NULL,
|
|
||||||
phase TEXT NOT NULL,
|
|
||||||
phase_order INTEGER NOT NULL,
|
|
||||||
execution_digest TEXT NOT NULL,
|
|
||||||
job_id TEXT NOT NULL,
|
|
||||||
repository TEXT NOT NULL,
|
|
||||||
base_oid TEXT NOT NULL,
|
|
||||||
expected_branch_oid TEXT NOT NULL,
|
|
||||||
target_branch TEXT NOT NULL,
|
|
||||||
generation INTEGER NOT NULL,
|
|
||||||
binding_sha256 TEXT NOT NULL,
|
|
||||||
payload_json TEXT NOT NULL,
|
|
||||||
payload_sha256 TEXT NOT NULL,
|
|
||||||
created_at TEXT NOT NULL,
|
|
||||||
PRIMARY KEY(task_id, attempt, sequence)
|
|
||||||
);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_checkpoints_task_latest
|
|
||||||
ON checkpoints(task_id, attempt DESC, sequence DESC);
|
|
||||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_checkpoints_exact_replay
|
|
||||||
ON checkpoints(task_id, attempt, phase, payload_sha256);
|
|
||||||
CREATE TABLE IF NOT EXISTS checkpoint_heads(
|
|
||||||
task_id TEXT PRIMARY KEY,
|
|
||||||
attempt INTEGER NOT NULL,
|
|
||||||
sequence INTEGER NOT NULL,
|
|
||||||
phase TEXT NOT NULL,
|
|
||||||
phase_order INTEGER NOT NULL,
|
|
||||||
execution_digest TEXT NOT NULL,
|
|
||||||
binding_sha256 TEXT NOT NULL,
|
|
||||||
payload_sha256 TEXT NOT NULL,
|
|
||||||
updated_at TEXT NOT NULL
|
|
||||||
);
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
|
|
||||||
def _connect(self) -> sqlite3.Connection:
|
|
||||||
connection = sqlite3.connect(self.path, timeout=30.0)
|
|
||||||
connection.row_factory = sqlite3.Row
|
|
||||||
connection.execute("PRAGMA foreign_keys=ON")
|
|
||||||
connection.execute("PRAGMA journal_mode=WAL")
|
|
||||||
connection.execute("PRAGMA synchronous=FULL")
|
|
||||||
connection.execute("PRAGMA busy_timeout=30000")
|
|
||||||
return connection
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _phase_order(phase: str) -> int:
|
|
||||||
try:
|
|
||||||
return PHASE_ORDER[phase]
|
|
||||||
except KeyError as exc:
|
|
||||||
raise CheckpointError("CHECKPOINT_PHASE_INVALID") from exc
|
|
||||||
|
|
||||||
def append(
|
|
||||||
self,
|
|
||||||
binding: CheckpointBinding,
|
|
||||||
phase: str,
|
|
||||||
payload: Mapping[str, Any],
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
order = self._phase_order(phase)
|
|
||||||
payload_value = dict(payload)
|
|
||||||
payload_sha = sha256_json(payload_value)
|
|
||||||
with self._connect() as connection:
|
|
||||||
connection.execute("BEGIN IMMEDIATE")
|
|
||||||
replay = connection.execute(
|
|
||||||
"""
|
|
||||||
SELECT * FROM checkpoints
|
|
||||||
WHERE task_id=? AND attempt=? AND phase=? AND payload_sha256=?
|
|
||||||
""",
|
|
||||||
(binding.task_id, binding.attempt, phase, payload_sha),
|
|
||||||
).fetchone()
|
|
||||||
if replay is not None:
|
|
||||||
connection.commit()
|
|
||||||
return dict(replay)
|
|
||||||
|
|
||||||
head = connection.execute(
|
|
||||||
"SELECT * FROM checkpoint_heads WHERE task_id=?",
|
|
||||||
(binding.task_id,),
|
|
||||||
).fetchone()
|
|
||||||
if head is not None:
|
|
||||||
head_attempt = int(head["attempt"])
|
|
||||||
head_order = int(head["phase_order"])
|
|
||||||
if binding.attempt < head_attempt:
|
|
||||||
raise CheckpointError("CHECKPOINT_ATTEMPT_REGRESSION")
|
|
||||||
if binding.attempt == head_attempt:
|
|
||||||
if head["execution_digest"] != binding.execution_digest:
|
|
||||||
raise CheckpointError("CHECKPOINT_EXECUTION_DIGEST_CONFLICT")
|
|
||||||
if head["binding_sha256"] != binding.binding_sha256:
|
|
||||||
raise CheckpointError("CHECKPOINT_SCOPE_MISMATCH_DENY")
|
|
||||||
if order < head_order and phase != "reconciliation_required":
|
|
||||||
raise CheckpointError("CHECKPOINT_PHASE_REGRESSION")
|
|
||||||
sequence = int(head["sequence"]) + 1
|
|
||||||
else:
|
|
||||||
sequence = 1
|
|
||||||
else:
|
|
||||||
sequence = 1
|
|
||||||
|
|
||||||
created_at = now()
|
|
||||||
connection.execute(
|
|
||||||
"""
|
|
||||||
INSERT INTO checkpoints(
|
|
||||||
task_id,attempt,sequence,phase,phase_order,execution_digest,job_id,
|
|
||||||
repository,base_oid,expected_branch_oid,target_branch,generation,
|
|
||||||
binding_sha256,payload_json,payload_sha256,created_at
|
|
||||||
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
|
||||||
""",
|
|
||||||
(
|
|
||||||
binding.task_id,
|
|
||||||
binding.attempt,
|
|
||||||
sequence,
|
|
||||||
phase,
|
|
||||||
order,
|
|
||||||
binding.execution_digest,
|
|
||||||
binding.job_id,
|
|
||||||
binding.repository,
|
|
||||||
binding.base_oid,
|
|
||||||
binding.expected_branch_oid,
|
|
||||||
binding.target_branch,
|
|
||||||
binding.generation,
|
|
||||||
binding.binding_sha256,
|
|
||||||
canonical(payload_value).decode("utf-8"),
|
|
||||||
payload_sha,
|
|
||||||
created_at,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
connection.execute(
|
|
||||||
"""
|
|
||||||
INSERT INTO checkpoint_heads(
|
|
||||||
task_id,attempt,sequence,phase,phase_order,execution_digest,
|
|
||||||
binding_sha256,payload_sha256,updated_at
|
|
||||||
) VALUES (?,?,?,?,?,?,?,?,?)
|
|
||||||
ON CONFLICT(task_id) DO UPDATE SET
|
|
||||||
attempt=excluded.attempt,
|
|
||||||
sequence=excluded.sequence,
|
|
||||||
phase=excluded.phase,
|
|
||||||
phase_order=excluded.phase_order,
|
|
||||||
execution_digest=excluded.execution_digest,
|
|
||||||
binding_sha256=excluded.binding_sha256,
|
|
||||||
payload_sha256=excluded.payload_sha256,
|
|
||||||
updated_at=excluded.updated_at
|
|
||||||
""",
|
|
||||||
(
|
|
||||||
binding.task_id,
|
|
||||||
binding.attempt,
|
|
||||||
sequence,
|
|
||||||
phase,
|
|
||||||
order,
|
|
||||||
binding.execution_digest,
|
|
||||||
binding.binding_sha256,
|
|
||||||
payload_sha,
|
|
||||||
created_at,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
connection.commit()
|
|
||||||
row = connection.execute(
|
|
||||||
"SELECT * FROM checkpoints WHERE task_id=? AND attempt=? AND sequence=?",
|
|
||||||
(binding.task_id, binding.attempt, sequence),
|
|
||||||
).fetchone()
|
|
||||||
return dict(row)
|
|
||||||
|
|
||||||
def latest(self, task_id: str) -> dict[str, Any] | None:
|
|
||||||
with self._connect() as connection:
|
|
||||||
row = connection.execute(
|
|
||||||
"""
|
|
||||||
SELECT * FROM checkpoints
|
|
||||||
WHERE task_id=?
|
|
||||||
ORDER BY attempt DESC, sequence DESC
|
|
||||||
LIMIT 1
|
|
||||||
""",
|
|
||||||
(task_id,),
|
|
||||||
).fetchone()
|
|
||||||
if row is None:
|
|
||||||
return None
|
|
||||||
value = dict(row)
|
|
||||||
value["payload"] = json.loads(value.pop("payload_json"))
|
|
||||||
return value
|
|
||||||
|
|
||||||
def latest_for_attempt(self, task_id: str, attempt: int) -> dict[str, Any] | None:
|
|
||||||
with self._connect() as connection:
|
|
||||||
row = connection.execute(
|
|
||||||
"""
|
|
||||||
SELECT * FROM checkpoints
|
|
||||||
WHERE task_id=? AND attempt=?
|
|
||||||
ORDER BY sequence DESC
|
|
||||||
LIMIT 1
|
|
||||||
""",
|
|
||||||
(task_id, attempt),
|
|
||||||
).fetchone()
|
|
||||||
if row is None:
|
|
||||||
return None
|
|
||||||
value = dict(row)
|
|
||||||
value["payload"] = json.loads(value.pop("payload_json"))
|
|
||||||
return value
|
|
||||||
|
|
||||||
def verify_latest_binding(self, binding: CheckpointBinding) -> dict[str, Any] | None:
|
|
||||||
value = self.latest(binding.task_id)
|
|
||||||
if value is None:
|
|
||||||
return None
|
|
||||||
if int(value["attempt"]) > binding.attempt:
|
|
||||||
raise CheckpointError("CHECKPOINT_FUTURE_ATTEMPT")
|
|
||||||
# A previous attempt may be reused only when the immutable execution/scope binding matches.
|
|
||||||
if value["execution_digest"] != binding.execution_digest:
|
|
||||||
return None
|
|
||||||
expected = CheckpointBinding(
|
|
||||||
task_id=value["task_id"],
|
|
||||||
attempt=int(value["attempt"]),
|
|
||||||
execution_digest=value["execution_digest"],
|
|
||||||
job_id=value["job_id"],
|
|
||||||
repository=value["repository"],
|
|
||||||
base_oid=value["base_oid"],
|
|
||||||
expected_branch_oid=value["expected_branch_oid"],
|
|
||||||
target_branch=value["target_branch"],
|
|
||||||
generation=int(value["generation"]),
|
|
||||||
)
|
|
||||||
current_scope = binding.material()
|
|
||||||
historical_scope = expected.material()
|
|
||||||
current_scope.pop("attempt", None)
|
|
||||||
historical_scope.pop("attempt", None)
|
|
||||||
if sha256_json(current_scope) != sha256_json(historical_scope):
|
|
||||||
raise CheckpointError("CHECKPOINT_SCOPE_MISMATCH_DENY")
|
|
||||||
return value
|
|
||||||
@@ -1,463 +0,0 @@
|
|||||||
"""Bounded read-only repository interface map for governed Online Agent analysis.
|
|
||||||
|
|
||||||
The compact map intentionally returns symbols and short source excerpts rather than a
|
|
||||||
source archive. It is designed for control-plane consumers that cannot safely ingest a
|
|
||||||
multi-megabyte repository snapshot.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import ast
|
|
||||||
import hashlib
|
|
||||||
import json
|
|
||||||
from pathlib import Path
|
|
||||||
import re
|
|
||||||
from typing import Any, Iterable, Mapping, Sequence
|
|
||||||
|
|
||||||
|
|
||||||
SCHEMA = "aios.engineering.interface-map.v1"
|
|
||||||
MAX_RESULT_BYTES = 120 * 1024
|
|
||||||
TARGET_RESULT_BYTES = 108 * 1024
|
|
||||||
MAX_MODULES = 40
|
|
||||||
MAX_SYMBOLS = 80
|
|
||||||
MAX_TESTS = 40
|
|
||||||
MAX_EXCERPTS = 25
|
|
||||||
MAX_EXCERPT_CHARS = 1200
|
|
||||||
MAX_FILE_BYTES = 2 * 1024 * 1024
|
|
||||||
|
|
||||||
ROOTS = (
|
|
||||||
"active-runtime/app",
|
|
||||||
"runtime/agent-os",
|
|
||||||
"tests",
|
|
||||||
"db/migrations",
|
|
||||||
"migrations",
|
|
||||||
"deploy/systemd",
|
|
||||||
"docs/task-plane",
|
|
||||||
"docs/superpowers/specs",
|
|
||||||
)
|
|
||||||
SUFFIXES = {".py", ".sql", ".service", ".md", ".json", ".yaml", ".yml"}
|
|
||||||
|
|
||||||
DEFAULT_TERMS = {
|
|
||||||
"agent", "worker", "fleet", "scheduler", "scheduling", "task", "claim", "lease",
|
|
||||||
"heartbeat", "capacity", "concurrency", "priority", "fairness", "queue", "retry",
|
|
||||||
"backoff", "circuit", "health", "liveness", "scope", "tenant", "project", "trace",
|
|
||||||
"message", "checkpoint", "recovery", "idempotency", "dispatch", "assignment",
|
|
||||||
}
|
|
||||||
FLEET_CONTRACT_TERMS: dict[str, tuple[str, ...]] = {
|
|
||||||
"FLEET_AGENT_REGISTRATION": ("agent", "register", "worker", "capability", "version"),
|
|
||||||
"FLEET_CAPACITY_SCHEDULING": ("capacity", "slot", "concurrency", "scheduler", "queue"),
|
|
||||||
"FLEET_FAIRNESS": ("fair", "priority", "queue", "tenant", "project", "starvation"),
|
|
||||||
"FLEET_FAULT_ISOLATION": ("failure", "health", "liveness", "timeout", "circuit", "poison"),
|
|
||||||
"FLEET_WORKER_REASSIGNMENT": ("reassign", "assignment", "claim", "lease", "retry", "worker"),
|
|
||||||
"FLEET_SCOPE_ISOLATION": ("tenant", "project", "scope", "conversation", "task", "run"),
|
|
||||||
"FLEET_IDEMPOTENT_DISPATCH": ("idempot", "dispatch", "claim", "lease", "result", "attempt"),
|
|
||||||
"FLEET_FAILURE_RECOVERY": ("checkpoint", "resume", "recover", "retry", "cleanup", "receipt"),
|
|
||||||
}
|
|
||||||
|
|
||||||
MODEL_GOVERNANCE_CONTRACT_TERMS: dict[str, tuple[str, ...]] = {
|
|
||||||
"MODEL_PROVIDER_REGISTRY": ("model", "provider", "registry", "endpoint", "capability"),
|
|
||||||
"MODEL_SELECTION_POLICY": ("model", "select", "route", "policy", "capability"),
|
|
||||||
"MODEL_VERSION_PINNING": ("model", "version", "pin", "revision", "snapshot"),
|
|
||||||
"MODEL_BUDGET_ACCOUNTING": ("budget", "token", "cost", "quota", "usage"),
|
|
||||||
"MODEL_FALLBACK_POLICY": ("fallback", "retry", "route", "provider", "model"),
|
|
||||||
"MODEL_SCOPE_ISOLATION": ("tenant", "project", "scope", "policy", "model"),
|
|
||||||
"MODEL_CREDENTIAL_BOUNDARY": ("credential", "secret", "provider", "tool", "token"),
|
|
||||||
"MODEL_TRACE_AUDIT": ("trace", "audit", "model", "provider", "request"),
|
|
||||||
"MODEL_EVALUATION_GATE": ("eval", "evaluation", "approval", "quality", "gate"),
|
|
||||||
"MODEL_ROUTING_RESILIENCE": ("route", "retry", "fallback", "timeout", "health"),
|
|
||||||
}
|
|
||||||
|
|
||||||
PROFILE_CLASSIFICATION = {
|
|
||||||
"fleet": "P2_INTERFACE_MAP_CAPTURED",
|
|
||||||
"model_governance": "MODEL_GOVERNANCE_INTERFACE_MAP_CAPTURED",
|
|
||||||
}
|
|
||||||
|
|
||||||
MODEL_PROFILE_MARKERS = {
|
|
||||||
"model", "provider", "llm", "inference", "routing", "router", "budget",
|
|
||||||
"token", "cost", "quota", "fallback", "evaluation", "eval", "approval",
|
|
||||||
"version", "pinning",
|
|
||||||
}
|
|
||||||
SENSITIVE_RE = re.compile(
|
|
||||||
r"(?i)(authorization\s*:\s*bearer\s+\S+|"
|
|
||||||
r"(?:password|passwd|secret|token|api[_-]?key)\s*[:=]\s*['\"]?[^'\"\s,;]+)"
|
|
||||||
)
|
|
||||||
SQL_TABLE_RE = re.compile(
|
|
||||||
r"(?is)\bcreate\s+table\s+(?:if\s+not\s+exists\s+)?[\"`[]?([A-Za-z_][A-Za-z0-9_.-]*)"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class CompactInterfaceMapError(RuntimeError):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def _sha(data: bytes) -> str:
|
|
||||||
return hashlib.sha256(data).hexdigest()
|
|
||||||
|
|
||||||
|
|
||||||
def _bounded_excerpt(text: str, start_line: int, end_line: int) -> str:
|
|
||||||
lines = text.splitlines()
|
|
||||||
start = max(0, start_line - 2)
|
|
||||||
end = min(len(lines), max(end_line + 1, start + 1))
|
|
||||||
excerpt = "\n".join(lines[start:end])
|
|
||||||
excerpt = SENSITIVE_RE.sub("<redacted-sensitive-source-pattern>", excerpt)
|
|
||||||
if len(excerpt) > MAX_EXCERPT_CHARS:
|
|
||||||
excerpt = excerpt[: MAX_EXCERPT_CHARS - 1] + "…"
|
|
||||||
return excerpt
|
|
||||||
|
|
||||||
|
|
||||||
def _signal_terms(signals: Sequence[str]) -> set[str]:
|
|
||||||
terms = set(DEFAULT_TERMS)
|
|
||||||
for signal in signals:
|
|
||||||
if not isinstance(signal, str):
|
|
||||||
continue
|
|
||||||
terms.update(
|
|
||||||
token.lower()
|
|
||||||
for token in re.findall(r"[A-Za-z][A-Za-z0-9_.-]{2,63}", signal)
|
|
||||||
)
|
|
||||||
return set(sorted(terms)[:96])
|
|
||||||
|
|
||||||
|
|
||||||
def _profile(source_signals: Sequence[str]) -> str:
|
|
||||||
tokens: set[str] = set()
|
|
||||||
for signal in source_signals:
|
|
||||||
if not isinstance(signal, str):
|
|
||||||
continue
|
|
||||||
tokens.update(
|
|
||||||
token.lower()
|
|
||||||
for token in re.findall(r"[A-Za-z][A-Za-z0-9_.-]{2,63}", signal)
|
|
||||||
)
|
|
||||||
model_score = len(tokens & MODEL_PROFILE_MARKERS)
|
|
||||||
return "model_governance" if model_score >= 2 else "fleet"
|
|
||||||
|
|
||||||
|
|
||||||
def _contract_terms(profile: str) -> Mapping[str, tuple[str, ...]]:
|
|
||||||
if profile == "model_governance":
|
|
||||||
return MODEL_GOVERNANCE_CONTRACT_TERMS
|
|
||||||
if profile == "fleet":
|
|
||||||
return FLEET_CONTRACT_TERMS
|
|
||||||
raise CompactInterfaceMapError("INTERFACE_MAP_PROFILE_INVALID")
|
|
||||||
|
|
||||||
|
|
||||||
def _candidate_files(root: Path, terms: set[str]) -> list[dict[str, Any]]:
|
|
||||||
root = root.resolve()
|
|
||||||
candidates: list[dict[str, Any]] = []
|
|
||||||
for relative_root in ROOTS:
|
|
||||||
directory = (root / relative_root).resolve(strict=False)
|
|
||||||
if directory == root or root not in directory.parents:
|
|
||||||
raise CompactInterfaceMapError("INTERFACE_MAP_SCOPE_INVALID")
|
|
||||||
if not directory.is_dir() or directory.is_symlink():
|
|
||||||
continue
|
|
||||||
for path in sorted(directory.rglob("*")):
|
|
||||||
if path.is_symlink() or not path.is_file() or path.suffix.lower() not in SUFFIXES:
|
|
||||||
continue
|
|
||||||
resolved = path.resolve(strict=True)
|
|
||||||
if root not in resolved.parents:
|
|
||||||
raise CompactInterfaceMapError("INTERFACE_MAP_PATH_ESCAPE")
|
|
||||||
data = path.read_bytes()
|
|
||||||
if len(data) > MAX_FILE_BYTES:
|
|
||||||
continue
|
|
||||||
relative = path.relative_to(root).as_posix()
|
|
||||||
text = data.decode("utf-8", "ignore")
|
|
||||||
searchable = (relative + "\n" + text[:384000]).lower()
|
|
||||||
matched = sorted(term for term in terms if term in searchable)
|
|
||||||
if not matched:
|
|
||||||
continue
|
|
||||||
path_bonus = sum(3 for term in terms if term in relative.lower())
|
|
||||||
test_bonus = 3 if relative.startswith("tests/") else 0
|
|
||||||
score = min(250, len(matched) + path_bonus + test_bonus)
|
|
||||||
candidates.append(
|
|
||||||
{
|
|
||||||
"path": relative,
|
|
||||||
"sha256": _sha(data),
|
|
||||||
"bytes": len(data),
|
|
||||||
"score": score,
|
|
||||||
"text": text,
|
|
||||||
"suffix": path.suffix.lower(),
|
|
||||||
}
|
|
||||||
)
|
|
||||||
candidates.sort(key=lambda item: (-item["score"], item["path"]))
|
|
||||||
return candidates
|
|
||||||
|
|
||||||
|
|
||||||
def _python_symbols(item: Mapping[str, Any], terms: set[str]) -> list[dict[str, Any]]:
|
|
||||||
try:
|
|
||||||
tree = ast.parse(str(item["text"]), filename=str(item["path"]))
|
|
||||||
except SyntaxError:
|
|
||||||
return []
|
|
||||||
result: list[dict[str, Any]] = []
|
|
||||||
|
|
||||||
class Visitor(ast.NodeVisitor):
|
|
||||||
def __init__(self) -> None:
|
|
||||||
self.stack: list[str] = []
|
|
||||||
|
|
||||||
def _record(self, node: ast.AST, kind: str, name: str) -> None:
|
|
||||||
qualified = ".".join([*self.stack, name]) if self.stack else name
|
|
||||||
haystack = f"{item['path']} {qualified}".lower()
|
|
||||||
score = sum(3 if term in haystack else 0 for term in terms)
|
|
||||||
if name.startswith("test_"):
|
|
||||||
score += 12
|
|
||||||
result.append(
|
|
||||||
{
|
|
||||||
"path": item["path"],
|
|
||||||
"symbol": qualified,
|
|
||||||
"kind": kind,
|
|
||||||
"line": int(getattr(node, "lineno", 1)),
|
|
||||||
"end_line": int(getattr(node, "end_lineno", getattr(node, "lineno", 1))),
|
|
||||||
"score": score + int(item["score"]),
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
def visit_ClassDef(self, node: ast.ClassDef) -> Any:
|
|
||||||
self._record(node, "class", node.name)
|
|
||||||
self.stack.append(node.name)
|
|
||||||
self.generic_visit(node)
|
|
||||||
self.stack.pop()
|
|
||||||
|
|
||||||
def visit_FunctionDef(self, node: ast.FunctionDef) -> Any:
|
|
||||||
self._record(node, "function", node.name)
|
|
||||||
self.stack.append(node.name)
|
|
||||||
self.generic_visit(node)
|
|
||||||
self.stack.pop()
|
|
||||||
|
|
||||||
def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> Any:
|
|
||||||
self._record(node, "async_function", node.name)
|
|
||||||
self.stack.append(node.name)
|
|
||||||
self.generic_visit(node)
|
|
||||||
self.stack.pop()
|
|
||||||
|
|
||||||
Visitor().visit(tree)
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def _surface_ref(symbol: Mapping[str, Any]) -> str:
|
|
||||||
return f"{symbol['path']}::{symbol['symbol']}"
|
|
||||||
|
|
||||||
|
|
||||||
def _contract_map(
|
|
||||||
symbols: Sequence[Mapping[str, Any]],
|
|
||||||
modules: Sequence[Mapping[str, Any]],
|
|
||||||
contract_terms: Mapping[str, tuple[str, ...]],
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
result: dict[str, Any] = {}
|
|
||||||
module_paths = [str(item["path"]) for item in modules]
|
|
||||||
for contract, terms in contract_terms.items():
|
|
||||||
scored: list[tuple[int, str]] = []
|
|
||||||
for symbol in symbols:
|
|
||||||
haystack = _surface_ref(symbol).lower()
|
|
||||||
score = sum(1 for term in terms if term in haystack)
|
|
||||||
if score:
|
|
||||||
scored.append((score, _surface_ref(symbol)))
|
|
||||||
scored.sort(key=lambda item: (-item[0], item[1]))
|
|
||||||
surfaces = [value for _, value in scored[:6]]
|
|
||||||
if not surfaces:
|
|
||||||
for path in module_paths:
|
|
||||||
score = sum(1 for term in terms if term in path.lower())
|
|
||||||
if score:
|
|
||||||
surfaces.append(path)
|
|
||||||
if len(surfaces) == 6:
|
|
||||||
break
|
|
||||||
result[contract] = {
|
|
||||||
"status": "PARTIAL" if surfaces else "GAP",
|
|
||||||
"surfaces": surfaces,
|
|
||||||
}
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def _gap_evidence(
|
|
||||||
symbols: Sequence[Mapping[str, Any]],
|
|
||||||
profile: str,
|
|
||||||
) -> tuple[list[str], list[str], list[str]]:
|
|
||||||
haystack = "\n".join(_surface_ref(item).lower() for item in symbols)
|
|
||||||
|
|
||||||
if profile == "model_governance":
|
|
||||||
invariants = [
|
|
||||||
"Online Task claim/heartbeat remains the authoritative worker ownership boundary.",
|
|
||||||
"Provider/model credentials remain outside uncredentialed planner/executor processes.",
|
|
||||||
"P0-A tenant/project scope and P0-C trace/audit lineage must constrain model routing.",
|
|
||||||
"P1 retry/checkpoint semantics remain authoritative for model-call execution recovery.",
|
|
||||||
]
|
|
||||||
failure_windows: list[str] = []
|
|
||||||
if "provider" not in haystack and "model" not in haystack:
|
|
||||||
failure_windows.append("No first-class model/provider registry surface detected.")
|
|
||||||
if "budget" not in haystack and "cost" not in haystack and "quota" not in haystack:
|
|
||||||
failure_windows.append("No explicit durable model budget/cost/quota accounting surface detected.")
|
|
||||||
if "fallback" not in haystack and "route" not in haystack:
|
|
||||||
failure_windows.append("No explicit model fallback/routing policy surface detected.")
|
|
||||||
if "eval" not in haystack and "approval" not in haystack:
|
|
||||||
failure_windows.append("No explicit evaluation/approval gate surface detected for model changes.")
|
|
||||||
if "version" not in haystack and "pin" not in haystack:
|
|
||||||
failure_windows.append("No explicit model-version pinning surface detected.")
|
|
||||||
desired = (
|
|
||||||
("trace", "P0-C trace/audit lineage"),
|
|
||||||
("scope", "P0-A tenant/project scope isolation"),
|
|
||||||
("credential", "P0-C credential governance"),
|
|
||||||
("checkpoint", "P1 durable execution recovery"),
|
|
||||||
("message", "P0-B MessageEnvelope/handoff"),
|
|
||||||
("retry", "P1 retry/idempotency"),
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
invariants = [
|
|
||||||
"Online Task claim/heartbeat remains the authoritative worker ownership boundary.",
|
|
||||||
"Governed mutation remains separated from read-only analysis and requires WriterLease.",
|
|
||||||
"P1 attempt-aware result idempotency and durable checkpoint recovery are reusable fleet primitives.",
|
|
||||||
"Scope/trace/message lineage must remain attached when work is scheduled or reassigned.",
|
|
||||||
]
|
|
||||||
failure_windows = []
|
|
||||||
if "scheduler" not in haystack:
|
|
||||||
failure_windows.append("No first-class durable fleet scheduler symbol detected; worker-local polling may still be the scheduling mechanism.")
|
|
||||||
if "capacity" not in haystack and "slot" not in haystack:
|
|
||||||
failure_windows.append("No explicit durable worker-capacity/slot primitive detected.")
|
|
||||||
if "fair" not in haystack and "starvation" not in haystack:
|
|
||||||
failure_windows.append("No explicit fairness/starvation policy symbol detected.")
|
|
||||||
if "circuit" not in haystack:
|
|
||||||
failure_windows.append("No explicit circuit-breaker primitive detected for repeated worker/task failures.")
|
|
||||||
if "reassign" not in haystack:
|
|
||||||
failure_windows.append("No explicit worker-reassignment primitive detected beyond lease expiry/retry semantics.")
|
|
||||||
desired = (
|
|
||||||
("checkpoint", "P1 durable checkpoint/resume"),
|
|
||||||
("message", "P0-B MessageEnvelope/handoff"),
|
|
||||||
("trace", "P0-C trace correlation"),
|
|
||||||
("scope", "P0-A scope isolation"),
|
|
||||||
("lease", "claim/WriterLease ownership"),
|
|
||||||
("heartbeat", "worker liveness"),
|
|
||||||
("idempot", "retry-safe dispatch/result"),
|
|
||||||
)
|
|
||||||
|
|
||||||
reuse_points: list[str] = []
|
|
||||||
for token, label in desired:
|
|
||||||
match = next((item for item in symbols if token in _surface_ref(item).lower()), None)
|
|
||||||
if match is not None:
|
|
||||||
reuse_points.append(f"{label}: {_surface_ref(match)}")
|
|
||||||
return invariants, failure_windows, reuse_points
|
|
||||||
|
|
||||||
|
|
||||||
def _trim_to_budget(value: dict[str, Any]) -> dict[str, Any]:
|
|
||||||
def size() -> int:
|
|
||||||
return len(json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8"))
|
|
||||||
|
|
||||||
while size() > TARGET_RESULT_BYTES and value["bounded_excerpts"]:
|
|
||||||
value["bounded_excerpts"].pop()
|
|
||||||
while size() > TARGET_RESULT_BYTES and len(value["implementation_map"]["functions"]) > 20:
|
|
||||||
value["implementation_map"]["functions"].pop()
|
|
||||||
while size() > TARGET_RESULT_BYTES and len(value["implementation_map"]["classes"]) > 15:
|
|
||||||
value["implementation_map"]["classes"].pop()
|
|
||||||
while size() > TARGET_RESULT_BYTES and len(value["implementation_map"]["tests"]) > 15:
|
|
||||||
value["implementation_map"]["tests"].pop()
|
|
||||||
while size() > TARGET_RESULT_BYTES and len(value["implementation_map"]["modules"]) > 20:
|
|
||||||
value["implementation_map"]["modules"].pop()
|
|
||||||
final_size = size()
|
|
||||||
if final_size > MAX_RESULT_BYTES:
|
|
||||||
raise CompactInterfaceMapError("INTERFACE_MAP_RESULT_TOO_LARGE")
|
|
||||||
value["result_bytes"] = final_size
|
|
||||||
return value
|
|
||||||
|
|
||||||
|
|
||||||
def build_compact_interface_map(
|
|
||||||
worktree: Path,
|
|
||||||
source_signals: Sequence[str],
|
|
||||||
*,
|
|
||||||
source_commit: str,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
root = Path(worktree)
|
|
||||||
if not root.is_dir() or root.is_symlink():
|
|
||||||
raise CompactInterfaceMapError("INTERFACE_MAP_WORKTREE_INVALID")
|
|
||||||
if re.fullmatch(r"[0-9a-f]{40}", source_commit) is None:
|
|
||||||
raise CompactInterfaceMapError("INTERFACE_MAP_SOURCE_COMMIT_INVALID")
|
|
||||||
|
|
||||||
profile = _profile(source_signals)
|
|
||||||
terms = _signal_terms(source_signals)
|
|
||||||
candidates = _candidate_files(root, terms)
|
|
||||||
if not candidates:
|
|
||||||
raise CompactInterfaceMapError("INTERFACE_MAP_EMPTY")
|
|
||||||
|
|
||||||
module_candidates = candidates[:MAX_MODULES]
|
|
||||||
modules = [
|
|
||||||
{
|
|
||||||
"path": item["path"],
|
|
||||||
"sha256": item["sha256"],
|
|
||||||
"bytes": item["bytes"],
|
|
||||||
"score": item["score"],
|
|
||||||
}
|
|
||||||
for item in module_candidates
|
|
||||||
]
|
|
||||||
|
|
||||||
symbols: list[dict[str, Any]] = []
|
|
||||||
tables: list[dict[str, Any]] = []
|
|
||||||
systemd_units: list[dict[str, Any]] = []
|
|
||||||
for item in candidates:
|
|
||||||
if item["suffix"] == ".py":
|
|
||||||
symbols.extend(_python_symbols(item, terms))
|
|
||||||
elif item["suffix"] == ".sql":
|
|
||||||
for table in SQL_TABLE_RE.findall(str(item["text"])):
|
|
||||||
tables.append({"path": item["path"], "table": table, "sha256": item["sha256"]})
|
|
||||||
elif item["suffix"] == ".service":
|
|
||||||
systemd_units.append({"path": item["path"], "unit": Path(str(item["path"])).name, "sha256": item["sha256"]})
|
|
||||||
|
|
||||||
symbols.sort(key=lambda item: (-int(item["score"]), str(item["path"]), str(item["symbol"])))
|
|
||||||
all_symbols = list(symbols)
|
|
||||||
test_symbols = [
|
|
||||||
item for item in all_symbols
|
|
||||||
if str(item["path"]).startswith("tests/") or str(item["symbol"]).split(".")[-1].startswith("test_")
|
|
||||||
][: min(MAX_TESTS, 24)]
|
|
||||||
non_test_symbols = [
|
|
||||||
item for item in all_symbols
|
|
||||||
if item not in test_symbols
|
|
||||||
][: max(0, MAX_SYMBOLS - len(test_symbols))]
|
|
||||||
symbols = [*test_symbols, *non_test_symbols]
|
|
||||||
symbols.sort(key=lambda item: (-int(item["score"]), str(item["path"]), str(item["symbol"])))
|
|
||||||
classes = [
|
|
||||||
{k: item[k] for k in ("path", "symbol", "line", "end_line")}
|
|
||||||
for item in symbols
|
|
||||||
if item["kind"] == "class"
|
|
||||||
]
|
|
||||||
functions = [
|
|
||||||
{k: item[k] for k in ("path", "symbol", "line", "end_line")}
|
|
||||||
for item in symbols
|
|
||||||
if item["kind"] in {"function", "async_function"} and not str(item["symbol"]).split(".")[-1].startswith("test_")
|
|
||||||
]
|
|
||||||
tests = [
|
|
||||||
{k: item[k] for k in ("path", "symbol", "line", "end_line")}
|
|
||||||
for item in symbols
|
|
||||||
if str(item["path"]).startswith("tests/") or str(item["symbol"]).split(".")[-1].startswith("test_")
|
|
||||||
][:MAX_TESTS]
|
|
||||||
|
|
||||||
excerpts: list[dict[str, Any]] = []
|
|
||||||
by_path = {str(item["path"]): item for item in candidates}
|
|
||||||
for symbol in symbols:
|
|
||||||
if len(excerpts) == MAX_EXCERPTS:
|
|
||||||
break
|
|
||||||
item = by_path.get(str(symbol["path"]))
|
|
||||||
if item is None:
|
|
||||||
continue
|
|
||||||
excerpt = _bounded_excerpt(str(item["text"]), int(symbol["line"]), int(symbol["end_line"]))
|
|
||||||
if not excerpt.strip():
|
|
||||||
continue
|
|
||||||
excerpts.append(
|
|
||||||
{
|
|
||||||
"path": symbol["path"],
|
|
||||||
"symbol": symbol["symbol"],
|
|
||||||
"sha256": item["sha256"],
|
|
||||||
"excerpt": excerpt,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
invariants, failure_windows, reuse_points = _gap_evidence(all_symbols, profile)
|
|
||||||
result = {
|
|
||||||
"schema": SCHEMA,
|
|
||||||
"profile": profile,
|
|
||||||
"classification": PROFILE_CLASSIFICATION[profile],
|
|
||||||
"source_commit": source_commit,
|
|
||||||
"implementation_map": {
|
|
||||||
"modules": modules,
|
|
||||||
"classes": classes[:MAX_SYMBOLS],
|
|
||||||
"functions": functions[:MAX_SYMBOLS],
|
|
||||||
"tables": tables[:40],
|
|
||||||
"systemd_units": systemd_units[:20],
|
|
||||||
"tests": tests,
|
|
||||||
"current_invariants": invariants,
|
|
||||||
"failure_windows": failure_windows,
|
|
||||||
"reuse_points": reuse_points,
|
|
||||||
},
|
|
||||||
"bounded_excerpts": excerpts,
|
|
||||||
"contracts": _contract_map(all_symbols, modules, _contract_terms(profile)),
|
|
||||||
}
|
|
||||||
return _trim_to_budget(result)
|
|
||||||
@@ -1,482 +0,0 @@
|
|||||||
"""Canonical governed execution projection for Online Agent Tasks.
|
|
||||||
|
|
||||||
The durable governed binding is authoritative. Natural-language task fields and
|
|
||||||
worker configuration are never allowed to invent repository, ref, generation, or
|
|
||||||
idempotency coordinates.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from dataclasses import asdict, dataclass
|
|
||||||
import hashlib
|
|
||||||
import json
|
|
||||||
import re
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any, Mapping, Sequence
|
|
||||||
import urllib.error
|
|
||||||
import urllib.request
|
|
||||||
|
|
||||||
|
|
||||||
EXECUTION_SCHEMA = "aios.online-agent.execution.v2"
|
|
||||||
PROJECTION_SCHEMA = "aios.online-agent.task-projection.v1"
|
|
||||||
OID_RE = re.compile(r"^[0-9a-f]{40}$")
|
|
||||||
TASK_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{2,179}$")
|
|
||||||
BRANCH_RE = re.compile(r"^agent/[A-Za-z0-9][A-Za-z0-9._/-]{2,179}$")
|
|
||||||
KEY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{7,159}$")
|
|
||||||
PROTECTED_PREFIXES = ("main", "master", "refs/", "tags/", "tag/", "release/", "baseline/", "production/")
|
|
||||||
MUTATION_ACTIONS = frozenset({"deterministic"})
|
|
||||||
READ_ONLY_ACTIONS = frozenset({"source_capture", "agent_runtime"})
|
|
||||||
SUPPORTED_ACTIONS = frozenset({"deterministic", "source_capture", "agent_runtime"})
|
|
||||||
AGENT_RUNTIME_ADAPTERS = frozenset({"hermes", "openclaw"})
|
|
||||||
MAX_AGENT_RUNTIME_INPUT_BYTES = 64 * 1024
|
|
||||||
|
|
||||||
|
|
||||||
class ExecutionProjectionError(RuntimeError):
|
|
||||||
"""A canonical fail-closed projection or validation error."""
|
|
||||||
|
|
||||||
|
|
||||||
class OnlineTaskProjectionClient:
|
|
||||||
"""Official HTTP projection client using only a control credential reference."""
|
|
||||||
|
|
||||||
def __init__(self, api_base: str, control_token_path: Path) -> None:
|
|
||||||
self.api_base = api_base.rstrip("/")
|
|
||||||
self.control_token_path = Path(control_token_path)
|
|
||||||
|
|
||||||
def _token(self) -> str:
|
|
||||||
path = self.control_token_path
|
|
||||||
if path.is_symlink() or not path.is_file():
|
|
||||||
_error("ONLINE_TASK_PROJECTION_CONTROL_UNAVAILABLE")
|
|
||||||
value = path.read_text(encoding="utf-8").strip()
|
|
||||||
if len(value) < 32:
|
|
||||||
_error("ONLINE_TASK_PROJECTION_CONTROL_INVALID")
|
|
||||||
return value
|
|
||||||
|
|
||||||
def create(
|
|
||||||
self,
|
|
||||||
projected_task: Mapping[str, Any],
|
|
||||||
*,
|
|
||||||
binding: "GovernedExecutionBinding",
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
body = dict(projected_task)
|
|
||||||
payload = _canonical(body).encode("utf-8")
|
|
||||||
request = urllib.request.Request(
|
|
||||||
self.api_base + "/v1/tasks",
|
|
||||||
method="POST",
|
|
||||||
data=payload,
|
|
||||||
headers={
|
|
||||||
"Authorization": "Bearer " + self._token(),
|
|
||||||
"Content-Type": "application/json",
|
|
||||||
"Accept": "application/json",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
with urllib.request.urlopen(request, timeout=15) as response:
|
|
||||||
result = json.loads(response.read().decode("utf-8"))
|
|
||||||
status = response.status
|
|
||||||
except urllib.error.HTTPError as exc:
|
|
||||||
raw = exc.read(8192).decode("utf-8", "replace")
|
|
||||||
try:
|
|
||||||
error = json.loads(raw).get("error")
|
|
||||||
except (AttributeError, json.JSONDecodeError):
|
|
||||||
error = "HTTP_ERROR"
|
|
||||||
raise ExecutionProjectionError(f"ONLINE_TASK_PROJECTION_FAILED:{exc.code}:{error}") from exc
|
|
||||||
except (OSError, urllib.error.URLError, TimeoutError) as exc:
|
|
||||||
raise ExecutionProjectionError("ONLINE_TASK_PROJECTION_UNAVAILABLE") from exc
|
|
||||||
if status not in {200, 201} or not isinstance(result, dict):
|
|
||||||
_error("ONLINE_TASK_PROJECTION_RESPONSE_INVALID")
|
|
||||||
if result.get("task_id") != binding.task_id or result.get("status") not in {"QUEUED", "VERIFYING", "SUCCEEDED"}:
|
|
||||||
_error("ONLINE_TASK_PROJECTION_RESPONSE_INVALID")
|
|
||||||
metadata = result.get("metadata")
|
|
||||||
if not isinstance(metadata, Mapping):
|
|
||||||
_error("ONLINE_TASK_PROJECTION_RESPONSE_INVALID")
|
|
||||||
returned_execution = validate_execution_envelope(
|
|
||||||
_object(metadata.get("execution"), "ONLINE_TASK_PROJECTION_RESPONSE_INVALID"),
|
|
||||||
binding=binding,
|
|
||||||
)
|
|
||||||
submitted_metadata = _object(body.get("metadata"), "ONLINE_TASK_PROJECTION_REQUEST_INVALID")
|
|
||||||
submitted_execution = validate_execution_envelope(
|
|
||||||
_object(submitted_metadata.get("execution"), "ONLINE_TASK_PROJECTION_REQUEST_INVALID"),
|
|
||||||
binding=binding,
|
|
||||||
)
|
|
||||||
if _canonical(returned_execution) != _canonical(submitted_execution):
|
|
||||||
_error("ONLINE_TASK_PROJECTION_RESPONSE_BINDING_INVALID")
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def _error(code: str) -> None:
|
|
||||||
raise ExecutionProjectionError(code)
|
|
||||||
|
|
||||||
|
|
||||||
def _object(value: object, code: str) -> dict[str, Any]:
|
|
||||||
if not isinstance(value, Mapping):
|
|
||||||
_error(code)
|
|
||||||
return dict(value)
|
|
||||||
|
|
||||||
|
|
||||||
def _string(value: object, code: str) -> str:
|
|
||||||
if not isinstance(value, str) or not value.strip():
|
|
||||||
_error(code)
|
|
||||||
return value.strip()
|
|
||||||
|
|
||||||
|
|
||||||
def _task_id(value: object) -> str:
|
|
||||||
value = _string(value, "TASK_ID_INVALID")
|
|
||||||
if TASK_RE.fullmatch(value) is None:
|
|
||||||
_error("TASK_ID_INVALID")
|
|
||||||
return value
|
|
||||||
|
|
||||||
|
|
||||||
def _oid(value: object, code: str) -> str:
|
|
||||||
value = _string(value, code)
|
|
||||||
if OID_RE.fullmatch(value) is None:
|
|
||||||
_error(code)
|
|
||||||
return value
|
|
||||||
|
|
||||||
|
|
||||||
def _branch(value: object) -> str:
|
|
||||||
value = _string(value, "TARGET_BRANCH_INVALID")
|
|
||||||
if BRANCH_RE.fullmatch(value) is None or value.startswith(PROTECTED_PREFIXES):
|
|
||||||
_error("TARGET_BRANCH_INVALID")
|
|
||||||
return value
|
|
||||||
|
|
||||||
|
|
||||||
def _generation(value: object) -> int:
|
|
||||||
if not isinstance(value, int) or isinstance(value, bool) or value < 1:
|
|
||||||
_error("GENERATION_INVALID")
|
|
||||||
return value
|
|
||||||
|
|
||||||
|
|
||||||
def _canonical(value: object) -> str:
|
|
||||||
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
|
||||||
|
|
||||||
|
|
||||||
def _digest(value: object) -> str:
|
|
||||||
return hashlib.sha256(_canonical(value).encode("utf-8")).hexdigest()
|
|
||||||
|
|
||||||
|
|
||||||
def _source_signals(value: object) -> tuple[str, ...]:
|
|
||||||
"""Normalize untrusted natural-language hints before they enter a signed envelope."""
|
|
||||||
if value is None:
|
|
||||||
return ()
|
|
||||||
if not isinstance(value, Sequence) or isinstance(value, (str, bytes)):
|
|
||||||
_error("SOURCE_CAPTURE_SIGNALS_INVALID")
|
|
||||||
result: list[str] = []
|
|
||||||
for item in value:
|
|
||||||
if not isinstance(item, str):
|
|
||||||
_error("SOURCE_CAPTURE_SIGNALS_INVALID")
|
|
||||||
text = " ".join(item.split()).strip()
|
|
||||||
if not text:
|
|
||||||
continue
|
|
||||||
# Projection may accept long task prose, but execution envelopes stay bounded.
|
|
||||||
text = text[:512].rstrip()
|
|
||||||
if text not in result:
|
|
||||||
result.append(text)
|
|
||||||
if len(result) == 16:
|
|
||||||
break
|
|
||||||
return tuple(result)
|
|
||||||
|
|
||||||
|
|
||||||
def _agent_runtime_input(value: object) -> dict[str, Any]:
|
|
||||||
if not isinstance(value, Mapping):
|
|
||||||
_error("AGENT_RUNTIME_INPUT_REQUIRED")
|
|
||||||
raw = dict(value)
|
|
||||||
allowed = {"goal", "instructions", "success_criteria", "constraints"}
|
|
||||||
if set(raw) - allowed:
|
|
||||||
_error("AGENT_RUNTIME_INPUT_FIELDS_INVALID")
|
|
||||||
goal = _string(raw.get("goal"), "AGENT_RUNTIME_GOAL_REQUIRED")
|
|
||||||
instructions = raw.get("instructions", "")
|
|
||||||
if not isinstance(instructions, str):
|
|
||||||
_error("AGENT_RUNTIME_INSTRUCTIONS_INVALID")
|
|
||||||
criteria = raw.get("success_criteria", [])
|
|
||||||
if not isinstance(criteria, list) or len(criteria) > 32:
|
|
||||||
_error("AGENT_RUNTIME_SUCCESS_CRITERIA_INVALID")
|
|
||||||
normalized_criteria: list[dict[str, str]] = []
|
|
||||||
for item in criteria:
|
|
||||||
if not isinstance(item, Mapping):
|
|
||||||
_error("AGENT_RUNTIME_SUCCESS_CRITERIA_INVALID")
|
|
||||||
criterion_id = _string(item.get("id"), "AGENT_RUNTIME_SUCCESS_CRITERIA_INVALID")
|
|
||||||
description = _string(
|
|
||||||
item.get("description"), "AGENT_RUNTIME_SUCCESS_CRITERIA_INVALID"
|
|
||||||
)
|
|
||||||
normalized_criteria.append(
|
|
||||||
{"id": criterion_id[:120], "description": description[:1000]}
|
|
||||||
)
|
|
||||||
constraints = raw.get("constraints", {})
|
|
||||||
if not isinstance(constraints, Mapping):
|
|
||||||
_error("AGENT_RUNTIME_CONSTRAINTS_INVALID")
|
|
||||||
sensitive_markers = ("token", "secret", "password", "credential", "authorization", "api_key", "apikey")
|
|
||||||
for key in constraints:
|
|
||||||
lowered = str(key).lower()
|
|
||||||
if any(marker in lowered for marker in sensitive_markers):
|
|
||||||
_error("AGENT_RUNTIME_SECRET_FIELD_DENY")
|
|
||||||
normalized = {
|
|
||||||
"goal": goal[:12000],
|
|
||||||
"instructions": instructions[:32000],
|
|
||||||
"success_criteria": normalized_criteria,
|
|
||||||
"constraints": dict(constraints),
|
|
||||||
}
|
|
||||||
if len(_canonical(normalized).encode("utf-8")) > MAX_AGENT_RUNTIME_INPUT_BYTES:
|
|
||||||
_error("AGENT_RUNTIME_INPUT_TOO_LARGE")
|
|
||||||
return normalized
|
|
||||||
|
|
||||||
|
|
||||||
def _capabilities(value: object) -> tuple[str, ...]:
|
|
||||||
if value is None:
|
|
||||||
return ()
|
|
||||||
if not isinstance(value, Sequence) or isinstance(value, (str, bytes)):
|
|
||||||
_error("CAPABILITY_SCOPE_INVALID")
|
|
||||||
result = tuple(sorted({_string(item, "CAPABILITY_SCOPE_INVALID") for item in value}))
|
|
||||||
if any(len(item) > 120 for item in result):
|
|
||||||
_error("CAPABILITY_SCOPE_INVALID")
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
|
||||||
class GovernedExecutionBinding:
|
|
||||||
task_id: str
|
|
||||||
job_id: str
|
|
||||||
repository: str
|
|
||||||
base_oid: str
|
|
||||||
expected_branch_oid: str
|
|
||||||
target_branch: str
|
|
||||||
branch_mode: str
|
|
||||||
generation: int
|
|
||||||
idempotency_key: str
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_mapping(cls, value: Mapping[str, Any]) -> "GovernedExecutionBinding":
|
|
||||||
data = dict(value)
|
|
||||||
task_id = _task_id(data.get("task_id"))
|
|
||||||
job_id = _string(data.get("job_id"), "JOB_ID_REQUIRED")
|
|
||||||
if TASK_RE.fullmatch(job_id) is None:
|
|
||||||
_error("JOB_ID_INVALID")
|
|
||||||
repository = _string(data.get("repository"), "REPOSITORY_INVALID")
|
|
||||||
if repository != "aios/aios":
|
|
||||||
_error("WRONG_REPOSITORY_ENVELOPE_DENY")
|
|
||||||
base_oid = _oid(data.get("base_oid"), "BASE_OID_INVALID")
|
|
||||||
expected = _oid(data.get("expected_branch_oid"), "EXPECTED_BRANCH_OID_INVALID")
|
|
||||||
target = _branch(data.get("target_branch"))
|
|
||||||
mode = _string(data.get("branch_mode"), "BRANCH_MODE_MISSING")
|
|
||||||
if mode not in {"CREATE_NEW", "CONTINUE_EXISTING"}:
|
|
||||||
_error("BRANCH_MODE_INVALID")
|
|
||||||
generation = _generation(data.get("generation"))
|
|
||||||
if mode == "CREATE_NEW" and expected != base_oid:
|
|
||||||
_error("EXPECTED_BASE_CONFLICT")
|
|
||||||
key = _string(data.get("idempotency_key"), "IDEMPOTENCY_KEY_INVALID")
|
|
||||||
if KEY_RE.fullmatch(key) is None:
|
|
||||||
_error("IDEMPOTENCY_KEY_INVALID")
|
|
||||||
return cls(task_id, job_id, repository, base_oid, expected, target, mode, generation, key)
|
|
||||||
|
|
||||||
def public(self) -> dict[str, Any]:
|
|
||||||
return asdict(self)
|
|
||||||
|
|
||||||
|
|
||||||
def binding_from_bootstrap(value: Mapping[str, Any]) -> GovernedExecutionBinding:
|
|
||||||
return GovernedExecutionBinding.from_mapping(value)
|
|
||||||
|
|
||||||
|
|
||||||
def validate_execution_envelope(
|
|
||||||
value: Mapping[str, Any],
|
|
||||||
*,
|
|
||||||
binding: GovernedExecutionBinding | None = None,
|
|
||||||
worker_capabilities: Sequence[str] | None = None,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
envelope = dict(value)
|
|
||||||
if envelope.get("schema") != EXECUTION_SCHEMA:
|
|
||||||
_error("EXECUTION_ENVELOPE_SCHEMA_VALIDATION")
|
|
||||||
candidate = GovernedExecutionBinding.from_mapping(envelope)
|
|
||||||
if binding is not None and candidate != binding:
|
|
||||||
comparisons = (
|
|
||||||
(candidate.task_id, binding.task_id, "TASK_BINDING_CONFLICT"),
|
|
||||||
(candidate.job_id, binding.job_id, "JOB_BINDING_CONFLICT"),
|
|
||||||
(candidate.repository, binding.repository, "WRONG_REPOSITORY_ENVELOPE_DENY"),
|
|
||||||
(candidate.base_oid, binding.base_oid, "WRONG_BASE_ENVELOPE_DENY"),
|
|
||||||
(candidate.expected_branch_oid, binding.expected_branch_oid, "WRONG_EXPECTED_HEAD_ENVELOPE_DENY"),
|
|
||||||
(candidate.target_branch, binding.target_branch, "WRONG_BRANCH_ENVELOPE_DENY"),
|
|
||||||
(candidate.generation, binding.generation, "WRONG_GENERATION_ENVELOPE_DENY"),
|
|
||||||
(candidate.branch_mode, binding.branch_mode, "BRANCH_MODE_BINDING_CONFLICT"),
|
|
||||||
(candidate.idempotency_key, binding.idempotency_key, "IDEMPOTENCY_BINDING_CONFLICT"),
|
|
||||||
)
|
|
||||||
for actual, expected, code in comparisons:
|
|
||||||
if actual != expected:
|
|
||||||
_error(code)
|
|
||||||
action = _string(envelope.get("action"), "EXECUTION_ACTION_MISSING")
|
|
||||||
if action not in SUPPORTED_ACTIONS:
|
|
||||||
_error("EXECUTION_ACTION_UNSUPPORTED")
|
|
||||||
capability_scope = _capabilities(envelope.get("capability_scope"))
|
|
||||||
if worker_capabilities is not None and not set(capability_scope).issubset(set(worker_capabilities)):
|
|
||||||
_error("CAPABILITY_MISMATCH_DENY")
|
|
||||||
workspace = _object(envelope.get("workspace"), "WORKSPACE_POLICY_REQUIRED")
|
|
||||||
if workspace.get("authority_git_dir") != "/data/aios-gitops/repo.git" or workspace.get("read_only_repository") is not True:
|
|
||||||
_error("WORKSPACE_POLICY_DENY")
|
|
||||||
governance = _object(envelope.get("governance"), "GOVERNANCE_POLICY_REQUIRED")
|
|
||||||
governed_coordinates = (
|
|
||||||
(governance.get("task_id"), candidate.task_id, "TASK_BINDING_CONFLICT"),
|
|
||||||
(governance.get("job_id"), candidate.job_id, "JOB_BINDING_CONFLICT"),
|
|
||||||
(governance.get("idempotency_key"), candidate.idempotency_key, "IDEMPOTENCY_BINDING_CONFLICT"),
|
|
||||||
)
|
|
||||||
for actual_coordinate, expected_coordinate, code in governed_coordinates:
|
|
||||||
if actual_coordinate != expected_coordinate:
|
|
||||||
_error(code)
|
|
||||||
commit_key = _string(governance.get("commit_idempotency_key"), "COMMIT_IDEMPOTENCY_KEY_INVALID")
|
|
||||||
if KEY_RE.fullmatch(commit_key) is None:
|
|
||||||
_error("COMMIT_IDEMPOTENCY_KEY_INVALID")
|
|
||||||
if governance.get("writer_lease") is not (action in MUTATION_ACTIONS):
|
|
||||||
_error("WRITER_LEASE_POLICY_DENY")
|
|
||||||
for field in ("main_write", "production_write", "tag_write", "baseline_write", "force_push", "protected_ref_bypass"):
|
|
||||||
if governance.get(field, False) is not False:
|
|
||||||
_error("POLICY_DENY")
|
|
||||||
changes = envelope.get("change_set")
|
|
||||||
verification = envelope.get("verification")
|
|
||||||
if not isinstance(changes, list) or not isinstance(verification, list):
|
|
||||||
_error("EXECUTION_OPERATIONS_INVALID")
|
|
||||||
if action == "source_capture":
|
|
||||||
if changes or verification:
|
|
||||||
_error("SOURCE_CAPTURE_MUTATION_DENY")
|
|
||||||
signals = list(_source_signals(envelope.get("source_signals") or []))
|
|
||||||
if envelope.get("source_signals") != signals:
|
|
||||||
_error("SOURCE_CAPTURE_SIGNALS_INVALID")
|
|
||||||
if envelope.get("runtime_input") not in (None, {}):
|
|
||||||
_error("SOURCE_CAPTURE_RUNTIME_INPUT_DENY")
|
|
||||||
elif action == "agent_runtime":
|
|
||||||
if changes or verification:
|
|
||||||
_error("AGENT_RUNTIME_MUTATION_DENY")
|
|
||||||
adapter = str(envelope.get("adapter") or "")
|
|
||||||
if adapter not in AGENT_RUNTIME_ADAPTERS:
|
|
||||||
_error("AGENT_RUNTIME_ADAPTER_INVALID")
|
|
||||||
runtime_input = _agent_runtime_input(envelope.get("runtime_input"))
|
|
||||||
if envelope.get("runtime_input") != runtime_input:
|
|
||||||
_error("AGENT_RUNTIME_INPUT_NOT_CANONICAL")
|
|
||||||
required_runtime_capability = f"agent.runtime.{adapter}"
|
|
||||||
if required_runtime_capability not in capability_scope:
|
|
||||||
_error("AGENT_RUNTIME_CAPABILITY_REQUIRED")
|
|
||||||
if envelope.get("source_signals") not in (None, []):
|
|
||||||
_error("AGENT_RUNTIME_SOURCE_SIGNALS_DENY")
|
|
||||||
elif not changes:
|
|
||||||
_error("CHANGE_SET_INVALID")
|
|
||||||
declared = str(envelope.get("execution_digest") or "")
|
|
||||||
material = {key: item for key, item in envelope.items() if key != "execution_digest"}
|
|
||||||
actual = _digest(material)
|
|
||||||
if declared != actual:
|
|
||||||
_error("EXECUTION_DIGEST_INVALID")
|
|
||||||
return {**material, "execution_digest": actual}
|
|
||||||
|
|
||||||
|
|
||||||
def project_execution_envelope(
|
|
||||||
binding: GovernedExecutionBinding,
|
|
||||||
execution_request: Mapping[str, Any] | None = None,
|
|
||||||
*,
|
|
||||||
required_capabilities: Sequence[str] = (),
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
request = dict(execution_request or {})
|
|
||||||
action = str(request.get("action") or "source_capture")
|
|
||||||
if action not in SUPPORTED_ACTIONS:
|
|
||||||
_error("EXECUTION_ACTION_UNSUPPORTED")
|
|
||||||
capabilities = _capabilities(request.get("capability_scope", required_capabilities))
|
|
||||||
governance = {
|
|
||||||
"task_id": binding.task_id,
|
|
||||||
"job_id": binding.job_id,
|
|
||||||
"idempotency_key": binding.idempotency_key,
|
|
||||||
"commit_idempotency_key": str(request.get("commit_idempotency_key") or f"{binding.task_id}-commit-g{binding.generation}"),
|
|
||||||
"writer_lease": action in MUTATION_ACTIONS,
|
|
||||||
"main_write": False,
|
|
||||||
"production_write": False,
|
|
||||||
"tag_write": False,
|
|
||||||
"baseline_write": False,
|
|
||||||
"force_push": False,
|
|
||||||
"protected_ref_bypass": False,
|
|
||||||
}
|
|
||||||
supplied_governance = request.get("governance")
|
|
||||||
if supplied_governance is not None:
|
|
||||||
governance.update(_object(supplied_governance, "GOVERNANCE_POLICY_REQUIRED"))
|
|
||||||
material: dict[str, Any] = {
|
|
||||||
"schema": EXECUTION_SCHEMA,
|
|
||||||
**binding.public(),
|
|
||||||
"action": action,
|
|
||||||
"adapter": str(
|
|
||||||
request.get("adapter")
|
|
||||||
or ("deterministic" if action == "deterministic" else "source_capture")
|
|
||||||
),
|
|
||||||
"capability_scope": list(capabilities),
|
|
||||||
"change_set": list(request.get("change_set") or []),
|
|
||||||
"verification": list(request.get("verification") or []),
|
|
||||||
"source_signals": list(_source_signals(request.get("source_signals") or [])) if action == "source_capture" else [],
|
|
||||||
"runtime_input": _agent_runtime_input(request.get("runtime_input")) if action == "agent_runtime" else {},
|
|
||||||
"governance": governance,
|
|
||||||
"workspace": {
|
|
||||||
"authority_git_dir": "/data/aios-gitops/repo.git",
|
|
||||||
"read_only_repository": True,
|
|
||||||
"cleanup_policy": "always",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
envelope = {**material, "execution_digest": _digest(material)}
|
|
||||||
return validate_execution_envelope(envelope, binding=binding)
|
|
||||||
|
|
||||||
|
|
||||||
def project_online_task(
|
|
||||||
binding: GovernedExecutionBinding,
|
|
||||||
task_request: Mapping[str, Any],
|
|
||||||
execution_request: Mapping[str, Any] | None = None,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
request = dict(task_request)
|
|
||||||
required = _capabilities(request.get("required_capabilities"))
|
|
||||||
execution_value = dict(execution_request or {})
|
|
||||||
if str(execution_value.get("action") or "source_capture") == "agent_runtime":
|
|
||||||
adapter = str(execution_value.get("adapter") or "")
|
|
||||||
if adapter not in AGENT_RUNTIME_ADAPTERS:
|
|
||||||
_error("AGENT_RUNTIME_ADAPTER_INVALID")
|
|
||||||
runtime_input = execution_value.get("runtime_input")
|
|
||||||
if runtime_input is None:
|
|
||||||
runtime_input = {
|
|
||||||
"goal": request.get("goal"),
|
|
||||||
"instructions": request.get("instructions") or "",
|
|
||||||
"success_criteria": request.get("success_criteria") or [],
|
|
||||||
"constraints": request.get("constraints") or {},
|
|
||||||
}
|
|
||||||
execution_value["runtime_input"] = runtime_input
|
|
||||||
execution = project_execution_envelope(
|
|
||||||
binding,
|
|
||||||
execution_value,
|
|
||||||
required_capabilities=required,
|
|
||||||
)
|
|
||||||
metadata = dict(request.get("metadata") or {})
|
|
||||||
if "execution" in metadata:
|
|
||||||
validate_execution_envelope(_object(metadata["execution"], "EXECUTION_ENVELOPE_INVALID"), binding=binding)
|
|
||||||
if metadata["execution"] != execution:
|
|
||||||
_error("EXECUTION_PROJECTION_CONFLICT")
|
|
||||||
metadata["execution"] = execution
|
|
||||||
metadata["execution_projection"] = {
|
|
||||||
"schema": PROJECTION_SCHEMA,
|
|
||||||
"task_id": binding.task_id,
|
|
||||||
"job_id": binding.job_id,
|
|
||||||
"execution_digest": execution["execution_digest"],
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
**request,
|
|
||||||
"task_id": binding.task_id,
|
|
||||||
"metadata": metadata,
|
|
||||||
"required_capabilities": list(required),
|
|
||||||
"idempotency_key": str(request.get("idempotency_key") or f"online-task:{binding.idempotency_key}"),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def resolve_task_execution(
|
|
||||||
task: Mapping[str, Any],
|
|
||||||
binding: GovernedExecutionBinding,
|
|
||||||
*,
|
|
||||||
worker_capabilities: Sequence[str] | None = None,
|
|
||||||
) -> tuple[dict[str, Any], str]:
|
|
||||||
metadata = task.get("metadata")
|
|
||||||
if isinstance(metadata, Mapping) and "execution" in metadata:
|
|
||||||
value = validate_execution_envelope(
|
|
||||||
_object(metadata["execution"], "EXECUTION_ENVELOPE_INVALID"),
|
|
||||||
binding=binding,
|
|
||||||
worker_capabilities=worker_capabilities,
|
|
||||||
)
|
|
||||||
return value, "persisted"
|
|
||||||
projected = project_execution_envelope(
|
|
||||||
binding,
|
|
||||||
{"action": "source_capture", "capability_scope": task.get("required_capabilities") or []},
|
|
||||||
required_capabilities=task.get("required_capabilities") or [],
|
|
||||||
)
|
|
||||||
validate_execution_envelope(projected, binding=binding, worker_capabilities=worker_capabilities)
|
|
||||||
return projected, "recovered_projection"
|
|
||||||
@@ -1,187 +0,0 @@
|
|||||||
"""Durable, authority-bound deterministic execution plans for Online Agent Tasks."""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from contextlib import contextmanager
|
|
||||||
from dataclasses import dataclass
|
|
||||||
import fcntl
|
|
||||||
import hashlib
|
|
||||||
import json
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any, Mapping
|
|
||||||
|
|
||||||
from aios_online_agent_execution import (
|
|
||||||
GovernedExecutionBinding,
|
|
||||||
project_execution_envelope,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class ExecutionPlanError(RuntimeError):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def _canonical(value: object) -> bytes:
|
|
||||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
|
||||||
|
|
||||||
|
|
||||||
def _digest(value: object) -> str:
|
|
||||||
return hashlib.sha256(_canonical(value)).hexdigest()
|
|
||||||
|
|
||||||
|
|
||||||
def _task_key(task_id: str) -> str:
|
|
||||||
readable = "".join(ch if ch.isalnum() or ch in "._-" else "-" for ch in task_id).strip("-")[:72] or "task"
|
|
||||||
return readable + "-" + hashlib.sha256(task_id.encode("utf-8")).hexdigest()[:24]
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class ExecutionPlanRecord:
|
|
||||||
task_id: str
|
|
||||||
job_id: str
|
|
||||||
source_head_oid: str
|
|
||||||
generation: int
|
|
||||||
idempotency_key: str
|
|
||||||
plan_digest: str
|
|
||||||
execution: dict[str, Any]
|
|
||||||
|
|
||||||
def public(self) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"schema": "aios.online-agent.plan-record.v1",
|
|
||||||
"task_id": self.task_id,
|
|
||||||
"job_id": self.job_id,
|
|
||||||
"source_head_oid": self.source_head_oid,
|
|
||||||
"generation": self.generation,
|
|
||||||
"idempotency_key": self.idempotency_key,
|
|
||||||
"plan_digest": self.plan_digest,
|
|
||||||
"execution": dict(self.execution),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class ExecutionPlanStore:
|
|
||||||
def __init__(self, root: Path) -> None:
|
|
||||||
self.root = Path(root)
|
|
||||||
self.root.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
def _path(self, task_id: str) -> Path:
|
|
||||||
return self.root / (_task_key(task_id) + ".json")
|
|
||||||
|
|
||||||
@contextmanager
|
|
||||||
def _lock(self, task_id: str):
|
|
||||||
path = self.root / (_task_key(task_id) + ".lock")
|
|
||||||
with path.open("a+b") as handle:
|
|
||||||
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
|
|
||||||
try:
|
|
||||||
yield
|
|
||||||
finally:
|
|
||||||
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _source_head(binding: GovernedExecutionBinding) -> str:
|
|
||||||
return binding.expected_branch_oid if binding.branch_mode == "CONTINUE_EXISTING" else binding.base_oid
|
|
||||||
|
|
||||||
def _read(self, task_id: str) -> ExecutionPlanRecord | None:
|
|
||||||
path = self._path(task_id)
|
|
||||||
if not path.is_file() or path.is_symlink():
|
|
||||||
return None
|
|
||||||
value = json.loads(path.read_text(encoding="utf-8"))
|
|
||||||
if not isinstance(value, Mapping) or value.get("schema") != "aios.online-agent.plan-record.v1":
|
|
||||||
raise ExecutionPlanError("EXECUTION_PLAN_RECORD_INVALID")
|
|
||||||
execution = value.get("execution")
|
|
||||||
if not isinstance(execution, Mapping):
|
|
||||||
raise ExecutionPlanError("EXECUTION_PLAN_RECORD_INVALID")
|
|
||||||
return ExecutionPlanRecord(
|
|
||||||
task_id=str(value.get("task_id") or ""),
|
|
||||||
job_id=str(value.get("job_id") or ""),
|
|
||||||
source_head_oid=str(value.get("source_head_oid") or ""),
|
|
||||||
generation=int(value.get("generation") or 0),
|
|
||||||
idempotency_key=str(value.get("idempotency_key") or ""),
|
|
||||||
plan_digest=str(value.get("plan_digest") or ""),
|
|
||||||
execution=dict(execution),
|
|
||||||
)
|
|
||||||
|
|
||||||
def register(
|
|
||||||
self,
|
|
||||||
binding: GovernedExecutionBinding,
|
|
||||||
request: Mapping[str, Any],
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
with self._lock(binding.task_id):
|
|
||||||
return self._register_locked(binding, request)
|
|
||||||
|
|
||||||
def _register_locked(
|
|
||||||
self,
|
|
||||||
binding: GovernedExecutionBinding,
|
|
||||||
request: Mapping[str, Any],
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
if request.get("schema") != "aios.online-agent.plan.v1":
|
|
||||||
raise ExecutionPlanError("EXECUTION_PLAN_SCHEMA_INVALID")
|
|
||||||
if request.get("task_id") != binding.task_id or request.get("job_id") != binding.job_id:
|
|
||||||
raise ExecutionPlanError("EXECUTION_PLAN_BINDING_CONFLICT")
|
|
||||||
if int(request.get("generation") or 0) != binding.generation:
|
|
||||||
raise ExecutionPlanError("EXECUTION_PLAN_GENERATION_CONFLICT")
|
|
||||||
source_head = str(request.get("source_head_oid") or "")
|
|
||||||
if source_head != self._source_head(binding):
|
|
||||||
raise ExecutionPlanError("EXECUTION_PLAN_SOURCE_HEAD_CONFLICT")
|
|
||||||
key = str(request.get("idempotency_key") or "")
|
|
||||||
if len(key) < 8:
|
|
||||||
raise ExecutionPlanError("EXECUTION_PLAN_IDEMPOTENCY_INVALID")
|
|
||||||
execution_request = request.get("execution")
|
|
||||||
if not isinstance(execution_request, Mapping) or execution_request.get("action") != "deterministic":
|
|
||||||
raise ExecutionPlanError("EXECUTION_PLAN_DETERMINISTIC_REQUIRED")
|
|
||||||
envelope = project_execution_envelope(binding, execution_request)
|
|
||||||
material = {
|
|
||||||
"task_id": binding.task_id,
|
|
||||||
"job_id": binding.job_id,
|
|
||||||
"source_head_oid": source_head,
|
|
||||||
"generation": binding.generation,
|
|
||||||
"idempotency_key": key,
|
|
||||||
"execution": dict(execution_request),
|
|
||||||
}
|
|
||||||
plan_digest = _digest(material)
|
|
||||||
existing = self._read(binding.task_id)
|
|
||||||
expected_previous = request.get("expected_previous_plan_digest")
|
|
||||||
if existing is not None:
|
|
||||||
if existing.idempotency_key == key:
|
|
||||||
if existing.plan_digest != plan_digest:
|
|
||||||
raise ExecutionPlanError("EXECUTION_PLAN_IDEMPOTENCY_CONFLICT")
|
|
||||||
return {**existing.public(), "idempotent": True, "execution_digest": envelope["execution_digest"]}
|
|
||||||
if expected_previous != existing.plan_digest:
|
|
||||||
raise ExecutionPlanError("EXECUTION_PLAN_REPLACEMENT_CAS_MISMATCH")
|
|
||||||
if existing.source_head_oid != source_head:
|
|
||||||
raise ExecutionPlanError("EXECUTION_PLAN_STALE_REPLACEMENT_DENY")
|
|
||||||
elif expected_previous not in (None, ""):
|
|
||||||
raise ExecutionPlanError("EXECUTION_PLAN_REPLACEMENT_CAS_MISMATCH")
|
|
||||||
|
|
||||||
record = ExecutionPlanRecord(
|
|
||||||
task_id=binding.task_id,
|
|
||||||
job_id=binding.job_id,
|
|
||||||
source_head_oid=source_head,
|
|
||||||
generation=binding.generation,
|
|
||||||
idempotency_key=key,
|
|
||||||
plan_digest=plan_digest,
|
|
||||||
execution=dict(execution_request),
|
|
||||||
)
|
|
||||||
path = self._path(binding.task_id)
|
|
||||||
temporary = path.with_suffix(".tmp")
|
|
||||||
temporary.write_text(
|
|
||||||
json.dumps(record.public(), sort_keys=True, separators=(",", ":")),
|
|
||||||
encoding="utf-8",
|
|
||||||
)
|
|
||||||
temporary.replace(path)
|
|
||||||
return {**record.public(), "idempotent": False, "execution_digest": envelope["execution_digest"]}
|
|
||||||
|
|
||||||
def resolve(self, binding: GovernedExecutionBinding) -> dict[str, Any] | None:
|
|
||||||
record = self._read(binding.task_id)
|
|
||||||
if record is None:
|
|
||||||
return None
|
|
||||||
if (
|
|
||||||
record.task_id != binding.task_id
|
|
||||||
or record.job_id != binding.job_id
|
|
||||||
or record.generation != binding.generation
|
|
||||||
):
|
|
||||||
raise ExecutionPlanError("EXECUTION_PLAN_BINDING_CONFLICT")
|
|
||||||
if record.source_head_oid != self._source_head(binding):
|
|
||||||
raise ExecutionPlanError("EXECUTION_PLAN_SOURCE_HEAD_DRIFT")
|
|
||||||
envelope = project_execution_envelope(binding, record.execution)
|
|
||||||
return {
|
|
||||||
"execution": dict(record.execution),
|
|
||||||
"plan_digest": record.plan_digest,
|
|
||||||
"execution_digest": envelope["execution_digest"],
|
|
||||||
}
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,462 +0,0 @@
|
|||||||
"""Self-healing availability guard for persistent Online Agent workers.
|
|
||||||
|
|
||||||
The guard never claims Tasks and never writes Task/Job/Git databases. It only:
|
|
||||||
- ensures required worker/executor systemd units are enabled + active;
|
|
||||||
- checks worker-owned liveness JSON written by the real poll/heartbeat loop;
|
|
||||||
- restarts an active worker only when its liveness is stale;
|
|
||||||
- honors an explicit maintenance inhibit file;
|
|
||||||
- writes a bounded status JSON under /run for observability.
|
|
||||||
|
|
||||||
Idle liveness is the successful poll loop itself. Busy liveness is refreshed by the
|
|
||||||
Task lease-heartbeat thread, so long-running Tasks are not mistaken for hung workers.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from dataclasses import asdict, dataclass
|
|
||||||
import argparse
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
from pathlib import Path
|
|
||||||
import shlex
|
|
||||||
import subprocess
|
|
||||||
import tempfile
|
|
||||||
import time
|
|
||||||
from typing import Any, Callable, Mapping, Sequence
|
|
||||||
|
|
||||||
SCHEMA = "aios.worker-availability.status.v1"
|
|
||||||
DEFAULT_WORKER_UNITS = (
|
|
||||||
"aios-online-agent-worker-20260815.service",
|
|
||||||
"aios-hermes-online-agent-worker.service",
|
|
||||||
"aios-openclaw-online-agent-worker.service",
|
|
||||||
)
|
|
||||||
DEFAULT_DEPENDENCY_UNITS = (
|
|
||||||
"aios-external-agent-executor.service",
|
|
||||||
)
|
|
||||||
DEFAULT_RUN_ROOT = Path("/run/aios-worker-availability")
|
|
||||||
DEFAULT_STATUS = DEFAULT_RUN_ROOT / "status.json"
|
|
||||||
DEFAULT_RECOVERY = DEFAULT_RUN_ROOT / "recovery.json"
|
|
||||||
DEFAULT_INHIBIT = DEFAULT_RUN_ROOT / "maintenance.lock"
|
|
||||||
DEFAULT_MAX_STALE_SECONDS = 90
|
|
||||||
DEFAULT_RESTART_COOLDOWN_SECONDS = 60
|
|
||||||
DEFAULT_STARTUP_WAIT_SECONDS = 10
|
|
||||||
ACCEPTED_ENABLED_STATES = frozenset({"enabled", "enabled-runtime", "static", "indirect", "generated"})
|
|
||||||
|
|
||||||
|
|
||||||
class AvailabilityGuardError(RuntimeError):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class UnitSnapshot:
|
|
||||||
unit: str
|
|
||||||
role: str
|
|
||||||
load_state: str
|
|
||||||
active_state: str
|
|
||||||
unit_file_state: str
|
|
||||||
enabled_ok: bool
|
|
||||||
liveness_path: str | None = None
|
|
||||||
liveness_state: str | None = None
|
|
||||||
last_seen_epoch: float | None = None
|
|
||||||
liveness_age_seconds: float | None = None
|
|
||||||
fresh: bool | None = None
|
|
||||||
action: str = "NONE"
|
|
||||||
detail: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class Systemd:
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
runner: Callable[..., subprocess.CompletedProcess[str]] | None = None,
|
|
||||||
) -> None:
|
|
||||||
self.runner = runner or subprocess.run
|
|
||||||
|
|
||||||
def _run(self, args: Sequence[str], *, check: bool = False) -> subprocess.CompletedProcess[str]:
|
|
||||||
result = self.runner(
|
|
||||||
["/usr/bin/systemctl", *args],
|
|
||||||
text=True,
|
|
||||||
capture_output=True,
|
|
||||||
check=False,
|
|
||||||
timeout=20,
|
|
||||||
)
|
|
||||||
if check and result.returncode != 0:
|
|
||||||
raise AvailabilityGuardError(
|
|
||||||
"SYSTEMD_COMMAND_FAILED:"
|
|
||||||
+ ":".join(args)
|
|
||||||
+ ":"
|
|
||||||
+ str(result.returncode)
|
|
||||||
)
|
|
||||||
return result
|
|
||||||
|
|
||||||
def property(self, unit: str, name: str) -> str:
|
|
||||||
result = self._run(["show", unit, f"--property={name}", "--value"])
|
|
||||||
if result.returncode != 0:
|
|
||||||
return ""
|
|
||||||
return result.stdout.strip()
|
|
||||||
|
|
||||||
def environment(self, unit: str) -> dict[str, str]:
|
|
||||||
text = self.property(unit, "Environment")
|
|
||||||
result: dict[str, str] = {}
|
|
||||||
for item in shlex.split(text):
|
|
||||||
if "=" not in item:
|
|
||||||
continue
|
|
||||||
key, value = item.split("=", 1)
|
|
||||||
result[key] = value
|
|
||||||
return result
|
|
||||||
|
|
||||||
def enable(self, unit: str) -> None:
|
|
||||||
self._run(["enable", unit], check=True)
|
|
||||||
|
|
||||||
def start(self, unit: str) -> None:
|
|
||||||
self._run(["start", unit], check=True)
|
|
||||||
|
|
||||||
def restart(self, unit: str) -> None:
|
|
||||||
self._run(["restart", unit], check=True)
|
|
||||||
|
|
||||||
|
|
||||||
def _atomic_json(path: Path, value: Mapping[str, Any]) -> None:
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True, mode=0o755)
|
|
||||||
if path.parent.is_symlink() or path.is_symlink():
|
|
||||||
raise AvailabilityGuardError("STATUS_PATH_SYMLINK_REJECTED")
|
|
||||||
fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
|
|
||||||
try:
|
|
||||||
os.fchmod(fd, 0o644)
|
|
||||||
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
|
||||||
json.dump(value, handle, ensure_ascii=False, sort_keys=True, indent=2)
|
|
||||||
handle.write("\n")
|
|
||||||
handle.flush()
|
|
||||||
os.fsync(handle.fileno())
|
|
||||||
os.replace(temporary, path)
|
|
||||||
os.chmod(path, 0o644)
|
|
||||||
finally:
|
|
||||||
if os.path.exists(temporary):
|
|
||||||
os.unlink(temporary)
|
|
||||||
|
|
||||||
|
|
||||||
def _load_json(path: Path) -> dict[str, Any] | None:
|
|
||||||
if path.is_symlink() or not path.is_file():
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
value = json.loads(path.read_text(encoding="utf-8"))
|
|
||||||
except (OSError, json.JSONDecodeError):
|
|
||||||
return None
|
|
||||||
return dict(value) if isinstance(value, Mapping) else None
|
|
||||||
|
|
||||||
|
|
||||||
def _state_root(environment: Mapping[str, str], unit: str) -> Path:
|
|
||||||
explicit = environment.get("AIOS_WORKER_STATE_ROOT")
|
|
||||||
if explicit:
|
|
||||||
return Path(explicit)
|
|
||||||
fallbacks = {
|
|
||||||
"aios-online-agent-worker-20260815.service": Path(
|
|
||||||
"/var/lib/aios-online-agent-worker-20260815"
|
|
||||||
),
|
|
||||||
"aios-hermes-online-agent-worker.service": Path(
|
|
||||||
"/var/lib/aios-online-agent-hermes"
|
|
||||||
),
|
|
||||||
"aios-openclaw-online-agent-worker.service": Path(
|
|
||||||
"/var/lib/aios-online-agent-openclaw"
|
|
||||||
),
|
|
||||||
}
|
|
||||||
if unit not in fallbacks:
|
|
||||||
raise AvailabilityGuardError("WORKER_STATE_ROOT_UNAVAILABLE:" + unit)
|
|
||||||
return fallbacks[unit]
|
|
||||||
|
|
||||||
|
|
||||||
def _liveness_path(environment: Mapping[str, str], unit: str) -> Path:
|
|
||||||
explicit = environment.get("AIOS_WORKER_LIVENESS_PATH")
|
|
||||||
if explicit:
|
|
||||||
return Path(explicit)
|
|
||||||
return _state_root(environment, unit) / "worker-liveness.json"
|
|
||||||
|
|
||||||
|
|
||||||
class AvailabilityGuard:
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
systemd: Systemd | None = None,
|
|
||||||
worker_units: Sequence[str] = DEFAULT_WORKER_UNITS,
|
|
||||||
dependency_units: Sequence[str] = DEFAULT_DEPENDENCY_UNITS,
|
|
||||||
status_path: Path = DEFAULT_STATUS,
|
|
||||||
recovery_path: Path = DEFAULT_RECOVERY,
|
|
||||||
inhibit_path: Path = DEFAULT_INHIBIT,
|
|
||||||
max_stale_seconds: int = DEFAULT_MAX_STALE_SECONDS,
|
|
||||||
restart_cooldown_seconds: int = DEFAULT_RESTART_COOLDOWN_SECONDS,
|
|
||||||
startup_wait_seconds: int = DEFAULT_STARTUP_WAIT_SECONDS,
|
|
||||||
now_fn: Callable[[], float] = time.time,
|
|
||||||
sleep_fn: Callable[[float], None] = time.sleep,
|
|
||||||
) -> None:
|
|
||||||
self.systemd = systemd or Systemd()
|
|
||||||
self.worker_units = tuple(worker_units)
|
|
||||||
self.dependency_units = tuple(dependency_units)
|
|
||||||
self.status_path = Path(status_path)
|
|
||||||
self.recovery_path = Path(recovery_path)
|
|
||||||
self.inhibit_path = Path(inhibit_path)
|
|
||||||
self.max_stale_seconds = int(max_stale_seconds)
|
|
||||||
self.restart_cooldown_seconds = int(restart_cooldown_seconds)
|
|
||||||
self.startup_wait_seconds = int(startup_wait_seconds)
|
|
||||||
self.now_fn = now_fn
|
|
||||||
self.sleep_fn = sleep_fn
|
|
||||||
if self.max_stale_seconds < 15:
|
|
||||||
raise AvailabilityGuardError("MAX_STALE_SECONDS_TOO_SMALL")
|
|
||||||
if self.restart_cooldown_seconds < 15:
|
|
||||||
raise AvailabilityGuardError("RESTART_COOLDOWN_TOO_SMALL")
|
|
||||||
if self.startup_wait_seconds < 0 or self.startup_wait_seconds > 60:
|
|
||||||
raise AvailabilityGuardError("STARTUP_WAIT_INVALID")
|
|
||||||
|
|
||||||
def _recovery_state(self) -> dict[str, float]:
|
|
||||||
value = _load_json(self.recovery_path) or {}
|
|
||||||
raw = value.get("last_restart_epoch")
|
|
||||||
if not isinstance(raw, Mapping):
|
|
||||||
return {}
|
|
||||||
result: dict[str, float] = {}
|
|
||||||
for unit, epoch in raw.items():
|
|
||||||
try:
|
|
||||||
result[str(unit)] = float(epoch)
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
continue
|
|
||||||
return result
|
|
||||||
|
|
||||||
def _save_recovery_state(self, state: Mapping[str, float]) -> None:
|
|
||||||
_atomic_json(
|
|
||||||
self.recovery_path,
|
|
||||||
{
|
|
||||||
"schema": "aios.worker-availability.recovery.v1",
|
|
||||||
"last_restart_epoch": dict(state),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
def _basic_snapshot(self, unit: str, role: str) -> tuple[str, str, str, bool]:
|
|
||||||
load = self.systemd.property(unit, "LoadState")
|
|
||||||
active = self.systemd.property(unit, "ActiveState")
|
|
||||||
enabled = self.systemd.property(unit, "UnitFileState")
|
|
||||||
return load, active, enabled, enabled in ACCEPTED_ENABLED_STATES
|
|
||||||
|
|
||||||
def _ensure_enabled_active(self, unit: str, role: str) -> tuple[str, bool]:
|
|
||||||
load, active, enabled, enabled_ok = self._basic_snapshot(unit, role)
|
|
||||||
if load != "loaded":
|
|
||||||
raise AvailabilityGuardError("REQUIRED_UNIT_NOT_LOADED:" + unit)
|
|
||||||
action = "NONE"
|
|
||||||
if not enabled_ok:
|
|
||||||
self.systemd.enable(unit)
|
|
||||||
action = "ENABLE"
|
|
||||||
if active != "active":
|
|
||||||
self.systemd.start(unit)
|
|
||||||
action = "START" if action == "NONE" else "ENABLE_START"
|
|
||||||
return action, active == "active"
|
|
||||||
|
|
||||||
def _wait_for_liveness(self, path: Path) -> dict[str, Any] | None:
|
|
||||||
deadline = self.now_fn() + self.startup_wait_seconds
|
|
||||||
while True:
|
|
||||||
value = _load_json(path)
|
|
||||||
if value is not None:
|
|
||||||
try:
|
|
||||||
epoch = float(value.get("last_seen_epoch"))
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
epoch = 0.0
|
|
||||||
if epoch > 0 and self.now_fn() - epoch <= self.max_stale_seconds:
|
|
||||||
return value
|
|
||||||
if self.now_fn() >= deadline:
|
|
||||||
return value
|
|
||||||
self.sleep_fn(0.5)
|
|
||||||
|
|
||||||
def run(self, *, heal: bool) -> dict[str, Any]:
|
|
||||||
now = self.now_fn()
|
|
||||||
if self.inhibit_path.exists():
|
|
||||||
result = {
|
|
||||||
"schema": SCHEMA,
|
|
||||||
"status": "MAINTENANCE_INHIBITED",
|
|
||||||
"generated_at_epoch": now,
|
|
||||||
"maintenance_inhibit": str(self.inhibit_path),
|
|
||||||
"workers": [],
|
|
||||||
"dependencies": [],
|
|
||||||
"all_fresh": False,
|
|
||||||
"actions": [],
|
|
||||||
}
|
|
||||||
_atomic_json(self.status_path, result)
|
|
||||||
return result
|
|
||||||
|
|
||||||
actions: list[dict[str, str]] = []
|
|
||||||
dependencies: list[UnitSnapshot] = []
|
|
||||||
for unit in self.dependency_units:
|
|
||||||
load, active, enabled, enabled_ok = self._basic_snapshot(unit, "dependency")
|
|
||||||
action = "NONE"
|
|
||||||
if heal:
|
|
||||||
if load != "loaded":
|
|
||||||
raise AvailabilityGuardError("REQUIRED_UNIT_NOT_LOADED:" + unit)
|
|
||||||
if not enabled_ok:
|
|
||||||
self.systemd.enable(unit)
|
|
||||||
action = "ENABLE"
|
|
||||||
if active != "active":
|
|
||||||
self.systemd.start(unit)
|
|
||||||
action = "START" if action == "NONE" else "ENABLE_START"
|
|
||||||
if action != "NONE":
|
|
||||||
actions.append({"unit": unit, "action": action})
|
|
||||||
load, active, enabled, enabled_ok = self._basic_snapshot(unit, "dependency")
|
|
||||||
dependencies.append(
|
|
||||||
UnitSnapshot(
|
|
||||||
unit=unit,
|
|
||||||
role="dependency",
|
|
||||||
load_state=load,
|
|
||||||
active_state=active,
|
|
||||||
unit_file_state=enabled,
|
|
||||||
enabled_ok=enabled_ok,
|
|
||||||
fresh=(active == "active"),
|
|
||||||
action=action,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
recovery = self._recovery_state()
|
|
||||||
workers: list[UnitSnapshot] = []
|
|
||||||
for unit in self.worker_units:
|
|
||||||
load, active, enabled, enabled_ok = self._basic_snapshot(unit, "worker")
|
|
||||||
environment = self.systemd.environment(unit) if load == "loaded" else {}
|
|
||||||
path = _liveness_path(environment, unit) if load == "loaded" else None
|
|
||||||
action = "NONE"
|
|
||||||
|
|
||||||
if heal:
|
|
||||||
if load != "loaded":
|
|
||||||
raise AvailabilityGuardError("REQUIRED_UNIT_NOT_LOADED:" + unit)
|
|
||||||
if not enabled_ok:
|
|
||||||
self.systemd.enable(unit)
|
|
||||||
action = "ENABLE"
|
|
||||||
if active != "active":
|
|
||||||
self.systemd.start(unit)
|
|
||||||
action = "START" if action == "NONE" else "ENABLE_START"
|
|
||||||
actions.append({"unit": unit, "action": action})
|
|
||||||
if path is not None:
|
|
||||||
self._wait_for_liveness(path)
|
|
||||||
load, active, enabled, enabled_ok = self._basic_snapshot(unit, "worker")
|
|
||||||
|
|
||||||
live = _load_json(path) if path is not None else None
|
|
||||||
last_seen: float | None = None
|
|
||||||
state: str | None = None
|
|
||||||
age: float | None = None
|
|
||||||
fresh = False
|
|
||||||
detail: str | None = None
|
|
||||||
if live is not None:
|
|
||||||
state = str(live.get("state") or "")
|
|
||||||
try:
|
|
||||||
last_seen = float(live.get("last_seen_epoch"))
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
last_seen = None
|
|
||||||
if last_seen is not None:
|
|
||||||
age = max(0.0, now - last_seen)
|
|
||||||
fresh = age <= self.max_stale_seconds
|
|
||||||
if live.get("agent_id") in (None, ""):
|
|
||||||
fresh = False
|
|
||||||
detail = "LIVENESS_AGENT_ID_MISSING"
|
|
||||||
else:
|
|
||||||
detail = "LIVENESS_MISSING"
|
|
||||||
|
|
||||||
if heal and active == "active" and not fresh:
|
|
||||||
previous = float(recovery.get(unit, 0.0))
|
|
||||||
if now - previous >= self.restart_cooldown_seconds:
|
|
||||||
self.systemd.restart(unit)
|
|
||||||
recovery[unit] = now
|
|
||||||
action = "RESTART_STALE"
|
|
||||||
actions.append({"unit": unit, "action": action})
|
|
||||||
if path is not None:
|
|
||||||
live = self._wait_for_liveness(path)
|
|
||||||
load, active, enabled, enabled_ok = self._basic_snapshot(unit, "worker")
|
|
||||||
if live is not None:
|
|
||||||
state = str(live.get("state") or "")
|
|
||||||
try:
|
|
||||||
last_seen = float(live.get("last_seen_epoch"))
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
last_seen = None
|
|
||||||
if last_seen is not None:
|
|
||||||
age = max(0.0, self.now_fn() - last_seen)
|
|
||||||
fresh = age <= self.max_stale_seconds
|
|
||||||
detail = None if fresh else "LIVENESS_STALE_AFTER_RESTART"
|
|
||||||
else:
|
|
||||||
detail = "RESTART_COOLDOWN_ACTIVE"
|
|
||||||
|
|
||||||
workers.append(
|
|
||||||
UnitSnapshot(
|
|
||||||
unit=unit,
|
|
||||||
role="worker",
|
|
||||||
load_state=load,
|
|
||||||
active_state=active,
|
|
||||||
unit_file_state=enabled,
|
|
||||||
enabled_ok=enabled_ok,
|
|
||||||
liveness_path=str(path) if path is not None else None,
|
|
||||||
liveness_state=state,
|
|
||||||
last_seen_epoch=last_seen,
|
|
||||||
liveness_age_seconds=age,
|
|
||||||
fresh=bool(active == "active" and enabled_ok and fresh),
|
|
||||||
action=action,
|
|
||||||
detail=detail,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
if heal:
|
|
||||||
self._save_recovery_state(recovery)
|
|
||||||
|
|
||||||
all_fresh = (
|
|
||||||
all(item.active_state == "active" and item.enabled_ok and item.fresh for item in workers)
|
|
||||||
and all(item.active_state == "active" for item in dependencies)
|
|
||||||
)
|
|
||||||
result = {
|
|
||||||
"schema": SCHEMA,
|
|
||||||
"status": "HEALTHY" if all_fresh else "DEGRADED",
|
|
||||||
"generated_at_epoch": self.now_fn(),
|
|
||||||
"max_stale_seconds": self.max_stale_seconds,
|
|
||||||
"maintenance_inhibit": str(self.inhibit_path),
|
|
||||||
"workers": [asdict(item) for item in workers],
|
|
||||||
"dependencies": [asdict(item) for item in dependencies],
|
|
||||||
"all_fresh": all_fresh,
|
|
||||||
"actions": actions,
|
|
||||||
}
|
|
||||||
_atomic_json(self.status_path, result)
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
mode = parser.add_mutually_exclusive_group(required=True)
|
|
||||||
mode.add_argument("--heal", action="store_true")
|
|
||||||
mode.add_argument("--check", action="store_true")
|
|
||||||
parser.add_argument("--status-file", default=str(DEFAULT_STATUS))
|
|
||||||
parser.add_argument("--recovery-file", default=str(DEFAULT_RECOVERY))
|
|
||||||
parser.add_argument("--inhibit-file", default=str(DEFAULT_INHIBIT))
|
|
||||||
parser.add_argument(
|
|
||||||
"--max-stale-seconds",
|
|
||||||
type=int,
|
|
||||||
default=int(
|
|
||||||
os.environ.get(
|
|
||||||
"AIOS_WORKER_AVAILABILITY_MAX_STALE_SECONDS",
|
|
||||||
str(DEFAULT_MAX_STALE_SECONDS),
|
|
||||||
)
|
|
||||||
),
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--restart-cooldown-seconds",
|
|
||||||
type=int,
|
|
||||||
default=int(
|
|
||||||
os.environ.get(
|
|
||||||
"AIOS_WORKER_AVAILABILITY_RESTART_COOLDOWN_SECONDS",
|
|
||||||
str(DEFAULT_RESTART_COOLDOWN_SECONDS),
|
|
||||||
)
|
|
||||||
),
|
|
||||||
)
|
|
||||||
args = parser.parse_args()
|
|
||||||
guard = AvailabilityGuard(
|
|
||||||
status_path=Path(args.status_file),
|
|
||||||
recovery_path=Path(args.recovery_file),
|
|
||||||
inhibit_path=Path(args.inhibit_file),
|
|
||||||
max_stale_seconds=args.max_stale_seconds,
|
|
||||||
restart_cooldown_seconds=args.restart_cooldown_seconds,
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
result = guard.run(heal=args.heal)
|
|
||||||
except AvailabilityGuardError as exc:
|
|
||||||
print(json.dumps({"status": "FAILED", "error": str(exc)}, sort_keys=True))
|
|
||||||
return 20
|
|
||||||
print(json.dumps(result, ensure_ascii=False, sort_keys=True))
|
|
||||||
if args.check and not result.get("all_fresh"):
|
|
||||||
return 10
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
@@ -964,7 +964,6 @@ def validate_writer_lease_request_v2(raw: Mapping[str, Any], probe: Any) -> dict
|
|||||||
required = {
|
required = {
|
||||||
"schema",
|
"schema",
|
||||||
"task_id",
|
"task_id",
|
||||||
"job_id",
|
|
||||||
"repository",
|
"repository",
|
||||||
"branch_mode",
|
"branch_mode",
|
||||||
"base_oid",
|
"base_oid",
|
||||||
@@ -977,7 +976,6 @@ def validate_writer_lease_request_v2(raw: Mapping[str, Any], probe: Any) -> dict
|
|||||||
raise ProvisioningError("LEASE_REQUEST_V2_FIELDS_REJECTED", 409)
|
raise ProvisioningError("LEASE_REQUEST_V2_FIELDS_REJECTED", 409)
|
||||||
request = WriterLeaseRequestV2(
|
request = WriterLeaseRequestV2(
|
||||||
task_id=str(raw["task_id"]),
|
task_id=str(raw["task_id"]),
|
||||||
job_id=str(raw["job_id"]),
|
|
||||||
repository=str(raw["repository"]),
|
repository=str(raw["repository"]),
|
||||||
branch_mode=str(raw["branch_mode"]),
|
branch_mode=str(raw["branch_mode"]),
|
||||||
base_oid=str(raw["base_oid"]),
|
base_oid=str(raw["base_oid"]),
|
||||||
@@ -990,7 +988,6 @@ def validate_writer_lease_request_v2(raw: Mapping[str, Any], probe: Any) -> dict
|
|||||||
return {
|
return {
|
||||||
"schema": WriterLeaseRequestV2.schema,
|
"schema": WriterLeaseRequestV2.schema,
|
||||||
"task_id": request.task_id,
|
"task_id": request.task_id,
|
||||||
"job_id": request.job_id,
|
|
||||||
"repository": request.repository,
|
"repository": request.repository,
|
||||||
"branch_mode": request.branch_mode,
|
"branch_mode": request.branch_mode,
|
||||||
"base_oid": request.base_oid,
|
"base_oid": request.base_oid,
|
||||||
|
|||||||
@@ -1,456 +0,0 @@
|
|||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import dataclasses
|
|
||||||
import datetime as dt
|
|
||||||
import hashlib
|
|
||||||
import json
|
|
||||||
import sqlite3
|
|
||||||
import re
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any, Iterable, Mapping
|
|
||||||
|
|
||||||
UTC = dt.timezone.utc
|
|
||||||
_ALLOWED_CONFIDENTIALITY = frozenset({"public", "tenant", "project", "private", "run"})
|
|
||||||
_PRIVILEGED = frozenset({"*", "admin", "system.admin", "message.admin", "task.admin"})
|
|
||||||
|
|
||||||
|
|
||||||
class HandoffError(RuntimeError):
|
|
||||||
code = "HANDOFF_ERROR"
|
|
||||||
|
|
||||||
|
|
||||||
class MessageScopeMismatch(HandoffError):
|
|
||||||
code = "MESSAGE_SCOPE_MISMATCH_DENY"
|
|
||||||
|
|
||||||
|
|
||||||
class ConfidentialityScopeDenied(HandoffError):
|
|
||||||
code = "MESSAGE_CONFIDENTIALITY_SCOPE_ENFORCE"
|
|
||||||
|
|
||||||
|
|
||||||
class MessageExpired(HandoffError):
|
|
||||||
code = "MESSAGE_EXPIRED"
|
|
||||||
|
|
||||||
|
|
||||||
class SequenceGap(HandoffError):
|
|
||||||
code = "MESSAGE_SEQUENCE_GAP_DETECT"
|
|
||||||
|
|
||||||
|
|
||||||
class SequenceConflict(HandoffError):
|
|
||||||
code = "MESSAGE_SEQUENCE_CONFLICT"
|
|
||||||
|
|
||||||
|
|
||||||
class IdempotencyConflict(HandoffError):
|
|
||||||
code = "MESSAGE_IDEMPOTENCY_CONFLICT"
|
|
||||||
|
|
||||||
|
|
||||||
def _utcnow() -> dt.datetime:
|
|
||||||
return dt.datetime.now(UTC)
|
|
||||||
|
|
||||||
|
|
||||||
def _iso(value: dt.datetime | str | None) -> str | None:
|
|
||||||
if value is None:
|
|
||||||
return None
|
|
||||||
parsed = dt.datetime.fromisoformat(value.replace("Z", "+00:00")) if isinstance(value, str) else value
|
|
||||||
if parsed.tzinfo is None:
|
|
||||||
parsed = parsed.replace(tzinfo=UTC)
|
|
||||||
return parsed.astimezone(UTC).isoformat()
|
|
||||||
|
|
||||||
|
|
||||||
def _canonical(value: Any) -> str:
|
|
||||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, default=str)
|
|
||||||
|
|
||||||
|
|
||||||
def _sha(value: Any) -> str:
|
|
||||||
return hashlib.sha256(_canonical(value).encode("utf-8")).hexdigest()
|
|
||||||
|
|
||||||
|
|
||||||
def _clean(value: Any) -> str | None:
|
|
||||||
if value is None:
|
|
||||||
return None
|
|
||||||
text = str(value).strip()
|
|
||||||
return text or None
|
|
||||||
|
|
||||||
|
|
||||||
def _tuple_scope(value: Iterable[str] | str | None) -> tuple[str, ...]:
|
|
||||||
if value is None:
|
|
||||||
return ()
|
|
||||||
if isinstance(value, str):
|
|
||||||
value = value.split(",")
|
|
||||||
return tuple(sorted({str(item).strip() for item in value if str(item).strip()}))
|
|
||||||
|
|
||||||
|
|
||||||
_SECRET_KEY = re.compile(r"(?i)(?:token|secret|password|private[_-]?key|api[_-]?key)")
|
|
||||||
_SECRET_VALUE = re.compile(r"(?i)bearer\s+[A-Za-z0-9._~+/-]{8,}|(?:token|secret|password|api[_-]?key)\s*[:=]\s*\S+")
|
|
||||||
_SAFE_REFERENCE = re.compile(r"^(?:systemd-credential|vault|secret-manager):[A-Za-z0-9][A-Za-z0-9._:/#-]{0,255}$")
|
|
||||||
|
|
||||||
|
|
||||||
def assert_payload_secret_safe(value: Any, *, key: str = "") -> None:
|
|
||||||
if isinstance(value, Mapping):
|
|
||||||
for child_key, child in value.items():
|
|
||||||
assert_payload_secret_safe(child, key=str(child_key))
|
|
||||||
return
|
|
||||||
if isinstance(value, (list, tuple)):
|
|
||||||
for child in value:
|
|
||||||
assert_payload_secret_safe(child, key=key)
|
|
||||||
return
|
|
||||||
if isinstance(value, (bytes, bytearray)):
|
|
||||||
raise ValueError("CREDENTIAL_SECRET_EXPOSURE_DENY")
|
|
||||||
if isinstance(value, str):
|
|
||||||
if _SECRET_VALUE.search(value):
|
|
||||||
raise ValueError("CREDENTIAL_SECRET_EXPOSURE_DENY")
|
|
||||||
if _SECRET_KEY.search(key) and value and _SAFE_REFERENCE.fullmatch(value) is None:
|
|
||||||
raise ValueError("CREDENTIAL_SECRET_EXPOSURE_DENY")
|
|
||||||
|
|
||||||
|
|
||||||
@dataclasses.dataclass(frozen=True)
|
|
||||||
class DeliveryContext:
|
|
||||||
tenant_id: str | None = None
|
|
||||||
user_id: str | None = None
|
|
||||||
project_id: str | None = None
|
|
||||||
conversation_id: str | None = None
|
|
||||||
run_id: str | None = None
|
|
||||||
capabilities: tuple[str, ...] = ()
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_actor(cls, actor: Any) -> "DeliveryContext":
|
|
||||||
def get(name: str) -> Any:
|
|
||||||
if isinstance(actor, Mapping):
|
|
||||||
return actor.get(name)
|
|
||||||
return getattr(actor, name, None)
|
|
||||||
return cls(
|
|
||||||
tenant_id=_clean(get("tenant_id")),
|
|
||||||
user_id=_clean(get("user_id")),
|
|
||||||
project_id=_clean(get("project_id")),
|
|
||||||
conversation_id=_clean(get("conversation_id")),
|
|
||||||
run_id=_clean(get("run_id")),
|
|
||||||
capabilities=_tuple_scope(get("capabilities") or get("capability_scope")),
|
|
||||||
)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def privileged(self) -> bool:
|
|
||||||
return bool(set(self.capabilities) & _PRIVILEGED)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclasses.dataclass(frozen=True)
|
|
||||||
class MessageEnvelope:
|
|
||||||
source: str
|
|
||||||
target: str
|
|
||||||
message_type: str
|
|
||||||
payload: Mapping[str, Any]
|
|
||||||
sequence: int
|
|
||||||
idempotency_key: str
|
|
||||||
tenant_id: str | None = None
|
|
||||||
user_id: str | None = None
|
|
||||||
project_id: str | None = None
|
|
||||||
conversation_id: str | None = None
|
|
||||||
task_id: str | None = None
|
|
||||||
run_id: str | None = None
|
|
||||||
correlation_id: str | None = None
|
|
||||||
causation_id: str | None = None
|
|
||||||
capability_scope: tuple[str, ...] = ()
|
|
||||||
confidentiality_scope: str = "private"
|
|
||||||
expires_at: str | None = None
|
|
||||||
created_at: str = dataclasses.field(default_factory=lambda: _iso(_utcnow()) or "")
|
|
||||||
envelope_id: str = ""
|
|
||||||
|
|
||||||
def __post_init__(self) -> None:
|
|
||||||
assert_payload_secret_safe(self.payload)
|
|
||||||
for field in ("source", "target", "message_type", "idempotency_key", "tenant_id", "user_id",
|
|
||||||
"project_id", "conversation_id", "task_id", "run_id", "correlation_id", "causation_id"):
|
|
||||||
object.__setattr__(self, field, _clean(getattr(self, field)) if field not in {
|
|
||||||
"source", "target", "message_type", "idempotency_key"
|
|
||||||
} else (_clean(getattr(self, field)) or ""))
|
|
||||||
object.__setattr__(self, "capability_scope", _tuple_scope(self.capability_scope))
|
|
||||||
object.__setattr__(self, "confidentiality_scope", (_clean(self.confidentiality_scope) or "private").lower())
|
|
||||||
object.__setattr__(self, "expires_at", _iso(self.expires_at))
|
|
||||||
object.__setattr__(self, "created_at", _iso(self.created_at) or "")
|
|
||||||
if not self.source or not self.target or not self.message_type or not self.idempotency_key:
|
|
||||||
raise ValueError("MESSAGE_ENVELOPE_IDENTITY_REQUIRED")
|
|
||||||
if not isinstance(self.sequence, int) or self.sequence < 1:
|
|
||||||
raise ValueError("MESSAGE_SEQUENCE_MUST_BE_POSITIVE")
|
|
||||||
if self.confidentiality_scope not in _ALLOWED_CONFIDENTIALITY:
|
|
||||||
raise ValueError("MESSAGE_CONFIDENTIALITY_SCOPE_INVALID")
|
|
||||||
if self.confidentiality_scope != "public" and self.tenant_id is None:
|
|
||||||
raise ValueError("MESSAGE_TENANT_REQUIRED")
|
|
||||||
if self.confidentiality_scope in {"project", "private", "run"} and self.project_id is None:
|
|
||||||
raise ValueError("MESSAGE_PROJECT_REQUIRED")
|
|
||||||
if self.confidentiality_scope in {"private", "run"} and self.user_id is None:
|
|
||||||
raise ValueError("MESSAGE_USER_REQUIRED")
|
|
||||||
if self.confidentiality_scope == "run" and self.run_id is None:
|
|
||||||
raise ValueError("MESSAGE_RUN_REQUIRED")
|
|
||||||
expected = stable_event_id(
|
|
||||||
source=self.source, target=self.target, message_type=self.message_type, payload=self.payload,
|
|
||||||
tenant_id=self.tenant_id, user_id=self.user_id, project_id=self.project_id,
|
|
||||||
conversation_id=self.conversation_id, task_id=self.task_id, run_id=self.run_id,
|
|
||||||
correlation_id=self.correlation_id, causation_id=self.causation_id,
|
|
||||||
sequence=self.sequence, idempotency_key=self.idempotency_key,
|
|
||||||
)
|
|
||||||
if self.envelope_id and self.envelope_id != expected:
|
|
||||||
raise ValueError("MESSAGE_ENVELOPE_ID_MISMATCH")
|
|
||||||
object.__setattr__(self, "envelope_id", expected)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def payload_sha256(self) -> str:
|
|
||||||
return _sha(self.payload)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def stream_key(self) -> str:
|
|
||||||
return _sha({
|
|
||||||
"tenant_id": self.tenant_id, "source": self.source, "target": self.target,
|
|
||||||
"conversation_id": self.conversation_id, "task_id": self.task_id, "run_id": self.run_id,
|
|
||||||
})
|
|
||||||
|
|
||||||
def is_expired(self, *, now: dt.datetime | None = None) -> bool:
|
|
||||||
if self.expires_at is None:
|
|
||||||
return False
|
|
||||||
return dt.datetime.fromisoformat(self.expires_at) <= (now or _utcnow()).astimezone(UTC)
|
|
||||||
|
|
||||||
def as_record(self) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"envelope_id": self.envelope_id, "stream_key": self.stream_key, "source": self.source,
|
|
||||||
"target": self.target, "message_type": self.message_type, "payload_json": _canonical(self.payload),
|
|
||||||
"payload_sha256": self.payload_sha256, "tenant_id": self.tenant_id, "user_id": self.user_id,
|
|
||||||
"project_id": self.project_id, "conversation_id": self.conversation_id, "task_id": self.task_id,
|
|
||||||
"run_id": self.run_id, "correlation_id": self.correlation_id,
|
|
||||||
"causation_id": self.causation_id, "sequence": self.sequence,
|
|
||||||
"capability_scope_json": _canonical(self.capability_scope),
|
|
||||||
"confidentiality_scope": self.confidentiality_scope, "idempotency_key": self.idempotency_key,
|
|
||||||
"expires_at": self.expires_at, "created_at": self.created_at,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def stable_event_id(*, source: Any, target: Any, message_type: Any, payload: Any,
|
|
||||||
tenant_id: Any = None, user_id: Any = None, project_id: Any = None,
|
|
||||||
conversation_id: Any = None, task_id: Any = None, run_id: Any = None,
|
|
||||||
correlation_id: Any = None, causation_id: Any = None,
|
|
||||||
sequence: Any = None, idempotency_key: Any = None) -> str:
|
|
||||||
assert_payload_secret_safe(payload)
|
|
||||||
material = {
|
|
||||||
"source": _clean(source), "target": _clean(target), "message_type": _clean(message_type),
|
|
||||||
"payload_sha256": _sha(payload), "tenant_id": _clean(tenant_id), "user_id": _clean(user_id),
|
|
||||||
"project_id": _clean(project_id), "conversation_id": _clean(conversation_id),
|
|
||||||
"task_id": _clean(task_id), "run_id": _clean(run_id), "causation_id": _clean(causation_id),
|
|
||||||
"sequence": sequence, "idempotency_key": _clean(idempotency_key),
|
|
||||||
}
|
|
||||||
# Preserve historical envelope IDs for legacy rows that predate correlation_id.
|
|
||||||
if _clean(correlation_id) is not None:
|
|
||||||
material["correlation_id"] = _clean(correlation_id)
|
|
||||||
return _sha(material)
|
|
||||||
|
|
||||||
|
|
||||||
def bind_envelope_scope(envelope: MessageEnvelope, actor: Any) -> MessageEnvelope:
|
|
||||||
ctx = DeliveryContext.from_actor(actor)
|
|
||||||
if ctx.privileged:
|
|
||||||
return envelope
|
|
||||||
if ctx.tenant_id is None:
|
|
||||||
if envelope.tenant_id is not None:
|
|
||||||
raise MessageScopeMismatch(MessageScopeMismatch.code)
|
|
||||||
return envelope
|
|
||||||
updates: dict[str, Any] = {}
|
|
||||||
for field in ("tenant_id", "user_id", "project_id", "conversation_id", "run_id"):
|
|
||||||
trusted = getattr(ctx, field)
|
|
||||||
requested = getattr(envelope, field)
|
|
||||||
if requested is not None and requested != trusted:
|
|
||||||
raise MessageScopeMismatch(f"{MessageScopeMismatch.code}:{field}")
|
|
||||||
if requested is None and trusted is not None:
|
|
||||||
updates[field] = trusted
|
|
||||||
if not updates:
|
|
||||||
return envelope
|
|
||||||
data = dataclasses.asdict(envelope)
|
|
||||||
data.update(updates)
|
|
||||||
data["envelope_id"] = ""
|
|
||||||
return MessageEnvelope(**data)
|
|
||||||
|
|
||||||
|
|
||||||
def authorize_delivery(actor: Any, envelope: MessageEnvelope) -> None:
|
|
||||||
ctx = DeliveryContext.from_actor(actor)
|
|
||||||
if ctx.privileged:
|
|
||||||
return
|
|
||||||
if not set(envelope.capability_scope).issubset(set(ctx.capabilities)):
|
|
||||||
raise ConfidentialityScopeDenied(f"{ConfidentialityScopeDenied.code}:capability")
|
|
||||||
if envelope.confidentiality_scope == "public":
|
|
||||||
return
|
|
||||||
if ctx.tenant_id != envelope.tenant_id:
|
|
||||||
raise MessageScopeMismatch(f"{MessageScopeMismatch.code}:tenant_id")
|
|
||||||
if envelope.confidentiality_scope in {"project", "private", "run"} and ctx.project_id != envelope.project_id:
|
|
||||||
raise ConfidentialityScopeDenied(f"{ConfidentialityScopeDenied.code}:project_id")
|
|
||||||
if envelope.confidentiality_scope in {"private", "run"} and ctx.user_id != envelope.user_id:
|
|
||||||
raise ConfidentialityScopeDenied(f"{ConfidentialityScopeDenied.code}:user_id")
|
|
||||||
if envelope.confidentiality_scope == "run" and ctx.run_id != envelope.run_id:
|
|
||||||
raise ConfidentialityScopeDenied(f"{ConfidentialityScopeDenied.code}:run_id")
|
|
||||||
|
|
||||||
|
|
||||||
_SCHEMA = """
|
|
||||||
CREATE TABLE IF NOT EXISTS message_handoff (
|
|
||||||
envelope_id TEXT PRIMARY KEY,
|
|
||||||
stream_key TEXT NOT NULL,
|
|
||||||
source TEXT NOT NULL,
|
|
||||||
target TEXT NOT NULL,
|
|
||||||
message_type TEXT NOT NULL,
|
|
||||||
payload_json TEXT NOT NULL,
|
|
||||||
payload_sha256 TEXT NOT NULL,
|
|
||||||
tenant_id TEXT,
|
|
||||||
user_id TEXT,
|
|
||||||
project_id TEXT,
|
|
||||||
conversation_id TEXT,
|
|
||||||
task_id TEXT,
|
|
||||||
run_id TEXT,
|
|
||||||
correlation_id TEXT,
|
|
||||||
causation_id TEXT,
|
|
||||||
sequence INTEGER NOT NULL CHECK(sequence >= 1),
|
|
||||||
capability_scope_json TEXT NOT NULL,
|
|
||||||
confidentiality_scope TEXT NOT NULL,
|
|
||||||
idempotency_key TEXT NOT NULL,
|
|
||||||
expires_at TEXT,
|
|
||||||
created_at TEXT NOT NULL,
|
|
||||||
status TEXT NOT NULL DEFAULT 'PENDING',
|
|
||||||
delivered_at TEXT,
|
|
||||||
acked_at TEXT,
|
|
||||||
UNIQUE(stream_key, idempotency_key),
|
|
||||||
UNIQUE(stream_key, sequence)
|
|
||||||
);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_message_handoff_target_status
|
|
||||||
ON message_handoff(target, status, created_at);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_message_handoff_scope
|
|
||||||
ON message_handoff(tenant_id, project_id, user_id, conversation_id, run_id);
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
class DurableHandoffStore:
|
|
||||||
def __init__(self, path: str | Path) -> None:
|
|
||||||
self.path = Path(path)
|
|
||||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
with self._connect() as conn:
|
|
||||||
conn.executescript(_SCHEMA)
|
|
||||||
columns = {str(row[1]) for row in conn.execute("PRAGMA table_info(message_handoff)")}
|
|
||||||
if "correlation_id" not in columns:
|
|
||||||
conn.execute("ALTER TABLE message_handoff ADD COLUMN correlation_id TEXT")
|
|
||||||
conn.execute(
|
|
||||||
"CREATE INDEX IF NOT EXISTS idx_message_handoff_lineage "
|
|
||||||
"ON message_handoff(correlation_id, causation_id)"
|
|
||||||
)
|
|
||||||
|
|
||||||
def _connect(self) -> sqlite3.Connection:
|
|
||||||
conn = sqlite3.connect(self.path, timeout=30.0)
|
|
||||||
conn.row_factory = sqlite3.Row
|
|
||||||
conn.execute("PRAGMA foreign_keys=ON")
|
|
||||||
conn.execute("PRAGMA busy_timeout=30000")
|
|
||||||
return conn
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _from_row(row: sqlite3.Row) -> MessageEnvelope:
|
|
||||||
return MessageEnvelope(
|
|
||||||
source=row["source"], target=row["target"], message_type=row["message_type"],
|
|
||||||
payload=json.loads(row["payload_json"]), tenant_id=row["tenant_id"], user_id=row["user_id"],
|
|
||||||
project_id=row["project_id"], conversation_id=row["conversation_id"], task_id=row["task_id"],
|
|
||||||
run_id=row["run_id"], correlation_id=row["correlation_id"],
|
|
||||||
causation_id=row["causation_id"], sequence=int(row["sequence"]),
|
|
||||||
capability_scope=tuple(json.loads(row["capability_scope_json"])),
|
|
||||||
confidentiality_scope=row["confidentiality_scope"], idempotency_key=row["idempotency_key"],
|
|
||||||
expires_at=row["expires_at"], created_at=row["created_at"], envelope_id=row["envelope_id"],
|
|
||||||
)
|
|
||||||
|
|
||||||
def put(self, envelope: MessageEnvelope) -> tuple[str, MessageEnvelope]:
|
|
||||||
if envelope.is_expired():
|
|
||||||
raise MessageExpired(MessageExpired.code)
|
|
||||||
record = envelope.as_record()
|
|
||||||
with self._connect() as conn:
|
|
||||||
conn.execute("BEGIN IMMEDIATE")
|
|
||||||
replay = conn.execute(
|
|
||||||
"SELECT * FROM message_handoff WHERE stream_key=? AND idempotency_key=?",
|
|
||||||
(envelope.stream_key, envelope.idempotency_key),
|
|
||||||
).fetchone()
|
|
||||||
if replay is not None:
|
|
||||||
existing = self._from_row(replay)
|
|
||||||
if existing.envelope_id != envelope.envelope_id or existing.payload_sha256 != envelope.payload_sha256:
|
|
||||||
raise IdempotencyConflict(IdempotencyConflict.code)
|
|
||||||
conn.commit()
|
|
||||||
return "REPLAY", existing
|
|
||||||
seq = conn.execute(
|
|
||||||
"SELECT * FROM message_handoff WHERE stream_key=? AND sequence=?",
|
|
||||||
(envelope.stream_key, envelope.sequence),
|
|
||||||
).fetchone()
|
|
||||||
if seq is not None:
|
|
||||||
existing = self._from_row(seq)
|
|
||||||
if existing.envelope_id == envelope.envelope_id:
|
|
||||||
conn.commit()
|
|
||||||
return "REPLAY", existing
|
|
||||||
raise SequenceConflict(SequenceConflict.code)
|
|
||||||
maximum = int(conn.execute(
|
|
||||||
"SELECT COALESCE(MAX(sequence),0) FROM message_handoff WHERE stream_key=?",
|
|
||||||
(envelope.stream_key,),
|
|
||||||
).fetchone()[0])
|
|
||||||
expected = maximum + 1
|
|
||||||
if envelope.sequence != expected:
|
|
||||||
raise SequenceGap(f"{SequenceGap.code}:expected={expected}:actual={envelope.sequence}")
|
|
||||||
columns = tuple(record)
|
|
||||||
placeholders = ",".join("?" for _ in columns)
|
|
||||||
conn.execute(
|
|
||||||
f"INSERT INTO message_handoff ({','.join(columns)}) VALUES ({placeholders})",
|
|
||||||
tuple(record[column] for column in columns),
|
|
||||||
)
|
|
||||||
conn.commit()
|
|
||||||
return "CREATED", envelope
|
|
||||||
|
|
||||||
def get(self, envelope_id: str, *, actor: Any) -> MessageEnvelope:
|
|
||||||
with self._connect() as conn:
|
|
||||||
row = conn.execute("SELECT * FROM message_handoff WHERE envelope_id=?", (envelope_id,)).fetchone()
|
|
||||||
if row is None:
|
|
||||||
raise KeyError(envelope_id)
|
|
||||||
envelope = self._from_row(row)
|
|
||||||
if envelope.is_expired():
|
|
||||||
conn.execute("UPDATE message_handoff SET status='EXPIRED' WHERE envelope_id=?", (envelope_id,))
|
|
||||||
conn.commit()
|
|
||||||
raise MessageExpired(MessageExpired.code)
|
|
||||||
authorize_delivery(actor, envelope)
|
|
||||||
return envelope
|
|
||||||
|
|
||||||
def pending(self, *, target: str, actor: Any, limit: int = 100) -> list[MessageEnvelope]:
|
|
||||||
if not 1 <= limit <= 1000:
|
|
||||||
raise ValueError("MESSAGE_PENDING_LIMIT_INVALID")
|
|
||||||
visible: list[MessageEnvelope] = []
|
|
||||||
with self._connect() as conn:
|
|
||||||
rows = conn.execute(
|
|
||||||
"SELECT * FROM message_handoff WHERE target=? AND status='PENDING' ORDER BY created_at, sequence LIMIT ?",
|
|
||||||
(target, limit),
|
|
||||||
).fetchall()
|
|
||||||
for row in rows:
|
|
||||||
envelope = self._from_row(row)
|
|
||||||
if envelope.is_expired():
|
|
||||||
conn.execute("UPDATE message_handoff SET status='EXPIRED' WHERE envelope_id=?", (envelope.envelope_id,))
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
authorize_delivery(actor, envelope)
|
|
||||||
except HandoffError:
|
|
||||||
continue
|
|
||||||
visible.append(envelope)
|
|
||||||
conn.commit()
|
|
||||||
return visible
|
|
||||||
|
|
||||||
def mark_delivered(self, envelope_id: str, *, actor: Any) -> MessageEnvelope:
|
|
||||||
envelope = self.get(envelope_id, actor=actor)
|
|
||||||
with self._connect() as conn:
|
|
||||||
conn.execute(
|
|
||||||
"UPDATE message_handoff SET status='DELIVERED', delivered_at=? "
|
|
||||||
"WHERE envelope_id=? AND status='PENDING'",
|
|
||||||
(_iso(_utcnow()), envelope_id),
|
|
||||||
)
|
|
||||||
conn.commit()
|
|
||||||
return envelope
|
|
||||||
|
|
||||||
def acknowledge(self, envelope_id: str, *, actor: Any) -> MessageEnvelope:
|
|
||||||
envelope = self.get(envelope_id, actor=actor)
|
|
||||||
with self._connect() as conn:
|
|
||||||
conn.execute(
|
|
||||||
"UPDATE message_handoff SET status='ACKED', acked_at=? WHERE envelope_id=?",
|
|
||||||
(_iso(_utcnow()), envelope_id),
|
|
||||||
)
|
|
||||||
conn.commit()
|
|
||||||
return envelope
|
|
||||||
|
|
||||||
|
|
||||||
class HandoffFabric:
|
|
||||||
def __init__(self, store: DurableHandoffStore) -> None:
|
|
||||||
self.store = store
|
|
||||||
|
|
||||||
def publish(self, envelope: MessageEnvelope, *, actor: Any | None = None) -> tuple[str, MessageEnvelope]:
|
|
||||||
return self.store.put(bind_envelope_scope(envelope, actor) if actor is not None else envelope)
|
|
||||||
@@ -1,68 +0,0 @@
|
|||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from typing import Any, Mapping
|
|
||||||
|
|
||||||
from message_handoff import MessageEnvelope
|
|
||||||
|
|
||||||
|
|
||||||
def _get(value: Any, *names: str, default: Any = None) -> Any:
|
|
||||||
for name in names:
|
|
||||||
if isinstance(value, Mapping) and name in value:
|
|
||||||
return value[name]
|
|
||||||
if hasattr(value, name):
|
|
||||||
return getattr(value, name)
|
|
||||||
return default
|
|
||||||
|
|
||||||
|
|
||||||
def agent_event_to_envelope(event: Any, *, target: str | None = None, sequence: int = 1,
|
|
||||||
idempotency_key: str | None = None, **scope: Any) -> MessageEnvelope:
|
|
||||||
payload = _get(event, "payload", "data", default={}) or {}
|
|
||||||
target = target or _get(event, "target", "recipient")
|
|
||||||
if not target:
|
|
||||||
raise ValueError("MESSAGE_TARGET_REQUIRED")
|
|
||||||
return MessageEnvelope(
|
|
||||||
source=str(_get(event, "source", "producer", "agent_id", default="agent-event")),
|
|
||||||
target=str(target),
|
|
||||||
message_type=str(_get(event, "event_type", "type", "message_type", default="agent.event")),
|
|
||||||
payload=dict(payload),
|
|
||||||
sequence=sequence,
|
|
||||||
idempotency_key=idempotency_key or str(_get(event, "id", "event_id", default="")) or f"agent-event:{sequence}",
|
|
||||||
causation_id=_get(event, "causation_id"),
|
|
||||||
**scope,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def taskplane_event_to_envelope(event: Any, *, target: str, sequence: int = 1,
|
|
||||||
idempotency_key: str | None = None, **scope: Any) -> MessageEnvelope:
|
|
||||||
data = _get(event, "data", "payload", default={}) or {}
|
|
||||||
event_id = str(_get(event, "id", "event_id", default=""))
|
|
||||||
return MessageEnvelope(
|
|
||||||
source=str(_get(event, "source", default="task-plane")),
|
|
||||||
target=target,
|
|
||||||
message_type=str(_get(event, "type", "event_type", default="task.event")),
|
|
||||||
payload=dict(data),
|
|
||||||
sequence=sequence,
|
|
||||||
idempotency_key=idempotency_key or event_id or f"task-event:{sequence}",
|
|
||||||
causation_id=event_id or None,
|
|
||||||
**scope,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def legacy_message_to_envelope(message: Any, *, target: str | None = None, sequence: int = 1,
|
|
||||||
idempotency_key: str | None = None, **scope: Any) -> MessageEnvelope:
|
|
||||||
payload = _get(message, "payload", "data", "body", "content", default={})
|
|
||||||
if not isinstance(payload, Mapping):
|
|
||||||
payload = {"value": payload}
|
|
||||||
target = target or _get(message, "target", "recipient", "receiver")
|
|
||||||
if not target:
|
|
||||||
raise ValueError("MESSAGE_TARGET_REQUIRED")
|
|
||||||
source = _get(message, "source", "sender", "agent_id", default="legacy-message-bus")
|
|
||||||
kind = _get(message, "message_type", "type", "kind", default="legacy.message")
|
|
||||||
identifier = _get(message, "id", "message_id", "event_id")
|
|
||||||
return MessageEnvelope(
|
|
||||||
source=str(source), target=str(target), message_type=str(kind), payload=dict(payload),
|
|
||||||
sequence=sequence,
|
|
||||||
idempotency_key=idempotency_key or (str(identifier) if identifier else f"legacy:{sequence}"),
|
|
||||||
causation_id=str(identifier) if identifier else None,
|
|
||||||
**scope,
|
|
||||||
)
|
|
||||||
@@ -11,7 +11,6 @@ from dataclasses import asdict, dataclass, field
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from .search_gateway_orchestrator import SearchGatewayOrchestrator
|
from .search_gateway_orchestrator import SearchGatewayOrchestrator
|
||||||
from .trace_tool_credential_governance import GovernanceActor, TraceContext
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
@@ -44,9 +43,6 @@ class SearchGatewayAgentAdapter:
|
|||||||
domain_profiles: list[str] | None = None,
|
domain_profiles: list[str] | None = None,
|
||||||
user_scope: str = "public",
|
user_scope: str = "public",
|
||||||
correlation_id: str | None = None,
|
correlation_id: str | None = None,
|
||||||
*,
|
|
||||||
actor: GovernanceActor | dict[str, Any] | None = None,
|
|
||||||
trace: TraceContext | None = None,
|
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
payload = {
|
payload = {
|
||||||
"query": query,
|
"query": query,
|
||||||
@@ -63,11 +59,7 @@ class SearchGatewayAgentAdapter:
|
|||||||
"session_profile": context.session_profile,
|
"session_profile": context.session_profile,
|
||||||
"correlation_id": correlation_id,
|
"correlation_id": correlation_id,
|
||||||
}
|
}
|
||||||
if actor is None and trace is None:
|
return self.orchestrator.execute(payload)
|
||||||
# Preserve adapter compatibility; the governed orchestrator itself
|
|
||||||
# still fails closed when trusted context is absent.
|
|
||||||
return self.orchestrator.execute(payload)
|
|
||||||
return self.orchestrator.execute(payload, actor=actor, trace=trace)
|
|
||||||
|
|
||||||
|
|
||||||
class OpenClawSearchClient(SearchGatewayAgentAdapter):
|
class OpenClawSearchClient(SearchGatewayAgentAdapter):
|
||||||
|
|||||||
@@ -13,18 +13,10 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from dataclasses import asdict, dataclass, field
|
from dataclasses import asdict, dataclass, field
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
import hashlib
|
|
||||||
import json
|
|
||||||
from typing import Any, Protocol
|
from typing import Any, Protocol
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
from .search_profile_resolver import DomainQueryProfileRegistry, SearchProfile, SearchProfileResolver
|
from .search_profile_resolver import DomainQueryProfileRegistry, SearchProfile, SearchProfileResolver
|
||||||
from .trace_tool_credential_governance import (
|
|
||||||
GovernanceActor,
|
|
||||||
GovernanceError,
|
|
||||||
GovernanceStore,
|
|
||||||
TraceContext,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def utc_now() -> str:
|
def utc_now() -> str:
|
||||||
@@ -164,25 +156,13 @@ class SearchPolicyGuard:
|
|||||||
|
|
||||||
|
|
||||||
class SearchGateway:
|
class SearchGateway:
|
||||||
def __init__(self, providers: list[SearchProvider] | None = None, *, governance_store: GovernanceStore | None = None):
|
def __init__(self, providers: list[SearchProvider] | None = None):
|
||||||
self.providers = providers or []
|
self.providers = providers or []
|
||||||
self.governance_store = governance_store
|
|
||||||
self.profile_resolver = SearchProfileResolver()
|
self.profile_resolver = SearchProfileResolver()
|
||||||
self.planner = QueryPlanner()
|
self.planner = QueryPlanner()
|
||||||
self.guard = SearchPolicyGuard()
|
self.guard = SearchPolicyGuard()
|
||||||
|
|
||||||
def search(self, request: SearchRequest, *, actor: GovernanceActor | dict[str, Any] | None = None, trace: TraceContext | None = None) -> SearchResponse:
|
def search(self, request: SearchRequest) -> SearchResponse:
|
||||||
trusted_actor: GovernanceActor | None = None
|
|
||||||
if self.governance_store is not None:
|
|
||||||
if actor is None or trace is None:
|
|
||||||
raise GovernanceError("TOOL_MCP_AUTHZ_DENY")
|
|
||||||
trusted_actor = GovernanceActor.from_value(actor)
|
|
||||||
if request.requester != trusted_actor.identity:
|
|
||||||
raise GovernanceError("TOOL_MCP_AUTHZ_DENY", field="requester")
|
|
||||||
if request.correlation_id is not None and request.correlation_id not in {
|
|
||||||
trace.trace_id, trace.correlation_id
|
|
||||||
}:
|
|
||||||
raise GovernanceError("TOOL_SCOPE_PROPAGATION", field="correlation_id")
|
|
||||||
request_id = request.correlation_id or f"search_{uuid4().hex[:12]}"
|
request_id = request.correlation_id or f"search_{uuid4().hex[:12]}"
|
||||||
profile = self.profile_resolver.resolve(request.group_profile, request.user_profile, request.session_profile)
|
profile = self.profile_resolver.resolve(request.group_profile, request.user_profile, request.session_profile)
|
||||||
expanded_queries = self.planner.expand(request, profile)
|
expanded_queries = self.planner.expand(request, profile)
|
||||||
@@ -204,60 +184,14 @@ class SearchGateway:
|
|||||||
attempts: list[SearchProviderAttempt] = []
|
attempts: list[SearchProviderAttempt] = []
|
||||||
providers = self._ordered_providers(profile)
|
providers = self._ordered_providers(profile)
|
||||||
for provider in providers:
|
for provider in providers:
|
||||||
if request.authorized_sources and provider.name not in request.authorized_sources:
|
|
||||||
attempts.append(SearchProviderAttempt(provider=provider.name, status="denied", reason="SOURCE_AUTHZ_DENY"))
|
|
||||||
continue
|
|
||||||
if request.user_scope != "public" and provider.name not in request.authorized_sources:
|
|
||||||
attempts.append(SearchProviderAttempt(provider=provider.name, status="denied", reason="SOURCE_AUTHZ_REQUIRED"))
|
|
||||||
continue
|
|
||||||
try:
|
try:
|
||||||
if self.governance_store is not None:
|
|
||||||
assert trusted_actor is not None and trace is not None
|
|
||||||
input_digest = hashlib.sha256(
|
|
||||||
json.dumps(
|
|
||||||
{"query": request.query, "intent": request.intent, "provider": provider.name},
|
|
||||||
sort_keys=True, separators=(",", ":"), ensure_ascii=False,
|
|
||||||
).encode("utf-8")
|
|
||||||
).hexdigest()
|
|
||||||
invocation_key = hashlib.sha256(
|
|
||||||
f"{trace.trace_id}:{trace.span_id}:{provider.name}:{input_digest}".encode("utf-8")
|
|
||||||
).hexdigest()
|
|
||||||
authorization = self.governance_store.authorize_tool(
|
|
||||||
actor=trusted_actor,
|
|
||||||
tool_name=f"search.{provider.name}",
|
|
||||||
trace=trace,
|
|
||||||
requested_scope=trace.scope,
|
|
||||||
input_digest=input_digest,
|
|
||||||
invocation_key=invocation_key,
|
|
||||||
)
|
|
||||||
if authorization.get("idempotent"):
|
|
||||||
raise GovernanceError("TOOL_INVOCATION_REPLAY")
|
|
||||||
provider_evidence, attempt = provider.search(request, expanded_queries)
|
provider_evidence, attempt = provider.search(request, expanded_queries)
|
||||||
if self.governance_store is not None:
|
|
||||||
result_digest = hashlib.sha256(
|
|
||||||
json.dumps(
|
|
||||||
{
|
|
||||||
"evidence": [item.to_dict() for item in provider_evidence],
|
|
||||||
"attempt": attempt.to_dict(),
|
|
||||||
},
|
|
||||||
sort_keys=True, separators=(",", ":"), ensure_ascii=False,
|
|
||||||
).encode("utf-8")
|
|
||||||
).hexdigest()
|
|
||||||
self.governance_store.record_tool_receipt(
|
|
||||||
invocation_key=invocation_key,
|
|
||||||
result_digest=result_digest,
|
|
||||||
actor=trusted_actor,
|
|
||||||
trace=trace,
|
|
||||||
)
|
|
||||||
evidence.extend(provider_evidence)
|
evidence.extend(provider_evidence)
|
||||||
attempts.append(attempt)
|
attempts.append(attempt)
|
||||||
if provider_evidence:
|
if provider_evidence:
|
||||||
break
|
break
|
||||||
except GovernanceError:
|
except Exception as exc:
|
||||||
raise
|
attempts.append(SearchProviderAttempt(provider=getattr(provider, "name", "unknown"), status="failed", reason=str(exc)))
|
||||||
except Exception:
|
|
||||||
# Provider exceptions may contain URLs, tokens, or local paths.
|
|
||||||
attempts.append(SearchProviderAttempt(provider=getattr(provider, "name", "unknown"), status="failed", reason="PROVIDER_FAILURE"))
|
|
||||||
|
|
||||||
if evidence:
|
if evidence:
|
||||||
return SearchResponse(
|
return SearchResponse(
|
||||||
|
|||||||
@@ -7,29 +7,27 @@ try:
|
|||||||
except Exception:
|
except Exception:
|
||||||
APIRouter = None
|
APIRouter = None
|
||||||
|
|
||||||
from .search_gateway import SearchRequest
|
from .search_gateway import SearchGateway, SearchRequest
|
||||||
from .search_gateway_orchestrator import SearchGatewayOrchestrator
|
|
||||||
from .trace_tool_credential_governance import GovernanceActor, TraceContext
|
|
||||||
from .search_gateway_dashboard import SearchGatewayDashboard
|
from .search_gateway_dashboard import SearchGatewayDashboard
|
||||||
|
|
||||||
|
|
||||||
orchestrator = SearchGatewayOrchestrator()
|
gateway = SearchGateway()
|
||||||
dashboard = SearchGatewayDashboard()
|
dashboard = SearchGatewayDashboard()
|
||||||
|
|
||||||
|
|
||||||
def search_gateway_request(payload: dict[str, Any], *, actor: GovernanceActor | dict[str, Any] | None = None, trace: TraceContext | None = None) -> dict[str, Any]:
|
def search_gateway_request(payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
return orchestrator.execute(payload, actor=actor, trace=trace)
|
request = SearchRequest(**payload)
|
||||||
|
return gateway.search(request).to_dict()
|
||||||
|
|
||||||
|
|
||||||
def get_search_gateway_status() -> dict[str, Any]:
|
def get_search_gateway_status() -> dict[str, Any]:
|
||||||
return {
|
return {
|
||||||
"module": "search-gateway",
|
"module": "search-gateway",
|
||||||
"status": "ready",
|
"status": "ready",
|
||||||
"providers_configured": len(orchestrator.gateway.providers),
|
"providers_configured": len(gateway.providers),
|
||||||
"policy_guard": "enabled",
|
"policy_guard": "enabled",
|
||||||
"profile_customization": "enabled",
|
"profile_customization": "enabled",
|
||||||
"fallback_required_when_provider_unavailable": True,
|
"fallback_required_when_provider_unavailable": True,
|
||||||
"governance": "required",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -3,49 +3,24 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from typing import Any
|
from typing import Any
|
||||||
import os
|
|
||||||
|
|
||||||
from .search_audit_store import SearchAuditStore
|
from .search_audit_store import SearchAuditStore
|
||||||
from .search_event_bridge import SearchEventBridge
|
from .search_event_bridge import SearchEventBridge
|
||||||
from .search_gateway import SearchGateway, SearchRequest
|
from .search_gateway import SearchGateway, SearchRequest
|
||||||
from .search_provider_registry import SearchProviderRegistry
|
from .search_provider_registry import SearchProviderRegistry
|
||||||
from .trace_tool_credential_governance import (
|
|
||||||
GovernanceActor,
|
|
||||||
GovernanceScope,
|
|
||||||
GovernanceStore,
|
|
||||||
ToolDescriptor,
|
|
||||||
TraceContext,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class SearchGatewayOrchestrator:
|
class SearchGatewayOrchestrator:
|
||||||
def __init__(self, *, governance_store: GovernanceStore | None = None):
|
def __init__(self):
|
||||||
self.registry = SearchProviderRegistry()
|
self.registry = SearchProviderRegistry()
|
||||||
self.audit = SearchAuditStore()
|
self.audit = SearchAuditStore()
|
||||||
self.events = SearchEventBridge()
|
self.events = SearchEventBridge()
|
||||||
self.governance = governance_store or GovernanceStore(
|
self.gateway = SearchGateway(self.registry.ordered())
|
||||||
os.environ.get("AIOS_SEARCH_GOVERNANCE_DB", "runtime/search-governance.sqlite3")
|
|
||||||
)
|
|
||||||
registrar = GovernanceActor(
|
|
||||||
"search-runtime-bootstrap", GovernanceScope("system"), ("system.admin",)
|
|
||||||
)
|
|
||||||
for provider in self.registry.ordered():
|
|
||||||
self.governance.register_tool(
|
|
||||||
ToolDescriptor(
|
|
||||||
f"search.{provider.name}", "tool", GovernanceScope("global"), ("tool.search",)
|
|
||||||
),
|
|
||||||
registrar,
|
|
||||||
)
|
|
||||||
self.gateway = SearchGateway(self.registry.ordered(), governance_store=self.governance)
|
|
||||||
|
|
||||||
def execute(self, payload: dict[str, Any], *, actor: GovernanceActor | dict[str, Any] | None = None, trace: TraceContext | None = None) -> dict[str, Any]:
|
def execute(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||||
if actor is None or trace is None:
|
|
||||||
from .trace_tool_credential_governance import GovernanceError
|
|
||||||
raise GovernanceError("TOOL_MCP_AUTHZ_DENY")
|
|
||||||
self.governance.record_trace(trace, actor)
|
|
||||||
request = SearchRequest(**payload)
|
request = SearchRequest(**payload)
|
||||||
self.events.search_started(request.to_dict())
|
self.events.search_started(request.to_dict())
|
||||||
response = self.gateway.search(request, actor=actor, trace=trace)
|
response = self.gateway.search(request)
|
||||||
result = response.to_dict()
|
result = response.to_dict()
|
||||||
for attempt in result.get("attempts", []):
|
for attempt in result.get("attempts", []):
|
||||||
self.events.provider_attempted(result["request_id"], attempt)
|
self.events.provider_attempted(result["request_id"], attempt)
|
||||||
@@ -60,7 +35,6 @@ class SearchGatewayOrchestrator:
|
|||||||
"registry": self.registry.status(),
|
"registry": self.registry.status(),
|
||||||
"audit": "enabled",
|
"audit": "enabled",
|
||||||
"events": "enabled",
|
"events": "enabled",
|
||||||
"governance": "required",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -34,7 +34,6 @@ from .models import (
|
|||||||
)
|
)
|
||||||
from .policy import PolicyEngine
|
from .policy import PolicyEngine
|
||||||
from .store import TaskPlaneStore
|
from .store import TaskPlaneStore
|
||||||
from .scope import bind_task_scope
|
|
||||||
from .worker_registry import WorkerRegistry
|
from .worker_registry import WorkerRegistry
|
||||||
|
|
||||||
|
|
||||||
@@ -69,54 +68,28 @@ class TaskPlaneService:
|
|||||||
self.delegation = DelegationEngine(store)
|
self.delegation = DelegationEngine(store)
|
||||||
self.store.probe_attestation_verifier = self.policy.probe_attestation_verifier
|
self.store.probe_attestation_verifier = self.policy.probe_attestation_verifier
|
||||||
|
|
||||||
def _authorized_task(self, task_id: str, actor: str, action: str) -> TaskRecord:
|
def create_task(self, body: dict[str, Any], idempotency_key: str | None = None) -> dict[str, Any]:
|
||||||
task = self.store.get_task(task_id, self.policy.actor_context(actor))
|
|
||||||
self.policy.assert_allowed(action, actor, task=task)
|
|
||||||
return task
|
|
||||||
|
|
||||||
def create_task(
|
|
||||||
self,
|
|
||||||
body: dict[str, Any],
|
|
||||||
idempotency_key: str | None = None,
|
|
||||||
*,
|
|
||||||
actor_identity: str | None = None,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
if body.get("parent_task_id"):
|
if body.get("parent_task_id"):
|
||||||
raise ForbiddenError("child tasks must use the delegate endpoint")
|
raise ForbiddenError("child tasks must use the delegate endpoint")
|
||||||
issuer = body.get("issuer", {})
|
issuer = body.get("issuer", {})
|
||||||
declared_actor = issuer.get("id", "anon") if isinstance(issuer, dict) else "anon"
|
actor = issuer.get("id", "anon") if isinstance(issuer, dict) else "anon"
|
||||||
actor = actor_identity or declared_actor
|
|
||||||
self.policy.assert_allowed("create", actor)
|
self.policy.assert_allowed("create", actor)
|
||||||
try:
|
try:
|
||||||
plan = TaskPlan.from_dict(body)
|
plan = TaskPlan.from_dict(body)
|
||||||
except (TypeError, ValueError) as exc:
|
except (TypeError, ValueError) as exc:
|
||||||
raise ValidationError("invalid task plan") from exc
|
raise ValidationError("invalid task plan") from exc
|
||||||
if actor_identity is not None:
|
|
||||||
plan = bind_task_scope(
|
|
||||||
plan, self.policy.actor_context(actor_identity), public_create=True
|
|
||||||
)
|
|
||||||
task = self.store.create_task_with_event(plan, idempotency_key=idempotency_key)
|
task = self.store.create_task_with_event(plan, idempotency_key=idempotency_key)
|
||||||
return task.to_dict()
|
return task.to_dict()
|
||||||
|
|
||||||
def list_tasks(self, actor: str | None = None) -> list[dict[str, Any]]:
|
def list_tasks(self, actor: str | None = None) -> list[dict[str, Any]]:
|
||||||
context = None
|
|
||||||
if actor is not None:
|
if actor is not None:
|
||||||
self.policy.assert_allowed("read", actor)
|
self.policy.assert_allowed("read", actor)
|
||||||
context = self.policy.actor_context(actor)
|
return [task.to_dict(include_task=False) for task in self.store.list_tasks()]
|
||||||
return [
|
|
||||||
task.to_dict(include_task=False)
|
|
||||||
for task in self.store.list_tasks(context)
|
|
||||||
]
|
|
||||||
|
|
||||||
def get_task(self, task_id: str, actor: str | None = None) -> dict[str, Any]:
|
def get_task(self, task_id: str, actor: str | None = None) -> dict[str, Any]:
|
||||||
context = None
|
|
||||||
if actor is not None:
|
if actor is not None:
|
||||||
self.policy.assert_allowed("read", actor)
|
self.policy.assert_allowed("read", actor, task=self.store.get_task(task_id))
|
||||||
context = self.policy.actor_context(actor)
|
return self.store.get_task(task_id).to_dict()
|
||||||
task = self.store.get_task(task_id, context)
|
|
||||||
if actor is not None:
|
|
||||||
self.policy.assert_allowed("read", actor, task=task)
|
|
||||||
return task.to_dict()
|
|
||||||
|
|
||||||
def get_worker(self, worker_id: str, actor: str | None = None) -> dict[str, Any]:
|
def get_worker(self, worker_id: str, actor: str | None = None) -> dict[str, Any]:
|
||||||
if actor is not None:
|
if actor is not None:
|
||||||
@@ -190,8 +163,9 @@ class TaskPlaneService:
|
|||||||
if not worker_id:
|
if not worker_id:
|
||||||
raise ValidationError("worker_id required")
|
raise ValidationError("worker_id required")
|
||||||
|
|
||||||
task = self._authorized_task(task_id, actor, "claim")
|
task = self.store.get_task(task_id)
|
||||||
self.policy.assert_readonly_worker_binding(actor, str(worker_id))
|
self.policy.assert_readonly_worker_binding(actor, str(worker_id))
|
||||||
|
self.policy.assert_allowed("claim", actor, task=task)
|
||||||
lease_seconds = _bounded_int(
|
lease_seconds = _bounded_int(
|
||||||
body.get("lease_seconds", 30), "lease_seconds", 1, 3600
|
body.get("lease_seconds", 30), "lease_seconds", 1, 3600
|
||||||
)
|
)
|
||||||
@@ -210,9 +184,10 @@ class TaskPlaneService:
|
|||||||
if not claim_id:
|
if not claim_id:
|
||||||
raise ValidationError("claim_id required")
|
raise ValidationError("claim_id required")
|
||||||
actor = body.get("actor_identity") or body.get("worker_identity") or "worker"
|
actor = body.get("actor_identity") or body.get("worker_identity") or "worker"
|
||||||
task = self._authorized_task(task_id, actor, "heartbeat")
|
task = self.store.get_task(task_id)
|
||||||
claim = self.store.get_claim(claim_id)
|
claim = self.store.get_claim(claim_id)
|
||||||
self.policy.assert_readonly_worker_binding(actor, claim.worker_id)
|
self.policy.assert_readonly_worker_binding(actor, claim.worker_id)
|
||||||
|
self.policy.assert_allowed("heartbeat", actor, task=task)
|
||||||
if claim.worker_identity != actor:
|
if claim.worker_identity != actor:
|
||||||
raise ForbiddenError("claim identity mismatch")
|
raise ForbiddenError("claim identity mismatch")
|
||||||
lease_seconds = _bounded_int(
|
lease_seconds = _bounded_int(
|
||||||
@@ -231,9 +206,10 @@ class TaskPlaneService:
|
|||||||
message = body.get("message")
|
message = body.get("message")
|
||||||
if progress is None:
|
if progress is None:
|
||||||
raise ValidationError("progress required")
|
raise ValidationError("progress required")
|
||||||
task = self._authorized_task(task_id, actor, "progress")
|
task = self.store.get_task(task_id)
|
||||||
claim = self.store.get_claim(claim_id)
|
claim = self.store.get_claim(claim_id)
|
||||||
self.policy.assert_readonly_worker_binding(actor, claim.worker_id)
|
self.policy.assert_readonly_worker_binding(actor, claim.worker_id)
|
||||||
|
self.policy.assert_allowed("progress", actor, task=task)
|
||||||
if task.task_plan.data_classification == "credential-secret":
|
if task.task_plan.data_classification == "credential-secret":
|
||||||
if message not in {None, ""} or contains_secret_text(stage):
|
if message not in {None, ""} or contains_secret_text(stage):
|
||||||
raise ValidationError(
|
raise ValidationError(
|
||||||
@@ -258,7 +234,7 @@ class TaskPlaneService:
|
|||||||
if not claim_id:
|
if not claim_id:
|
||||||
raise ValidationError("claim_id required")
|
raise ValidationError("claim_id required")
|
||||||
actor = body.get("actor_identity") or body.get("worker_identity") or "worker"
|
actor = body.get("actor_identity") or body.get("worker_identity") or "worker"
|
||||||
task = self._authorized_task(task_id, actor, "result")
|
task = self.store.get_task(task_id)
|
||||||
|
|
||||||
status = body.get("status", "COMPLETED")
|
status = body.get("status", "COMPLETED")
|
||||||
outputs = body.get("outputs", [])
|
outputs = body.get("outputs", [])
|
||||||
@@ -271,6 +247,7 @@ class TaskPlaneService:
|
|||||||
raise ValidationError("evidence items must be objects")
|
raise ValidationError("evidence items must be objects")
|
||||||
claim = self.store.get_claim(claim_id)
|
claim = self.store.get_claim(claim_id)
|
||||||
self.policy.assert_readonly_worker_binding(actor, claim.worker_id)
|
self.policy.assert_readonly_worker_binding(actor, claim.worker_id)
|
||||||
|
self.policy.assert_allowed("result", actor, task=task)
|
||||||
if claim.worker_identity != actor:
|
if claim.worker_identity != actor:
|
||||||
raise ForbiddenError("claim identity mismatch")
|
raise ForbiddenError("claim identity mismatch")
|
||||||
if task.task_plan.data_classification == "credential-secret":
|
if task.task_plan.data_classification == "credential-secret":
|
||||||
@@ -349,7 +326,8 @@ class TaskPlaneService:
|
|||||||
reason = body.get("reason", "failed")
|
reason = body.get("reason", "failed")
|
||||||
actor = body.get("actor_identity") or body.get("worker_identity") or "worker"
|
actor = body.get("actor_identity") or body.get("worker_identity") or "worker"
|
||||||
self.policy.assert_allowed("fail", actor)
|
self.policy.assert_allowed("fail", actor)
|
||||||
task = self._authorized_task(task_id, actor, "fail")
|
task = self.store.get_task(task_id)
|
||||||
|
self.policy.assert_allowed("fail", actor, task=task)
|
||||||
claim = self.store.get_claim(claim_id)
|
claim = self.store.get_claim(claim_id)
|
||||||
if claim.worker_identity != actor:
|
if claim.worker_identity != actor:
|
||||||
raise ForbiddenError("claim identity mismatch")
|
raise ForbiddenError("claim identity mismatch")
|
||||||
@@ -361,7 +339,8 @@ class TaskPlaneService:
|
|||||||
actor = body.get("actor_identity") or body.get("worker_identity") or "owner"
|
actor = body.get("actor_identity") or body.get("worker_identity") or "owner"
|
||||||
self.policy.assert_allowed("cancel", actor)
|
self.policy.assert_allowed("cancel", actor)
|
||||||
reason = body.get("reason", "cancelled")
|
reason = body.get("reason", "cancelled")
|
||||||
task = self._authorized_task(task_id, actor, "cancel")
|
task = self.store.get_task(task_id)
|
||||||
|
self.policy.assert_allowed("cancel", actor, task=task)
|
||||||
updated = self.store.cancel_task_with_event(
|
updated = self.store.cancel_task_with_event(
|
||||||
task_id, task.revision, reason=reason
|
task_id, task.revision, reason=reason
|
||||||
)
|
)
|
||||||
@@ -370,7 +349,8 @@ class TaskPlaneService:
|
|||||||
def delegate(self, task_id: str, body: dict[str, Any]) -> dict[str, Any]:
|
def delegate(self, task_id: str, body: dict[str, Any]) -> dict[str, Any]:
|
||||||
actor = body.get("actor_identity") or "owner"
|
actor = body.get("actor_identity") or "owner"
|
||||||
self.policy.assert_allowed("delegate", actor)
|
self.policy.assert_allowed("delegate", actor)
|
||||||
parent = self._authorized_task(task_id, actor, "delegate")
|
parent = self.store.get_task(task_id)
|
||||||
|
self.policy.assert_allowed("delegate", actor, task=parent)
|
||||||
if "task" in body:
|
if "task" in body:
|
||||||
try:
|
try:
|
||||||
child_plan = TaskPlan.from_dict(body["task"])
|
child_plan = TaskPlan.from_dict(body["task"])
|
||||||
@@ -384,31 +364,25 @@ class TaskPlaneService:
|
|||||||
except (TypeError, ValueError) as exc:
|
except (TypeError, ValueError) as exc:
|
||||||
raise ValidationError("invalid child task plan") from exc
|
raise ValidationError("invalid child task plan") from exc
|
||||||
child_plan.parent_task_id = task_id
|
child_plan.parent_task_id = task_id
|
||||||
child_plan = bind_task_scope(
|
|
||||||
child_plan, self.policy.actor_context(actor), public_create=False
|
|
||||||
)
|
|
||||||
child = self.delegation.delegate(task_id, child_plan, caller=actor)
|
child = self.delegation.delegate(task_id, child_plan, caller=actor)
|
||||||
return child.to_dict()
|
return child.to_dict()
|
||||||
|
|
||||||
def get_events(self, task_id: str, actor: str | None = None) -> list[dict[str, Any]]:
|
def get_events(self, task_id: str, actor: str | None = None) -> list[dict[str, Any]]:
|
||||||
if actor is not None:
|
if actor is not None:
|
||||||
self._authorized_task(task_id, actor, "audit")
|
self.policy.assert_allowed("audit", actor, task=self.store.get_task(task_id))
|
||||||
else:
|
self.store.get_task(task_id)
|
||||||
self.store.get_task(task_id)
|
|
||||||
return self.store.list_events(task_id)
|
return self.store.list_events(task_id)
|
||||||
|
|
||||||
def get_evidence(self, task_id: str, actor: str | None = None) -> list[dict[str, Any]]:
|
def get_evidence(self, task_id: str, actor: str | None = None) -> list[dict[str, Any]]:
|
||||||
if actor is not None:
|
if actor is not None:
|
||||||
self._authorized_task(task_id, actor, "audit")
|
self.policy.assert_allowed("audit", actor, task=self.store.get_task(task_id))
|
||||||
else:
|
self.store.get_task(task_id)
|
||||||
self.store.get_task(task_id)
|
|
||||||
return self.evidence.list(task_id)
|
return self.evidence.list(task_id)
|
||||||
|
|
||||||
def get_children(self, task_id: str, actor: str | None = None) -> list[str]:
|
def get_children(self, task_id: str, actor: str | None = None) -> list[str]:
|
||||||
if actor is not None:
|
if actor is not None:
|
||||||
self._authorized_task(task_id, actor, "read")
|
self.policy.assert_allowed("read", actor, task=self.store.get_task(task_id))
|
||||||
else:
|
self.store.get_task(task_id)
|
||||||
self.store.get_task(task_id)
|
|
||||||
return self.store.list_children(task_id)
|
return self.store.list_children(task_id)
|
||||||
|
|
||||||
|
|
||||||
@@ -532,11 +506,7 @@ class TaskPlaneRequestHandler(BaseHTTPRequestHandler):
|
|||||||
):
|
):
|
||||||
raise ForbiddenError("issuer identity mismatch")
|
raise ForbiddenError("issuer identity mismatch")
|
||||||
body["issuer"] = {"type": "authenticated", "id": identity}
|
body["issuer"] = {"type": "authenticated", "id": identity}
|
||||||
return 201, self.service.create_task(
|
return 201, self.service.create_task(body or {}, headers.get("idempotency-key"))
|
||||||
body or {},
|
|
||||||
headers.get("idempotency-key"),
|
|
||||||
actor_identity=identity,
|
|
||||||
)
|
|
||||||
if body.get("actor_identity") not in {None, identity}:
|
if body.get("actor_identity") not in {None, identity}:
|
||||||
raise ForbiddenError("actor identity mismatch")
|
raise ForbiddenError("actor identity mismatch")
|
||||||
body["actor_identity"] = identity
|
body["actor_identity"] = identity
|
||||||
|
|||||||
@@ -106,8 +106,3 @@ class DurableOutboxDispatcher:
|
|||||||
receipt=str(receipt) if receipt else None,
|
receipt=str(receipt) if receipt else None,
|
||||||
error=None if receipt else f"{adapter.name} receipt unavailable",
|
error=None if receipt else f"{adapter.name} receipt unavailable",
|
||||||
)
|
)
|
||||||
|
|
||||||
# R4.2 P0-B compatibility adapter.
|
|
||||||
def to_handoff_envelope(event, *, target, sequence=1, idempotency_key=None, **scope):
|
|
||||||
from message_handoff_adapters import taskplane_event_to_envelope
|
|
||||||
return taskplane_event_to_envelope(event, target=target, sequence=sequence, idempotency_key=idempotency_key, **scope)
|
|
||||||
|
|||||||
@@ -1,911 +0,0 @@
|
|||||||
"""Durable multi-agent fleet scheduling for TaskPlane.
|
|
||||||
|
|
||||||
The fleet layer decides *which* eligible worker should receive *which* queued task.
|
|
||||||
It never replaces TaskPlane claim/lease ownership: callers must still perform the
|
|
||||||
authoritative TaskPlane claim CAS after a reservation is returned.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from contextlib import contextmanager
|
|
||||||
from dataclasses import dataclass
|
|
||||||
import hashlib
|
|
||||||
import json
|
|
||||||
import math
|
|
||||||
import sqlite3
|
|
||||||
import threading
|
|
||||||
import time
|
|
||||||
from typing import Any, Iterable, Mapping, Sequence
|
|
||||||
|
|
||||||
DEFAULT_RESERVATION_SECONDS = 30
|
|
||||||
DEFAULT_CIRCUIT_THRESHOLD = 3
|
|
||||||
DEFAULT_CIRCUIT_SECONDS = 30
|
|
||||||
MAX_CIRCUIT_SECONDS = 15 * 60
|
|
||||||
MAX_SLOTS = 128
|
|
||||||
|
|
||||||
|
|
||||||
class FleetError(RuntimeError):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class FleetConflict(FleetError):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def _canonical(value: object) -> bytes:
|
|
||||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
|
||||||
|
|
||||||
|
|
||||||
def _sha(value: object) -> str:
|
|
||||||
return hashlib.sha256(_canonical(value)).hexdigest()
|
|
||||||
|
|
||||||
|
|
||||||
def _text(value: object, default: str = "") -> str:
|
|
||||||
if value is None:
|
|
||||||
return default
|
|
||||||
return str(value).strip() or default
|
|
||||||
|
|
||||||
|
|
||||||
def _int(value: object, default: int, minimum: int, maximum: int) -> int:
|
|
||||||
if value is None:
|
|
||||||
return default
|
|
||||||
if isinstance(value, bool):
|
|
||||||
raise FleetError("FLEET_INTEGER_INVALID")
|
|
||||||
try:
|
|
||||||
result = int(value)
|
|
||||||
except (TypeError, ValueError) as exc:
|
|
||||||
raise FleetError("FLEET_INTEGER_INVALID") from exc
|
|
||||||
if result < minimum or result > maximum:
|
|
||||||
raise FleetError("FLEET_INTEGER_OUT_OF_RANGE")
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def _float(value: object, default: float, minimum: float, maximum: float) -> float:
|
|
||||||
if value is None:
|
|
||||||
return default
|
|
||||||
try:
|
|
||||||
result = float(value)
|
|
||||||
except (TypeError, ValueError) as exc:
|
|
||||||
raise FleetError("FLEET_NUMBER_INVALID") from exc
|
|
||||||
if not math.isfinite(result) or result < minimum or result > maximum:
|
|
||||||
raise FleetError("FLEET_NUMBER_OUT_OF_RANGE")
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def _mapping(value: object) -> Mapping[str, Any]:
|
|
||||||
if isinstance(value, Mapping):
|
|
||||||
return value
|
|
||||||
if hasattr(value, "to_dict"):
|
|
||||||
data = value.to_dict()
|
|
||||||
if isinstance(data, Mapping):
|
|
||||||
return data
|
|
||||||
if hasattr(value, "__dict__"):
|
|
||||||
return vars(value)
|
|
||||||
return {}
|
|
||||||
|
|
||||||
|
|
||||||
def _attr(value: object, name: str, default: Any = None) -> Any:
|
|
||||||
if isinstance(value, Mapping):
|
|
||||||
return value.get(name, default)
|
|
||||||
return getattr(value, name, default)
|
|
||||||
|
|
||||||
|
|
||||||
def _task_plan(task: object) -> object:
|
|
||||||
return _attr(task, "task_plan", _attr(task, "plan", task))
|
|
||||||
|
|
||||||
|
|
||||||
def _task_id(task: object) -> str:
|
|
||||||
return _text(_attr(task, "task_id", _attr(_task_plan(task), "task_id", "")))
|
|
||||||
|
|
||||||
|
|
||||||
def _task_status(task: object) -> str:
|
|
||||||
return _text(_attr(task, "status", "")).upper()
|
|
||||||
|
|
||||||
|
|
||||||
def _task_revision(task: object) -> int:
|
|
||||||
return _int(_attr(task, "revision", 0), 0, 0, 2**31 - 1)
|
|
||||||
|
|
||||||
|
|
||||||
def _task_created_at(task: object) -> float:
|
|
||||||
value = _attr(task, "created_at", _attr(task, "updated_at", 0.0))
|
|
||||||
try:
|
|
||||||
return float(value or 0.0)
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
return 0.0
|
|
||||||
|
|
||||||
|
|
||||||
def _task_required_capabilities(task: object) -> frozenset[str]:
|
|
||||||
plan = _task_plan(task)
|
|
||||||
value = _attr(plan, "required_capabilities", ())
|
|
||||||
if not isinstance(value, Sequence) or isinstance(value, (str, bytes)):
|
|
||||||
return frozenset()
|
|
||||||
return frozenset(_text(item) for item in value if _text(item))
|
|
||||||
|
|
||||||
|
|
||||||
def _task_metadata(task: object) -> Mapping[str, Any]:
|
|
||||||
plan = _task_plan(task)
|
|
||||||
value = _attr(plan, "metadata", {})
|
|
||||||
return value if isinstance(value, Mapping) else {}
|
|
||||||
|
|
||||||
|
|
||||||
def _task_scope_key(task: object) -> str:
|
|
||||||
plan = _task_plan(task)
|
|
||||||
scope_type = _text(_attr(plan, "scope_type", "legacy"), "legacy")
|
|
||||||
tenant = _text(_attr(plan, "tenant_id", ""))
|
|
||||||
project = _text(_attr(plan, "project_id", ""))
|
|
||||||
user = _text(_attr(plan, "user_id", ""))
|
|
||||||
if scope_type == "tenant":
|
|
||||||
return "tenant:" + tenant + "|project:" + (project or "*")
|
|
||||||
if scope_type == "system":
|
|
||||||
return "system"
|
|
||||||
return "legacy:" + (user or "*")
|
|
||||||
|
|
||||||
|
|
||||||
def _worker_id(worker: object) -> str:
|
|
||||||
return _text(_attr(worker, "worker_id", ""))
|
|
||||||
|
|
||||||
|
|
||||||
def _worker_capabilities(worker: object) -> frozenset[str]:
|
|
||||||
value = _attr(worker, "capabilities", ())
|
|
||||||
if isinstance(value, Mapping):
|
|
||||||
value = value.keys()
|
|
||||||
if not isinstance(value, Iterable) or isinstance(value, (str, bytes)):
|
|
||||||
return frozenset()
|
|
||||||
return frozenset(_text(item) for item in value if _text(item))
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class FleetWorkerPolicy:
|
|
||||||
worker_id: str
|
|
||||||
max_slots: int = 1
|
|
||||||
failure_domain: str = "default"
|
|
||||||
weight: float = 1.0
|
|
||||||
labels: Mapping[str, str] | None = None
|
|
||||||
revision: int = 1
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class FleetDispatchDecision:
|
|
||||||
reservation_id: str
|
|
||||||
task_id: str
|
|
||||||
worker_id: str
|
|
||||||
task_revision: int
|
|
||||||
scope_key: str
|
|
||||||
failure_domain: str
|
|
||||||
reserved_until: float
|
|
||||||
score: tuple[float, float, float, str]
|
|
||||||
|
|
||||||
def to_dict(self) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"reservation_id": self.reservation_id,
|
|
||||||
"task_id": self.task_id,
|
|
||||||
"worker_id": self.worker_id,
|
|
||||||
"task_revision": self.task_revision,
|
|
||||||
"scope_key": self.scope_key,
|
|
||||||
"failure_domain": self.failure_domain,
|
|
||||||
"reserved_until": self.reserved_until,
|
|
||||||
"score": list(self.score),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class FleetStateStore:
|
|
||||||
"""Small durable fleet journal colocated in the existing TaskPlane SQLite database."""
|
|
||||||
|
|
||||||
def __init__(self, connection: sqlite3.Connection, lock: threading.RLock | None = None) -> None:
|
|
||||||
self.conn = connection
|
|
||||||
self.lock = lock or threading.RLock()
|
|
||||||
self._savepoint_counter = 0
|
|
||||||
self._ensure_schema()
|
|
||||||
|
|
||||||
@contextmanager
|
|
||||||
def _transaction(self):
|
|
||||||
with self.lock:
|
|
||||||
self._savepoint_counter += 1
|
|
||||||
name = f"fleet_{threading.get_ident()}_{self._savepoint_counter}"
|
|
||||||
self.conn.execute(f"SAVEPOINT {name}")
|
|
||||||
try:
|
|
||||||
yield
|
|
||||||
except Exception:
|
|
||||||
self.conn.execute(f"ROLLBACK TO {name}")
|
|
||||||
self.conn.execute(f"RELEASE {name}")
|
|
||||||
raise
|
|
||||||
else:
|
|
||||||
self.conn.execute(f"RELEASE {name}")
|
|
||||||
|
|
||||||
def _ensure_schema(self) -> None:
|
|
||||||
with self.lock:
|
|
||||||
self.conn.executescript(
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS fleet_worker_capacity (
|
|
||||||
worker_id TEXT PRIMARY KEY,
|
|
||||||
max_slots INTEGER NOT NULL,
|
|
||||||
failure_domain TEXT NOT NULL,
|
|
||||||
weight REAL NOT NULL,
|
|
||||||
labels_json TEXT NOT NULL,
|
|
||||||
revision INTEGER NOT NULL,
|
|
||||||
updated_at REAL NOT NULL
|
|
||||||
);
|
|
||||||
CREATE TABLE IF NOT EXISTS fleet_scope_state (
|
|
||||||
scope_key TEXT PRIMARY KEY,
|
|
||||||
virtual_finish REAL NOT NULL,
|
|
||||||
dispatch_count INTEGER NOT NULL,
|
|
||||||
last_dispatched_at REAL,
|
|
||||||
revision INTEGER NOT NULL
|
|
||||||
);
|
|
||||||
CREATE TABLE IF NOT EXISTS fleet_dispatches (
|
|
||||||
reservation_id TEXT PRIMARY KEY,
|
|
||||||
task_id TEXT NOT NULL UNIQUE,
|
|
||||||
worker_id TEXT NOT NULL,
|
|
||||||
task_revision INTEGER NOT NULL,
|
|
||||||
scope_key TEXT NOT NULL,
|
|
||||||
failure_domain TEXT NOT NULL,
|
|
||||||
state TEXT NOT NULL,
|
|
||||||
reserved_until REAL NOT NULL,
|
|
||||||
claim_id TEXT,
|
|
||||||
created_at REAL NOT NULL,
|
|
||||||
updated_at REAL NOT NULL
|
|
||||||
);
|
|
||||||
CREATE INDEX IF NOT EXISTS ix_fleet_dispatch_worker_state
|
|
||||||
ON fleet_dispatches(worker_id, state, reserved_until);
|
|
||||||
CREATE TABLE IF NOT EXISTS fleet_worker_failures (
|
|
||||||
worker_id TEXT PRIMARY KEY,
|
|
||||||
consecutive_failures INTEGER NOT NULL,
|
|
||||||
circuit_open_until REAL,
|
|
||||||
last_failure_class TEXT,
|
|
||||||
revision INTEGER NOT NULL,
|
|
||||||
updated_at REAL NOT NULL
|
|
||||||
);
|
|
||||||
CREATE TABLE IF NOT EXISTS fleet_task_worker_failures (
|
|
||||||
task_id TEXT NOT NULL,
|
|
||||||
worker_id TEXT NOT NULL,
|
|
||||||
failure_count INTEGER NOT NULL,
|
|
||||||
blocked_until REAL,
|
|
||||||
last_failure_class TEXT,
|
|
||||||
revision INTEGER NOT NULL,
|
|
||||||
updated_at REAL NOT NULL,
|
|
||||||
PRIMARY KEY(task_id, worker_id)
|
|
||||||
);
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
|
|
||||||
def configure_worker(
|
|
||||||
self,
|
|
||||||
worker_id: str,
|
|
||||||
*,
|
|
||||||
max_slots: int = 1,
|
|
||||||
failure_domain: str = "default",
|
|
||||||
weight: float = 1.0,
|
|
||||||
labels: Mapping[str, str] | None = None,
|
|
||||||
expected_revision: int | None = None,
|
|
||||||
now: float | None = None,
|
|
||||||
) -> FleetWorkerPolicy:
|
|
||||||
worker_id = _text(worker_id)
|
|
||||||
if not worker_id:
|
|
||||||
raise FleetError("FLEET_WORKER_ID_REQUIRED")
|
|
||||||
max_slots = _int(max_slots, 1, 1, MAX_SLOTS)
|
|
||||||
weight = _float(weight, 1.0, 0.05, 100.0)
|
|
||||||
failure_domain = _text(failure_domain, "default")
|
|
||||||
normalized_labels = {
|
|
||||||
_text(k): _text(v)
|
|
||||||
for k, v in dict(labels or {}).items()
|
|
||||||
if _text(k) and _text(v)
|
|
||||||
}
|
|
||||||
now = float(time.time() if now is None else now)
|
|
||||||
with self._transaction():
|
|
||||||
row = self.conn.execute(
|
|
||||||
"SELECT revision FROM fleet_worker_capacity WHERE worker_id=?",
|
|
||||||
(worker_id,),
|
|
||||||
).fetchone()
|
|
||||||
current = int(row[0]) if row else 0
|
|
||||||
if expected_revision is not None and current != int(expected_revision):
|
|
||||||
raise FleetConflict("FLEET_WORKER_POLICY_CAS_MISMATCH")
|
|
||||||
revision = current + 1
|
|
||||||
self.conn.execute(
|
|
||||||
"""
|
|
||||||
INSERT INTO fleet_worker_capacity(
|
|
||||||
worker_id,max_slots,failure_domain,weight,labels_json,revision,updated_at
|
|
||||||
) VALUES (?,?,?,?,?,?,?)
|
|
||||||
ON CONFLICT(worker_id) DO UPDATE SET
|
|
||||||
max_slots=excluded.max_slots,
|
|
||||||
failure_domain=excluded.failure_domain,
|
|
||||||
weight=excluded.weight,
|
|
||||||
labels_json=excluded.labels_json,
|
|
||||||
revision=excluded.revision,
|
|
||||||
updated_at=excluded.updated_at
|
|
||||||
""",
|
|
||||||
(
|
|
||||||
worker_id,
|
|
||||||
max_slots,
|
|
||||||
failure_domain,
|
|
||||||
weight,
|
|
||||||
json.dumps(normalized_labels, sort_keys=True, separators=(",", ":")),
|
|
||||||
revision,
|
|
||||||
now,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
return FleetWorkerPolicy(
|
|
||||||
worker_id, max_slots, failure_domain, weight, normalized_labels, revision
|
|
||||||
)
|
|
||||||
|
|
||||||
def worker_policy(self, worker_id: str) -> FleetWorkerPolicy:
|
|
||||||
worker_id = _text(worker_id)
|
|
||||||
with self.lock:
|
|
||||||
row = self.conn.execute(
|
|
||||||
"""
|
|
||||||
SELECT worker_id,max_slots,failure_domain,weight,labels_json,revision
|
|
||||||
FROM fleet_worker_capacity WHERE worker_id=?
|
|
||||||
""",
|
|
||||||
(worker_id,),
|
|
||||||
).fetchone()
|
|
||||||
if row is None:
|
|
||||||
return FleetWorkerPolicy(worker_id=worker_id)
|
|
||||||
return FleetWorkerPolicy(
|
|
||||||
worker_id=str(row[0]),
|
|
||||||
max_slots=int(row[1]),
|
|
||||||
failure_domain=str(row[2]),
|
|
||||||
weight=float(row[3]),
|
|
||||||
labels=json.loads(row[4] or "{}"),
|
|
||||||
revision=int(row[5]),
|
|
||||||
)
|
|
||||||
|
|
||||||
def _cleanup_expired(self, now: float) -> None:
|
|
||||||
self.conn.execute(
|
|
||||||
"""
|
|
||||||
UPDATE fleet_dispatches
|
|
||||||
SET state='EXPIRED', updated_at=?
|
|
||||||
WHERE state='RESERVED' AND reserved_until<=?
|
|
||||||
""",
|
|
||||||
(now, now),
|
|
||||||
)
|
|
||||||
|
|
||||||
def reserved_slots(self, worker_id: str, now: float) -> int:
|
|
||||||
with self.lock:
|
|
||||||
self._cleanup_expired(now)
|
|
||||||
row = self.conn.execute(
|
|
||||||
"""
|
|
||||||
SELECT COUNT(*) FROM fleet_dispatches
|
|
||||||
WHERE worker_id=? AND state='RESERVED' AND reserved_until>?
|
|
||||||
""",
|
|
||||||
(worker_id, now),
|
|
||||||
).fetchone()
|
|
||||||
return int(row[0] if row else 0)
|
|
||||||
|
|
||||||
def scope_virtual_finish(self, scope_key: str) -> float:
|
|
||||||
with self.lock:
|
|
||||||
row = self.conn.execute(
|
|
||||||
"SELECT virtual_finish FROM fleet_scope_state WHERE scope_key=?",
|
|
||||||
(scope_key,),
|
|
||||||
).fetchone()
|
|
||||||
return float(row[0]) if row else 0.0
|
|
||||||
|
|
||||||
def reserve(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
task_id: str,
|
|
||||||
worker_id: str,
|
|
||||||
task_revision: int,
|
|
||||||
scope_key: str,
|
|
||||||
failure_domain: str,
|
|
||||||
score: tuple[float, float, float, str],
|
|
||||||
reservation_seconds: int,
|
|
||||||
now: float,
|
|
||||||
scope_weight: float,
|
|
||||||
) -> FleetDispatchDecision:
|
|
||||||
reservation_seconds = _int(
|
|
||||||
reservation_seconds, DEFAULT_RESERVATION_SECONDS, 1, 300
|
|
||||||
)
|
|
||||||
reservation_id = "fleet-" + _sha(
|
|
||||||
{
|
|
||||||
"task_id": task_id,
|
|
||||||
"worker_id": worker_id,
|
|
||||||
"task_revision": task_revision,
|
|
||||||
"scope_key": scope_key,
|
|
||||||
}
|
|
||||||
)[:32]
|
|
||||||
reserved_until = now + reservation_seconds
|
|
||||||
with self._transaction():
|
|
||||||
self._cleanup_expired(now)
|
|
||||||
existing = self.conn.execute(
|
|
||||||
"""
|
|
||||||
SELECT reservation_id,worker_id,task_revision,scope_key,failure_domain,
|
|
||||||
reserved_until,state
|
|
||||||
FROM fleet_dispatches WHERE task_id=?
|
|
||||||
""",
|
|
||||||
(task_id,),
|
|
||||||
).fetchone()
|
|
||||||
# Capacity is rechecked inside the same durable reservation transaction.
|
|
||||||
# This closes the cross-thread/process race between eligibility scoring and reserve.
|
|
||||||
policy_row = self.conn.execute(
|
|
||||||
"SELECT max_slots FROM fleet_worker_capacity WHERE worker_id=?",
|
|
||||||
(worker_id,),
|
|
||||||
).fetchone()
|
|
||||||
max_slots = int(policy_row[0]) if policy_row else 1
|
|
||||||
if existing and str(existing[6]) in {"RESERVED", "CLAIMED"} and float(existing[5]) > now:
|
|
||||||
if str(existing[1]) != worker_id or int(existing[2]) != task_revision:
|
|
||||||
raise FleetConflict("FLEET_TASK_ALREADY_RESERVED")
|
|
||||||
return FleetDispatchDecision(
|
|
||||||
str(existing[0]), task_id, worker_id, task_revision, str(existing[3]),
|
|
||||||
str(existing[4]), float(existing[5]), score
|
|
||||||
)
|
|
||||||
|
|
||||||
claim_columns = {
|
|
||||||
str(row[1])
|
|
||||||
for row in self.conn.execute("PRAGMA table_info(claims)").fetchall()
|
|
||||||
}
|
|
||||||
active_claims = 0
|
|
||||||
if {"worker_id", "status"}.issubset(claim_columns):
|
|
||||||
sql = "SELECT COUNT(*) FROM claims WHERE worker_id=? AND status='ACTIVE'"
|
|
||||||
params: list[Any] = [worker_id]
|
|
||||||
if "lease_expires_at" in claim_columns:
|
|
||||||
sql += " AND lease_expires_at>?"
|
|
||||||
params.append(now)
|
|
||||||
row = self.conn.execute(sql, tuple(params)).fetchone()
|
|
||||||
active_claims = int(row[0] if row else 0)
|
|
||||||
row = self.conn.execute(
|
|
||||||
"""
|
|
||||||
SELECT COUNT(*) FROM fleet_dispatches
|
|
||||||
WHERE worker_id=? AND task_id<>?
|
|
||||||
AND state='RESERVED' AND reserved_until>?
|
|
||||||
""",
|
|
||||||
(worker_id, task_id, now),
|
|
||||||
).fetchone()
|
|
||||||
active_reservations = int(row[0] if row else 0)
|
|
||||||
if active_claims + active_reservations >= max_slots:
|
|
||||||
raise FleetConflict("FLEET_WORKER_CAPACITY_EXHAUSTED")
|
|
||||||
|
|
||||||
self.conn.execute(
|
|
||||||
"""
|
|
||||||
INSERT INTO fleet_dispatches(
|
|
||||||
reservation_id,task_id,worker_id,task_revision,scope_key,failure_domain,
|
|
||||||
state,reserved_until,claim_id,created_at,updated_at
|
|
||||||
) VALUES (?,?,?,?,?,?,'RESERVED',?,NULL,?,?)
|
|
||||||
ON CONFLICT(task_id) DO UPDATE SET
|
|
||||||
reservation_id=excluded.reservation_id,
|
|
||||||
worker_id=excluded.worker_id,
|
|
||||||
task_revision=excluded.task_revision,
|
|
||||||
scope_key=excluded.scope_key,
|
|
||||||
failure_domain=excluded.failure_domain,
|
|
||||||
state='RESERVED',
|
|
||||||
reserved_until=excluded.reserved_until,
|
|
||||||
claim_id=NULL,
|
|
||||||
updated_at=excluded.updated_at
|
|
||||||
""",
|
|
||||||
(
|
|
||||||
reservation_id, task_id, worker_id, task_revision, scope_key,
|
|
||||||
failure_domain, reserved_until, now, now,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
row = self.conn.execute(
|
|
||||||
"SELECT virtual_finish,dispatch_count,revision FROM fleet_scope_state WHERE scope_key=?",
|
|
||||||
(scope_key,),
|
|
||||||
).fetchone()
|
|
||||||
current_finish = float(row[0]) if row else 0.0
|
|
||||||
dispatch_count = int(row[1]) if row else 0
|
|
||||||
revision = int(row[2]) if row else 0
|
|
||||||
next_finish = max(current_finish, 0.0) + 1.0 / max(scope_weight, 0.05)
|
|
||||||
self.conn.execute(
|
|
||||||
"""
|
|
||||||
INSERT INTO fleet_scope_state(
|
|
||||||
scope_key,virtual_finish,dispatch_count,last_dispatched_at,revision
|
|
||||||
) VALUES (?,?,?,?,?)
|
|
||||||
ON CONFLICT(scope_key) DO UPDATE SET
|
|
||||||
virtual_finish=excluded.virtual_finish,
|
|
||||||
dispatch_count=excluded.dispatch_count,
|
|
||||||
last_dispatched_at=excluded.last_dispatched_at,
|
|
||||||
revision=excluded.revision
|
|
||||||
""",
|
|
||||||
(scope_key, next_finish, dispatch_count + 1, now, revision + 1),
|
|
||||||
)
|
|
||||||
return FleetDispatchDecision(
|
|
||||||
reservation_id, task_id, worker_id, task_revision, scope_key,
|
|
||||||
failure_domain, reserved_until, score
|
|
||||||
)
|
|
||||||
|
|
||||||
def bind_claim(self, reservation_id: str, claim_id: str, *, now: float | None = None) -> None:
|
|
||||||
now = float(time.time() if now is None else now)
|
|
||||||
with self._transaction():
|
|
||||||
row = self.conn.execute(
|
|
||||||
"SELECT state FROM fleet_dispatches WHERE reservation_id=?",
|
|
||||||
(reservation_id,),
|
|
||||||
).fetchone()
|
|
||||||
if row is None:
|
|
||||||
raise FleetError("FLEET_RESERVATION_NOT_FOUND")
|
|
||||||
if str(row[0]) not in {"RESERVED", "CLAIMED"}:
|
|
||||||
raise FleetConflict("FLEET_RESERVATION_NOT_ACTIVE")
|
|
||||||
self.conn.execute(
|
|
||||||
"""
|
|
||||||
UPDATE fleet_dispatches
|
|
||||||
SET state='CLAIMED', claim_id=?, updated_at=?
|
|
||||||
WHERE reservation_id=?
|
|
||||||
""",
|
|
||||||
(claim_id, now, reservation_id),
|
|
||||||
)
|
|
||||||
|
|
||||||
def release(
|
|
||||||
self,
|
|
||||||
task_id: str,
|
|
||||||
*,
|
|
||||||
state: str = "RELEASED",
|
|
||||||
now: float | None = None,
|
|
||||||
) -> None:
|
|
||||||
if state not in {"RELEASED", "COMPLETED", "FAILED", "EXPIRED"}:
|
|
||||||
raise FleetError("FLEET_RESERVATION_STATE_INVALID")
|
|
||||||
now = float(time.time() if now is None else now)
|
|
||||||
with self._transaction():
|
|
||||||
self.conn.execute(
|
|
||||||
"UPDATE fleet_dispatches SET state=?,updated_at=? WHERE task_id=?",
|
|
||||||
(state, now, task_id),
|
|
||||||
)
|
|
||||||
|
|
||||||
def worker_circuit_open(self, worker_id: str, now: float) -> bool:
|
|
||||||
with self.lock:
|
|
||||||
row = self.conn.execute(
|
|
||||||
"SELECT circuit_open_until FROM fleet_worker_failures WHERE worker_id=?",
|
|
||||||
(worker_id,),
|
|
||||||
).fetchone()
|
|
||||||
return bool(row and row[0] is not None and float(row[0]) > now)
|
|
||||||
|
|
||||||
def task_worker_blocked(self, task_id: str, worker_id: str, now: float) -> bool:
|
|
||||||
with self.lock:
|
|
||||||
row = self.conn.execute(
|
|
||||||
"""
|
|
||||||
SELECT blocked_until FROM fleet_task_worker_failures
|
|
||||||
WHERE task_id=? AND worker_id=?
|
|
||||||
""",
|
|
||||||
(task_id, worker_id),
|
|
||||||
).fetchone()
|
|
||||||
return bool(row and row[0] is not None and float(row[0]) > now)
|
|
||||||
|
|
||||||
def record_worker_failure(
|
|
||||||
self,
|
|
||||||
worker_id: str,
|
|
||||||
failure_class: str,
|
|
||||||
*,
|
|
||||||
threshold: int = DEFAULT_CIRCUIT_THRESHOLD,
|
|
||||||
base_circuit_seconds: int = DEFAULT_CIRCUIT_SECONDS,
|
|
||||||
now: float | None = None,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
now = float(time.time() if now is None else now)
|
|
||||||
threshold = _int(threshold, DEFAULT_CIRCUIT_THRESHOLD, 1, 20)
|
|
||||||
base_circuit_seconds = _int(base_circuit_seconds, DEFAULT_CIRCUIT_SECONDS, 1, 3600)
|
|
||||||
with self._transaction():
|
|
||||||
row = self.conn.execute(
|
|
||||||
"""
|
|
||||||
SELECT consecutive_failures,revision FROM fleet_worker_failures
|
|
||||||
WHERE worker_id=?
|
|
||||||
""",
|
|
||||||
(worker_id,),
|
|
||||||
).fetchone()
|
|
||||||
failures = (int(row[0]) if row else 0) + 1
|
|
||||||
revision = (int(row[1]) if row else 0) + 1
|
|
||||||
open_until = None
|
|
||||||
if failures >= threshold:
|
|
||||||
exponent = min(failures - threshold, 8)
|
|
||||||
open_until = now + min(
|
|
||||||
MAX_CIRCUIT_SECONDS, base_circuit_seconds * (2**exponent)
|
|
||||||
)
|
|
||||||
self.conn.execute(
|
|
||||||
"""
|
|
||||||
INSERT INTO fleet_worker_failures(
|
|
||||||
worker_id,consecutive_failures,circuit_open_until,last_failure_class,
|
|
||||||
revision,updated_at
|
|
||||||
) VALUES (?,?,?,?,?,?)
|
|
||||||
ON CONFLICT(worker_id) DO UPDATE SET
|
|
||||||
consecutive_failures=excluded.consecutive_failures,
|
|
||||||
circuit_open_until=excluded.circuit_open_until,
|
|
||||||
last_failure_class=excluded.last_failure_class,
|
|
||||||
revision=excluded.revision,
|
|
||||||
updated_at=excluded.updated_at
|
|
||||||
""",
|
|
||||||
(worker_id, failures, open_until, _text(failure_class, "unknown"), revision, now),
|
|
||||||
)
|
|
||||||
return {
|
|
||||||
"worker_id": worker_id,
|
|
||||||
"consecutive_failures": failures,
|
|
||||||
"circuit_open_until": open_until,
|
|
||||||
"revision": revision,
|
|
||||||
}
|
|
||||||
|
|
||||||
def record_worker_success(self, worker_id: str, *, now: float | None = None) -> None:
|
|
||||||
now = float(time.time() if now is None else now)
|
|
||||||
with self._transaction():
|
|
||||||
self.conn.execute(
|
|
||||||
"""
|
|
||||||
INSERT INTO fleet_worker_failures(
|
|
||||||
worker_id,consecutive_failures,circuit_open_until,last_failure_class,
|
|
||||||
revision,updated_at
|
|
||||||
) VALUES (?,0,NULL,NULL,1,?)
|
|
||||||
ON CONFLICT(worker_id) DO UPDATE SET
|
|
||||||
consecutive_failures=0,
|
|
||||||
circuit_open_until=NULL,
|
|
||||||
last_failure_class=NULL,
|
|
||||||
revision=revision+1,
|
|
||||||
updated_at=excluded.updated_at
|
|
||||||
""",
|
|
||||||
(worker_id, now),
|
|
||||||
)
|
|
||||||
|
|
||||||
def record_task_worker_failure(
|
|
||||||
self,
|
|
||||||
task_id: str,
|
|
||||||
worker_id: str,
|
|
||||||
failure_class: str,
|
|
||||||
*,
|
|
||||||
quarantine_seconds: int = 60,
|
|
||||||
now: float | None = None,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
now = float(time.time() if now is None else now)
|
|
||||||
quarantine_seconds = _int(quarantine_seconds, 60, 1, 3600)
|
|
||||||
with self._transaction():
|
|
||||||
row = self.conn.execute(
|
|
||||||
"""
|
|
||||||
SELECT failure_count,revision FROM fleet_task_worker_failures
|
|
||||||
WHERE task_id=? AND worker_id=?
|
|
||||||
""",
|
|
||||||
(task_id, worker_id),
|
|
||||||
).fetchone()
|
|
||||||
count = (int(row[0]) if row else 0) + 1
|
|
||||||
revision = (int(row[1]) if row else 0) + 1
|
|
||||||
blocked_until = now + quarantine_seconds
|
|
||||||
self.conn.execute(
|
|
||||||
"""
|
|
||||||
INSERT INTO fleet_task_worker_failures(
|
|
||||||
task_id,worker_id,failure_count,blocked_until,last_failure_class,
|
|
||||||
revision,updated_at
|
|
||||||
) VALUES (?,?,?,?,?,?,?)
|
|
||||||
ON CONFLICT(task_id,worker_id) DO UPDATE SET
|
|
||||||
failure_count=excluded.failure_count,
|
|
||||||
blocked_until=excluded.blocked_until,
|
|
||||||
last_failure_class=excluded.last_failure_class,
|
|
||||||
revision=excluded.revision,
|
|
||||||
updated_at=excluded.updated_at
|
|
||||||
""",
|
|
||||||
(
|
|
||||||
task_id, worker_id, count, blocked_until,
|
|
||||||
_text(failure_class, "unknown"), revision, now,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
return {
|
|
||||||
"task_id": task_id,
|
|
||||||
"worker_id": worker_id,
|
|
||||||
"failure_count": count,
|
|
||||||
"blocked_until": blocked_until,
|
|
||||||
"revision": revision,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class FleetScheduler:
|
|
||||||
"""Deterministic capacity/fairness scheduler over existing TaskPlane records."""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
store: Any,
|
|
||||||
workers: Any,
|
|
||||||
*,
|
|
||||||
now_fn: Any = time.time,
|
|
||||||
reservation_seconds: int = DEFAULT_RESERVATION_SECONDS,
|
|
||||||
) -> None:
|
|
||||||
self.store = store
|
|
||||||
self.workers = workers
|
|
||||||
self.now_fn = now_fn
|
|
||||||
self.reservation_seconds = _int(
|
|
||||||
reservation_seconds, DEFAULT_RESERVATION_SECONDS, 1, 300
|
|
||||||
)
|
|
||||||
connection = getattr(store, "conn", None)
|
|
||||||
if not isinstance(connection, sqlite3.Connection):
|
|
||||||
raise FleetError("FLEET_TASKPLANE_SQLITE_CONNECTION_REQUIRED")
|
|
||||||
lock = getattr(store, "_lock", None)
|
|
||||||
self.state = FleetStateStore(connection, lock)
|
|
||||||
|
|
||||||
def configure_worker(self, worker_id: str, **kwargs: Any) -> FleetWorkerPolicy:
|
|
||||||
return self.state.configure_worker(worker_id, **kwargs)
|
|
||||||
|
|
||||||
def _online(self, worker_id: str, now: float) -> bool:
|
|
||||||
method = getattr(self.workers, "is_online", None)
|
|
||||||
if not callable(method):
|
|
||||||
raise FleetError("FLEET_WORKER_REGISTRY_ONLINE_CHECK_REQUIRED")
|
|
||||||
try:
|
|
||||||
return bool(method(worker_id, now=now))
|
|
||||||
except TypeError:
|
|
||||||
return bool(method(worker_id))
|
|
||||||
|
|
||||||
def _worker_record(self, worker_id: str) -> object:
|
|
||||||
getter = getattr(self.store, "get_worker", None)
|
|
||||||
if not callable(getter):
|
|
||||||
raise FleetError("FLEET_WORKER_LOOKUP_REQUIRED")
|
|
||||||
return getter(worker_id)
|
|
||||||
|
|
||||||
def _active_claims(self, worker_id: str, now: float) -> int:
|
|
||||||
# The TaskPlane store owns one SQLite connection in the current runtime.
|
|
||||||
# Fleet reads must share its lock just like fleet writes do, otherwise two
|
|
||||||
# concurrent pullers can issue overlapping operations on the same handle.
|
|
||||||
with self.state.lock:
|
|
||||||
columns = {
|
|
||||||
str(row[1])
|
|
||||||
for row in self.store.conn.execute("PRAGMA table_info(claims)").fetchall()
|
|
||||||
}
|
|
||||||
required = {"worker_id", "status"}
|
|
||||||
if not required.issubset(columns):
|
|
||||||
raise FleetError("FLEET_CLAIMS_SCHEMA_UNSUPPORTED")
|
|
||||||
sql = "SELECT COUNT(*) FROM claims WHERE worker_id=? AND status='ACTIVE'"
|
|
||||||
params: list[Any] = [worker_id]
|
|
||||||
if "lease_expires_at" in columns:
|
|
||||||
sql += " AND lease_expires_at>?"
|
|
||||||
params.append(now)
|
|
||||||
row = self.store.conn.execute(sql, tuple(params)).fetchone()
|
|
||||||
return int(row[0] if row else 0)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _labels_match(policy: FleetWorkerPolicy, task: object) -> bool:
|
|
||||||
metadata = _task_metadata(task)
|
|
||||||
affinity = metadata.get("fleet_affinity_labels") or {}
|
|
||||||
if affinity and not isinstance(affinity, Mapping):
|
|
||||||
return False
|
|
||||||
labels = dict(policy.labels or {})
|
|
||||||
for key, value in dict(affinity).items():
|
|
||||||
if labels.get(_text(key)) != _text(value):
|
|
||||||
return False
|
|
||||||
avoid = metadata.get("fleet_avoid_failure_domains") or []
|
|
||||||
if isinstance(avoid, Sequence) and not isinstance(avoid, (str, bytes)):
|
|
||||||
if policy.failure_domain in {_text(item) for item in avoid}:
|
|
||||||
return False
|
|
||||||
return True
|
|
||||||
|
|
||||||
def _eligible_worker(self, worker_id: str, task: object, now: float) -> tuple[bool, int, FleetWorkerPolicy]:
|
|
||||||
if not self._online(worker_id, now):
|
|
||||||
return False, 0, self.state.worker_policy(worker_id)
|
|
||||||
worker = self._worker_record(worker_id)
|
|
||||||
policy = self.state.worker_policy(worker_id)
|
|
||||||
if self.state.worker_circuit_open(worker_id, now):
|
|
||||||
return False, 0, policy
|
|
||||||
if self.state.task_worker_blocked(_task_id(task), worker_id, now):
|
|
||||||
return False, 0, policy
|
|
||||||
if not self._labels_match(policy, task):
|
|
||||||
return False, 0, policy
|
|
||||||
required = _task_required_capabilities(task)
|
|
||||||
capabilities = _worker_capabilities(worker)
|
|
||||||
if required and not required.issubset(capabilities):
|
|
||||||
return False, 0, policy
|
|
||||||
active = self._active_claims(worker_id, now)
|
|
||||||
reserved = self.state.reserved_slots(worker_id, now)
|
|
||||||
load = active + reserved
|
|
||||||
if load >= policy.max_slots:
|
|
||||||
return False, load, policy
|
|
||||||
return True, load, policy
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _task_priority(task: object, now: float) -> float:
|
|
||||||
metadata = _task_metadata(task)
|
|
||||||
base = _int(metadata.get("fleet_priority"), 0, -100, 100)
|
|
||||||
created = _task_created_at(task)
|
|
||||||
waited = max(0.0, now - created) if created > 0 else 0.0
|
|
||||||
aging = min(100.0, waited / 60.0)
|
|
||||||
return float(base) + aging
|
|
||||||
|
|
||||||
def _task_score(self, task: object, now: float) -> tuple[float, float, float, str]:
|
|
||||||
scope = _task_scope_key(task)
|
|
||||||
virtual_finish = self.state.scope_virtual_finish(scope)
|
|
||||||
priority = self._task_priority(task, now)
|
|
||||||
return (virtual_finish, -priority, _task_created_at(task), _task_id(task))
|
|
||||||
|
|
||||||
def _tasks(self, actor_context: Any = None) -> list[object]:
|
|
||||||
lister = getattr(self.store, "list_tasks", None)
|
|
||||||
if not callable(lister):
|
|
||||||
raise FleetError("FLEET_TASK_LIST_REQUIRED")
|
|
||||||
try:
|
|
||||||
tasks = lister(actor_context)
|
|
||||||
except TypeError:
|
|
||||||
tasks = lister()
|
|
||||||
return [
|
|
||||||
task for task in tasks
|
|
||||||
if _task_status(task) in {"CREATED", "QUEUED"}
|
|
||||||
and _task_id(task)
|
|
||||||
]
|
|
||||||
|
|
||||||
def rank_for_worker(
|
|
||||||
self,
|
|
||||||
worker_id: str,
|
|
||||||
*,
|
|
||||||
actor_context: Any = None,
|
|
||||||
now: float | None = None,
|
|
||||||
) -> list[object]:
|
|
||||||
"""Return fleet-ranked tasks without reserving or claiming anything."""
|
|
||||||
now = float(self.now_fn() if now is None else now)
|
|
||||||
worker_id = _text(worker_id)
|
|
||||||
if not worker_id:
|
|
||||||
return []
|
|
||||||
ranked: list[object] = []
|
|
||||||
for task in self._tasks(actor_context):
|
|
||||||
eligible, _, _ = self._eligible_worker(worker_id, task, now)
|
|
||||||
if eligible:
|
|
||||||
ranked.append(task)
|
|
||||||
return sorted(ranked, key=lambda task: self._task_score(task, now))
|
|
||||||
|
|
||||||
def reserve_for_worker(
|
|
||||||
self,
|
|
||||||
worker_id: str,
|
|
||||||
task: object,
|
|
||||||
*,
|
|
||||||
now: float | None = None,
|
|
||||||
) -> FleetDispatchDecision:
|
|
||||||
"""Reserve exactly one already-ranked task, rechecking eligibility and capacity."""
|
|
||||||
now = float(self.now_fn() if now is None else now)
|
|
||||||
worker_id = _text(worker_id)
|
|
||||||
eligible, _, policy = self._eligible_worker(worker_id, task, now)
|
|
||||||
if not eligible:
|
|
||||||
raise FleetConflict("FLEET_WORKER_NOT_ELIGIBLE")
|
|
||||||
metadata = _task_metadata(task)
|
|
||||||
scope_weight = _float(metadata.get("fleet_scope_weight"), 1.0, 0.05, 100.0)
|
|
||||||
return self.state.reserve(
|
|
||||||
task_id=_task_id(task),
|
|
||||||
worker_id=worker_id,
|
|
||||||
task_revision=_task_revision(task),
|
|
||||||
scope_key=_task_scope_key(task),
|
|
||||||
failure_domain=policy.failure_domain,
|
|
||||||
score=self._task_score(task, now),
|
|
||||||
reservation_seconds=self.reservation_seconds,
|
|
||||||
now=now,
|
|
||||||
scope_weight=scope_weight,
|
|
||||||
)
|
|
||||||
|
|
||||||
def select(
|
|
||||||
self,
|
|
||||||
worker_ids: Sequence[str],
|
|
||||||
*,
|
|
||||||
actor_context: Any = None,
|
|
||||||
limit: int = 1,
|
|
||||||
now: float | None = None,
|
|
||||||
) -> list[FleetDispatchDecision]:
|
|
||||||
now = float(self.now_fn() if now is None else now)
|
|
||||||
limit = _int(limit, 1, 1, 100)
|
|
||||||
worker_ids = tuple(dict.fromkeys(_text(item) for item in worker_ids if _text(item)))
|
|
||||||
tasks = sorted(self._tasks(actor_context), key=lambda task: self._task_score(task, now))
|
|
||||||
decisions: list[FleetDispatchDecision] = []
|
|
||||||
local_load: dict[str, int] = {}
|
|
||||||
for task in tasks:
|
|
||||||
if len(decisions) >= limit:
|
|
||||||
break
|
|
||||||
candidates: list[tuple[int, float, str, FleetWorkerPolicy]] = []
|
|
||||||
for worker_id in worker_ids:
|
|
||||||
eligible, load, policy = self._eligible_worker(worker_id, task, now)
|
|
||||||
if not eligible:
|
|
||||||
continue
|
|
||||||
effective_load = local_load.get(worker_id, load)
|
|
||||||
if effective_load >= policy.max_slots:
|
|
||||||
continue
|
|
||||||
candidates.append(
|
|
||||||
(
|
|
||||||
effective_load,
|
|
||||||
-policy.weight,
|
|
||||||
worker_id,
|
|
||||||
policy,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
if not candidates:
|
|
||||||
continue
|
|
||||||
candidates.sort(key=lambda item: (item[0], item[1], item[2]))
|
|
||||||
_, _, worker_id, policy = candidates[0]
|
|
||||||
score = self._task_score(task, now)
|
|
||||||
metadata = _task_metadata(task)
|
|
||||||
scope_weight = _float(metadata.get("fleet_scope_weight"), 1.0, 0.05, 100.0)
|
|
||||||
try:
|
|
||||||
decision = self.state.reserve(
|
|
||||||
task_id=_task_id(task),
|
|
||||||
worker_id=worker_id,
|
|
||||||
task_revision=_task_revision(task),
|
|
||||||
scope_key=_task_scope_key(task),
|
|
||||||
failure_domain=policy.failure_domain,
|
|
||||||
score=score,
|
|
||||||
reservation_seconds=self.reservation_seconds,
|
|
||||||
now=now,
|
|
||||||
scope_weight=scope_weight,
|
|
||||||
)
|
|
||||||
except FleetConflict:
|
|
||||||
continue
|
|
||||||
local_load[worker_id] = local_load.get(
|
|
||||||
worker_id,
|
|
||||||
self._active_claims(worker_id, now)
|
|
||||||
+ self.state.reserved_slots(worker_id, now)
|
|
||||||
- 1,
|
|
||||||
) + 1
|
|
||||||
decisions.append(decision)
|
|
||||||
return decisions
|
|
||||||
@@ -13,8 +13,6 @@ from dataclasses import dataclass, field
|
|||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from .scope import normalize_task_scope
|
|
||||||
|
|
||||||
|
|
||||||
SENSITIVE_KEYS = {
|
SENSITIVE_KEYS = {
|
||||||
"secret",
|
"secret",
|
||||||
@@ -68,12 +66,6 @@ TASK_PLAN_KEYS = {
|
|||||||
"allowed_input_paths",
|
"allowed_input_paths",
|
||||||
"allowed_output_paths",
|
"allowed_output_paths",
|
||||||
"output_policy",
|
"output_policy",
|
||||||
"scope_type",
|
|
||||||
"tenant_id",
|
|
||||||
"user_id",
|
|
||||||
"project_id",
|
|
||||||
"conversation_id",
|
|
||||||
"run_id",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -245,12 +237,6 @@ class TaskPlan:
|
|||||||
schema_version: int = 1
|
schema_version: int = 1
|
||||||
task_id: str | None = None
|
task_id: str | None = None
|
||||||
task_type: str = "code.change"
|
task_type: str = "code.change"
|
||||||
scope_type: str = "legacy"
|
|
||||||
tenant_id: str | None = None
|
|
||||||
user_id: str | None = None
|
|
||||||
project_id: str | None = None
|
|
||||||
conversation_id: str | None = None
|
|
||||||
run_id: str | None = None
|
|
||||||
issuer: dict[str, Any] = field(default_factory=dict)
|
issuer: dict[str, Any] = field(default_factory=dict)
|
||||||
intent: str | None = None
|
intent: str | None = None
|
||||||
required_capabilities: list[str] = field(default_factory=list)
|
required_capabilities: list[str] = field(default_factory=list)
|
||||||
@@ -325,17 +311,10 @@ class TaskPlan:
|
|||||||
raise ValueError("credential-secret requires metadata-only logging")
|
raise ValueError("credential-secret requires metadata-only logging")
|
||||||
if output_policy != "hash-count-status-only":
|
if output_policy != "hash-count-status-only":
|
||||||
raise ValueError("credential-secret requires restricted output")
|
raise ValueError("credential-secret requires restricted output")
|
||||||
scope = normalize_task_scope(data)
|
|
||||||
return cls(
|
return cls(
|
||||||
schema_version=schema_version,
|
schema_version=schema_version,
|
||||||
task_id=validate_identifier(data["task_id"], "task_id"),
|
task_id=validate_identifier(data["task_id"], "task_id"),
|
||||||
task_type=validate_type(data.get("task_type", "code.change"), "task_type"),
|
task_type=validate_type(data.get("task_type", "code.change"), "task_type"),
|
||||||
scope_type=scope.scope_type,
|
|
||||||
tenant_id=scope.tenant_id,
|
|
||||||
user_id=scope.user_id,
|
|
||||||
project_id=scope.project_id,
|
|
||||||
conversation_id=scope.conversation_id,
|
|
||||||
run_id=scope.run_id,
|
|
||||||
issuer=issuer,
|
issuer=issuer,
|
||||||
intent=data.get("intent"),
|
intent=data.get("intent"),
|
||||||
required_capabilities=_normalize_list(data.get("required_capabilities", [])),
|
required_capabilities=_normalize_list(data.get("required_capabilities", [])),
|
||||||
@@ -369,12 +348,6 @@ class TaskPlan:
|
|||||||
"schema_version": self.schema_version,
|
"schema_version": self.schema_version,
|
||||||
"task_id": self.task_id,
|
"task_id": self.task_id,
|
||||||
"task_type": self.task_type,
|
"task_type": self.task_type,
|
||||||
"scope_type": self.scope_type,
|
|
||||||
"tenant_id": self.tenant_id,
|
|
||||||
"user_id": self.user_id,
|
|
||||||
"project_id": self.project_id,
|
|
||||||
"conversation_id": self.conversation_id,
|
|
||||||
"run_id": self.run_id,
|
|
||||||
"issuer": self.issuer,
|
"issuer": self.issuer,
|
||||||
"intent": self.intent,
|
"intent": self.intent,
|
||||||
"required_capabilities": list(self.required_capabilities),
|
"required_capabilities": list(self.required_capabilities),
|
||||||
|
|||||||
@@ -3,11 +3,10 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any, Callable, Mapping
|
from typing import Callable
|
||||||
|
|
||||||
from .errors import ForbiddenError, LockedError
|
from .errors import ForbiddenError, LockedError
|
||||||
from .models import ProbeEvidence, ResultEnvelope, TaskRecord
|
from .models import ProbeEvidence, ResultEnvelope, TaskRecord
|
||||||
from .scope import TaskActorContext, actor_context as build_actor_context
|
|
||||||
|
|
||||||
|
|
||||||
RISK_RANK = {"low": 1, "medium": 2, "high": 3}
|
RISK_RANK = {"low": 1, "medium": 2, "high": 3}
|
||||||
@@ -94,10 +93,6 @@ class PolicyEngine:
|
|||||||
result_signature_verifier: Callable[[ResultEnvelope], bool] | None = None,
|
result_signature_verifier: Callable[[ResultEnvelope], bool] | None = None,
|
||||||
probe_attestation_verifier: Callable[[ProbeEvidence], bool] | None = None,
|
probe_attestation_verifier: Callable[[ProbeEvidence], bool] | None = None,
|
||||||
identity_capabilities: dict[str, set[str]] | None = None,
|
identity_capabilities: dict[str, set[str]] | None = None,
|
||||||
identity_actor_contexts: Mapping[
|
|
||||||
str, TaskActorContext | Mapping[str, Any]
|
|
||||||
]
|
|
||||||
| None = None,
|
|
||||||
admin_identities: set[str] | None = None,
|
admin_identities: set[str] | None = None,
|
||||||
readonly_worker_execution_enabled: bool = False,
|
readonly_worker_execution_enabled: bool = False,
|
||||||
readonly_worker_profiles: list[ReadonlyWorkerProfile] | None = None,
|
readonly_worker_profiles: list[ReadonlyWorkerProfile] | None = None,
|
||||||
@@ -114,26 +109,12 @@ class PolicyEngine:
|
|||||||
identity: set(capabilities)
|
identity: set(capabilities)
|
||||||
for identity, capabilities in (identity_capabilities or {}).items()
|
for identity, capabilities in (identity_capabilities or {}).items()
|
||||||
}
|
}
|
||||||
self.identity_actor_contexts = dict(identity_actor_contexts or {})
|
|
||||||
self.admin_identities = set(admin_identities or ())
|
self.admin_identities = set(admin_identities or ())
|
||||||
self.readonly_worker_execution_enabled = readonly_worker_execution_enabled
|
self.readonly_worker_execution_enabled = readonly_worker_execution_enabled
|
||||||
self.readonly_worker_profiles = {
|
self.readonly_worker_profiles = {
|
||||||
profile.identity: profile for profile in (readonly_worker_profiles or [])
|
profile.identity: profile for profile in (readonly_worker_profiles or [])
|
||||||
}
|
}
|
||||||
|
|
||||||
def actor_context(self, identity: str) -> TaskActorContext:
|
|
||||||
configured = self.identity_actor_contexts.get(
|
|
||||||
identity, self.identity_actor_contexts.get("*")
|
|
||||||
)
|
|
||||||
capabilities = set(
|
|
||||||
self.identity_capabilities.get(
|
|
||||||
identity, self.identity_capabilities.get("*", set())
|
|
||||||
)
|
|
||||||
)
|
|
||||||
if identity in self.admin_identities:
|
|
||||||
capabilities.add("task.admin")
|
|
||||||
return build_actor_context(identity, configured, capabilities)
|
|
||||||
|
|
||||||
def _readonly_worker_allows(
|
def _readonly_worker_allows(
|
||||||
self, action: str, actor: str, task: TaskRecord | None
|
self, action: str, actor: str, task: TaskRecord | None
|
||||||
) -> bool:
|
) -> bool:
|
||||||
|
|||||||
@@ -74,203 +74,3 @@ class TaskScheduler:
|
|||||||
operation_digest=task.task_plan.operation_digest(),
|
operation_digest=task.task_plan.operation_digest(),
|
||||||
)
|
)
|
||||||
return claim
|
return claim
|
||||||
|
|
||||||
# R4.2 P2 durable multi-agent fleet scheduling public surface.
|
|
||||||
from .fleet import (
|
|
||||||
FleetConflict,
|
|
||||||
FleetDispatchDecision,
|
|
||||||
FleetError,
|
|
||||||
FleetScheduler,
|
|
||||||
FleetStateStore,
|
|
||||||
FleetWorkerPolicy,
|
|
||||||
)
|
|
||||||
# R4.2 P2-B fleet-aware pull scheduling.
|
|
||||||
#
|
|
||||||
# The original TaskScheduler remains the security oracle and claim-CAS implementation.
|
|
||||||
# Fleet ordering only controls the order presented to that oracle.
|
|
||||||
import inspect as _fleet_inspect
|
|
||||||
import threading as _fleet_threading
|
|
||||||
from collections.abc import Mapping as _FleetMapping
|
|
||||||
|
|
||||||
from .worker_registry import WorkerRegistry as _FleetWorkerRegistry
|
|
||||||
from .errors import NotFoundError as _FleetNotFoundError
|
|
||||||
|
|
||||||
_LegacyTaskScheduler = TaskScheduler
|
|
||||||
_LegacyScheduleParameters = _fleet_inspect.signature(
|
|
||||||
_LegacyTaskScheduler.schedule_for_worker
|
|
||||||
).parameters
|
|
||||||
_LegacyNextParameters = _fleet_inspect.signature(
|
|
||||||
_LegacyTaskScheduler.next_queued_task
|
|
||||||
).parameters
|
|
||||||
|
|
||||||
|
|
||||||
class _FleetOrderedStoreProxy:
|
|
||||||
def __init__(self, delegate, tasks):
|
|
||||||
self._delegate = delegate
|
|
||||||
self._tasks = tuple(tasks)
|
|
||||||
|
|
||||||
def list_tasks(self, *args, **kwargs):
|
|
||||||
return list(self._tasks)
|
|
||||||
|
|
||||||
def __getattr__(self, name):
|
|
||||||
return getattr(self._delegate, name)
|
|
||||||
|
|
||||||
|
|
||||||
def _fleet_claim_field(claim, name, default=None):
|
|
||||||
if isinstance(claim, _FleetMapping):
|
|
||||||
return claim.get(name, default)
|
|
||||||
return getattr(claim, name, default)
|
|
||||||
|
|
||||||
|
|
||||||
class TaskScheduler(_LegacyTaskScheduler):
|
|
||||||
"""Fleet-integrated pull scheduler preserving legacy security and claim semantics."""
|
|
||||||
|
|
||||||
def __init__(self, *args, **kwargs):
|
|
||||||
super().__init__(*args, **kwargs)
|
|
||||||
store = getattr(self, "store", args[0] if args else None)
|
|
||||||
if store is None:
|
|
||||||
raise FleetError("FLEET_SCHEDULER_STORE_REQUIRED")
|
|
||||||
registry = getattr(self, "workers", None)
|
|
||||||
if registry is None:
|
|
||||||
registry = _FleetWorkerRegistry(store)
|
|
||||||
self.fleet_scheduler = FleetScheduler(store, registry)
|
|
||||||
self._fleet_local = _fleet_threading.local()
|
|
||||||
|
|
||||||
def _fleet_legacy_next(self, worker_id, tasks, *args, **kwargs):
|
|
||||||
proxy = _FleetOrderedStoreProxy(self.store, tasks)
|
|
||||||
legacy = _LegacyTaskScheduler(proxy)
|
|
||||||
legacy_kwargs = dict(kwargs)
|
|
||||||
if "actor_context" not in _LegacyNextParameters:
|
|
||||||
legacy_kwargs.pop("actor_context", None)
|
|
||||||
return legacy.next_queued_task(worker_id, *args, **legacy_kwargs)
|
|
||||||
|
|
||||||
def next_queued_task(self, worker_id, *args, **kwargs):
|
|
||||||
actor_context = kwargs.get(
|
|
||||||
"actor_context", getattr(self._fleet_local, "actor_context", None)
|
|
||||||
)
|
|
||||||
|
|
||||||
# Preserve legacy fail-closed worker checks before fleet ordering.
|
|
||||||
probe_kwargs = dict(kwargs)
|
|
||||||
if "actor_context" not in _LegacyNextParameters:
|
|
||||||
probe_kwargs.pop("actor_context", None)
|
|
||||||
legacy_probe = _LegacyTaskScheduler(self.store).next_queued_task(
|
|
||||||
worker_id, *args, **probe_kwargs
|
|
||||||
)
|
|
||||||
if legacy_probe is None:
|
|
||||||
return None
|
|
||||||
|
|
||||||
ranked = self.fleet_scheduler.rank_for_worker(
|
|
||||||
worker_id, actor_context=actor_context
|
|
||||||
)
|
|
||||||
# Preserve FleetScheduler fairness order. The legacy scheduler is used
|
|
||||||
# only as a fail-closed security oracle: present one fleet-ranked task
|
|
||||||
# at a time so its internal sort cannot reorder across scopes.
|
|
||||||
# Keep the legacy priority dimension ahead of Fleet fairness for
|
|
||||||
# compatibility with existing TaskPlane scheduling contracts; the
|
|
||||||
# Fleet order remains authoritative among equal-priority tasks.
|
|
||||||
def _legacy_priority(task):
|
|
||||||
plan = getattr(task, "task_plan", task)
|
|
||||||
metadata = getattr(plan, "metadata", {}) or {}
|
|
||||||
if not isinstance(metadata, _FleetMapping):
|
|
||||||
return 0
|
|
||||||
try:
|
|
||||||
return int(metadata.get("priority", 0))
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
return 0
|
|
||||||
|
|
||||||
ranked = sorted(ranked, key=lambda task: -_legacy_priority(task))
|
|
||||||
for ranked_candidate in ranked:
|
|
||||||
try:
|
|
||||||
candidate = self._fleet_legacy_next(
|
|
||||||
worker_id, [ranked_candidate], *args, **kwargs
|
|
||||||
)
|
|
||||||
except _FleetNotFoundError:
|
|
||||||
continue
|
|
||||||
if candidate is None:
|
|
||||||
continue
|
|
||||||
|
|
||||||
ranked_id = getattr(
|
|
||||||
ranked_candidate,
|
|
||||||
"task_id",
|
|
||||||
getattr(getattr(ranked_candidate, "task_plan", None), "task_id", None),
|
|
||||||
)
|
|
||||||
candidate_id = getattr(
|
|
||||||
candidate,
|
|
||||||
"task_id",
|
|
||||||
getattr(getattr(candidate, "task_plan", None), "task_id", None),
|
|
||||||
)
|
|
||||||
if candidate_id != ranked_id:
|
|
||||||
raise FleetConflict("FLEET_LEGACY_ORACLE_ORDER_VIOLATION")
|
|
||||||
|
|
||||||
try:
|
|
||||||
decision = self.fleet_scheduler.reserve_for_worker(
|
|
||||||
worker_id, candidate
|
|
||||||
)
|
|
||||||
except FleetConflict:
|
|
||||||
continue
|
|
||||||
self._fleet_local.pending = decision
|
|
||||||
return candidate
|
|
||||||
return None
|
|
||||||
|
|
||||||
def schedule_for_worker(self, worker_id, *args, **kwargs):
|
|
||||||
actor_context = kwargs.get("actor_context")
|
|
||||||
legacy_kwargs = dict(kwargs)
|
|
||||||
if "actor_context" not in _LegacyScheduleParameters:
|
|
||||||
legacy_kwargs.pop("actor_context", None)
|
|
||||||
self._fleet_local.actor_context = actor_context
|
|
||||||
self._fleet_local.pending = None
|
|
||||||
try:
|
|
||||||
claim = _LegacyTaskScheduler.schedule_for_worker(
|
|
||||||
self, worker_id, *args, **legacy_kwargs
|
|
||||||
)
|
|
||||||
decision = getattr(self._fleet_local, "pending", None)
|
|
||||||
if claim is None:
|
|
||||||
if decision is not None:
|
|
||||||
self.fleet_scheduler.state.release(
|
|
||||||
decision.task_id, state="RELEASED"
|
|
||||||
)
|
|
||||||
return None
|
|
||||||
if decision is not None:
|
|
||||||
claim_id = _fleet_claim_field(claim, "claim_id")
|
|
||||||
claim_task_id = _fleet_claim_field(claim, "task_id", decision.task_id)
|
|
||||||
if not claim_id:
|
|
||||||
self.fleet_scheduler.state.release(
|
|
||||||
decision.task_id, state="RELEASED"
|
|
||||||
)
|
|
||||||
raise FleetError("FLEET_CLAIM_ID_REQUIRED")
|
|
||||||
if claim_task_id != decision.task_id:
|
|
||||||
self.fleet_scheduler.state.release(
|
|
||||||
decision.task_id, state="RELEASED"
|
|
||||||
)
|
|
||||||
raise FleetConflict("FLEET_CLAIM_TASK_MISMATCH")
|
|
||||||
self.fleet_scheduler.state.bind_claim(
|
|
||||||
decision.reservation_id, str(claim_id)
|
|
||||||
)
|
|
||||||
return claim
|
|
||||||
except Exception:
|
|
||||||
decision = getattr(self._fleet_local, "pending", None)
|
|
||||||
if decision is not None:
|
|
||||||
self.fleet_scheduler.state.release(
|
|
||||||
decision.task_id, state="RELEASED"
|
|
||||||
)
|
|
||||||
raise
|
|
||||||
finally:
|
|
||||||
self._fleet_local.pending = None
|
|
||||||
self._fleet_local.actor_context = None
|
|
||||||
|
|
||||||
def record_worker_failure(self, worker_id, failure_class, **kwargs):
|
|
||||||
return self.fleet_scheduler.state.record_worker_failure(
|
|
||||||
worker_id, failure_class, **kwargs
|
|
||||||
)
|
|
||||||
|
|
||||||
def record_worker_success(self, worker_id, **kwargs):
|
|
||||||
return self.fleet_scheduler.state.record_worker_success(
|
|
||||||
worker_id, **kwargs
|
|
||||||
)
|
|
||||||
|
|
||||||
def quarantine_task_worker(
|
|
||||||
self, task_id, worker_id, failure_class, **kwargs
|
|
||||||
):
|
|
||||||
return self.fleet_scheduler.state.record_task_worker_failure(
|
|
||||||
task_id, worker_id, failure_class, **kwargs
|
|
||||||
)
|
|
||||||
|
|||||||
@@ -1,181 +0,0 @@
|
|||||||
"""Trusted TaskPlane scope model and SQL authorization helpers."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from dataclasses import dataclass, replace
|
|
||||||
from typing import Any, Mapping
|
|
||||||
|
|
||||||
from .errors import ForbiddenError, ValidationError
|
|
||||||
|
|
||||||
SCOPE_FIELDS = ("tenant_id", "user_id", "project_id", "conversation_id", "run_id")
|
|
||||||
PRIVILEGED_CAPABILITIES = frozenset(
|
|
||||||
{"*", "admin", "system.admin", "task.admin", "task.read.all"}
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _value(value: Any, field: str) -> Any:
|
|
||||||
if hasattr(value, "task_plan"):
|
|
||||||
value = value.task_plan
|
|
||||||
if isinstance(value, Mapping):
|
|
||||||
return value.get(field)
|
|
||||||
return getattr(value, field, None)
|
|
||||||
|
|
||||||
|
|
||||||
def _text(value: Any) -> str | None:
|
|
||||||
if value is None:
|
|
||||||
return None
|
|
||||||
result = str(value).strip()
|
|
||||||
return result or None
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class TaskScope:
|
|
||||||
scope_type: str
|
|
||||||
tenant_id: str | None = None
|
|
||||||
user_id: str | None = None
|
|
||||||
project_id: str | None = None
|
|
||||||
conversation_id: str | None = None
|
|
||||||
run_id: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class TaskActorContext:
|
|
||||||
identity: str
|
|
||||||
scope_type: str = "legacy"
|
|
||||||
tenant_id: str | None = None
|
|
||||||
user_id: str | None = None
|
|
||||||
project_id: str | None = None
|
|
||||||
conversation_id: str | None = None
|
|
||||||
run_id: str | None = None
|
|
||||||
capabilities: frozenset[str] = frozenset()
|
|
||||||
|
|
||||||
|
|
||||||
def normalize_task_scope(value: Any) -> TaskScope:
|
|
||||||
fields = {field: _text(_value(value, field)) for field in SCOPE_FIELDS}
|
|
||||||
declared = _text(_value(value, "scope_type"))
|
|
||||||
if declared is None:
|
|
||||||
declared = "tenant" if fields["tenant_id"] else "legacy"
|
|
||||||
if declared == "tenant":
|
|
||||||
if fields["tenant_id"] is None:
|
|
||||||
raise ValidationError(
|
|
||||||
"tenant scope requires tenant_id", code="TASK_SCOPE_TENANT_ID_REQUIRED"
|
|
||||||
)
|
|
||||||
elif declared == "legacy":
|
|
||||||
if any(fields.values()):
|
|
||||||
raise ValidationError(
|
|
||||||
"legacy scope cannot contain scoped ownership",
|
|
||||||
code="TASK_SCOPE_LEGACY_WITH_SCOPED_FIELDS",
|
|
||||||
)
|
|
||||||
elif declared == "system":
|
|
||||||
if any(fields[field] for field in ("tenant_id", "user_id", "project_id", "conversation_id")):
|
|
||||||
raise ValidationError(
|
|
||||||
"system scope cannot contain tenant ownership",
|
|
||||||
code="TASK_SCOPE_SYSTEM_WITH_TENANT_FIELDS",
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
raise ValidationError("invalid task scope type", code="TASK_SCOPE_TYPE_INVALID")
|
|
||||||
return TaskScope(declared, **fields)
|
|
||||||
|
|
||||||
|
|
||||||
def actor_is_privileged(actor: TaskActorContext) -> bool:
|
|
||||||
return bool(actor.capabilities & PRIVILEGED_CAPABILITIES)
|
|
||||||
|
|
||||||
|
|
||||||
def actor_context(
|
|
||||||
identity: str,
|
|
||||||
configured: TaskActorContext | Mapping[str, Any] | None,
|
|
||||||
capabilities: set[str] | frozenset[str],
|
|
||||||
) -> TaskActorContext:
|
|
||||||
if isinstance(configured, TaskActorContext):
|
|
||||||
context = configured
|
|
||||||
else:
|
|
||||||
data = dict(configured or {})
|
|
||||||
data.pop("identity", None)
|
|
||||||
data.pop("capabilities", None)
|
|
||||||
context = TaskActorContext(identity=str(identity), **data)
|
|
||||||
scope = normalize_task_scope(context)
|
|
||||||
return replace(
|
|
||||||
context,
|
|
||||||
identity=str(identity),
|
|
||||||
scope_type=scope.scope_type,
|
|
||||||
tenant_id=scope.tenant_id,
|
|
||||||
user_id=scope.user_id,
|
|
||||||
project_id=scope.project_id,
|
|
||||||
conversation_id=scope.conversation_id,
|
|
||||||
run_id=scope.run_id,
|
|
||||||
capabilities=frozenset(str(item) for item in capabilities),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def bind_task_scope(plan: Any, actor: TaskActorContext, *, public_create: bool) -> Any:
|
|
||||||
requested = normalize_task_scope(plan)
|
|
||||||
if actor_is_privileged(actor):
|
|
||||||
trusted = normalize_task_scope(actor)
|
|
||||||
if requested.scope_type == "legacy" and trusted.scope_type != "legacy":
|
|
||||||
plan.scope_type = trusted.scope_type
|
|
||||||
for field in SCOPE_FIELDS:
|
|
||||||
setattr(plan, field, getattr(actor, field))
|
|
||||||
return plan
|
|
||||||
trusted = normalize_task_scope(actor)
|
|
||||||
if public_create and trusted.scope_type == "legacy":
|
|
||||||
raise ForbiddenError(
|
|
||||||
"authenticated identity has no tenant scope",
|
|
||||||
code="TASK_ACTOR_SCOPE_UNAVAILABLE",
|
|
||||||
)
|
|
||||||
if trusted.scope_type != "tenant":
|
|
||||||
if requested.scope_type == "legacy":
|
|
||||||
plan.scope_type = trusted.scope_type
|
|
||||||
for field in SCOPE_FIELDS:
|
|
||||||
setattr(plan, field, getattr(actor, field))
|
|
||||||
return plan
|
|
||||||
if requested.scope_type != trusted.scope_type:
|
|
||||||
raise ForbiddenError("task scope mismatch", code="TASK_SCOPE_MISMATCH_DENY")
|
|
||||||
return plan
|
|
||||||
if requested.scope_type not in {"legacy", "tenant"}:
|
|
||||||
raise ForbiddenError("task scope mismatch", code="TASK_SCOPE_MISMATCH_DENY")
|
|
||||||
updates: dict[str, Any] = {"scope_type": "tenant"}
|
|
||||||
for field in SCOPE_FIELDS:
|
|
||||||
declared = getattr(requested, field)
|
|
||||||
authoritative = getattr(actor, field)
|
|
||||||
if declared is not None and declared != authoritative:
|
|
||||||
raise ForbiddenError(
|
|
||||||
"task scope mismatch",
|
|
||||||
code="TASK_SCOPE_MISMATCH_DENY",
|
|
||||||
details={"field": field},
|
|
||||||
)
|
|
||||||
updates[field] = authoritative
|
|
||||||
for field, item in updates.items():
|
|
||||||
setattr(plan, field, item)
|
|
||||||
return plan
|
|
||||||
|
|
||||||
|
|
||||||
def actor_scope_sql(actor: TaskActorContext | None) -> tuple[str, tuple[Any, ...]]:
|
|
||||||
if actor is None or actor_is_privileged(actor):
|
|
||||||
return "", ()
|
|
||||||
scope = normalize_task_scope(actor)
|
|
||||||
if scope.scope_type == "legacy":
|
|
||||||
return "scope_type = 'legacy'", ()
|
|
||||||
if scope.scope_type == "system":
|
|
||||||
return "scope_type = 'system'", ()
|
|
||||||
clauses = ["scope_type = 'tenant'", "tenant_id = ?"]
|
|
||||||
params: list[Any] = [scope.tenant_id]
|
|
||||||
for field in ("user_id", "project_id", "conversation_id", "run_id"):
|
|
||||||
value = getattr(scope, field)
|
|
||||||
if value is None:
|
|
||||||
clauses.append(f"{field} IS NULL")
|
|
||||||
else:
|
|
||||||
clauses.append(f"({field} IS NULL OR {field} = ?)")
|
|
||||||
params.append(value)
|
|
||||||
return " AND ".join(clauses), tuple(params)
|
|
||||||
|
|
||||||
|
|
||||||
def task_scope_values(value: Any) -> tuple[str | None, ...]:
|
|
||||||
scope = normalize_task_scope(value)
|
|
||||||
return (
|
|
||||||
scope.scope_type,
|
|
||||||
scope.tenant_id,
|
|
||||||
scope.user_id,
|
|
||||||
scope.project_id,
|
|
||||||
scope.conversation_id,
|
|
||||||
scope.run_id,
|
|
||||||
)
|
|
||||||
@@ -32,7 +32,6 @@ from .models import (
|
|||||||
redact_value,
|
redact_value,
|
||||||
validate_identifier,
|
validate_identifier,
|
||||||
)
|
)
|
||||||
from .scope import TaskActorContext, actor_scope_sql, task_scope_values
|
|
||||||
|
|
||||||
|
|
||||||
EventEnvelope = tuple[str, str, str, str, str, float]
|
EventEnvelope = tuple[str, str, str, str, str, float]
|
||||||
@@ -135,12 +134,6 @@ class TaskPlaneStore:
|
|||||||
status TEXT NOT NULL,
|
status TEXT NOT NULL,
|
||||||
task_json TEXT NOT NULL,
|
task_json TEXT NOT NULL,
|
||||||
task_type TEXT NOT NULL,
|
task_type TEXT NOT NULL,
|
||||||
scope_type TEXT NOT NULL DEFAULT 'legacy',
|
|
||||||
tenant_id TEXT,
|
|
||||||
user_id TEXT,
|
|
||||||
project_id TEXT,
|
|
||||||
conversation_id TEXT,
|
|
||||||
run_id TEXT,
|
|
||||||
root_task_id TEXT NOT NULL,
|
root_task_id TEXT NOT NULL,
|
||||||
parent_task_id TEXT,
|
parent_task_id TEXT,
|
||||||
correlation_id TEXT,
|
correlation_id TEXT,
|
||||||
@@ -279,21 +272,6 @@ class TaskPlaneStore:
|
|||||||
)
|
)
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
self._ensure_columns(
|
|
||||||
"tasks",
|
|
||||||
{
|
|
||||||
"scope_type": "TEXT NOT NULL DEFAULT 'legacy'",
|
|
||||||
"tenant_id": "TEXT",
|
|
||||||
"user_id": "TEXT",
|
|
||||||
"project_id": "TEXT",
|
|
||||||
"conversation_id": "TEXT",
|
|
||||||
"run_id": "TEXT",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
self.conn.execute(
|
|
||||||
"UPDATE tasks SET scope_type='legacy' "
|
|
||||||
"WHERE scope_type IS NULL OR TRIM(scope_type)=''"
|
|
||||||
)
|
|
||||||
self._ensure_columns(
|
self._ensure_columns(
|
||||||
"workers",
|
"workers",
|
||||||
{
|
{
|
||||||
@@ -321,14 +299,6 @@ class TaskPlaneStore:
|
|||||||
)
|
)
|
||||||
self.conn.execute("CREATE INDEX IF NOT EXISTS ix_tasks_status ON tasks(status)")
|
self.conn.execute("CREATE INDEX IF NOT EXISTS ix_tasks_status ON tasks(status)")
|
||||||
self.conn.execute("CREATE INDEX IF NOT EXISTS ix_tasks_parent ON tasks(parent_task_id)")
|
self.conn.execute("CREATE INDEX IF NOT EXISTS ix_tasks_parent ON tasks(parent_task_id)")
|
||||||
self.conn.execute(
|
|
||||||
"CREATE INDEX IF NOT EXISTS ix_tasks_scope_tenant_project "
|
|
||||||
"ON tasks(scope_type, tenant_id, project_id)"
|
|
||||||
)
|
|
||||||
self.conn.execute(
|
|
||||||
"CREATE INDEX IF NOT EXISTS ix_tasks_scope_owner "
|
|
||||||
"ON tasks(scope_type, tenant_id, user_id, conversation_id, run_id)"
|
|
||||||
)
|
|
||||||
self.conn.execute("CREATE INDEX IF NOT EXISTS ix_workers_status ON workers(status)")
|
self.conn.execute("CREATE INDEX IF NOT EXISTS ix_workers_status ON workers(status)")
|
||||||
self.conn.execute("CREATE INDEX IF NOT EXISTS ix_claims_task ON claims(task_id)")
|
self.conn.execute("CREATE INDEX IF NOT EXISTS ix_claims_task ON claims(task_id)")
|
||||||
self.conn.execute(
|
self.conn.execute(
|
||||||
@@ -377,42 +347,6 @@ class TaskPlaneStore:
|
|||||||
raise NotFoundError("task")
|
raise NotFoundError("task")
|
||||||
task_json = json.loads(row["task_json"])
|
task_json = json.loads(row["task_json"])
|
||||||
task_plan = TaskPlan.from_dict(task_json)
|
task_plan = TaskPlan.from_dict(task_json)
|
||||||
stored_scope = tuple(
|
|
||||||
row[field]
|
|
||||||
for field in (
|
|
||||||
"scope_type",
|
|
||||||
"tenant_id",
|
|
||||||
"user_id",
|
|
||||||
"project_id",
|
|
||||||
"conversation_id",
|
|
||||||
"run_id",
|
|
||||||
)
|
|
||||||
)
|
|
||||||
json_has_scope = any(
|
|
||||||
field in task_json
|
|
||||||
for field in (
|
|
||||||
"scope_type",
|
|
||||||
"tenant_id",
|
|
||||||
"user_id",
|
|
||||||
"project_id",
|
|
||||||
"conversation_id",
|
|
||||||
"run_id",
|
|
||||||
)
|
|
||||||
)
|
|
||||||
if json_has_scope and stored_scope != task_scope_values(task_plan):
|
|
||||||
raise ConflictError(
|
|
||||||
"task scope persistence mismatch",
|
|
||||||
code="TASK_SCOPE_PERSISTENCE_MISMATCH",
|
|
||||||
)
|
|
||||||
if not json_has_scope:
|
|
||||||
(
|
|
||||||
task_plan.scope_type,
|
|
||||||
task_plan.tenant_id,
|
|
||||||
task_plan.user_id,
|
|
||||||
task_plan.project_id,
|
|
||||||
task_plan.conversation_id,
|
|
||||||
task_plan.run_id,
|
|
||||||
) = stored_scope
|
|
||||||
task_plan.parent_task_id = row["parent_task_id"]
|
task_plan.parent_task_id = row["parent_task_id"]
|
||||||
task_plan.correlation_id = row["correlation_id"]
|
task_plan.correlation_id = row["correlation_id"]
|
||||||
return TaskRecord(
|
return TaskRecord(
|
||||||
@@ -455,7 +389,6 @@ class TaskPlaneStore:
|
|||||||
task_id = task_plan.task_id or f"task-{uuid.uuid4()}"
|
task_id = task_plan.task_id or f"task-{uuid.uuid4()}"
|
||||||
task_plan.task_id = task_id
|
task_plan.task_id = task_id
|
||||||
task_json = redact_value(task_plan.to_dict(redact=False))
|
task_json = redact_value(task_plan.to_dict(redact=False))
|
||||||
scope_values = task_scope_values(task_plan)
|
|
||||||
now = _now()
|
now = _now()
|
||||||
with self._transaction():
|
with self._transaction():
|
||||||
if idempotency_key:
|
if idempotency_key:
|
||||||
@@ -464,11 +397,7 @@ class TaskPlaneStore:
|
|||||||
(idempotency_key,),
|
(idempotency_key,),
|
||||||
).fetchone()
|
).fetchone()
|
||||||
if existing:
|
if existing:
|
||||||
existing_plan = TaskPlan.from_dict(
|
if _canonical(json.loads(existing["task_json"])) != _canonical(task_json):
|
||||||
json.loads(existing["task_json"])
|
|
||||||
)
|
|
||||||
existing_json = redact_value(existing_plan.to_dict(redact=False))
|
|
||||||
if _canonical(existing_json) != _canonical(task_json):
|
|
||||||
raise ConflictError("idempotency key payload mismatch")
|
raise ConflictError("idempotency key payload mismatch")
|
||||||
return self.get_task(existing["task_id"])
|
return self.get_task(existing["task_id"])
|
||||||
root_task_id = task_id
|
root_task_id = task_id
|
||||||
@@ -483,18 +412,14 @@ class TaskPlaneStore:
|
|||||||
self.conn.execute(
|
self.conn.execute(
|
||||||
"""
|
"""
|
||||||
INSERT INTO tasks
|
INSERT INTO tasks
|
||||||
(task_id, revision, status, task_json, task_type,
|
(task_id, revision, status, task_json, task_type, root_task_id, parent_task_id, correlation_id, idempotency_key, created_at, updated_at, failure_reason)
|
||||||
scope_type, tenant_id, user_id, project_id, conversation_id, run_id,
|
VALUES (?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL)
|
||||||
root_task_id, parent_task_id, correlation_id, idempotency_key,
|
|
||||||
created_at, updated_at, failure_reason)
|
|
||||||
VALUES (?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL)
|
|
||||||
""",
|
""",
|
||||||
(
|
(
|
||||||
task_id,
|
task_id,
|
||||||
"CREATED",
|
"CREATED",
|
||||||
json.dumps(task_json, ensure_ascii=False),
|
json.dumps(task_json, ensure_ascii=False),
|
||||||
task_plan.task_type,
|
task_plan.task_type,
|
||||||
*scope_values,
|
|
||||||
root_task_id,
|
root_task_id,
|
||||||
task_plan.parent_task_id,
|
task_plan.parent_task_id,
|
||||||
task_plan.correlation_id,
|
task_plan.correlation_id,
|
||||||
@@ -526,30 +451,21 @@ class TaskPlaneStore:
|
|||||||
)
|
)
|
||||||
return self.get_task(task.task_id)
|
return self.get_task(task.task_id)
|
||||||
|
|
||||||
def get_task(
|
def get_task(self, task_id: str) -> TaskRecord:
|
||||||
self, task_id: str, actor_context: TaskActorContext | None = None
|
|
||||||
) -> TaskRecord:
|
|
||||||
task_id = _validated_id(task_id, "task_id")
|
task_id = _validated_id(task_id, "task_id")
|
||||||
predicate, scope_params = actor_scope_sql(actor_context)
|
|
||||||
sql = "SELECT * FROM tasks WHERE task_id=?"
|
|
||||||
if predicate:
|
|
||||||
sql += f" AND ({predicate})"
|
|
||||||
with self._lock:
|
with self._lock:
|
||||||
row = self.conn.execute(sql, (task_id, *scope_params)).fetchone()
|
row = self.conn.execute(
|
||||||
|
"SELECT * FROM tasks WHERE task_id=?", (task_id,)
|
||||||
|
).fetchone()
|
||||||
if row is None:
|
if row is None:
|
||||||
raise NotFoundError(f"task {task_id} not found")
|
raise NotFoundError(f"task {task_id} not found")
|
||||||
return self._row_to_task(row)
|
return self._row_to_task(row)
|
||||||
|
|
||||||
def list_tasks(
|
def list_tasks(self) -> list[TaskRecord]:
|
||||||
self, actor_context: TaskActorContext | None = None
|
|
||||||
) -> list[TaskRecord]:
|
|
||||||
predicate, scope_params = actor_scope_sql(actor_context)
|
|
||||||
sql = "SELECT * FROM tasks"
|
|
||||||
if predicate:
|
|
||||||
sql += f" WHERE ({predicate})"
|
|
||||||
sql += " ORDER BY created_at DESC"
|
|
||||||
with self._lock:
|
with self._lock:
|
||||||
rows = self.conn.execute(sql, scope_params).fetchall()
|
rows = self.conn.execute(
|
||||||
|
"SELECT * FROM tasks ORDER BY created_at DESC"
|
||||||
|
).fetchall()
|
||||||
return [self._row_to_task(row) for row in rows]
|
return [self._row_to_task(row) for row in rows]
|
||||||
|
|
||||||
def get_task_budget(self, task_id: str) -> Budget:
|
def get_task_budget(self, task_id: str) -> Budget:
|
||||||
|
|||||||
@@ -1,759 +0,0 @@
|
|||||||
"""Scope-bound governance for traces, Tool/MCP calls, and credentials.
|
|
||||||
|
|
||||||
This module deliberately stores references and audit metadata only. Secret
|
|
||||||
material remains in the configured credential provider boundary.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from dataclasses import asdict, dataclass, field
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
from contextlib import contextmanager
|
|
||||||
import hashlib
|
|
||||||
import json
|
|
||||||
import re
|
|
||||||
import sqlite3
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any, Iterable, Mapping
|
|
||||||
|
|
||||||
|
|
||||||
SCOPE_FIELDS = ("tenant_id", "user_id", "project_id", "conversation_id", "run_id")
|
|
||||||
PRIVILEGED_CAPABILITIES = frozenset({"*", "admin", "system.admin", "governance.admin"})
|
|
||||||
REFERENCE_RE = re.compile(r"^(systemd-credential|vault|secret-manager):[A-Za-z0-9][A-Za-z0-9._:/#-]{0,255}$")
|
|
||||||
IDENTIFIER_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,179}$")
|
|
||||||
|
|
||||||
|
|
||||||
class GovernanceError(RuntimeError):
|
|
||||||
def __init__(self, code: str, *, field: str | None = None) -> None:
|
|
||||||
super().__init__(code if field is None else f"{code}:{field}")
|
|
||||||
self.code = code
|
|
||||||
self.field = field
|
|
||||||
|
|
||||||
|
|
||||||
def _clean(value: Any) -> str | None:
|
|
||||||
if value is None:
|
|
||||||
return None
|
|
||||||
text = str(value).strip()
|
|
||||||
return text or None
|
|
||||||
|
|
||||||
|
|
||||||
def _scope_tuple(value: Iterable[str] | str | None) -> tuple[str, ...]:
|
|
||||||
if value is None:
|
|
||||||
return ()
|
|
||||||
if isinstance(value, str):
|
|
||||||
value = value.split(",")
|
|
||||||
return tuple(sorted({_clean(item) for item in value if _clean(item)}))
|
|
||||||
|
|
||||||
|
|
||||||
def _canonical(value: Any) -> str:
|
|
||||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, default=str)
|
|
||||||
|
|
||||||
|
|
||||||
def _sha(value: Any) -> str:
|
|
||||||
return hashlib.sha256(_canonical(value).encode("utf-8")).hexdigest()
|
|
||||||
|
|
||||||
|
|
||||||
def _now() -> str:
|
|
||||||
return datetime.now(timezone.utc).isoformat()
|
|
||||||
|
|
||||||
|
|
||||||
def _identifier(value: Any, code: str) -> str:
|
|
||||||
text = _clean(value)
|
|
||||||
if text is None or IDENTIFIER_RE.fullmatch(text) is None:
|
|
||||||
raise GovernanceError(code)
|
|
||||||
return text
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class GovernanceScope:
|
|
||||||
"""Explicit resource scope; system/global resources never get fake tenants."""
|
|
||||||
|
|
||||||
scope_type: str
|
|
||||||
tenant_id: str | None = None
|
|
||||||
user_id: str | None = None
|
|
||||||
project_id: str | None = None
|
|
||||||
conversation_id: str | None = None
|
|
||||||
run_id: str | None = None
|
|
||||||
|
|
||||||
def __post_init__(self) -> None:
|
|
||||||
object.__setattr__(self, "scope_type", (_clean(self.scope_type) or "").lower())
|
|
||||||
for name in SCOPE_FIELDS:
|
|
||||||
object.__setattr__(self, name, _clean(getattr(self, name)))
|
|
||||||
if self.scope_type == "tenant":
|
|
||||||
if self.tenant_id is None:
|
|
||||||
raise GovernanceError("GOVERNANCE_TENANT_ID_REQUIRED")
|
|
||||||
elif self.scope_type in {"system", "global"}:
|
|
||||||
if any(getattr(self, name) is not None for name in SCOPE_FIELDS):
|
|
||||||
raise GovernanceError("GOVERNANCE_NON_TENANT_SCOPE_INVALID")
|
|
||||||
elif self.scope_type == "legacy":
|
|
||||||
if any(getattr(self, name) is not None for name in SCOPE_FIELDS):
|
|
||||||
raise GovernanceError("GOVERNANCE_LEGACY_SCOPE_INVALID")
|
|
||||||
else:
|
|
||||||
raise GovernanceError("GOVERNANCE_SCOPE_TYPE_INVALID")
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_value(cls, value: Any) -> "GovernanceScope":
|
|
||||||
def get(name: str) -> Any:
|
|
||||||
return value.get(name) if isinstance(value, Mapping) else getattr(value, name, None)
|
|
||||||
|
|
||||||
declared = _clean(get("scope_type"))
|
|
||||||
tenant_id = _clean(get("tenant_id"))
|
|
||||||
return cls(
|
|
||||||
scope_type=declared or ("tenant" if tenant_id else "legacy"),
|
|
||||||
tenant_id=tenant_id,
|
|
||||||
user_id=get("user_id"),
|
|
||||||
project_id=get("project_id"),
|
|
||||||
conversation_id=get("conversation_id"),
|
|
||||||
run_id=get("run_id"),
|
|
||||||
)
|
|
||||||
|
|
||||||
def as_dict(self) -> dict[str, Any]:
|
|
||||||
return asdict(self)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class GovernanceActor:
|
|
||||||
identity: str
|
|
||||||
scope: GovernanceScope
|
|
||||||
capabilities: tuple[str, ...] = ()
|
|
||||||
|
|
||||||
def __post_init__(self) -> None:
|
|
||||||
object.__setattr__(self, "identity", _identifier(self.identity, "GOVERNANCE_ACTOR_INVALID"))
|
|
||||||
object.__setattr__(self, "capabilities", _scope_tuple(self.capabilities))
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_value(cls, value: Any) -> "GovernanceActor":
|
|
||||||
if isinstance(value, cls):
|
|
||||||
return value
|
|
||||||
if not isinstance(value, Mapping):
|
|
||||||
raise GovernanceError("GOVERNANCE_ACTOR_INVALID")
|
|
||||||
return cls(
|
|
||||||
identity=value.get("identity") or value.get("actor_id") or value.get("user_id"),
|
|
||||||
scope=GovernanceScope.from_value(value),
|
|
||||||
capabilities=_scope_tuple(value.get("capabilities") or value.get("capability_scope")),
|
|
||||||
)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def privileged(self) -> bool:
|
|
||||||
return bool(set(self.capabilities) & PRIVILEGED_CAPABILITIES)
|
|
||||||
|
|
||||||
|
|
||||||
def authorize_scope(actor: GovernanceActor | Mapping[str, Any], resource: GovernanceScope | Mapping[str, Any]) -> None:
|
|
||||||
actor = GovernanceActor.from_value(actor)
|
|
||||||
resource = resource if isinstance(resource, GovernanceScope) else GovernanceScope.from_value(resource)
|
|
||||||
if actor.privileged:
|
|
||||||
return
|
|
||||||
if resource.scope_type in {"system", "global"}:
|
|
||||||
raise GovernanceError("GOVERNANCE_SYSTEM_SCOPE_AUTHZ_DENY")
|
|
||||||
if resource.scope_type == "legacy":
|
|
||||||
if actor.scope.scope_type != "legacy":
|
|
||||||
raise GovernanceError("GOVERNANCE_LEGACY_SCOPE_AUTHZ_DENY")
|
|
||||||
return
|
|
||||||
if actor.scope.scope_type != "tenant":
|
|
||||||
raise GovernanceError("GOVERNANCE_SCOPE_MISMATCH_DENY", field="scope_type")
|
|
||||||
for name in SCOPE_FIELDS:
|
|
||||||
expected = getattr(resource, name)
|
|
||||||
actual = getattr(actor.scope, name)
|
|
||||||
if expected is not None and expected != actual:
|
|
||||||
raise GovernanceError("GOVERNANCE_SCOPE_MISMATCH_DENY", field=name)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class TraceContext:
|
|
||||||
trace_id: str
|
|
||||||
span_id: str
|
|
||||||
scope: GovernanceScope
|
|
||||||
parent_span_id: str | None = None
|
|
||||||
correlation_id: str | None = None
|
|
||||||
causation_id: str | None = None
|
|
||||||
task_id: str | None = None
|
|
||||||
claim_id: str | None = None
|
|
||||||
message_id: str | None = None
|
|
||||||
|
|
||||||
def __post_init__(self) -> None:
|
|
||||||
object.__setattr__(self, "trace_id", _identifier(self.trace_id, "TRACE_ID_INVALID"))
|
|
||||||
object.__setattr__(self, "span_id", _identifier(self.span_id, "TRACE_SPAN_ID_INVALID"))
|
|
||||||
for name in ("parent_span_id", "correlation_id", "causation_id", "task_id", "claim_id", "message_id"):
|
|
||||||
value = _clean(getattr(self, name))
|
|
||||||
if value is not None and IDENTIFIER_RE.fullmatch(value) is None:
|
|
||||||
raise GovernanceError("TRACE_LINEAGE_IDENTIFIER_INVALID", field=name)
|
|
||||||
object.__setattr__(self, name, value)
|
|
||||||
|
|
||||||
def child(self, *, span_id: str, actor: GovernanceActor | Mapping[str, Any], **lineage: Any) -> "TraceContext":
|
|
||||||
authorize_scope(actor, self.scope)
|
|
||||||
for name in ("trace_id", "task_id", "claim_id", "message_id", "correlation_id"):
|
|
||||||
supplied = _clean(lineage.get(name))
|
|
||||||
expected = getattr(self, name)
|
|
||||||
if supplied is not None and expected is not None and supplied != expected:
|
|
||||||
raise GovernanceError("TRACE_LINEAGE_CONTINUITY", field=name)
|
|
||||||
requested_scope = lineage.pop("scope", self.scope)
|
|
||||||
requested_scope = requested_scope if isinstance(requested_scope, GovernanceScope) else GovernanceScope.from_value(requested_scope)
|
|
||||||
if requested_scope != self.scope:
|
|
||||||
raise GovernanceError("TRACE_SCOPE_MISMATCH_DENY")
|
|
||||||
return TraceContext(
|
|
||||||
trace_id=self.trace_id,
|
|
||||||
span_id=span_id,
|
|
||||||
parent_span_id=self.span_id,
|
|
||||||
correlation_id=self.correlation_id,
|
|
||||||
causation_id=_clean(lineage.get("causation_id")) or self.span_id,
|
|
||||||
task_id=self.task_id,
|
|
||||||
claim_id=self.claim_id,
|
|
||||||
message_id=self.message_id,
|
|
||||||
scope=self.scope,
|
|
||||||
)
|
|
||||||
|
|
||||||
def as_dict(self) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"trace_id": self.trace_id,
|
|
||||||
"span_id": self.span_id,
|
|
||||||
"parent_span_id": self.parent_span_id,
|
|
||||||
"correlation_id": self.correlation_id,
|
|
||||||
"causation_id": self.causation_id,
|
|
||||||
"task_id": self.task_id,
|
|
||||||
"claim_id": self.claim_id,
|
|
||||||
"message_id": self.message_id,
|
|
||||||
**self.scope.as_dict(),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def bind_trace_lineage(
|
|
||||||
*,
|
|
||||||
actor: GovernanceActor | Mapping[str, Any],
|
|
||||||
task: Any,
|
|
||||||
message: Any,
|
|
||||||
trace_id: str,
|
|
||||||
span_id: str,
|
|
||||||
parent: TraceContext | None = None,
|
|
||||||
) -> TraceContext:
|
|
||||||
actor = GovernanceActor.from_value(actor)
|
|
||||||
task_scope = GovernanceScope.from_value(task)
|
|
||||||
message_scope = GovernanceScope.from_value(message)
|
|
||||||
if task_scope != message_scope:
|
|
||||||
raise GovernanceError("TRACE_SCOPE_MISMATCH_DENY")
|
|
||||||
authorize_scope(actor, task_scope)
|
|
||||||
|
|
||||||
def get(value: Any, name: str) -> Any:
|
|
||||||
return value.get(name) if isinstance(value, Mapping) else getattr(value, name, None)
|
|
||||||
|
|
||||||
task_id = _clean(get(task, "task_id"))
|
|
||||||
message_task = _clean(get(message, "task_id"))
|
|
||||||
if task_id and message_task and task_id != message_task:
|
|
||||||
raise GovernanceError("TRACE_LINEAGE_CONTINUITY", field="task_id")
|
|
||||||
context = TraceContext(
|
|
||||||
trace_id=trace_id,
|
|
||||||
span_id=span_id,
|
|
||||||
parent_span_id=parent.span_id if parent else None,
|
|
||||||
correlation_id=_clean(get(task, "correlation_id")) or _clean(get(message, "correlation_id")),
|
|
||||||
causation_id=_clean(get(message, "causation_id")) or (parent.span_id if parent else None),
|
|
||||||
task_id=task_id or message_task,
|
|
||||||
claim_id=_clean(get(task, "claim_id")) or _clean(get(message, "claim_id")),
|
|
||||||
message_id=_clean(get(message, "envelope_id")) or _clean(get(message, "id")),
|
|
||||||
scope=task_scope,
|
|
||||||
)
|
|
||||||
if parent is not None:
|
|
||||||
if parent.trace_id != context.trace_id or parent.scope != context.scope:
|
|
||||||
raise GovernanceError("TRACE_LINEAGE_CONTINUITY")
|
|
||||||
if parent.task_id and context.task_id and parent.task_id != context.task_id:
|
|
||||||
raise GovernanceError("TRACE_LINEAGE_CONTINUITY", field="task_id")
|
|
||||||
return context
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class ToolDescriptor:
|
|
||||||
name: str
|
|
||||||
kind: str
|
|
||||||
scope: GovernanceScope
|
|
||||||
required_capabilities: tuple[str, ...] = ()
|
|
||||||
credential_ref: str | None = None
|
|
||||||
|
|
||||||
def __post_init__(self) -> None:
|
|
||||||
object.__setattr__(self, "name", _identifier(self.name, "TOOL_NAME_INVALID"))
|
|
||||||
object.__setattr__(self, "kind", (_clean(self.kind) or "").lower())
|
|
||||||
if self.kind not in {"tool", "mcp"}:
|
|
||||||
raise GovernanceError("TOOL_KIND_INVALID")
|
|
||||||
object.__setattr__(self, "required_capabilities", _scope_tuple(self.required_capabilities))
|
|
||||||
if self.credential_ref is not None and REFERENCE_RE.fullmatch(self.credential_ref) is None:
|
|
||||||
raise GovernanceError("CREDENTIAL_SECRET_EXPOSURE_DENY")
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class CredentialGrant:
|
|
||||||
grant_id: str
|
|
||||||
credential_ref: str
|
|
||||||
scope: GovernanceScope
|
|
||||||
operations: tuple[str, ...]
|
|
||||||
task_id: str | None = None
|
|
||||||
tool_name: str | None = None
|
|
||||||
expires_at: str | None = None
|
|
||||||
|
|
||||||
def __post_init__(self) -> None:
|
|
||||||
object.__setattr__(self, "grant_id", _identifier(self.grant_id, "CREDENTIAL_GRANT_ID_INVALID"))
|
|
||||||
if REFERENCE_RE.fullmatch(self.credential_ref or "") is None:
|
|
||||||
raise GovernanceError("CREDENTIAL_SECRET_EXPOSURE_DENY")
|
|
||||||
object.__setattr__(self, "operations", _scope_tuple(self.operations))
|
|
||||||
if not self.operations:
|
|
||||||
raise GovernanceError("CREDENTIAL_OPERATIONS_REQUIRED")
|
|
||||||
if self.expires_at is not None:
|
|
||||||
parsed = datetime.fromisoformat(self.expires_at.replace("Z", "+00:00"))
|
|
||||||
if parsed.tzinfo is None:
|
|
||||||
raise GovernanceError("CREDENTIAL_EXPIRY_INVALID")
|
|
||||||
|
|
||||||
def authorize(
|
|
||||||
self,
|
|
||||||
actor: GovernanceActor | Mapping[str, Any],
|
|
||||||
*,
|
|
||||||
operation: str,
|
|
||||||
task_id: str | None = None,
|
|
||||||
tool_name: str | None = None,
|
|
||||||
now: datetime | None = None,
|
|
||||||
) -> str:
|
|
||||||
try:
|
|
||||||
authorize_scope(actor, self.scope)
|
|
||||||
except GovernanceError as exc:
|
|
||||||
raise GovernanceError("CREDENTIAL_SCOPE_DENY", field=exc.field) from exc
|
|
||||||
if operation not in self.operations:
|
|
||||||
raise GovernanceError("CREDENTIAL_SCOPE_DENY", field="operation")
|
|
||||||
if self.task_id is not None and self.task_id != _clean(task_id):
|
|
||||||
raise GovernanceError("CREDENTIAL_SCOPE_DENY", field="task_id")
|
|
||||||
if self.tool_name is not None and self.tool_name != _clean(tool_name):
|
|
||||||
raise GovernanceError("CREDENTIAL_SCOPE_DENY", field="tool_name")
|
|
||||||
if self.expires_at is not None:
|
|
||||||
current = (now or datetime.now(timezone.utc)).astimezone(timezone.utc)
|
|
||||||
expiry = datetime.fromisoformat(self.expires_at.replace("Z", "+00:00")).astimezone(timezone.utc)
|
|
||||||
if current >= expiry:
|
|
||||||
raise GovernanceError("CREDENTIAL_SCOPE_DENY", field="expires_at")
|
|
||||||
return self.credential_ref
|
|
||||||
|
|
||||||
def public(self) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"grant_id": self.grant_id,
|
|
||||||
"credential_ref": self.credential_ref,
|
|
||||||
"operations": list(self.operations),
|
|
||||||
"task_id": self.task_id,
|
|
||||||
"tool_name": self.tool_name,
|
|
||||||
"expires_at": self.expires_at,
|
|
||||||
**self.scope.as_dict(),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
_SCHEMA = """
|
|
||||||
CREATE TABLE IF NOT EXISTS governance_schema_metadata (
|
|
||||||
singleton INTEGER PRIMARY KEY CHECK(singleton=1),
|
|
||||||
schema_version INTEGER NOT NULL
|
|
||||||
);
|
|
||||||
CREATE TABLE IF NOT EXISTS governed_tools (
|
|
||||||
name TEXT PRIMARY KEY,
|
|
||||||
kind TEXT NOT NULL,
|
|
||||||
scope_json TEXT NOT NULL,
|
|
||||||
required_capabilities_json TEXT NOT NULL,
|
|
||||||
credential_ref TEXT,
|
|
||||||
descriptor_sha256 TEXT NOT NULL
|
|
||||||
);
|
|
||||||
CREATE TABLE IF NOT EXISTS governed_trace_spans (
|
|
||||||
trace_id TEXT NOT NULL,
|
|
||||||
span_id TEXT NOT NULL,
|
|
||||||
parent_span_id TEXT,
|
|
||||||
context_json TEXT NOT NULL,
|
|
||||||
context_sha256 TEXT NOT NULL,
|
|
||||||
created_at TEXT NOT NULL,
|
|
||||||
PRIMARY KEY(trace_id, span_id)
|
|
||||||
);
|
|
||||||
CREATE TABLE IF NOT EXISTS governed_credential_grants (
|
|
||||||
grant_id TEXT PRIMARY KEY,
|
|
||||||
grant_json TEXT NOT NULL,
|
|
||||||
grant_sha256 TEXT NOT NULL,
|
|
||||||
created_at TEXT NOT NULL
|
|
||||||
);
|
|
||||||
CREATE TABLE IF NOT EXISTS governed_tool_invocations (
|
|
||||||
invocation_key TEXT PRIMARY KEY,
|
|
||||||
request_sha256 TEXT NOT NULL,
|
|
||||||
tool_name TEXT NOT NULL,
|
|
||||||
task_id TEXT NOT NULL,
|
|
||||||
claim_id TEXT NOT NULL,
|
|
||||||
trace_id TEXT NOT NULL,
|
|
||||||
span_id TEXT NOT NULL,
|
|
||||||
status TEXT NOT NULL,
|
|
||||||
receipt_sha256 TEXT,
|
|
||||||
created_at TEXT NOT NULL,
|
|
||||||
updated_at TEXT NOT NULL
|
|
||||||
);
|
|
||||||
CREATE TABLE IF NOT EXISTS governance_audit_evidence (
|
|
||||||
sequence INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
||||||
evidence_id TEXT NOT NULL UNIQUE,
|
|
||||||
action TEXT NOT NULL,
|
|
||||||
actor_identity TEXT NOT NULL,
|
|
||||||
trace_id TEXT,
|
|
||||||
span_id TEXT,
|
|
||||||
resource_id TEXT NOT NULL,
|
|
||||||
outcome TEXT NOT NULL,
|
|
||||||
details_json TEXT NOT NULL,
|
|
||||||
previous_sha256 TEXT NOT NULL,
|
|
||||||
evidence_sha256 TEXT NOT NULL,
|
|
||||||
created_at TEXT NOT NULL
|
|
||||||
);
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
class GovernanceStore:
|
|
||||||
def __init__(self, path: str | Path) -> None:
|
|
||||||
self.path = Path(path)
|
|
||||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
with self._connection() as connection:
|
|
||||||
has_metadata = connection.execute(
|
|
||||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='governance_schema_metadata'"
|
|
||||||
).fetchone()
|
|
||||||
if has_metadata:
|
|
||||||
row = connection.execute(
|
|
||||||
"SELECT schema_version FROM governance_schema_metadata WHERE singleton=1"
|
|
||||||
).fetchone()
|
|
||||||
if row is not None and int(row[0]) != 1:
|
|
||||||
raise GovernanceError("GOVERNANCE_SCHEMA_VERSION_UNSUPPORTED")
|
|
||||||
connection.executescript(_SCHEMA)
|
|
||||||
row = connection.execute(
|
|
||||||
"SELECT schema_version FROM governance_schema_metadata WHERE singleton=1"
|
|
||||||
).fetchone()
|
|
||||||
if row is None:
|
|
||||||
connection.execute(
|
|
||||||
"INSERT INTO governance_schema_metadata(singleton,schema_version) VALUES(1,1)"
|
|
||||||
)
|
|
||||||
connection.commit()
|
|
||||||
|
|
||||||
def _connect(self) -> sqlite3.Connection:
|
|
||||||
connection = sqlite3.connect(self.path, timeout=30.0)
|
|
||||||
connection.row_factory = sqlite3.Row
|
|
||||||
connection.execute("PRAGMA foreign_keys=ON")
|
|
||||||
connection.execute("PRAGMA busy_timeout=30000")
|
|
||||||
return connection
|
|
||||||
|
|
||||||
@contextmanager
|
|
||||||
def _connection(self):
|
|
||||||
connection = self._connect()
|
|
||||||
try:
|
|
||||||
yield connection
|
|
||||||
connection.commit()
|
|
||||||
except Exception:
|
|
||||||
connection.rollback()
|
|
||||||
raise
|
|
||||||
finally:
|
|
||||||
connection.close()
|
|
||||||
|
|
||||||
def _audit(
|
|
||||||
self,
|
|
||||||
connection: sqlite3.Connection,
|
|
||||||
*,
|
|
||||||
action: str,
|
|
||||||
actor: GovernanceActor,
|
|
||||||
resource_id: str,
|
|
||||||
outcome: str,
|
|
||||||
trace: TraceContext | None,
|
|
||||||
details: Mapping[str, Any],
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
previous = connection.execute(
|
|
||||||
"SELECT evidence_sha256 FROM governance_audit_evidence ORDER BY sequence DESC LIMIT 1"
|
|
||||||
).fetchone()
|
|
||||||
previous_sha = str(previous[0]) if previous else ""
|
|
||||||
created_at = _now()
|
|
||||||
material = {
|
|
||||||
"action": action,
|
|
||||||
"actor_identity": actor.identity,
|
|
||||||
"trace_id": trace.trace_id if trace else None,
|
|
||||||
"span_id": trace.span_id if trace else None,
|
|
||||||
"resource_id": resource_id,
|
|
||||||
"outcome": outcome,
|
|
||||||
"details": dict(details),
|
|
||||||
"previous_sha256": previous_sha,
|
|
||||||
"created_at": created_at,
|
|
||||||
}
|
|
||||||
digest = _sha(material)
|
|
||||||
evidence_id = "gev-" + digest[:32]
|
|
||||||
connection.execute(
|
|
||||||
"""INSERT INTO governance_audit_evidence
|
|
||||||
(evidence_id,action,actor_identity,trace_id,span_id,resource_id,outcome,
|
|
||||||
details_json,previous_sha256,evidence_sha256,created_at)
|
|
||||||
VALUES (?,?,?,?,?,?,?,?,?,?,?)""",
|
|
||||||
(
|
|
||||||
evidence_id, action, actor.identity, material["trace_id"], material["span_id"],
|
|
||||||
resource_id, outcome, _canonical(material["details"]), previous_sha, digest, created_at,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
return {"evidence_id": evidence_id, "evidence_sha256": digest, **material}
|
|
||||||
|
|
||||||
def record_trace(self, trace: TraceContext, actor: GovernanceActor | Mapping[str, Any]) -> dict[str, Any]:
|
|
||||||
actor = GovernanceActor.from_value(actor)
|
|
||||||
authorize_scope(actor, trace.scope)
|
|
||||||
context = trace.as_dict()
|
|
||||||
digest = _sha(context)
|
|
||||||
with self._connection() as connection:
|
|
||||||
existing = connection.execute(
|
|
||||||
"SELECT context_sha256 FROM governed_trace_spans WHERE trace_id=? AND span_id=?",
|
|
||||||
(trace.trace_id, trace.span_id),
|
|
||||||
).fetchone()
|
|
||||||
if existing is not None and existing[0] != digest:
|
|
||||||
raise GovernanceError("TRACE_LINEAGE_CONFLICT")
|
|
||||||
if existing is not None:
|
|
||||||
return {"context": context, "context_sha256": digest, "idempotent": True}
|
|
||||||
if trace.parent_span_id:
|
|
||||||
parent = connection.execute(
|
|
||||||
"SELECT context_json FROM governed_trace_spans WHERE trace_id=? AND span_id=?",
|
|
||||||
(trace.trace_id, trace.parent_span_id),
|
|
||||||
).fetchone()
|
|
||||||
if parent is None:
|
|
||||||
raise GovernanceError("TRACE_LINEAGE_CONTINUITY", field="parent_span_id")
|
|
||||||
parent_context = json.loads(parent[0])
|
|
||||||
if GovernanceScope.from_value(parent_context) != trace.scope:
|
|
||||||
raise GovernanceError("TRACE_SCOPE_MISMATCH_DENY")
|
|
||||||
connection.execute(
|
|
||||||
"INSERT OR IGNORE INTO governed_trace_spans VALUES (?,?,?,?,?,?)",
|
|
||||||
(trace.trace_id, trace.span_id, trace.parent_span_id, _canonical(context), digest, _now()),
|
|
||||||
)
|
|
||||||
evidence = self._audit(
|
|
||||||
connection, action="TRACE_RECORDED", actor=actor,
|
|
||||||
resource_id=f"{trace.trace_id}:{trace.span_id}", outcome="ALLOW", trace=trace,
|
|
||||||
details={"context_sha256": digest},
|
|
||||||
)
|
|
||||||
return {"context": context, "context_sha256": digest, "evidence": evidence}
|
|
||||||
|
|
||||||
def register_tool(self, descriptor: ToolDescriptor, actor: GovernanceActor | Mapping[str, Any]) -> dict[str, Any]:
|
|
||||||
actor = GovernanceActor.from_value(actor)
|
|
||||||
if not actor.privileged:
|
|
||||||
raise GovernanceError("TOOL_MCP_AUTHZ_DENY")
|
|
||||||
public = {
|
|
||||||
"name": descriptor.name,
|
|
||||||
"kind": descriptor.kind,
|
|
||||||
"required_capabilities": list(descriptor.required_capabilities),
|
|
||||||
"credential_ref": descriptor.credential_ref,
|
|
||||||
**descriptor.scope.as_dict(),
|
|
||||||
}
|
|
||||||
digest = _sha(public)
|
|
||||||
with self._connection() as connection:
|
|
||||||
existing = connection.execute(
|
|
||||||
"SELECT descriptor_sha256 FROM governed_tools WHERE name=?", (descriptor.name,)
|
|
||||||
).fetchone()
|
|
||||||
if existing is not None and existing[0] != digest:
|
|
||||||
raise GovernanceError("TOOL_DESCRIPTOR_CONFLICT")
|
|
||||||
if existing is not None:
|
|
||||||
return {"descriptor": public, "descriptor_sha256": digest, "idempotent": True}
|
|
||||||
connection.execute(
|
|
||||||
"INSERT OR IGNORE INTO governed_tools VALUES (?,?,?,?,?,?)",
|
|
||||||
(descriptor.name, descriptor.kind, _canonical(descriptor.scope.as_dict()),
|
|
||||||
_canonical(descriptor.required_capabilities), descriptor.credential_ref, digest),
|
|
||||||
)
|
|
||||||
evidence = self._audit(
|
|
||||||
connection, action="TOOL_REGISTERED", actor=actor, resource_id=descriptor.name,
|
|
||||||
outcome="ALLOW", trace=None, details={"descriptor_sha256": digest},
|
|
||||||
)
|
|
||||||
return {"descriptor": public, "descriptor_sha256": digest, "evidence": evidence}
|
|
||||||
|
|
||||||
def authorize_tool(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
actor: GovernanceActor | Mapping[str, Any],
|
|
||||||
tool_name: str,
|
|
||||||
trace: TraceContext,
|
|
||||||
requested_scope: GovernanceScope | Mapping[str, Any],
|
|
||||||
input_digest: str,
|
|
||||||
invocation_key: str,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
actor = GovernanceActor.from_value(actor)
|
|
||||||
invocation_key = _identifier(invocation_key, "TOOL_INVOCATION_KEY_INVALID")
|
|
||||||
if not re.fullmatch(r"[0-9a-f]{64}", input_digest):
|
|
||||||
raise GovernanceError("TOOL_INPUT_DIGEST_INVALID")
|
|
||||||
if trace.task_id is None or trace.claim_id is None:
|
|
||||||
raise GovernanceError("TOOL_MCP_AUTHZ_DENY", field="active_claim")
|
|
||||||
requested_scope = requested_scope if isinstance(requested_scope, GovernanceScope) else GovernanceScope.from_value(requested_scope)
|
|
||||||
if requested_scope != trace.scope:
|
|
||||||
raise GovernanceError("TOOL_SCOPE_PROPAGATION")
|
|
||||||
authorize_scope(actor, requested_scope)
|
|
||||||
with self._connection() as connection:
|
|
||||||
persisted_trace = connection.execute(
|
|
||||||
"SELECT context_sha256 FROM governed_trace_spans WHERE trace_id=? AND span_id=?",
|
|
||||||
(trace.trace_id, trace.span_id),
|
|
||||||
).fetchone()
|
|
||||||
if persisted_trace is None or persisted_trace[0] != _sha(trace.as_dict()):
|
|
||||||
raise GovernanceError("TOOL_MCP_AUTHZ_DENY", field="trace")
|
|
||||||
row = connection.execute("SELECT * FROM governed_tools WHERE name=?", (tool_name,)).fetchone()
|
|
||||||
if row is None:
|
|
||||||
raise GovernanceError("TOOL_MCP_AUTHZ_DENY")
|
|
||||||
descriptor_scope = GovernanceScope.from_value(json.loads(row["scope_json"]))
|
|
||||||
if descriptor_scope.scope_type == "tenant" and descriptor_scope != requested_scope:
|
|
||||||
raise GovernanceError("TOOL_MCP_AUTHZ_DENY")
|
|
||||||
required = set(json.loads(row["required_capabilities_json"]))
|
|
||||||
if not required.issubset(set(actor.capabilities)):
|
|
||||||
raise GovernanceError("TOOL_MCP_AUTHZ_DENY")
|
|
||||||
request_sha = _sha({
|
|
||||||
"tool_name": tool_name, "input_digest": input_digest,
|
|
||||||
"task_id": trace.task_id, "claim_id": trace.claim_id,
|
|
||||||
"trace_id": trace.trace_id, "span_id": trace.span_id,
|
|
||||||
"scope": requested_scope.as_dict(),
|
|
||||||
})
|
|
||||||
replay = connection.execute(
|
|
||||||
"SELECT * FROM governed_tool_invocations WHERE invocation_key=?",
|
|
||||||
(invocation_key,),
|
|
||||||
).fetchone()
|
|
||||||
if replay is not None:
|
|
||||||
if replay["request_sha256"] != request_sha:
|
|
||||||
raise GovernanceError("TOOL_INVOCATION_IDEMPOTENCY_CONFLICT")
|
|
||||||
if replay["status"] != "VERIFIED":
|
|
||||||
raise GovernanceError("TOOL_INVOCATION_RECONCILIATION_REQUIRED")
|
|
||||||
return {
|
|
||||||
"tool_name": tool_name, "trace_id": trace.trace_id,
|
|
||||||
"span_id": trace.span_id, "invocation_key": invocation_key,
|
|
||||||
"status": "VERIFIED", "receipt_sha256": replay["receipt_sha256"],
|
|
||||||
"idempotent": True,
|
|
||||||
}
|
|
||||||
created_at = _now()
|
|
||||||
connection.execute(
|
|
||||||
"""INSERT INTO governed_tool_invocations
|
|
||||||
(invocation_key,request_sha256,tool_name,task_id,claim_id,trace_id,span_id,
|
|
||||||
status,receipt_sha256,created_at,updated_at)
|
|
||||||
VALUES(?,?,?,?,?,?,?,?,?,?,?)""",
|
|
||||||
(invocation_key, request_sha, tool_name, trace.task_id, trace.claim_id,
|
|
||||||
trace.trace_id, trace.span_id, "AUTHORIZED", None, created_at, created_at),
|
|
||||||
)
|
|
||||||
evidence = self._audit(
|
|
||||||
connection, action="TOOL_AUTHORIZED", actor=actor, resource_id=tool_name,
|
|
||||||
outcome="ALLOW", trace=trace,
|
|
||||||
details={"input_sha256": input_digest, "scope_sha256": _sha(requested_scope.as_dict()),
|
|
||||||
"invocation_key": invocation_key},
|
|
||||||
)
|
|
||||||
return {
|
|
||||||
"tool_name": tool_name,
|
|
||||||
"trace_id": trace.trace_id,
|
|
||||||
"span_id": trace.span_id,
|
|
||||||
"invocation_key": invocation_key,
|
|
||||||
"status": "AUTHORIZED",
|
|
||||||
"credential_required": row["credential_ref"] is not None,
|
|
||||||
"evidence": evidence,
|
|
||||||
}
|
|
||||||
|
|
||||||
def record_tool_receipt(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
invocation_key: str,
|
|
||||||
result_digest: str,
|
|
||||||
actor: GovernanceActor | Mapping[str, Any],
|
|
||||||
trace: TraceContext,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
actor = GovernanceActor.from_value(actor)
|
|
||||||
invocation_key = _identifier(invocation_key, "TOOL_INVOCATION_KEY_INVALID")
|
|
||||||
if not re.fullmatch(r"[0-9a-f]{64}", result_digest):
|
|
||||||
raise GovernanceError("TOOL_RECEIPT_DIGEST_INVALID")
|
|
||||||
authorize_scope(actor, trace.scope)
|
|
||||||
with self._connection() as connection:
|
|
||||||
row = connection.execute(
|
|
||||||
"SELECT * FROM governed_tool_invocations WHERE invocation_key=?",
|
|
||||||
(invocation_key,),
|
|
||||||
).fetchone()
|
|
||||||
if row is None or row["trace_id"] != trace.trace_id or row["span_id"] != trace.span_id:
|
|
||||||
raise GovernanceError("TOOL_RECEIPT_BINDING_INVALID")
|
|
||||||
if row["status"] == "VERIFIED":
|
|
||||||
if row["receipt_sha256"] != result_digest:
|
|
||||||
raise GovernanceError("TOOL_RECEIPT_CONFLICT")
|
|
||||||
return {"invocation_key": invocation_key, "status": "VERIFIED", "idempotent": True}
|
|
||||||
if row["status"] != "AUTHORIZED":
|
|
||||||
raise GovernanceError("TOOL_INVOCATION_RECONCILIATION_REQUIRED")
|
|
||||||
connection.execute(
|
|
||||||
"UPDATE governed_tool_invocations SET status='VERIFIED',receipt_sha256=?,updated_at=? "
|
|
||||||
"WHERE invocation_key=? AND status='AUTHORIZED'",
|
|
||||||
(result_digest, _now(), invocation_key),
|
|
||||||
)
|
|
||||||
evidence = self._audit(
|
|
||||||
connection, action="TOOL_RESULT_VERIFIED", actor=actor,
|
|
||||||
resource_id=row["tool_name"], outcome="ALLOW", trace=trace,
|
|
||||||
details={"invocation_key": invocation_key, "result_sha256": result_digest},
|
|
||||||
)
|
|
||||||
return {"invocation_key": invocation_key, "status": "VERIFIED", "evidence": evidence}
|
|
||||||
|
|
||||||
def store_grant(self, grant: CredentialGrant, actor: GovernanceActor | Mapping[str, Any]) -> dict[str, Any]:
|
|
||||||
actor = GovernanceActor.from_value(actor)
|
|
||||||
if not actor.privileged:
|
|
||||||
raise GovernanceError("CREDENTIAL_SCOPE_DENY")
|
|
||||||
public = grant.public()
|
|
||||||
digest = _sha(public)
|
|
||||||
with self._connection() as connection:
|
|
||||||
existing = connection.execute(
|
|
||||||
"SELECT grant_sha256 FROM governed_credential_grants WHERE grant_id=?", (grant.grant_id,)
|
|
||||||
).fetchone()
|
|
||||||
if existing is not None and existing[0] != digest:
|
|
||||||
raise GovernanceError("CREDENTIAL_GRANT_CONFLICT")
|
|
||||||
connection.execute(
|
|
||||||
"INSERT OR IGNORE INTO governed_credential_grants VALUES (?,?,?,?)",
|
|
||||||
(grant.grant_id, _canonical(public), digest, _now()),
|
|
||||||
)
|
|
||||||
evidence = self._audit(
|
|
||||||
connection, action="CREDENTIAL_GRANT_STORED", actor=actor,
|
|
||||||
resource_id=grant.grant_id, outcome="ALLOW", trace=None,
|
|
||||||
details={"grant_sha256": digest},
|
|
||||||
)
|
|
||||||
return {"grant": public, "grant_sha256": digest, "evidence": evidence}
|
|
||||||
|
|
||||||
def authorize_credential(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
grant_id: str,
|
|
||||||
actor: GovernanceActor | Mapping[str, Any],
|
|
||||||
trace: TraceContext,
|
|
||||||
operation: str,
|
|
||||||
task_id: str | None,
|
|
||||||
tool_name: str | None,
|
|
||||||
now: datetime | None = None,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
actor = GovernanceActor.from_value(actor)
|
|
||||||
with self._connection() as connection:
|
|
||||||
row = connection.execute(
|
|
||||||
"SELECT grant_json FROM governed_credential_grants WHERE grant_id=?", (grant_id,)
|
|
||||||
).fetchone()
|
|
||||||
if row is None:
|
|
||||||
raise GovernanceError("CREDENTIAL_SCOPE_DENY")
|
|
||||||
raw = json.loads(row[0])
|
|
||||||
grant = CredentialGrant(
|
|
||||||
grant_id=raw["grant_id"], credential_ref=raw["credential_ref"],
|
|
||||||
scope=GovernanceScope.from_value(raw), operations=tuple(raw["operations"]),
|
|
||||||
task_id=raw.get("task_id"), tool_name=raw.get("tool_name"),
|
|
||||||
expires_at=raw.get("expires_at"),
|
|
||||||
)
|
|
||||||
if trace.scope != grant.scope:
|
|
||||||
raise GovernanceError("CREDENTIAL_SCOPE_DENY", field="trace_scope")
|
|
||||||
reference = grant.authorize(
|
|
||||||
actor, operation=operation, task_id=task_id, tool_name=tool_name, now=now
|
|
||||||
)
|
|
||||||
evidence = self._audit(
|
|
||||||
connection, action="CREDENTIAL_AUTHORIZED", actor=actor,
|
|
||||||
resource_id=grant_id, outcome="ALLOW", trace=trace,
|
|
||||||
details={"operation": operation, "task_id": task_id, "tool_name": tool_name,
|
|
||||||
"credential_ref_sha256": _sha(reference)},
|
|
||||||
)
|
|
||||||
return {"credential_ref": reference, "evidence": evidence}
|
|
||||||
|
|
||||||
def verify_audit_chain(self, required_actions: Iterable[str] = ()) -> bool:
|
|
||||||
ordered_actions: list[str] = []
|
|
||||||
previous = ""
|
|
||||||
with self._connection() as connection:
|
|
||||||
rows = connection.execute("SELECT * FROM governance_audit_evidence ORDER BY sequence").fetchall()
|
|
||||||
for row in rows:
|
|
||||||
details = json.loads(row["details_json"])
|
|
||||||
material = {
|
|
||||||
"action": row["action"], "actor_identity": row["actor_identity"],
|
|
||||||
"trace_id": row["trace_id"], "span_id": row["span_id"],
|
|
||||||
"resource_id": row["resource_id"], "outcome": row["outcome"],
|
|
||||||
"details": details, "previous_sha256": row["previous_sha256"],
|
|
||||||
"created_at": row["created_at"],
|
|
||||||
}
|
|
||||||
if row["previous_sha256"] != previous or row["evidence_sha256"] != _sha(material):
|
|
||||||
return False
|
|
||||||
previous = row["evidence_sha256"]
|
|
||||||
ordered_actions.append(row["action"])
|
|
||||||
position = 0
|
|
||||||
for required in required_actions:
|
|
||||||
try:
|
|
||||||
position = ordered_actions.index(required, position) + 1
|
|
||||||
except ValueError:
|
|
||||||
return False
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
|
||||||
"CredentialGrant", "GovernanceActor", "GovernanceError", "GovernanceScope",
|
|
||||||
"GovernanceStore", "ToolDescriptor", "TraceContext", "authorize_scope",
|
|
||||||
"bind_trace_lineage",
|
|
||||||
]
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
providers:
|
|
||||||
oneapi:
|
|
||||||
name: oneapi
|
|
||||||
base_url: http://172.18.0.2:3000/v1
|
|
||||||
key_env: OPENAI_API_KEY
|
|
||||||
default_model: deepseek-v4-flash
|
|
||||||
model:
|
|
||||||
provider: oneapi
|
|
||||||
name: deepseek-v4-flash
|
|
||||||
@@ -1,13 +0,0 @@
|
|||||||
{
|
|
||||||
"argv": [
|
|
||||||
"/usr/local/bin/hermes",
|
|
||||||
"run",
|
|
||||||
"--non-interactive"
|
|
||||||
],
|
|
||||||
"input_mode": "stdin_text",
|
|
||||||
"max_output_bytes": 131072,
|
|
||||||
"output_mode": "stdout_text",
|
|
||||||
"runtime": "hermes",
|
|
||||||
"schema": "aios.external-agent-runtime.profile.v1",
|
|
||||||
"timeout_seconds": 900
|
|
||||||
}
|
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
{
|
|
||||||
"argv": [
|
|
||||||
"/opt/aios-hermes-agent/hermes-agent/.venv/bin/hermes",
|
|
||||||
"-z",
|
|
||||||
"{prompt}",
|
|
||||||
"--provider",
|
|
||||||
"oneapi",
|
|
||||||
"-m",
|
|
||||||
"deepseek-v4-flash",
|
|
||||||
"--no-restore-cwd"
|
|
||||||
],
|
|
||||||
"credential_env": {
|
|
||||||
"OPENAI_API_KEY": {
|
|
||||||
"credential": "aios-external-agent-oneapi-env",
|
|
||||||
"format": "dotenv",
|
|
||||||
"key": "ONEAPI_REGULAR_TOKEN"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"environment": {
|
|
||||||
"HERMES_HOME": "/etc/aios/external-agent-runtime.d/hermes-home"
|
|
||||||
},
|
|
||||||
"input_mode": "stdin_text",
|
|
||||||
"max_output_bytes": 131072,
|
|
||||||
"output_mode": "stdout_text",
|
|
||||||
"runtime": "hermes",
|
|
||||||
"schema": "aios.external-agent-runtime.profile.v1",
|
|
||||||
"timeout_seconds": 180
|
|
||||||
}
|
|
||||||
@@ -1,41 +0,0 @@
|
|||||||
{
|
|
||||||
"tools": {
|
|
||||||
"profile": "minimal"
|
|
||||||
},
|
|
||||||
"memory": {
|
|
||||||
"search": {
|
|
||||||
"enabled": false
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"agents": {
|
|
||||||
"defaults": {
|
|
||||||
"workspace": "/var/lib/aios-external-agent-executor/openclaw-workspace",
|
|
||||||
"model": {
|
|
||||||
"primary": "openai/deepseek-v4-flash"
|
|
||||||
},
|
|
||||||
"thinkingDefault": "off"
|
|
||||||
},
|
|
||||||
"entries": {
|
|
||||||
"main": {
|
|
||||||
"default": true,
|
|
||||||
"workspace": "/var/lib/aios-external-agent-executor/openclaw-workspace"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"models": {
|
|
||||||
"providers": {
|
|
||||||
"openai": {
|
|
||||||
"baseUrl": "http://172.18.0.2:3000/v1",
|
|
||||||
"api": "openai-completions",
|
|
||||||
"models": [
|
|
||||||
{
|
|
||||||
"id": "deepseek-v4-flash",
|
|
||||||
"name": "deepseek-v4-flash",
|
|
||||||
"contextWindow": 32768,
|
|
||||||
"maxTokens": 256
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,13 +0,0 @@
|
|||||||
{
|
|
||||||
"argv": [
|
|
||||||
"/usr/local/bin/openclaw",
|
|
||||||
"run",
|
|
||||||
"--non-interactive"
|
|
||||||
],
|
|
||||||
"input_mode": "stdin_text",
|
|
||||||
"max_output_bytes": 131072,
|
|
||||||
"output_mode": "stdout_text",
|
|
||||||
"runtime": "openclaw",
|
|
||||||
"schema": "aios.external-agent-runtime.profile.v1",
|
|
||||||
"timeout_seconds": 900
|
|
||||||
}
|
|
||||||
@@ -1,37 +0,0 @@
|
|||||||
{
|
|
||||||
"argv": [
|
|
||||||
"/usr/bin/node",
|
|
||||||
"/opt/aios/openclaw-engineering/v2026.7.2-beta.7/openclaw.mjs",
|
|
||||||
"agent",
|
|
||||||
"--local",
|
|
||||||
"--json",
|
|
||||||
"--agent",
|
|
||||||
"main",
|
|
||||||
"--thinking",
|
|
||||||
"off",
|
|
||||||
"--message",
|
|
||||||
"{prompt}",
|
|
||||||
"--model",
|
|
||||||
"openai/deepseek-v4-flash",
|
|
||||||
"--timeout",
|
|
||||||
"180"
|
|
||||||
],
|
|
||||||
"credential_env": {
|
|
||||||
"OPENAI_API_KEY": {
|
|
||||||
"credential": "aios-external-agent-oneapi-env",
|
|
||||||
"format": "dotenv",
|
|
||||||
"key": "ONEAPI_REGULAR_TOKEN"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"environment": {
|
|
||||||
"OPENCLAW_CONFIG_PATH": "/etc/aios/external-agent-runtime.d/openclaw-config.json",
|
|
||||||
"OPENCLAW_HOME": "/var/lib/aios-external-agent-executor/openclaw-home",
|
|
||||||
"OPENCLAW_STATE_DIR": "/var/lib/aios-external-agent-executor/openclaw-state"
|
|
||||||
},
|
|
||||||
"input_mode": "stdin_text",
|
|
||||||
"max_output_bytes": 131072,
|
|
||||||
"output_mode": "stdout_json",
|
|
||||||
"runtime": "openclaw",
|
|
||||||
"schema": "aios.external-agent-runtime.profile.v1",
|
|
||||||
"timeout_seconds": 240
|
|
||||||
}
|
|
||||||
@@ -1,36 +0,0 @@
|
|||||||
-- R4.2 P0-B durable Message/Handoff Fabric.
|
|
||||||
-- Forward-only. The existing 20260707 message_outbox remains a compatibility source.
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS message_handoff (
|
|
||||||
envelope_id TEXT PRIMARY KEY,
|
|
||||||
stream_key TEXT NOT NULL,
|
|
||||||
source TEXT NOT NULL,
|
|
||||||
target TEXT NOT NULL,
|
|
||||||
message_type TEXT NOT NULL,
|
|
||||||
payload_json TEXT NOT NULL,
|
|
||||||
payload_sha256 TEXT NOT NULL,
|
|
||||||
tenant_id TEXT,
|
|
||||||
user_id TEXT,
|
|
||||||
project_id TEXT,
|
|
||||||
conversation_id TEXT,
|
|
||||||
task_id TEXT,
|
|
||||||
run_id TEXT,
|
|
||||||
causation_id TEXT,
|
|
||||||
sequence INTEGER NOT NULL CHECK(sequence >= 1),
|
|
||||||
capability_scope_json TEXT NOT NULL,
|
|
||||||
confidentiality_scope TEXT NOT NULL,
|
|
||||||
idempotency_key TEXT NOT NULL,
|
|
||||||
expires_at TEXT,
|
|
||||||
created_at TEXT NOT NULL,
|
|
||||||
status TEXT NOT NULL DEFAULT 'PENDING',
|
|
||||||
delivered_at TEXT,
|
|
||||||
acked_at TEXT,
|
|
||||||
UNIQUE(stream_key, idempotency_key),
|
|
||||||
UNIQUE(stream_key, sequence)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_message_handoff_target_status
|
|
||||||
ON message_handoff(target, status, created_at);
|
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_message_handoff_scope
|
|
||||||
ON message_handoff(tenant_id, project_id, user_id, conversation_id, run_id);
|
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
# Non-secret staging configuration for the OpenClaw engineering backend.
|
||||||
|
#
|
||||||
|
# Discover the exact binary with `command -v openclaw`.
|
||||||
|
# Do not point this at the chat/WeChat Gateway HTTP endpoint.
|
||||||
|
AIOS_AGENT_RUNTIME_BACKEND_ARGV_JSON='["/usr/bin/python3","/usr/local/lib/aios-agent-runtime-broker/openclaw_engineering_backend.py"]'
|
||||||
|
|
||||||
|
# Required: exact absolute path discovered on hw.imruyi.top.
|
||||||
|
AIOS_OPENCLAW_BIN=/REPLACE/WITH/ABSOLUTE/OPENCLAW/PATH
|
||||||
|
|
||||||
|
# Required: provider/model verified on the installed OpenClaw build.
|
||||||
|
AIOS_OPENCLAW_MODEL=REPLACE_WITH_VERIFIED_PROVIDER_MODEL
|
||||||
|
|
||||||
|
# Non-secret map: provider environment variable -> systemd credential file name.
|
||||||
|
# Example after separately projecting an encrypted credential:
|
||||||
|
# AIOS_OPENCLAW_PROVIDER_CREDENTIAL_MAP_JSON='{"OPENAI_API_KEY":"openai_api_key"}'
|
||||||
|
AIOS_OPENCLAW_PROVIDER_CREDENTIAL_MAP_JSON='{}'
|
||||||
|
|
||||||
|
AIOS_OPENCLAW_EXEC_TIMEOUT_SECONDS=1800
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
if [ "$(id -u)" -ne 0 ]; then
|
||||||
|
echo "must run as root" >&2
|
||||||
|
exit 20
|
||||||
|
fi
|
||||||
|
|
||||||
|
SRC_ROOT="${1:-.}"
|
||||||
|
WORKER_SRC="$SRC_ROOT/services/aios-autonomous-agent-worker"
|
||||||
|
BROKER_SRC="$SRC_ROOT/services/aios-agent-runtime-broker"
|
||||||
|
UNIT_SRC="$SRC_ROOT/deploy/systemd"
|
||||||
|
|
||||||
|
grep -Fq 'AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:19998' \
|
||||||
|
"$UNIT_SRC/aios-autonomous-agent-worker.service" || {
|
||||||
|
echo "staging bridge binding missing" >&2
|
||||||
|
exit 21
|
||||||
|
}
|
||||||
|
|
||||||
|
getent group aios-agent-runtime >/dev/null 2>&1 || groupadd --system aios-agent-runtime
|
||||||
|
id aios-agent-runtime >/dev/null 2>&1 || \
|
||||||
|
useradd --system --gid aios-agent-runtime --home-dir /nonexistent --shell /usr/sbin/nologin aios-agent-runtime
|
||||||
|
id aios-autonomous-agent-worker >/dev/null 2>&1 || \
|
||||||
|
useradd --system --home-dir /nonexistent --shell /usr/sbin/nologin aios-autonomous-agent-worker
|
||||||
|
usermod -a -G aios-agent-runtime aios-autonomous-agent-worker
|
||||||
|
|
||||||
|
install -d -m 0755 /usr/local/lib/aios-autonomous-agent-worker
|
||||||
|
install -d -m 0755 /usr/local/lib/aios-agent-runtime-broker
|
||||||
|
install -d -m 0750 -o aios-autonomous-agent-worker -g aios-autonomous-agent-worker \
|
||||||
|
/var/lib/aios-autonomous-agent-worker
|
||||||
|
install -d -m 0770 -o aios-autonomous-agent-worker -g aios-agent-runtime \
|
||||||
|
/var/lib/aios-autonomous-agent-worker/workspaces
|
||||||
|
|
||||||
|
for file in autonomous_worker.py intent_runtime.py runtime_client.py register.py runtime_selftest.py; do
|
||||||
|
install -m 0755 "$WORKER_SRC/$file" "/usr/local/lib/aios-autonomous-agent-worker/$file"
|
||||||
|
done
|
||||||
|
install -m 0755 "$BROKER_SRC/broker.py" /usr/local/lib/aios-agent-runtime-broker/broker.py
|
||||||
|
install -m 0755 "$BROKER_SRC/openclaw_engineering_backend.py" /usr/local/lib/aios-agent-runtime-broker/openclaw_engineering_backend.py
|
||||||
|
install -m 0644 "$BROKER_SRC/RUNTIME_CONTRACT.md" /usr/local/lib/aios-agent-runtime-broker/RUNTIME_CONTRACT.md
|
||||||
|
|
||||||
|
install -m 0644 "$UNIT_SRC/aios-autonomous-agent-worker.service" /etc/systemd/system/
|
||||||
|
install -m 0644 "$UNIT_SRC/aios-autonomous-agent-register.service" /etc/systemd/system/
|
||||||
|
install -m 0644 "$UNIT_SRC/aios-agent-runtime-broker@.service" /etc/systemd/system/
|
||||||
|
|
||||||
|
systemctl daemon-reload
|
||||||
|
|
||||||
|
cat <<'EOF'
|
||||||
|
Files installed. Services were NOT started or enabled.
|
||||||
|
|
||||||
|
Required before activation:
|
||||||
|
1. Discover `openclaw` with `command -v openclaw`.
|
||||||
|
2. Verify `openclaw agent exec --help` exposes the required headless coding
|
||||||
|
options on the installed version.
|
||||||
|
3. Create /etc/aios/agent-runtime-openclaw.conf from the non-secret example.
|
||||||
|
4. Project provider/model credentials into the OpenClaw broker service with a
|
||||||
|
separate systemd encrypted credential. Never reuse the Online Agent Task
|
||||||
|
token or writer lease.
|
||||||
|
5. Start only `aios-agent-runtime-broker@openclaw.service`.
|
||||||
|
6. Run runtime_selftest.py and require a real worktree mutation with unchanged
|
||||||
|
Git HEAD.
|
||||||
|
7. Only after self-test PASS register the autonomous identity and enable
|
||||||
|
continuous polling.
|
||||||
|
|
||||||
|
The autonomous worker is staging-only and rejects any Governed Bridge other
|
||||||
|
than http://127.0.0.1:19998.
|
||||||
|
EOF
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS isolated %i engineering runtime broker
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=aios-agent-runtime
|
||||||
|
Group=aios-agent-runtime
|
||||||
|
Environment=PYTHONUNBUFFERED=1
|
||||||
|
Environment=AIOS_AGENT_RUNTIME_ADAPTER=%i
|
||||||
|
Environment=AIOS_AGENT_RUNTIME_SOCKET=/run/aios-agent-runtime/%i.sock
|
||||||
|
Environment=AIOS_AGENT_RUNTIME_WORKSPACE_ROOT=/var/lib/aios-autonomous-agent-worker/workspaces
|
||||||
|
Environment=AIOS_AGENT_RUNTIME_ALLOWED_PEER_USER=aios-autonomous-agent-worker
|
||||||
|
Environment=AIOS_AGENT_RUNTIME_CONNECTION_TIMEOUT_SECONDS=15
|
||||||
|
Environment=AIOS_AGENT_RUNTIME_BACKEND_TIMEOUT_SECONDS=3600
|
||||||
|
EnvironmentFile=/etc/aios/agent-runtime-%i.conf
|
||||||
|
ExecStart=/usr/bin/python3 -B /usr/local/lib/aios-agent-runtime-broker/broker.py
|
||||||
|
Restart=always
|
||||||
|
RestartSec=5
|
||||||
|
RuntimeDirectory=aios-agent-runtime
|
||||||
|
RuntimeDirectoryMode=0770
|
||||||
|
UMask=0007
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
ProtectKernelTunables=true
|
||||||
|
ProtectKernelModules=true
|
||||||
|
ProtectControlGroups=true
|
||||||
|
LockPersonality=true
|
||||||
|
RestrictSUIDSGID=true
|
||||||
|
RestrictRealtime=true
|
||||||
|
RestrictNamespaces=true
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
||||||
|
ReadWritePaths=/var/lib/aios-autonomous-agent-worker/workspaces /run/aios-agent-runtime
|
||||||
|
InaccessiblePaths=-/data/aios-gitops/repo.git -/etc/credstore.encrypted/aios-autonomous-agent-worker
|
||||||
|
CapabilityBoundingSet=
|
||||||
|
AmbientCapabilities=
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
+5
@@ -0,0 +1,5 @@
|
|||||||
|
# Example only. Choose the credential name/provider used by the verified model.
|
||||||
|
# The encrypted credential file contains secret material and must never be committed.
|
||||||
|
[Service]
|
||||||
|
# Example:
|
||||||
|
# LoadCredentialEncrypted=openai_api_key:/etc/credstore.encrypted/aios-agent-runtime-openclaw/openai_api_key
|
||||||
@@ -1,37 +0,0 @@
|
|||||||
[Unit]
|
|
||||||
Description=AIOS owner-controlled Task execution-rebind API
|
|
||||||
After=aios-agent-task-api.service
|
|
||||||
Wants=aios-agent-task-api.service
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=simple
|
|
||||||
User=aios-agent-task
|
|
||||||
Group=aios-agent-task
|
|
||||||
LoadCredential=agent_task_control_token:/etc/aios-gitops/agent-task-control.token
|
|
||||||
EnvironmentFile=/etc/aios-gitops/agent-task-execution-rebind.env
|
|
||||||
Environment=AIOS_AGENT_TASK_DB=/var/lib/aios-agent-task/state.sqlite3
|
|
||||||
Environment=AIOS_AGENT_TASK_CONTROL_TOKEN_FILE=%d/agent_task_control_token
|
|
||||||
Environment=AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892
|
|
||||||
Environment=AIOS_TASK_EXECUTION_REBIND_HOST=127.0.0.1
|
|
||||||
Environment=AIOS_TASK_EXECUTION_REBIND_PORT=18942
|
|
||||||
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_agent_task_execution_rebind.py
|
|
||||||
Restart=on-failure
|
|
||||||
RestartSec=2
|
|
||||||
NoNewPrivileges=true
|
|
||||||
PrivateTmp=true
|
|
||||||
ProtectSystem=strict
|
|
||||||
ProtectHome=true
|
|
||||||
ProtectKernelTunables=true
|
|
||||||
ProtectKernelModules=true
|
|
||||||
ProtectControlGroups=true
|
|
||||||
LockPersonality=true
|
|
||||||
RestrictSUIDSGID=true
|
|
||||||
RestrictRealtime=true
|
|
||||||
RestrictNamespaces=true
|
|
||||||
MemoryDenyWriteExecute=true
|
|
||||||
RestrictAddressFamilies=AF_UNIX AF_INET
|
|
||||||
ReadWritePaths=/var/lib/aios-agent-task
|
|
||||||
UMask=0077
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
||||||
@@ -1,36 +0,0 @@
|
|||||||
[Unit]
|
|
||||||
Description=AIOS owner-controlled Task retry-extension API
|
|
||||||
After=aios-agent-task-api.service
|
|
||||||
Wants=aios-agent-task-api.service
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=simple
|
|
||||||
User=aios-agent-task
|
|
||||||
Group=aios-agent-task
|
|
||||||
LoadCredential=agent_task_control_token:/etc/aios-gitops/agent-task-control.token
|
|
||||||
EnvironmentFile=/etc/aios-gitops/agent-task-retry-extension.env
|
|
||||||
Environment=AIOS_AGENT_TASK_DB=/var/lib/aios-agent-task/state.sqlite3
|
|
||||||
Environment=AIOS_AGENT_TASK_CONTROL_TOKEN_FILE=%d/agent_task_control_token
|
|
||||||
Environment=AIOS_TASK_RETRY_HOST=127.0.0.1
|
|
||||||
Environment=AIOS_TASK_RETRY_PORT=18941
|
|
||||||
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_agent_task_retry_extension.py
|
|
||||||
Restart=on-failure
|
|
||||||
RestartSec=2
|
|
||||||
NoNewPrivileges=true
|
|
||||||
PrivateTmp=true
|
|
||||||
ProtectSystem=strict
|
|
||||||
ProtectHome=true
|
|
||||||
ProtectKernelTunables=true
|
|
||||||
ProtectKernelModules=true
|
|
||||||
ProtectControlGroups=true
|
|
||||||
LockPersonality=true
|
|
||||||
RestrictSUIDSGID=true
|
|
||||||
RestrictRealtime=true
|
|
||||||
RestrictNamespaces=true
|
|
||||||
MemoryDenyWriteExecute=true
|
|
||||||
RestrictAddressFamilies=AF_UNIX AF_INET
|
|
||||||
ReadWritePaths=/var/lib/aios-agent-task
|
|
||||||
UMask=0077
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS Autonomous Intent Agent register/recover
|
||||||
|
After=network-online.target aios-agent-task-api.service aios-agent-runtime-broker@openclaw.service
|
||||||
|
Wants=network-online.target
|
||||||
|
Requires=aios-agent-runtime-broker@openclaw.service
|
||||||
|
Before=aios-autonomous-agent-worker.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
RemainAfterExit=yes
|
||||||
|
User=root
|
||||||
|
Group=root
|
||||||
|
LoadCredential=bridge_control_token:/run/credentials/aios-gitops-model-api.service/agent_task_control_token
|
||||||
|
Environment=AIOS_AGENT_TASK_API_URL=http://127.0.0.1:18940
|
||||||
|
Environment=AIOS_AGENT_ID=online-engineering-worker-cn-network-mkt-autonomous-20260815
|
||||||
|
Environment=AIOS_AGENT_VERSION=2026-08-15.4-autonomous-intent
|
||||||
|
Environment=AIOS_WORKER_REGISTRATION_STATE=/var/lib/aios-autonomous-agent-worker/registration
|
||||||
|
Environment=AIOS_WORKER_ENCRYPTED_CREDENTIAL=/etc/credstore.encrypted/aios-autonomous-agent-worker/agent_token
|
||||||
|
Environment=AIOS_HERMES_BROKER_SOCKET=/run/aios-agent-runtime/hermes.sock
|
||||||
|
Environment=AIOS_OPENCLAW_BROKER_SOCKET=/run/aios-agent-runtime/openclaw.sock
|
||||||
|
ExecStart=/usr/bin/python3 -B /usr/local/lib/aios-autonomous-agent-worker/register.py
|
||||||
|
TimeoutStartSec=120
|
||||||
|
UMask=0077
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
ProtectKernelTunables=true
|
||||||
|
ProtectKernelModules=true
|
||||||
|
ProtectControlGroups=true
|
||||||
|
LockPersonality=true
|
||||||
|
RestrictSUIDSGID=true
|
||||||
|
RestrictRealtime=true
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
||||||
|
ReadWritePaths=/etc/credstore.encrypted /var/lib/aios-autonomous-agent-worker
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS Autonomous Intent Engineering Worker (staging)
|
||||||
|
After=network-online.target aios-agent-task-api.service aios-agent-runtime-broker@openclaw.service aios-autonomous-agent-register.service
|
||||||
|
Wants=network-online.target aios-agent-runtime-broker@openclaw.service
|
||||||
|
Requires=aios-autonomous-agent-register.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=aios-autonomous-agent-worker
|
||||||
|
Group=aios-autonomous-agent-worker
|
||||||
|
SupplementaryGroups=aios-agent-runtime gitops
|
||||||
|
LoadCredentialEncrypted=agent_token:/etc/credstore.encrypted/aios-autonomous-agent-worker/agent_token
|
||||||
|
Environment=PYTHONUNBUFFERED=1
|
||||||
|
Environment=AIOS_AGENT_TASK_API_URL=http://127.0.0.1:18940
|
||||||
|
Environment=AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:19998
|
||||||
|
Environment=AIOS_AGENT_ID=online-engineering-worker-cn-network-mkt-autonomous-20260815
|
||||||
|
Environment=AIOS_AGENT_VERSION=2026-08-15.4-autonomous-intent
|
||||||
|
Environment=AIOS_AUTONOMOUS_REPOSITORY=aios/aios
|
||||||
|
Environment=AIOS_AUTONOMOUS_BASE_REF=refs/heads/main
|
||||||
|
Environment=AIOS_AUTHORITY_GIT_DIR=/data/aios-gitops/repo.git
|
||||||
|
Environment=AIOS_AUTONOMOUS_STATE_ROOT=/var/lib/aios-autonomous-agent-worker
|
||||||
|
Environment=AIOS_AUTONOMOUS_WORKSPACE_ROOT=/var/lib/aios-autonomous-agent-worker/workspaces
|
||||||
|
Environment=AIOS_AUTONOMOUS_POLL_SECONDS=5
|
||||||
|
Environment=AIOS_AUTONOMOUS_HEARTBEAT_SECONDS=10
|
||||||
|
Environment=AIOS_AUTONOMOUS_LEASE_SECONDS=300
|
||||||
|
Environment=AIOS_AUTONOMOUS_RUNTIME_TIMEOUT_SECONDS=3600
|
||||||
|
Environment=AIOS_AUTONOMOUS_DEFAULT_ADAPTER=openclaw
|
||||||
|
Environment=AIOS_AGENT_RUNTIME_SHARED_GROUP=aios-agent-runtime
|
||||||
|
Environment=AIOS_HERMES_BROKER_SOCKET=/run/aios-agent-runtime/hermes.sock
|
||||||
|
Environment=AIOS_OPENCLAW_BROKER_SOCKET=/run/aios-agent-runtime/openclaw.sock
|
||||||
|
ExecStart=/usr/bin/python3 -B /usr/local/lib/aios-autonomous-agent-worker/autonomous_worker.py
|
||||||
|
Restart=always
|
||||||
|
RestartSec=5
|
||||||
|
TimeoutStopSec=30
|
||||||
|
UMask=0027
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
ProtectKernelTunables=true
|
||||||
|
ProtectKernelModules=true
|
||||||
|
ProtectControlGroups=true
|
||||||
|
LockPersonality=true
|
||||||
|
RestrictSUIDSGID=true
|
||||||
|
RestrictRealtime=true
|
||||||
|
RestrictNamespaces=true
|
||||||
|
MemoryDenyWriteExecute=true
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
||||||
|
ReadOnlyPaths=/data/aios-gitops/repo.git
|
||||||
|
ReadWritePaths=/var/lib/aios-autonomous-agent-worker
|
||||||
|
InaccessiblePaths=-/etc/credstore.encrypted
|
||||||
|
CapabilityBoundingSet=
|
||||||
|
AmbientCapabilities=
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -1,40 +0,0 @@
|
|||||||
[Unit]
|
|
||||||
Description=AIOS credential-isolated external Agent runtime executor
|
|
||||||
After=network-online.target
|
|
||||||
Wants=network-online.target
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=simple
|
|
||||||
User=aios-engineering-executor
|
|
||||||
Group=aios-executor-exchange
|
|
||||||
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_external_agent_runtime.py --root /var/lib/aios-external-agent-executor --profile-root /etc/aios/external-agent-runtime.d
|
|
||||||
Restart=always
|
|
||||||
RestartSec=2
|
|
||||||
NoNewPrivileges=true
|
|
||||||
PrivateTmp=true
|
|
||||||
ProtectSystem=strict
|
|
||||||
ProtectHome=true
|
|
||||||
ProtectKernelTunables=true
|
|
||||||
ProtectKernelModules=true
|
|
||||||
ProtectControlGroups=true
|
|
||||||
LockPersonality=true
|
|
||||||
RestrictSUIDSGID=true
|
|
||||||
RestrictRealtime=true
|
|
||||||
RestrictNamespaces=true
|
|
||||||
MemoryDenyWriteExecute=true
|
|
||||||
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
|
||||||
IPAddressDeny=127.0.0.0/8
|
|
||||||
IPAddressDeny=::1/128
|
|
||||||
IPAddressAllow=172.18.0.2/32
|
|
||||||
LoadCredential=aios-external-agent-oneapi-env:/data/agent/secrets/oneapi-model-ready.env
|
|
||||||
InaccessiblePaths=/etc/aios-gitops
|
|
||||||
InaccessiblePaths=/data/aios-gitops
|
|
||||||
InaccessiblePaths=/data/agent/secrets
|
|
||||||
ReadOnlyPaths=/etc/aios/external-agent-runtime.d
|
|
||||||
ReadOnlyPaths=/opt/aios-hermes-agent
|
|
||||||
ReadOnlyPaths=/opt/aios/openclaw-engineering
|
|
||||||
ReadWritePaths=/var/lib/aios-external-agent-executor
|
|
||||||
UMask=0027
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
||||||
@@ -1,48 +0,0 @@
|
|||||||
[Unit]
|
|
||||||
Description=AIOS Hermes Online Agent Worker
|
|
||||||
After=aios-agent-task-api.service aios-chatgpt-bridge-v2-staging.service aios-external-agent-executor.service
|
|
||||||
Wants=aios-agent-task-api.service aios-chatgpt-bridge-v2-staging.service aios-external-agent-executor.service
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=simple
|
|
||||||
User=aios-engineering-worker
|
|
||||||
Group=aios-engineering-worker
|
|
||||||
SupplementaryGroups=gitops aios-executor-exchange
|
|
||||||
LoadCredentialEncrypted=agent_token:/etc/credstore.encrypted/aios-online-agent-worker-20260815/agent_token
|
|
||||||
Environment=PYTHONUNBUFFERED=1
|
|
||||||
Environment=AIOS_AGENT_TASK_API_URL=http://127.0.0.1:18940
|
|
||||||
Environment=AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892
|
|
||||||
Environment=AIOS_AGENT_ID=online-hermes-worker-cn-network-mkt
|
|
||||||
Environment=AIOS_AGENT_TYPE=external-agent-runtime-worker
|
|
||||||
Environment=AIOS_AGENT_VERSION=2026-08-18.1-hermes-runtime
|
|
||||||
Environment=AIOS_WORKER_CAPABILITIES=agent.runtime.hermes
|
|
||||||
Environment=AIOS_WORKER_ALLOWED_ADAPTERS=hermes
|
|
||||||
Environment=AIOS_AUTHORITY_GIT_DIR=/data/aios-gitops/repo.git
|
|
||||||
Environment=AIOS_EXTERNAL_AGENT_EXECUTOR_ROOT=/var/lib/aios-external-agent-executor
|
|
||||||
Environment=AIOS_WORKER_STATE_ROOT=/var/lib/aios-online-agent-hermes
|
|
||||||
Environment=AIOS_WORKER_CHECKPOINT_DB=/var/lib/aios-online-agent-hermes/checkpoints.sqlite3
|
|
||||||
Environment=AIOS_WORKER_WORKSPACE_ROOT=/var/lib/aios-online-agent-hermes/workspaces
|
|
||||||
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_online_agent_worker.py
|
|
||||||
Restart=always
|
|
||||||
RestartSec=5
|
|
||||||
TimeoutStopSec=20
|
|
||||||
NoNewPrivileges=true
|
|
||||||
PrivateTmp=true
|
|
||||||
ProtectSystem=strict
|
|
||||||
ProtectHome=true
|
|
||||||
ProtectKernelTunables=true
|
|
||||||
ProtectKernelModules=true
|
|
||||||
ProtectControlGroups=true
|
|
||||||
LockPersonality=true
|
|
||||||
RestrictSUIDSGID=true
|
|
||||||
RestrictRealtime=true
|
|
||||||
RestrictNamespaces=true
|
|
||||||
MemoryDenyWriteExecute=true
|
|
||||||
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
|
||||||
ReadOnlyPaths=/data/aios-gitops/repo.git
|
|
||||||
ReadWritePaths=/var/lib/aios-online-agent-hermes
|
|
||||||
ReadWritePaths=/var/lib/aios-external-agent-executor
|
|
||||||
UMask=0027
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
||||||
@@ -1,33 +0,0 @@
|
|||||||
[Unit]
|
|
||||||
Description=AIOS uncredentialed deterministic Planner/Executor
|
|
||||||
After=local-fs.target
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=simple
|
|
||||||
User=aios-engineering-executor
|
|
||||||
Group=aios-executor-exchange
|
|
||||||
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_isolated_executor.py --root /var/lib/aios-online-agent-executor
|
|
||||||
Restart=always
|
|
||||||
RestartSec=2
|
|
||||||
NoNewPrivileges=true
|
|
||||||
PrivateTmp=true
|
|
||||||
PrivateNetwork=true
|
|
||||||
ProtectSystem=strict
|
|
||||||
ProtectHome=true
|
|
||||||
ProtectKernelTunables=true
|
|
||||||
ProtectKernelModules=true
|
|
||||||
ProtectControlGroups=true
|
|
||||||
LockPersonality=true
|
|
||||||
RestrictSUIDSGID=true
|
|
||||||
RestrictRealtime=true
|
|
||||||
RestrictNamespaces=true
|
|
||||||
MemoryDenyWriteExecute=true
|
|
||||||
RestrictAddressFamilies=AF_UNIX AF_INET
|
|
||||||
IPAddressDeny=any
|
|
||||||
IPAddressAllow=localhost
|
|
||||||
InaccessiblePaths=/run/credentials
|
|
||||||
ReadWritePaths=/var/lib/aios-online-agent-executor
|
|
||||||
UMask=0027
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
||||||
+14
-14
@@ -1,30 +1,30 @@
|
|||||||
[Unit]
|
[Unit]
|
||||||
Description=AIOS governed Online Agent Worker
|
Description=AIOS generic governed Online Agent Worker (2026-08-15)
|
||||||
After=aios-agent-task-api.service aios-chatgpt-bridge-v2-staging.service
|
After=aios-agent-task-api.service aios-online-agent-worker-register-20260815.service
|
||||||
Wants=aios-agent-task-api.service aios-chatgpt-bridge-v2-staging.service
|
Wants=aios-online-agent-worker-register-20260815.service
|
||||||
|
|
||||||
[Service]
|
[Service]
|
||||||
Type=simple
|
Type=simple
|
||||||
User=aios-engineering-worker
|
User=aios-engineering-worker
|
||||||
Group=aios-engineering-worker
|
Group=aios-engineering-worker
|
||||||
SupplementaryGroups=gitops aios-executor-exchange
|
SupplementaryGroups=gitops
|
||||||
LoadCredentialEncrypted=agent_token:/etc/credstore.encrypted/aios-online-agent-worker-20260815/agent_token
|
LoadCredentialEncrypted=agent_token:/etc/credstore.encrypted/aios-online-agent-worker-20260815/agent_token
|
||||||
Environment=PYTHONUNBUFFERED=1
|
Environment=PYTHONUNBUFFERED=1
|
||||||
Environment=AIOS_AGENT_TASK_API_URL=http://127.0.0.1:18940
|
Environment=AIOS_AGENT_TASK_API_URL=http://127.0.0.1:18940
|
||||||
Environment=AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892
|
Environment=AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892
|
||||||
Environment=AIOS_AGENT_ID=online-engineering-worker
|
Environment=AIOS_AGENT_ID=online-engineering-worker-cn-network-mkt-generic-20260815
|
||||||
Environment=AIOS_AGENT_TYPE=trusted-online-engineering-worker
|
Environment=AIOS_AGENT_TYPE=trusted-online-engineering-worker
|
||||||
Environment=AIOS_AGENT_VERSION=2026-08-17.2-durable
|
Environment=AIOS_AGENT_VERSION=2026-08-15.3-generic
|
||||||
Environment=AIOS_WORKER_CAPABILITIES=git.read,git.test,governed.write.branch,governed.write.file
|
|
||||||
Environment=AIOS_AUTHORITY_GIT_DIR=/data/aios-gitops/repo.git
|
Environment=AIOS_AUTHORITY_GIT_DIR=/data/aios-gitops/repo.git
|
||||||
Environment=AIOS_ISOLATED_EXECUTOR_ROOT=/var/lib/aios-online-agent-executor
|
Environment=AIOS_WORKER_STATE_ROOT=/var/lib/aios-online-agent-worker-20260815
|
||||||
Environment=AIOS_WORKER_STATE_ROOT=/var/lib/aios-online-agent-worker
|
Environment=AIOS_WORKER_WORKSPACE_ROOT=/var/lib/aios-online-agent-worker-20260815/workspaces
|
||||||
Environment=AIOS_WORKER_CHECKPOINT_DB=/var/lib/aios-online-agent-worker/checkpoints.sqlite3
|
Environment=AIOS_WORKER_POLL_SECONDS=5
|
||||||
Environment=AIOS_WORKER_WORKSPACE_ROOT=/var/lib/aios-online-agent-worker/workspaces
|
Environment=AIOS_WORKER_HEARTBEAT_SECONDS=10
|
||||||
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_online_agent_worker.py
|
Environment=AIOS_WORKER_LEASE_SECONDS=300
|
||||||
|
ExecStart=/usr/bin/python3 -B /usr/local/lib/aios-online-agent-worker-20260815/online_agent_worker.py
|
||||||
Restart=always
|
Restart=always
|
||||||
RestartSec=5
|
RestartSec=5
|
||||||
TimeoutStopSec=20
|
TimeoutStopSec=15
|
||||||
NoNewPrivileges=true
|
NoNewPrivileges=true
|
||||||
PrivateTmp=true
|
PrivateTmp=true
|
||||||
ProtectSystem=strict
|
ProtectSystem=strict
|
||||||
@@ -39,7 +39,7 @@ RestrictNamespaces=true
|
|||||||
MemoryDenyWriteExecute=true
|
MemoryDenyWriteExecute=true
|
||||||
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
||||||
ReadOnlyPaths=/data/aios-gitops/repo.git
|
ReadOnlyPaths=/data/aios-gitops/repo.git
|
||||||
ReadWritePaths=/var/lib/aios-online-agent-worker
|
ReadWritePaths=/var/lib/aios-online-agent-worker-20260815
|
||||||
UMask=0027
|
UMask=0027
|
||||||
|
|
||||||
[Install]
|
[Install]
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS generic Online Agent register-or-recover (2026-08-15)
|
||||||
|
After=network-online.target aios-gitops-model-api.service aios-agent-task-api.service
|
||||||
|
Wants=network-online.target
|
||||||
|
Before=aios-online-agent-worker-20260815.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
RemainAfterExit=yes
|
||||||
|
User=root
|
||||||
|
Group=root
|
||||||
|
LoadCredential=bridge_control_token:/run/credentials/aios-gitops-model-api.service/agent_task_control_token
|
||||||
|
Environment=AIOS_AGENT_TASK_API_URL=http://127.0.0.1:18940
|
||||||
|
Environment=AIOS_AGENT_ID=online-engineering-worker-cn-network-mkt-generic-20260815
|
||||||
|
Environment=AIOS_AGENT_TYPE=trusted-online-engineering-worker
|
||||||
|
Environment=AIOS_AGENT_VERSION=2026-08-15.3-generic
|
||||||
|
Environment=AIOS_WORKER_ENCRYPTED_CREDENTIAL=/etc/credstore.encrypted/aios-online-agent-worker-20260815/agent_token
|
||||||
|
Environment=AIOS_WORKER_REGISTRATION_STATE=/var/lib/aios-online-agent-worker-20260815/registration
|
||||||
|
ExecStart=/usr/bin/python3 -B /usr/local/lib/aios-online-agent-worker-20260815/register.py
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
ProtectKernelTunables=true
|
||||||
|
ProtectKernelModules=true
|
||||||
|
ProtectControlGroups=true
|
||||||
|
LockPersonality=true
|
||||||
|
RestrictSUIDSGID=true
|
||||||
|
RestrictRealtime=true
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
||||||
|
ReadWritePaths=/etc/credstore.encrypted /var/lib/aios-online-agent-worker-20260815
|
||||||
|
UMask=0077
|
||||||
|
TimeoutStartSec=120
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -1,48 +0,0 @@
|
|||||||
[Unit]
|
|
||||||
Description=AIOS OpenClaw Online Agent Worker
|
|
||||||
After=aios-agent-task-api.service aios-chatgpt-bridge-v2-staging.service aios-external-agent-executor.service
|
|
||||||
Wants=aios-agent-task-api.service aios-chatgpt-bridge-v2-staging.service aios-external-agent-executor.service
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=simple
|
|
||||||
User=aios-engineering-worker
|
|
||||||
Group=aios-engineering-worker
|
|
||||||
SupplementaryGroups=gitops aios-executor-exchange
|
|
||||||
LoadCredentialEncrypted=agent_token:/etc/credstore.encrypted/aios-online-agent-worker-20260815/agent_token
|
|
||||||
Environment=PYTHONUNBUFFERED=1
|
|
||||||
Environment=AIOS_AGENT_TASK_API_URL=http://127.0.0.1:18940
|
|
||||||
Environment=AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892
|
|
||||||
Environment=AIOS_AGENT_ID=online-openclaw-worker-cn-network-mkt
|
|
||||||
Environment=AIOS_AGENT_TYPE=external-agent-runtime-worker
|
|
||||||
Environment=AIOS_AGENT_VERSION=2026-08-18.1-openclaw-runtime
|
|
||||||
Environment=AIOS_WORKER_CAPABILITIES=agent.runtime.openclaw
|
|
||||||
Environment=AIOS_WORKER_ALLOWED_ADAPTERS=openclaw
|
|
||||||
Environment=AIOS_AUTHORITY_GIT_DIR=/data/aios-gitops/repo.git
|
|
||||||
Environment=AIOS_EXTERNAL_AGENT_EXECUTOR_ROOT=/var/lib/aios-external-agent-executor
|
|
||||||
Environment=AIOS_WORKER_STATE_ROOT=/var/lib/aios-online-agent-openclaw
|
|
||||||
Environment=AIOS_WORKER_CHECKPOINT_DB=/var/lib/aios-online-agent-openclaw/checkpoints.sqlite3
|
|
||||||
Environment=AIOS_WORKER_WORKSPACE_ROOT=/var/lib/aios-online-agent-openclaw/workspaces
|
|
||||||
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_online_agent_worker.py
|
|
||||||
Restart=always
|
|
||||||
RestartSec=5
|
|
||||||
TimeoutStopSec=20
|
|
||||||
NoNewPrivileges=true
|
|
||||||
PrivateTmp=true
|
|
||||||
ProtectSystem=strict
|
|
||||||
ProtectHome=true
|
|
||||||
ProtectKernelTunables=true
|
|
||||||
ProtectKernelModules=true
|
|
||||||
ProtectControlGroups=true
|
|
||||||
LockPersonality=true
|
|
||||||
RestrictSUIDSGID=true
|
|
||||||
RestrictRealtime=true
|
|
||||||
RestrictNamespaces=true
|
|
||||||
MemoryDenyWriteExecute=true
|
|
||||||
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
|
||||||
ReadOnlyPaths=/data/aios-gitops/repo.git
|
|
||||||
ReadWritePaths=/var/lib/aios-online-agent-openclaw
|
|
||||||
ReadWritePaths=/var/lib/aios-external-agent-executor
|
|
||||||
UMask=0027
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
||||||
@@ -1,27 +0,0 @@
|
|||||||
[Unit]
|
|
||||||
Description=AIOS persistent Online Agent worker availability guard
|
|
||||||
After=local-fs.target
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=oneshot
|
|
||||||
User=root
|
|
||||||
Group=root
|
|
||||||
Environment=PYTHONUNBUFFERED=1
|
|
||||||
Environment=AIOS_WORKER_AVAILABILITY_MAX_STALE_SECONDS=90
|
|
||||||
Environment=AIOS_WORKER_AVAILABILITY_RESTART_COOLDOWN_SECONDS=60
|
|
||||||
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_worker_availability_guard.py --heal
|
|
||||||
NoNewPrivileges=true
|
|
||||||
PrivateTmp=true
|
|
||||||
ProtectSystem=strict
|
|
||||||
ProtectHome=true
|
|
||||||
ProtectKernelTunables=true
|
|
||||||
ProtectKernelModules=true
|
|
||||||
ProtectControlGroups=true
|
|
||||||
LockPersonality=true
|
|
||||||
RestrictSUIDSGID=true
|
|
||||||
RestrictRealtime=true
|
|
||||||
RestrictNamespaces=true
|
|
||||||
MemoryDenyWriteExecute=true
|
|
||||||
RestrictAddressFamilies=AF_UNIX
|
|
||||||
ReadWritePaths=/run/aios-worker-availability
|
|
||||||
UMask=0022
|
|
||||||
@@ -1,13 +0,0 @@
|
|||||||
[Unit]
|
|
||||||
Description=Periodically ensure AIOS Online Agent workers remain available
|
|
||||||
|
|
||||||
[Timer]
|
|
||||||
OnBootSec=15s
|
|
||||||
OnUnitActiveSec=30s
|
|
||||||
AccuracySec=5s
|
|
||||||
RandomizedDelaySec=3s
|
|
||||||
Persistent=true
|
|
||||||
Unit=aios-worker-availability-guard.service
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=timers.target
|
|
||||||
@@ -1,4 +0,0 @@
|
|||||||
g aios-executor-exchange -
|
|
||||||
u aios-engineering-executor - "AIOS isolated engineering executor" /var/lib/aios-online-agent-executor /usr/sbin/nologin
|
|
||||||
m aios-engineering-executor aios-executor-exchange
|
|
||||||
m aios-engineering-worker aios-executor-exchange
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
d /var/lib/aios-external-agent-executor 2770 aios-engineering-executor aios-executor-exchange -
|
|
||||||
d /var/lib/aios-external-agent-executor/requests 2770 aios-engineering-worker aios-executor-exchange -
|
|
||||||
d /var/lib/aios-external-agent-executor/results 2770 aios-engineering-executor aios-executor-exchange -
|
|
||||||
d /var/lib/aios-external-agent-executor/failed 2770 aios-engineering-executor aios-executor-exchange -
|
|
||||||
d /var/lib/aios-external-agent-executor/openclaw-workspace 0750 aios-engineering-executor aios-executor-exchange -
|
|
||||||
d /var/lib/aios-external-agent-executor/openclaw-state 0750 aios-engineering-executor aios-executor-exchange -
|
|
||||||
d /var/lib/aios-external-agent-executor/openclaw-home 0750 aios-engineering-executor aios-executor-exchange -
|
|
||||||
d /var/lib/aios-online-agent-hermes 0750 aios-engineering-worker aios-engineering-worker -
|
|
||||||
d /var/lib/aios-online-agent-hermes/workspaces 0750 aios-engineering-worker aios-engineering-worker -
|
|
||||||
d /var/lib/aios-online-agent-openclaw 0750 aios-engineering-worker aios-engineering-worker -
|
|
||||||
d /var/lib/aios-online-agent-openclaw/workspaces 0750 aios-engineering-worker aios-engineering-worker -
|
|
||||||
d /etc/aios/external-agent-runtime.d 0755 root root -
|
|
||||||
@@ -1,5 +0,0 @@
|
|||||||
d /var/lib/aios-online-agent-executor 2770 aios-engineering-executor aios-executor-exchange -
|
|
||||||
d /var/lib/aios-online-agent-executor/requests 2770 aios-engineering-worker aios-executor-exchange -
|
|
||||||
d /var/lib/aios-online-agent-executor/results 2770 aios-engineering-executor aios-executor-exchange -
|
|
||||||
d /var/lib/aios-online-agent-executor/failed 2770 aios-engineering-executor aios-executor-exchange -
|
|
||||||
d /var/lib/aios-online-agent-executor/sources 2770 aios-engineering-worker aios-executor-exchange -
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
d /run/aios-worker-availability 0755 root root -
|
|
||||||
@@ -0,0 +1,110 @@
|
|||||||
|
# Autonomous Intent Worker v2
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
A Custom GPT or other trusted producer creates a prose engineering task with:
|
||||||
|
|
||||||
|
- `goal`
|
||||||
|
- `instructions`
|
||||||
|
- `success_criteria`
|
||||||
|
- `required_capabilities`
|
||||||
|
|
||||||
|
A long-running autonomous worker claims only tasks explicitly marked with:
|
||||||
|
|
||||||
|
- `intent.task`
|
||||||
|
- `engineering.execute`
|
||||||
|
|
||||||
|
and only when the requested engineering runtime capability is actually
|
||||||
|
available, e.g. `agent.runtime.hermes`.
|
||||||
|
|
||||||
|
The runtime edits a disposable worktree. The worker independently inspects the
|
||||||
|
diff, scans for secrets, reruns verification, and persists through staging
|
||||||
|
Governed GitOps.
|
||||||
|
|
||||||
|
## Security domains
|
||||||
|
|
||||||
|
The Online Agent worker owns task/lease/governed-write credentials.
|
||||||
|
|
||||||
|
The runtime broker is a separate service identity. It authorizes the worker
|
||||||
|
over Unix `SO_PEERCRED`, opens the worktree using `O_PATH|O_NOFOLLOW`, and
|
||||||
|
passes a pinned `/proc/self/fd/<fd>` path to the engineering backend. The
|
||||||
|
backend never receives the Agent Task token or writer lease.
|
||||||
|
|
||||||
|
The worker rejects:
|
||||||
|
- a runtime-created Git commit (`HEAD != base_oid`);
|
||||||
|
- direct authoritative ref mutation/push;
|
||||||
|
- sensitive credential/key paths;
|
||||||
|
- obvious secret material;
|
||||||
|
- non-staging Governed Bridge endpoints.
|
||||||
|
|
||||||
|
## Staging boundary
|
||||||
|
|
||||||
|
`autonomous_worker.py` intentionally rejects any Governed Write API base other
|
||||||
|
than:
|
||||||
|
|
||||||
|
`http://127.0.0.1:19998`
|
||||||
|
|
||||||
|
Production/main/tag/baseline/force-push writes remain outside this worker.
|
||||||
|
|
||||||
|
## Migration protection
|
||||||
|
|
||||||
|
The worker does not infer intent from prose. Historical tasks without
|
||||||
|
`intent.task` are skipped before claim, so they do not lose an attempt.
|
||||||
|
|
||||||
|
The bootstrap task
|
||||||
|
`implement-autonomous-intent-worker-v2-20260815` currently lacks the intent
|
||||||
|
marker and is therefore intentionally not claimable by this worker. That is a
|
||||||
|
control-plane/bootstrap concern, not a reason to weaken the runtime rule.
|
||||||
|
|
||||||
|
## Runtime requirement
|
||||||
|
|
||||||
|
A chat/WeChat Hermes endpoint is not enough. A compliant engineering backend
|
||||||
|
must:
|
||||||
|
1. accept `aios.online-agent.runtime-request.v1`;
|
||||||
|
2. inspect and mutate the supplied disposable worktree;
|
||||||
|
3. leave Git HEAD unchanged;
|
||||||
|
4. return `aios.online-agent.runtime-result.v1`.
|
||||||
|
|
||||||
|
The exact installed Hermes/Codex interface is host-specific and must be
|
||||||
|
discovered before configuring the broker. No CLI flags are invented here.
|
||||||
|
|
||||||
|
## Activation gate
|
||||||
|
|
||||||
|
Do not register or enable the autonomous consumer until:
|
||||||
|
1. a real engineering backend is found;
|
||||||
|
2. broker is configured with that backend;
|
||||||
|
3. `runtime_selftest.py` reports PASS;
|
||||||
|
4. backend has no Agent Task token or writer lease.
|
||||||
|
|
||||||
|
After activation, complete D/E/F/G:
|
||||||
|
- D: native intent task, no `metadata.execution`;
|
||||||
|
- E: goal-only engineering task, runtime finds files/root cause;
|
||||||
|
- F: unavailable runtime capability remains unclaimed;
|
||||||
|
- G: restart during real runtime execution, no duplicate commit/branch.
|
||||||
|
|
||||||
|
|
||||||
|
## OpenClaw backend candidate
|
||||||
|
|
||||||
|
A current OpenClaw installation can be used as an autonomous engineering
|
||||||
|
runtime without making OpenClaw itself an AIOS Task API client.
|
||||||
|
|
||||||
|
The bundled wrapper uses the official headless coding entry point:
|
||||||
|
|
||||||
|
```text
|
||||||
|
openclaw agent exec
|
||||||
|
--message-file -
|
||||||
|
--cwd <pinned-disposable-worktree>
|
||||||
|
--model <verified-provider/model>
|
||||||
|
--auth-env-only
|
||||||
|
--timeout <seconds>
|
||||||
|
--json
|
||||||
|
```
|
||||||
|
|
||||||
|
Reference used for implementation:
|
||||||
|
`https://docs.openclaw.ai/cli/agent`
|
||||||
|
|
||||||
|
The wrapper maps the stable OpenClaw JSON envelope into
|
||||||
|
`aios.online-agent.runtime-result.v1`. The AIOS worker remains responsible for
|
||||||
|
claim/heartbeat, independent diff verification and Governed GitOps.
|
||||||
|
|
||||||
|
OpenClaw is advertised as `agent.runtime.openclaw`, not as Hermes or Codex.
|
||||||
@@ -240,114 +240,6 @@ components:
|
|||||||
additionalProperties: false
|
additionalProperties: false
|
||||||
properties:
|
properties:
|
||||||
candidate_id: {type: string}
|
candidate_id: {type: string}
|
||||||
BridgeWriterLeaseRecoveryRequest:
|
|
||||||
type: object
|
|
||||||
additionalProperties: false
|
|
||||||
required: [schema, task_id, repository, branch_mode, base_oid, expected_branch_oid, target_branch, generation, lease_seconds]
|
|
||||||
properties:
|
|
||||||
schema:
|
|
||||||
type: string
|
|
||||||
enum: [aios.task-writer-lease.v2, aios.task-writer-lease-request.v2]
|
|
||||||
task_id: {type: string, minLength: 1, maxLength: 256}
|
|
||||||
job_id: {type: string, pattern: '^job-[A-Za-z0-9][A-Za-z0-9._-]{7,255}$'}
|
|
||||||
repository: {type: string, const: aios/aios}
|
|
||||||
branch_mode: {type: string, enum: [CREATE_NEW, CONTINUE_EXISTING]}
|
|
||||||
base_oid: {type: string, pattern: '^[0-9a-f]{40}$'}
|
|
||||||
expected_branch_oid: {type: string, pattern: '^[0-9a-f]{40}$'}
|
|
||||||
target_branch: {type: string, pattern: '^agent/[A-Za-z0-9][A-Za-z0-9._/-]{2,179}$'}
|
|
||||||
generation: {type: integer, minimum: 1}
|
|
||||||
lease_seconds: {type: integer, minimum: 1, maximum: 86400}
|
|
||||||
BridgeWriterLeaseResponse:
|
|
||||||
type: object
|
|
||||||
additionalProperties: false
|
|
||||||
required: [schema, task_id, job_id, repository, branch_mode, base_oid, expected_branch_oid, target_branch, generation, lease_seconds, create_target_branch, worktree_mode, cas_ref_update_required]
|
|
||||||
properties:
|
|
||||||
schema: {type: string, enum: [aios.task-writer-lease.v2]}
|
|
||||||
task_id: {type: string}
|
|
||||||
job_id: {type: string, pattern: '^job-[A-Za-z0-9][A-Za-z0-9._-]{7,255}$'}
|
|
||||||
repository: {type: string, const: aios/aios}
|
|
||||||
branch_mode: {type: string, enum: [CREATE_NEW, CONTINUE_EXISTING]}
|
|
||||||
base_oid: {type: string, pattern: '^[0-9a-f]{40}$'}
|
|
||||||
expected_branch_oid: {type: string, pattern: '^[0-9a-f]{40}$'}
|
|
||||||
target_branch: {type: string, pattern: '^agent/[A-Za-z0-9][A-Za-z0-9._/-]{2,179}$'}
|
|
||||||
generation: {type: integer, minimum: 1}
|
|
||||||
lease_seconds: {type: integer, minimum: 1, maximum: 86400}
|
|
||||||
create_target_branch: {type: boolean}
|
|
||||||
worktree_mode: {type: string, enum: [detached_at_base_oid, detached_at_expected_head]}
|
|
||||||
cas_ref_update_required: {type: boolean, const: true}
|
|
||||||
lease_expires_at: {type: string, format: date-time}
|
|
||||||
BridgeWriterLeaseRecoveryResponse:
|
|
||||||
type: object
|
|
||||||
additionalProperties: false
|
|
||||||
required: [schema, task_id, job_id, repository, branch_mode, base_oid, expected_branch_oid, target_branch, generation, lease_seconds, lease_expires_at, create_target_branch, worktree_mode, cas_ref_update_required, recovered, bootstrapped, idempotent, runtime_rehydrated, durable_branch_lease_present]
|
|
||||||
properties:
|
|
||||||
schema: {type: string, enum: [aios.task-writer-lease.v2]}
|
|
||||||
task_id: {type: string}
|
|
||||||
job_id: {type: string, pattern: '^job-[A-Za-z0-9][A-Za-z0-9._-]{7,255}$'}
|
|
||||||
repository: {type: string, const: aios/aios}
|
|
||||||
branch_mode: {type: string, enum: [CREATE_NEW, CONTINUE_EXISTING]}
|
|
||||||
base_oid: {type: string, pattern: '^[0-9a-f]{40}$'}
|
|
||||||
expected_branch_oid: {type: string, pattern: '^[0-9a-f]{40}$'}
|
|
||||||
target_branch: {type: string, pattern: '^agent/[A-Za-z0-9][A-Za-z0-9._/-]{2,179}$'}
|
|
||||||
generation: {type: integer, minimum: 1}
|
|
||||||
lease_seconds: {type: integer, minimum: 1, maximum: 86400}
|
|
||||||
lease_expires_at: {type: string, format: date-time}
|
|
||||||
create_target_branch: {type: boolean, const: false}
|
|
||||||
worktree_mode: {type: string, enum: [detached_at_base_oid, detached_at_expected_head]}
|
|
||||||
cas_ref_update_required: {type: boolean, const: true}
|
|
||||||
recovered: {type: boolean}
|
|
||||||
bootstrapped: {type: boolean}
|
|
||||||
idempotent: {type: boolean}
|
|
||||||
runtime_rehydrated: {type: boolean}
|
|
||||||
durable_branch_lease_present: {type: boolean}
|
|
||||||
BridgeCommitFileRequest:
|
|
||||||
type: object
|
|
||||||
additionalProperties: false
|
|
||||||
required: [task_id, repository, expected_branch_oid, target_branch, generation, idempotency_key, path, content_base64, message]
|
|
||||||
properties:
|
|
||||||
task_id: {type: string, minLength: 1}
|
|
||||||
job_id: {type: string, pattern: '^job-[A-Za-z0-9][A-Za-z0-9._-]{7,255}$'}
|
|
||||||
repository: {type: string, const: aios/aios}
|
|
||||||
expected_branch_oid: {type: string, pattern: '^[0-9a-f]{40}$'}
|
|
||||||
target_branch: {type: string, pattern: '^agent/[A-Za-z0-9][A-Za-z0-9._/-]{2,179}$'}
|
|
||||||
generation: {type: integer, minimum: 1}
|
|
||||||
idempotency_key: {type: string, minLength: 1, maxLength: 256}
|
|
||||||
path: {type: string, minLength: 1, maxLength: 512}
|
|
||||||
content_base64: {type: string, format: byte}
|
|
||||||
message: {type: string, minLength: 1, maxLength: 200}
|
|
||||||
BridgeCommitFilesRequest:
|
|
||||||
type: object
|
|
||||||
additionalProperties: false
|
|
||||||
required: [task_id, repository, expected_branch_oid, target_branch, generation, idempotency_key, files, message]
|
|
||||||
properties:
|
|
||||||
task_id: {type: string, minLength: 1}
|
|
||||||
job_id: {type: string, pattern: '^job-[A-Za-z0-9][A-Za-z0-9._-]{7,255}$'}
|
|
||||||
repository: {type: string, const: aios/aios}
|
|
||||||
expected_branch_oid: {type: string, pattern: '^[0-9a-f]{40}$'}
|
|
||||||
target_branch: {type: string, pattern: '^agent/[A-Za-z0-9][A-Za-z0-9._/-]{2,179}$'}
|
|
||||||
generation: {type: integer, minimum: 1}
|
|
||||||
idempotency_key: {type: string, minLength: 1, maxLength: 256}
|
|
||||||
files: {type: object, additionalProperties: {type: string, format: byte}, minProperties: 1}
|
|
||||||
message: {type: string, minLength: 1, maxLength: 200}
|
|
||||||
BridgeCommitResponse:
|
|
||||||
type: object
|
|
||||||
additionalProperties: true
|
|
||||||
required: [job_id, commit_oid, commit_visible, branch_mode, target_branch, generation]
|
|
||||||
properties:
|
|
||||||
job_id: {type: string, pattern: '^job-[A-Za-z0-9][A-Za-z0-9._-]{7,255}$'}
|
|
||||||
commit_oid: {type: string, pattern: '^[0-9a-f]{40}$'}
|
|
||||||
commit_visible: {type: boolean}
|
|
||||||
branch_mode: {type: string, enum: [CONTINUE_EXISTING]}
|
|
||||||
target_branch: {type: string, pattern: '^agent/[A-Za-z0-9][A-Za-z0-9._/-]{2,179}$'}
|
|
||||||
generation: {type: integer, minimum: 1}
|
|
||||||
BridgeGovernedCapabilities:
|
|
||||||
type: object
|
|
||||||
required: [bridge_schema_version, governed_writes_enabled, operations, operation_contracts]
|
|
||||||
properties:
|
|
||||||
bridge_schema_version: {type: string, const: aios.chatgpt-gitops-bridge.v2}
|
|
||||||
governed_writes_enabled: {type: boolean, const: true}
|
|
||||||
operations: {type: array, items: {type: string}}
|
|
||||||
operation_contracts: {type: object, additionalProperties: true}
|
|
||||||
paths:
|
paths:
|
||||||
/createReleaseIntent:
|
/createReleaseIntent:
|
||||||
post:
|
post:
|
||||||
@@ -675,118 +567,3 @@ paths:
|
|||||||
'200': {description: Safe GitOps plan}
|
'200': {description: Safe GitOps plan}
|
||||||
'401': {description: Unauthorized}
|
'401': {description: Unauthorized}
|
||||||
x-openai-isConsequential: true
|
x-openai-isConsequential: true
|
||||||
/v2/capabilities:
|
|
||||||
get:
|
|
||||||
operationId: getGovernedWriteCapabilities
|
|
||||||
summary: Read the governed Bridge v2 capability contract
|
|
||||||
responses:
|
|
||||||
'200':
|
|
||||||
description: Governed operations and their request/response contracts
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema: {$ref: '#/components/schemas/BridgeGovernedCapabilities'}
|
|
||||||
/v2/writer-lease:
|
|
||||||
post:
|
|
||||||
operationId: acquireWriterLease
|
|
||||||
summary: Acquire the exact governed writer lease for a task generation
|
|
||||||
security: [{bearerAuth: []}]
|
|
||||||
requestBody:
|
|
||||||
required: true
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema: {$ref: '#/components/schemas/BridgeWriterLeaseRecoveryRequest'}
|
|
||||||
responses:
|
|
||||||
'200':
|
|
||||||
description: Exact detached worktree lease binding
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema: {$ref: '#/components/schemas/BridgeWriterLeaseResponse'}
|
|
||||||
'400': {description: Invalid lease binding}
|
|
||||||
'409': {description: Branch, base, or generation conflict}
|
|
||||||
x-openai-isConsequential: true
|
|
||||||
/v2/writer-lease/bootstrap:
|
|
||||||
post:
|
|
||||||
operationId: bootstrapWriterLease
|
|
||||||
summary: Bootstrap a governed writer runtime without creating a durable lease
|
|
||||||
description: >
|
|
||||||
Staging-only runtime recovery. It validates the exact task, repository,
|
|
||||||
base OID, expected branch OID, target branch, and generation. It must
|
|
||||||
not create a branch, insert a branch lease, or consume a generation.
|
|
||||||
security: [{bearerAuth: []}]
|
|
||||||
requestBody:
|
|
||||||
required: true
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema: {$ref: '#/components/schemas/BridgeWriterLeaseRecoveryRequest'}
|
|
||||||
responses:
|
|
||||||
'200':
|
|
||||||
description: Runtime binding bootstrapped or returned idempotently
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema: {$ref: '#/components/schemas/BridgeWriterLeaseRecoveryResponse'}
|
|
||||||
'400': {description: Invalid lease binding}
|
|
||||||
'409': {description: A matching durable lease already exists or binding conflicts}
|
|
||||||
'503': {description: Runtime or durable state unavailable}
|
|
||||||
x-openai-isConsequential: true
|
|
||||||
/v2/writer-lease/renew:
|
|
||||||
post:
|
|
||||||
operationId: renewWriterLease
|
|
||||||
summary: Recover an expired governed writer lease without changing its binding
|
|
||||||
description: >
|
|
||||||
Renews only the current task generation and exact repository, base OID,
|
|
||||||
expected branch OID, and target branch. Durable renewal is revision-CAS
|
|
||||||
protected; runtime rehydration is reported separately.
|
|
||||||
security: [{bearerAuth: []}]
|
|
||||||
requestBody:
|
|
||||||
required: true
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema: {$ref: '#/components/schemas/BridgeWriterLeaseRecoveryRequest'}
|
|
||||||
responses:
|
|
||||||
'200':
|
|
||||||
description: Lease recovered or idempotently observed as already active
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema: {$ref: '#/components/schemas/BridgeWriterLeaseRecoveryResponse'}
|
|
||||||
'400': {description: Invalid schema, generation, or lease duration}
|
|
||||||
'409': {description: Task, base, repository, branch, generation, or durable lease conflict}
|
|
||||||
'503': {description: Durable job store or runtime unavailable}
|
|
||||||
x-openai-isConsequential: true
|
|
||||||
/v2/commit-file:
|
|
||||||
post:
|
|
||||||
operationId: commitFileGoverned
|
|
||||||
summary: Commit one file through the governed writer lease
|
|
||||||
security: [{bearerAuth: []}]
|
|
||||||
requestBody:
|
|
||||||
required: true
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema: {$ref: '#/components/schemas/BridgeCommitFileRequest'}
|
|
||||||
responses:
|
|
||||||
'200':
|
|
||||||
description: Authority and remote visibility receipt
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema: {$ref: '#/components/schemas/BridgeCommitResponse'}
|
|
||||||
'409': {description: Lease, expected-head, idempotency, or repository conflict}
|
|
||||||
'503': {description: Reconciliation required}
|
|
||||||
x-openai-isConsequential: true
|
|
||||||
/v2/commit-files:
|
|
||||||
post:
|
|
||||||
operationId: commitFilesGoverned
|
|
||||||
summary: Atomically commit a bounded set of files through the governed writer lease
|
|
||||||
security: [{bearerAuth: []}]
|
|
||||||
requestBody:
|
|
||||||
required: true
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema: {$ref: '#/components/schemas/BridgeCommitFilesRequest'}
|
|
||||||
responses:
|
|
||||||
'200':
|
|
||||||
description: Authority and remote visibility receipt
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema: {$ref: '#/components/schemas/BridgeCommitResponse'}
|
|
||||||
'409': {description: Lease, expected-head, idempotency, or repository conflict}
|
|
||||||
'503': {description: Reconciliation required}
|
|
||||||
x-openai-isConsequential: true
|
|
||||||
|
|||||||
@@ -1,54 +0,0 @@
|
|||||||
# R4.2 P0-A TaskPlane Scope Isolation Design
|
|
||||||
|
|
||||||
Status: approved candidate design. This document is not evidence of a protected-main merge or production deployment.
|
|
||||||
|
|
||||||
## Boundary
|
|
||||||
|
|
||||||
- Task: `r42-p0a-taskplane-scope-isolation-20260816`
|
|
||||||
- Governed Job: `job-3fd743b51971c6007923699c1233f84c`
|
|
||||||
- Repository: `aios/aios`
|
|
||||||
- Base: `0eb6e1a864edfdaeef37b438991707a110e21558`
|
|
||||||
- Target branch: `agent/r42-p0a-taskplane-scope-isolation-20260816`
|
|
||||||
- Generation: `1`
|
|
||||||
|
|
||||||
The implementation reuses this exact boundary. It does not create a replacement Task, Job, branch, generation, or idempotency key.
|
|
||||||
|
|
||||||
## Model and invariants
|
|
||||||
|
|
||||||
`TaskPlan` gains first-class `scope_type`, `tenant_id`, `user_id`, `project_id`, `conversation_id`, and `run_id` fields. The fields participate in accepted-key validation, parsing, serialization, persistence, and authorization.
|
|
||||||
|
|
||||||
Historical rows without scope data normalize to `scope_type=legacy`; no tenant is inferred. Tenant-scoped tasks require a non-empty `tenant_id`. System/global resources use an explicit `scope_type=system` invariant and do not receive a fabricated tenant. A task may add user, project, conversation, and run ownership within its tenant.
|
|
||||||
|
|
||||||
## Trusted ActorContext
|
|
||||||
|
|
||||||
TaskPlane defines a small immutable `TaskActorContext` used only for authorization. The HTTP authenticator returns an identity. `PolicyEngine` resolves that identity through server-owned policy configuration into the effective context and capabilities. Request fields such as `tenant_id`, `user_id`, `group_id`, and `role` are declarations only and never expand authority.
|
|
||||||
|
|
||||||
Public create, list, and get paths always use the resolved context. Internal compatibility calls may omit a context and retain explicit legacy behavior, but the HTTP adapter never uses that path.
|
|
||||||
|
|
||||||
## Persistence
|
|
||||||
|
|
||||||
SQLite remains the current runtime. `_create_schema()` performs an idempotent forward migration that adds the six scope columns and indexes, then backfills only missing `scope_type` values to `legacy`.
|
|
||||||
|
|
||||||
New writes persist scope in both the canonical `task_json` representation and first-class columns. Reads accept historical JSON without scope, using legacy columns, but reject a new-row mismatch between JSON and columns. SQL list/get predicates are parameterized and applied before row hydration.
|
|
||||||
|
|
||||||
PostgreSQL remains a separate migration target and is not claimed by this change.
|
|
||||||
|
|
||||||
## Authorization flow
|
|
||||||
|
|
||||||
1. HTTP bearer authentication yields an identity.
|
|
||||||
2. `PolicyEngine` verifies capability and resolves the identity to `TaskActorContext`.
|
|
||||||
3. Create binds the requested scope to the trusted context and rejects mismatches.
|
|
||||||
4. List/get pass the trusted context to `TaskPlaneStore`.
|
|
||||||
5. Store applies tenant and ownership predicates in SQL; privileged capabilities may explicitly cross scope.
|
|
||||||
|
|
||||||
Tenant actors cannot read legacy tasks, another tenant, or mismatched owned resources. Legacy actors can only read legacy tasks. System actors require explicit privileged capability for cross-scope access.
|
|
||||||
|
|
||||||
## Failure behavior
|
|
||||||
|
|
||||||
Missing tenant scope, forged scope, unknown scope types, JSON/column divergence, and unauthorized list/get operations fail closed with canonical TaskPlane errors. Public get does not reveal whether an out-of-scope task exists.
|
|
||||||
|
|
||||||
## Verification
|
|
||||||
|
|
||||||
Tests exercise real model round trips, SQLite migration/restart, Service/Policy/Store wiring, tenant A versus tenant B list/get denial, ownership filtering, forged create payload rejection, legacy isolation, privileged access, and SQL parameterization. Existing TaskPlane API, RBAC, recovery/security, and security-contract suites remain mandatory.
|
|
||||||
|
|
||||||
The candidate is complete only after real `TASK_CLAIMED`, `TASK_HEARTBEAT`, exact WriterLease, governed commit, OID convergence, result submission, and `TASK_VERIFIED`. Protected main and production remain unchanged.
|
|
||||||
@@ -1,90 +0,0 @@
|
|||||||
# R4.2 P0-C Trace / Tool-MCP / Credential Governance
|
|
||||||
|
|
||||||
## Status boundary
|
|
||||||
|
|
||||||
This change is a candidate implementation on the governed P0-C branch. It does
|
|
||||||
not promote protected `main`, deploy Production, or assert that all R4.2 target
|
|
||||||
architecture is complete.
|
|
||||||
|
|
||||||
## Scope model
|
|
||||||
|
|
||||||
Governance resources use an explicit `scope_type`:
|
|
||||||
|
|
||||||
- `tenant` requires `tenant_id` and may narrow to user, project,
|
|
||||||
conversation, and run;
|
|
||||||
- `system` and `global` prohibit tenant ownership fields;
|
|
||||||
- `legacy` preserves unscoped historical records without inventing ownership.
|
|
||||||
|
|
||||||
System and global tools therefore never receive a synthetic tenant merely to
|
|
||||||
satisfy a tenant-oriented schema.
|
|
||||||
|
|
||||||
## Trace lineage
|
|
||||||
|
|
||||||
`TaskPlan` and `MessageEnvelope` scope is bound to a `TraceContext`. The trace
|
|
||||||
records `trace_id`, `span_id`, parent span, correlation, causation, task,
|
|
||||||
message, and the canonical P0-A scope. Child spans must keep the parent's trace,
|
|
||||||
task, and scope. Privileged visibility does not permit cross-scope trace
|
|
||||||
splicing.
|
|
||||||
|
|
||||||
`MessageEnvelope.correlation_id` is persisted and identity-bound. Existing
|
|
||||||
handoff databases are upgraded in place with an idempotent column migration;
|
|
||||||
legacy rows retain their prior event IDs because a null correlation field is
|
|
||||||
not added to the historical identity material.
|
|
||||||
|
|
||||||
Observability keeps its legacy presentation fields, but new governed records
|
|
||||||
also carry immutable trace and scope fields. Scoped records require a trusted
|
|
||||||
server actor for queries. A repeated execution ID is an idempotent replay only
|
|
||||||
when the complete record matches; conflicting overwrites are denied.
|
|
||||||
|
|
||||||
## Tool and MCP governance
|
|
||||||
|
|
||||||
`ToolDescriptor` binds a tool or MCP server to:
|
|
||||||
|
|
||||||
- an explicit resource scope;
|
|
||||||
- required actor capabilities;
|
|
||||||
- an optional credential reference.
|
|
||||||
|
|
||||||
Invocation authorization requires exact propagation of the trace scope and a
|
|
||||||
capability superset. Unknown tools, cross-tenant calls, scope substitution, and
|
|
||||||
non-privileged access to system/global resources fail closed. Every successful
|
|
||||||
registration and invocation appends hash-chained audit evidence.
|
|
||||||
|
|
||||||
## Credential governance
|
|
||||||
|
|
||||||
Credentials are represented only by references whose schemes are
|
|
||||||
`systemd-credential`, `vault`, or `secret-manager`. Raw bearer tokens, passwords,
|
|
||||||
and arbitrary values are rejected. Grants are bound to scope, operation, task,
|
|
||||||
tool, and expiry. Public/audit records contain a reference hash where disclosure
|
|
||||||
is unnecessary; secret material is never persisted by this module.
|
|
||||||
|
|
||||||
The existing runtime credential broker remains backward compatible for system
|
|
||||||
leases, adds explicit tenant/system/global invariants, and rejects providers
|
|
||||||
that attempt to return raw token/secret/password environment values instead of
|
|
||||||
file/reference indirection.
|
|
||||||
|
|
||||||
## Durable evidence
|
|
||||||
|
|
||||||
The SQLite governance store is restart-safe and idempotently initializes four
|
|
||||||
tables: governed tools, trace spans, credential grants, and audit evidence.
|
|
||||||
Audit entries form a SHA-256 chain. Acceptance requires the chain to verify and
|
|
||||||
contain the required actions; missing evidence is not treated as success.
|
|
||||||
|
|
||||||
## Acceptance tests
|
|
||||||
|
|
||||||
The candidate must pass:
|
|
||||||
|
|
||||||
- `TRACE_LINEAGE_CONTINUITY`;
|
|
||||||
- `TRACE_SCOPE_MISMATCH_DENY`;
|
|
||||||
- `TOOL_MCP_AUTHZ_DENY`;
|
|
||||||
- `TOOL_SCOPE_PROPAGATION`;
|
|
||||||
- `CREDENTIAL_SCOPE_DENY`;
|
|
||||||
- `CREDENTIAL_SECRET_EXPOSURE_DENY`;
|
|
||||||
- `AUDIT_EVIDENCE_COMPLETENESS`;
|
|
||||||
- P0-A scope and P0-B handoff regression suites;
|
|
||||||
- Python compilation, `git diff --check`, and a diff-scoped secret scan.
|
|
||||||
|
|
||||||
The repository-wide baseline currently has seven unrelated failures in the
|
|
||||||
isolated host environment (five stale continuation-contract fixtures and two
|
|
||||||
host-permission fixtures). The same seven failures reproduce at the untouched
|
|
||||||
`a642b045775b9154a00ae78543037fbef4fb447b` base; they are not suppressed or
|
|
||||||
modified by P0-C.
|
|
||||||
@@ -1,75 +0,0 @@
|
|||||||
# R4.2 P1 — Durable Online Agent Execution
|
|
||||||
|
|
||||||
Status: candidate implementation. This document does not imply protected-main promotion or Production deployment.
|
|
||||||
|
|
||||||
## State model
|
|
||||||
|
|
||||||
The generic worker uses two local durable layers:
|
|
||||||
|
|
||||||
1. `tasks/<task_id>.json`: atomic fast-state cache for compatibility and quick restart.
|
|
||||||
2. `checkpoints.sqlite3`: append-only attempt/phase journal with `synchronous=FULL`.
|
|
||||||
|
|
||||||
Checkpoint identity is bound to:
|
|
||||||
|
|
||||||
- task_id
|
|
||||||
- attempt
|
|
||||||
- execution_digest
|
|
||||||
- job_id
|
|
||||||
- repository
|
|
||||||
- base_oid
|
|
||||||
- expected_branch_oid
|
|
||||||
- target_branch
|
|
||||||
- generation
|
|
||||||
|
|
||||||
A checkpoint from another repository/branch/generation is never reused.
|
|
||||||
|
|
||||||
Phases are monotonic within an attempt:
|
|
||||||
|
|
||||||
`claimed → workspace_prepared → verification_passed → source_capture_done|commit_done → result_pending → result_submitted`
|
|
||||||
|
|
||||||
`reconciliation_required` is an explicit recovery phase.
|
|
||||||
|
|
||||||
## Attempt-aware result idempotency
|
|
||||||
|
|
||||||
Result submission idempotency keys include the official Task attempt and the execution digest. The same
|
|
||||||
attempt reuses the same key after a timeout/restart; a later official attempt cannot collide with an
|
|
||||||
earlier result.
|
|
||||||
|
|
||||||
Before submitting a result the worker durably writes `result_pending`, including the result hash and
|
|
||||||
idempotency key. After a successful or exact-idempotency-reconciled API response it writes
|
|
||||||
`result_submitted`.
|
|
||||||
|
|
||||||
A 409 is accepted only if the Task API already contains the exact same result body and the Task is
|
|
||||||
VERIFYING/SUCCEEDED. A conflicting result remains a hard failure.
|
|
||||||
|
|
||||||
## Resume rules
|
|
||||||
|
|
||||||
- `source_capture_done` and `result_pending` may be reused across later official attempts only when the
|
|
||||||
execution/scope binding is identical. This avoids repeating read-only source capture.
|
|
||||||
- A deterministic task that crashed before governed commit replays from a new clean detached worktree.
|
|
||||||
This is safe because there is no external side effect yet.
|
|
||||||
- `commit_done` is never recommitted. Recovery requires exact Job receipt OID convergence plus exactly
|
|
||||||
one ACTIVE WriterLease whose `current_head_oid` equals the governed commit.
|
|
||||||
- delayed Git visibility remains `RECONCILIATION_REQUIRED`, preserving an explicit recovery entry.
|
|
||||||
|
|
||||||
## Crash cleanup
|
|
||||||
|
|
||||||
On worker startup, stale `task-*` workspaces under the configured workspace root are removed before
|
|
||||||
polling begins. The path boundary is verified before deletion.
|
|
||||||
|
|
||||||
## Contracts
|
|
||||||
|
|
||||||
P1 gates include:
|
|
||||||
|
|
||||||
- CHECKPOINT_DURABLE_WRITE
|
|
||||||
- RESUME_FROM_CHECKPOINT
|
|
||||||
- RETRY_IDEMPOTENT_NO_DUPLICATE_SIDE_EFFECT
|
|
||||||
- CRASH_RECOVERY_CONVERGENCE
|
|
||||||
- WORKER_RESTART_RESUME
|
|
||||||
- CHECKPOINT_SCOPE_ISOLATION
|
|
||||||
- RESULT_SUBMISSION_REPLAY_SAFE
|
|
||||||
- WORKSPACE_CLEANUP_RECOVERY
|
|
||||||
- LEASE_RENEWAL_RECOVERY
|
|
||||||
|
|
||||||
WriterLease and governed Job authority remain in the existing Bridge/GitOps durable stores; the worker
|
|
||||||
checkpoint DB is not a competing authority for Git state.
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
# R4.2 P2 Agent Fleet / Multi-Agent Scheduling / Fault Isolation
|
|
||||||
|
|
||||||
P2 adds a durable fleet decision layer inside TaskPlane without replacing TaskPlane claim/lease
|
|
||||||
ownership. Worker registration/heartbeat remains authoritative for liveness. TaskPlane claim CAS
|
|
||||||
remains authoritative for ownership. Probe-before-reassignment remains mandatory after an expired
|
|
||||||
lease. P1 checkpoint/result replay remains authoritative after worker restart.
|
|
||||||
|
|
||||||
`FleetScheduler` adds durable worker capacity slots, weighted scope fairness, starvation aging,
|
|
||||||
affinity/anti-affinity labels, worker circuit breakers, per-task/worker quarantine and idempotent
|
|
||||||
dispatch reservations. Fleet state is colocated in the existing TaskPlane SQLite database and is
|
|
||||||
created idempotently; no new daemon or database is introduced.
|
|
||||||
|
|
||||||
A fleet reservation is only a scheduling decision. It does not claim a Task. Callers must perform
|
|
||||||
the normal TaskPlane claim and may then bind the returned claim ID to the reservation. This preserves
|
|
||||||
the existing ownership and recovery invariants.
|
|
||||||
|
|
||||||
Failure recovery never bypasses probe evidence. When TaskPlane recovery places work back in a
|
|
||||||
claimable state, P2 may quarantine the failed worker and prefer another worker/failure domain.
|
|
||||||
@@ -1,79 +0,0 @@
|
|||||||
# R4.2 P2 Compact Interface Map Runtime
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
The persistent Online Agent already supports the canonical read-only `source_capture` action, but
|
|
||||||
its `SourceCaptureExecutor` always returns an `aios.engineering.source-snapshot.v2` tar.gz+base64
|
|
||||||
payload. That is correct for implementation source transfer, but too large for control-plane
|
|
||||||
consumers that only need a bounded implementation map.
|
|
||||||
|
|
||||||
P2 adds a new **adapter**, not a new execution action:
|
|
||||||
|
|
||||||
```text
|
|
||||||
action=source_capture
|
|
||||||
adapter=compact_interface_map
|
|
||||||
```
|
|
||||||
|
|
||||||
This preserves the existing execution-v2 schema and Bridge policy:
|
|
||||||
|
|
||||||
- read-only repository;
|
|
||||||
- no change set;
|
|
||||||
- no verification commands;
|
|
||||||
- `writer_lease=false`;
|
|
||||||
- no branch or commit;
|
|
||||||
- existing capability and execution-digest validation.
|
|
||||||
|
|
||||||
## Compact map contract
|
|
||||||
|
|
||||||
`aios_online_agent_compact_map.py` scans only approved repository roots and source suffixes. It
|
|
||||||
returns:
|
|
||||||
|
|
||||||
- at most 40 relevant modules;
|
|
||||||
- at most 80 selected Python symbols in total;
|
|
||||||
- at most 40 test symbols;
|
|
||||||
- at most 25 excerpts;
|
|
||||||
- at most 1200 characters per excerpt;
|
|
||||||
- SQL table names;
|
|
||||||
- systemd unit names;
|
|
||||||
- current invariants, failure windows and reuse points;
|
|
||||||
- P2 fleet contract-to-surface mappings.
|
|
||||||
|
|
||||||
The result is hard-bounded to 120 KiB and contains no source archive or `payload_b64`.
|
|
||||||
|
|
||||||
Source excerpts apply a conservative sensitive-pattern redaction before result construction.
|
|
||||||
|
|
||||||
## P2 contracts
|
|
||||||
|
|
||||||
The map supplies evidence surfaces for:
|
|
||||||
|
|
||||||
- `FLEET_AGENT_REGISTRATION`
|
|
||||||
- `FLEET_CAPACITY_SCHEDULING`
|
|
||||||
- `FLEET_FAIRNESS`
|
|
||||||
- `FLEET_FAULT_ISOLATION`
|
|
||||||
- `FLEET_WORKER_REASSIGNMENT`
|
|
||||||
- `FLEET_SCOPE_ISOLATION`
|
|
||||||
- `FLEET_IDEMPOTENT_DISPATCH`
|
|
||||||
- `FLEET_FAILURE_RECOVERY`
|
|
||||||
|
|
||||||
Missing first-class scheduling/fleet primitives are reported as gaps rather than invented.
|
|
||||||
|
|
||||||
## Result
|
|
||||||
|
|
||||||
When the adapter is selected, the worker returns:
|
|
||||||
|
|
||||||
```text
|
|
||||||
status=SUCCEEDED
|
|
||||||
classification=P2_INTERFACE_MAP_CAPTURED
|
|
||||||
execution_mode=compact_interface_map
|
|
||||||
writer_lease=NONE
|
|
||||||
```
|
|
||||||
|
|
||||||
The existing `source_capture_done` / P1 checkpoint/retry path is reused, so the compact analysis
|
|
||||||
inherits attempt-aware result idempotency and restart-safe result submission.
|
|
||||||
|
|
||||||
## Governance
|
|
||||||
|
|
||||||
This capability is runtime infrastructure only. The implementation candidate may use a governed
|
|
||||||
branch/WriterLease for its own code change, but any task executed through `compact_interface_map`
|
|
||||||
is read-only and must not create a WriterLease, branch, commit, protected-ref write, or Production
|
|
||||||
side effect.
|
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
# R4.2 P2-B Fleet Order Preservation Repair
|
|
||||||
|
|
||||||
P2-C live acceptance exposed a real integration defect in the verified P2-B candidate. The fleet
|
|
||||||
scheduler correctly produced durable cross-scope fairness order, but the integration passed the
|
|
||||||
entire ranked list to the legacy scheduler. The legacy scheduler then applied its own priority /
|
|
||||||
created_at / task_id sort and silently discarded fleet ordering.
|
|
||||||
|
|
||||||
The repair preserves the legacy scheduler as the fail-closed security oracle while changing how it
|
|
||||||
is called: one fleet-ranked candidate is presented at a time. A candidate rejected by legacy
|
|
||||||
security/policy advances to the next fleet candidate. A candidate returned by the oracle must have
|
|
||||||
the same task ID as the candidate presented, otherwise execution fails closed.
|
|
||||||
|
|
||||||
Task ownership remains unchanged: the original schedule_for_worker/claim_task CAS remains
|
|
||||||
authoritative, and a fleet reservation is still only bound after the claim succeeds.
|
|
||||||
|
|
||||||
This repair does not modify FleetScheduler state semantics, capacity, WriterLease, checkpoint,
|
|
||||||
protected refs or Production behavior.
|
|
||||||
@@ -1,23 +0,0 @@
|
|||||||
# R4.2 P2-B Fleet Runtime Integration
|
|
||||||
|
|
||||||
P2-B connects the verified P2 fleet primitives to the existing TaskPlane pull scheduler without
|
|
||||||
replacing its security or ownership logic.
|
|
||||||
|
|
||||||
The integration uses two stages. `FleetScheduler.rank_for_worker()` performs only read-only
|
|
||||||
capacity/fairness/fault-isolation ranking. The original `TaskScheduler.next_queued_task()` is then
|
|
||||||
run over that ordered list and remains the fail-closed security oracle for worker liveness,
|
|
||||||
capabilities, data classification, execution zone and other existing policy checks. Only a task
|
|
||||||
accepted by the original scheduler receives a short durable fleet reservation.
|
|
||||||
|
|
||||||
The original `TaskScheduler.schedule_for_worker()` remains the sole claim path and continues to call
|
|
||||||
the existing TaskPlane `claim_task` CAS. After a successful claim, the claim ID is bound to the
|
|
||||||
fleet reservation. A claim error releases the reservation. A fleet reservation is never treated as
|
|
||||||
task ownership.
|
|
||||||
|
|
||||||
P2-B also makes the reservation capacity check atomic inside the fleet SQLite transaction so two
|
|
||||||
concurrent pullers cannot oversubscribe the same worker slot. Circuit and task/worker quarantine
|
|
||||||
state are enforced on the real pull path. Explicit execution-failure signals can be recorded through
|
|
||||||
the integrated scheduler's failure/success/quarantine methods; policy/claim failures are not
|
|
||||||
automatically mislabeled as worker crashes.
|
|
||||||
|
|
||||||
No new daemon, database or production write path is introduced.
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
# Generic Compact Interface-Map Profiles
|
|
||||||
|
|
||||||
The compact interface-map runtime is a reusable read-only control-plane capability. It keeps the
|
|
||||||
canonical `source_capture` action and `compact_interface_map` adapter, but selects a bounded semantic
|
|
||||||
profile from the task's source signals.
|
|
||||||
|
|
||||||
The existing `fleet` profile remains backward compatible and returns
|
|
||||||
`P2_INTERFACE_MAP_CAPTURED` with the `FLEET_*` contracts.
|
|
||||||
|
|
||||||
The new `model_governance` profile is selected when model/provider/routing/budget/evaluation terms
|
|
||||||
are present. It returns `MODEL_GOVERNANCE_INTERFACE_MAP_CAPTURED` and model-specific contracts for
|
|
||||||
provider registry, selection policy, version pinning, budget accounting, fallback, scope isolation,
|
|
||||||
credential boundary, trace/audit, evaluation gates and routing resilience.
|
|
||||||
|
|
||||||
Profile and classification are emitted by the compact-map module and revalidated by the worker
|
|
||||||
before result submission. Unknown or mismatched profile/classification pairs fail closed.
|
|
||||||
|
|
||||||
No new Bridge action, daemon, database, WriterLease, mutation path or credential surface is added.
|
|
||||||
@@ -1,142 +0,0 @@
|
|||||||
# Hermes / OpenClaw Online Agent Runtime
|
|
||||||
|
|
||||||
## Goal
|
|
||||||
|
|
||||||
Promote Hermes and OpenClaw from shadow-only adapter descriptions to real Online Agent workers
|
|
||||||
without creating a second Task/claim protocol.
|
|
||||||
|
|
||||||
Both workers reuse the existing P1/P2 Online Agent lifecycle:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Online Task projected
|
|
||||||
→ capability-filtered poll
|
|
||||||
→ automatic claim
|
|
||||||
→ heartbeat
|
|
||||||
→ durable checkpoint
|
|
||||||
→ external Agent execution
|
|
||||||
→ attempt-aware result submission
|
|
||||||
→ official verification
|
|
||||||
```
|
|
||||||
|
|
||||||
They do **not** change the existing TaskPlane `OpenClawAdapter/HermesAdapter` shadow contract. Those
|
|
||||||
adapters remain side-effect-free compatibility surfaces.
|
|
||||||
|
|
||||||
## Execution action
|
|
||||||
|
|
||||||
The canonical execution envelope adds a read-only action:
|
|
||||||
|
|
||||||
```text
|
|
||||||
action=agent_runtime
|
|
||||||
adapter=hermes | openclaw
|
|
||||||
```
|
|
||||||
|
|
||||||
The action requires capability `agent.runtime.<adapter>`, has an empty change set, no verification
|
|
||||||
commands, and `writer_lease=false`.
|
|
||||||
|
|
||||||
`project_online_task()` copies only bounded task fields into the signed execution envelope:
|
|
||||||
|
|
||||||
- goal;
|
|
||||||
- instructions;
|
|
||||||
- success criteria;
|
|
||||||
- non-secret constraints.
|
|
||||||
|
|
||||||
The task may not choose a command, executable, URL, credential path, environment, or shell string.
|
|
||||||
|
|
||||||
## Worker separation
|
|
||||||
|
|
||||||
The generic engineering worker defaults to:
|
|
||||||
|
|
||||||
```text
|
|
||||||
AIOS_WORKER_ALLOWED_ADAPTERS=deterministic,source_capture,compact_interface_map,codex
|
|
||||||
```
|
|
||||||
|
|
||||||
so it skips Hermes/OpenClaw tasks before claim.
|
|
||||||
|
|
||||||
Dedicated services advertise:
|
|
||||||
|
|
||||||
```text
|
|
||||||
Hermes:
|
|
||||||
AIOS_WORKER_CAPABILITIES=agent.runtime.hermes
|
|
||||||
AIOS_WORKER_ALLOWED_ADAPTERS=hermes
|
|
||||||
|
|
||||||
OpenClaw:
|
|
||||||
AIOS_WORKER_CAPABILITIES=agent.runtime.openclaw
|
|
||||||
AIOS_WORKER_ALLOWED_ADAPTERS=openclaw
|
|
||||||
```
|
|
||||||
|
|
||||||
This makes adapter selection a pre-claim capability/fleet decision instead of a post-claim failure.
|
|
||||||
|
|
||||||
## Credential-isolated executor
|
|
||||||
|
|
||||||
The Online Agent worker itself still owns only Task API/Bridge coordination credentials. It writes
|
|
||||||
a hash-bound request to `/var/lib/aios-external-agent-executor`.
|
|
||||||
|
|
||||||
`aios-external-agent-executor.service` runs as the existing uncredentialed engineering executor user.
|
|
||||||
It has:
|
|
||||||
|
|
||||||
- one provider-only `LoadCredential` projection for the model-ready OneAPI environment;
|
|
||||||
- `/etc/aios-gitops`, `/data/aios-gitops`, and the original provider-secret directory inaccessible;
|
|
||||||
- no repository path;
|
|
||||||
- no shell execution;
|
|
||||||
- loopback denied, preventing direct access to the AIOS Task/Bridge control plane;
|
|
||||||
- only the staging OneAPI container address allowed for the installed Agent runtime/provider;
|
|
||||||
- only the exchange directory writable.
|
|
||||||
|
|
||||||
The root-owned runtime profile in `/etc/aios/external-agent-runtime.d/<runtime>.json` decides the
|
|
||||||
absolute executable, fixed non-secret environment, provider-only credential mapping, and I/O mode.
|
|
||||||
A task cannot override any of these values. The profile may select `OPENAI_API_KEY` only from the
|
|
||||||
systemd credential named `aios-external-agent-oneapi-env`; it cannot name a credential, inject an
|
|
||||||
environment variable, or carry an inline secret. The runtime executor parses only the selected
|
|
||||||
`ONEAPI_REGULAR_TOKEN` entry and does not expose the source file or other provider tiers to the
|
|
||||||
child.
|
|
||||||
|
|
||||||
## Runtime profile
|
|
||||||
|
|
||||||
A profile supports:
|
|
||||||
|
|
||||||
- `argv`: fixed argv, absolute allowlisted executable;
|
|
||||||
- `input_mode`: `stdin_text`, `stdin_json`, or `file_json`;
|
|
||||||
- `output_mode`: `stdout_text`, `stdout_json`, or `file_json`;
|
|
||||||
- timeout;
|
|
||||||
- bounded output size.
|
|
||||||
|
|
||||||
`{input_json}` and `{output_json}` placeholders are allowed only for file modes. `{prompt}` is a
|
|
||||||
root-profile-only argv placeholder populated from the bounded, signed task projection.
|
|
||||||
|
|
||||||
The repository includes the staging profile templates validated against the installed Hermes
|
|
||||||
v0.20.0 and OpenClaw v2026.7.2-beta.7 CLIs. Before activation, the Agent must re-run the actual
|
|
||||||
`--help`/version probes and install the profiles and static configs root-owned. If the installed
|
|
||||||
CLI differs, update only the root-owned profile/config projection and record the observed command;
|
|
||||||
do not modify the Task contract to compensate.
|
|
||||||
|
|
||||||
The provider projection is staging-only and uses the existing model-ready OneAPI source. It is not
|
|
||||||
an AIOS Task, Bridge, GitOps, SSH, or Writer credential. Live activation remains blocked if the
|
|
||||||
model-ready source cannot be projected through systemd or if the selected provider credential is
|
|
||||||
missing/quota-invalid. This worker never passes API keys through task metadata or arbitrary
|
|
||||||
environment variables.
|
|
||||||
|
|
||||||
## Retry/restart
|
|
||||||
|
|
||||||
The external request ID is deterministic over Task/Job/execution digest/runtime. The exchange keeps
|
|
||||||
the request/result across worker restart. A repeated attempt with the exact execution digest reuses
|
|
||||||
the durable external result instead of invoking the Agent twice.
|
|
||||||
|
|
||||||
The normal P1 result idempotency/checkpoint path remains authoritative for Task result submission.
|
|
||||||
|
|
||||||
## Result
|
|
||||||
|
|
||||||
Successful execution returns:
|
|
||||||
|
|
||||||
```text
|
|
||||||
status=SUCCEEDED
|
|
||||||
classification=EXTERNAL_AGENT_RUNTIME_SUCCEEDED
|
|
||||||
execution_mode=agent_runtime
|
|
||||||
agent_runtime=hermes | openclaw
|
|
||||||
writer_lease=NONE
|
|
||||||
changed_files=[]
|
|
||||||
credential_material_exposed=false
|
|
||||||
control_plane_access_granted_to_runtime=false
|
|
||||||
```
|
|
||||||
|
|
||||||
This first runtime version is intentionally read-only with respect to the repository. Engineering
|
|
||||||
changes continue to use assistant-authored deterministic plans and governed WriterLease/commit.
|
|
||||||
@@ -1,36 +0,0 @@
|
|||||||
# Owner-Controlled Same-Binding Execution Rebind
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
`aios_agent_task_execution_rebind.py` closes a projection/plan race without replacing the Task,
|
|
||||||
Governed Job, target branch, generation, or idempotency lineage.
|
|
||||||
|
|
||||||
The only supported transition is:
|
|
||||||
|
|
||||||
```text
|
|
||||||
official NEEDS_REWORK
|
|
||||||
→ Task QUEUED with no active lease
|
|
||||||
→ persisted source_capture envelope
|
|
||||||
→ canonical Bridge resolves an already-registered deterministic plan
|
|
||||||
→ CAS-bound metadata.execution + execution_projection replacement
|
|
||||||
→ audit event
|
|
||||||
→ normal worker claim
|
|
||||||
```
|
|
||||||
|
|
||||||
The service rejects arbitrary metadata patches. It derives the new envelope from canonical Bridge
|
|
||||||
`/v2/execution-envelope`, validates every governed binding coordinate against the persisted envelope,
|
|
||||||
requires `source_capture → deterministic`, requires exact old/new execution digests supplied by the
|
|
||||||
Owner recovery request, and increments the Task revision under `BEGIN IMMEDIATE`.
|
|
||||||
|
|
||||||
It records `TASK_EXECUTION_REBIND_AUTHORIZED` and an idempotent `task_execution_rebinds` receipt.
|
|
||||||
No Task/Job/branch replacement is created and the worker still performs the real claim/heartbeat,
|
|
||||||
WriterLease and governed commit.
|
|
||||||
|
|
||||||
The endpoint is loopback-only:
|
|
||||||
|
|
||||||
`POST /v1/tasks/{task_id}/execution-rebind`
|
|
||||||
|
|
||||||
and requires the existing Owner control credential plus
|
|
||||||
`AIOS_AGENT_TASK_EXECUTION_REBIND_AUTHORIZATION_ID`.
|
|
||||||
|
|
||||||
Direct/manual Task database edits remain forbidden; the service is the audited mutation boundary.
|
|
||||||
@@ -1,212 +0,0 @@
|
|||||||
# AIOS Worker Availability Runtime
|
|
||||||
|
|
||||||
## Priority
|
|
||||||
|
|
||||||
This runtime exists to make the already verified Online Agent execution path continuously usable
|
|
||||||
without repeated human intervention.
|
|
||||||
|
|
||||||
The system already proved that generic, Hermes and OpenClaw workers can claim and complete real
|
|
||||||
Online Tasks. The remaining availability defect was operational: evidence-freeze workflows left the
|
|
||||||
worker services stopped, while registry `ACTIVE` records remained stale.
|
|
||||||
|
|
||||||
The availability contract therefore separates:
|
|
||||||
|
|
||||||
```text
|
|
||||||
AUTO_CLAIM_MECHANISM=VERIFIED
|
|
||||||
CURRENT_WORKER_AVAILABILITY=<fresh runtime state>
|
|
||||||
```
|
|
||||||
|
|
||||||
A historical successful claim is not current liveness.
|
|
||||||
|
|
||||||
## Source of truth for idle liveness
|
|
||||||
|
|
||||||
An idle worker may have no Task lease heartbeat. Therefore Task heartbeat alone cannot prove that
|
|
||||||
the worker is continuously polling.
|
|
||||||
|
|
||||||
`aios_online_agent_worker.py` now writes:
|
|
||||||
|
|
||||||
```text
|
|
||||||
<AIOS_WORKER_STATE_ROOT>/worker-liveness.json
|
|
||||||
```
|
|
||||||
|
|
||||||
on:
|
|
||||||
|
|
||||||
- process start: `STARTING`;
|
|
||||||
- every successful poll: `POLL_OK`;
|
|
||||||
- poll failure: `POLL_FAILED`;
|
|
||||||
- every successful Task lease heartbeat while executing: `BUSY`.
|
|
||||||
|
|
||||||
The file contains:
|
|
||||||
|
|
||||||
- agent ID/type/version;
|
|
||||||
- process PID;
|
|
||||||
- monotonically increasing local sequence;
|
|
||||||
- `last_seen_epoch`;
|
|
||||||
- poll/heartbeat intervals;
|
|
||||||
- current Task/stage while busy.
|
|
||||||
|
|
||||||
The liveness file is worker-owned observability state only. It does not write or mutate Task, Job,
|
|
||||||
Git, WriterLease or registry databases.
|
|
||||||
|
|
||||||
## Self-healing guard
|
|
||||||
|
|
||||||
`aios-worker-availability-guard.timer` invokes a root-owned oneshot guard every 30 seconds.
|
|
||||||
|
|
||||||
The guard manages:
|
|
||||||
|
|
||||||
```text
|
|
||||||
aios-online-agent-worker-20260815.service
|
|
||||||
aios-hermes-online-agent-worker.service
|
|
||||||
aios-openclaw-online-agent-worker.service
|
|
||||||
```
|
|
||||||
|
|
||||||
and the required external runtime dependency:
|
|
||||||
|
|
||||||
```text
|
|
||||||
aios-external-agent-executor.service
|
|
||||||
```
|
|
||||||
|
|
||||||
For each worker:
|
|
||||||
|
|
||||||
1. missing systemd unit → fail closed;
|
|
||||||
2. disabled → `systemctl enable`;
|
|
||||||
3. inactive → `systemctl start`;
|
|
||||||
4. active but liveness older than 90 seconds → restart;
|
|
||||||
5. repeated stale restart within 60 seconds → do not restart again; report degraded.
|
|
||||||
|
|
||||||
Long-running Tasks are safe because the Task lease-heartbeat thread refreshes `BUSY` liveness.
|
|
||||||
|
|
||||||
The guard never claims a Task and never touches Task/Job/Git databases.
|
|
||||||
|
|
||||||
## Maintenance inhibit
|
|
||||||
|
|
||||||
Planned maintenance creates:
|
|
||||||
|
|
||||||
```text
|
|
||||||
/run/aios-worker-availability/maintenance.lock
|
|
||||||
```
|
|
||||||
|
|
||||||
While this file exists the guard performs no start/restart actions and reports:
|
|
||||||
|
|
||||||
```text
|
|
||||||
MAINTENANCE_INHIBITED
|
|
||||||
```
|
|
||||||
|
|
||||||
Maintenance tooling must remove the file when done. This prevents the self-healer from fighting
|
|
||||||
intentional service changes.
|
|
||||||
|
|
||||||
## Status
|
|
||||||
|
|
||||||
The guard writes:
|
|
||||||
|
|
||||||
```text
|
|
||||||
/run/aios-worker-availability/status.json
|
|
||||||
```
|
|
||||||
|
|
||||||
with current worker/dependency states, liveness ages and recovery actions.
|
|
||||||
|
|
||||||
A worker is currently available only when:
|
|
||||||
|
|
||||||
```text
|
|
||||||
systemd load_state=loaded
|
|
||||||
systemd active_state=active
|
|
||||||
unit enabled/static
|
|
||||||
worker-liveness.json present
|
|
||||||
last_seen_age <= 90s
|
|
||||||
```
|
|
||||||
|
|
||||||
Registry `ACTIVE` without fresh poll/busy liveness is not sufficient.
|
|
||||||
|
|
||||||
## Boot/restart behavior
|
|
||||||
|
|
||||||
The timer is:
|
|
||||||
|
|
||||||
```text
|
|
||||||
OnBootSec=15s
|
|
||||||
OnUnitActiveSec=30s
|
|
||||||
Persistent=true
|
|
||||||
```
|
|
||||||
|
|
||||||
Therefore reboot, manual stop left behind by another workflow, process crash, and hung poll loop are
|
|
||||||
all repaired without requiring a human to notice and intervene.
|
|
||||||
|
|
||||||
Normal worker units retain their own `Restart=always`; the availability guard handles the cases that
|
|
||||||
`Restart=always` cannot, especially a deliberate `systemctl stop` left after an evidence workflow.
|
|
||||||
|
|
||||||
## Acceptance
|
|
||||||
|
|
||||||
The live acceptance requires all of the following:
|
|
||||||
|
|
||||||
```text
|
|
||||||
guard timer active/enabled
|
|
||||||
external executor active/enabled
|
|
||||||
generic worker active/enabled + fresh liveness
|
|
||||||
Hermes worker active/enabled + fresh liveness
|
|
||||||
OpenClaw worker active/enabled + fresh liveness
|
|
||||||
```
|
|
||||||
|
|
||||||
Then a new read-only Online Task is projected through canonical Bridge `18892`.
|
|
||||||
|
|
||||||
Success requires:
|
|
||||||
|
|
||||||
```text
|
|
||||||
TASK_CREATED
|
|
||||||
→ generic worker auto claim
|
|
||||||
→ TASK_HEARTBEAT
|
|
||||||
→ RESULT_SUBMITTED
|
|
||||||
→ official TASK_VERIFIED(SUCCEEDED)
|
|
||||||
```
|
|
||||||
|
|
||||||
After the Task, the services must remain active and liveness must continue advancing. The acceptance
|
|
||||||
also stops a worker intentionally once and proves the guard starts it again with a new fresh
|
|
||||||
liveness sequence.
|
|
||||||
|
|
||||||
Only after these gates may current status be upgraded from:
|
|
||||||
|
|
||||||
```text
|
|
||||||
OVERALL=PARTIAL
|
|
||||||
```
|
|
||||||
|
|
||||||
to:
|
|
||||||
|
|
||||||
```text
|
|
||||||
PERSISTENT_WORKER_LIVENESS=VERIFIED
|
|
||||||
NEW_TASK_AUTO_CONSUMPTION=VERIFIED
|
|
||||||
WORKER_AVAILABILITY_RUNTIME=VERIFIED_CANDIDATE
|
|
||||||
```
|
|
||||||
|
|
||||||
Protected main and Production remain out of scope.
|
|
||||||
|
|
||||||
|
|
||||||
## Forced source-capture projection compatibility
|
|
||||||
|
|
||||||
The deployed canonical Bridge currently projects a default `source_capture` envelope even when a
|
|
||||||
governed binding request asks for binding-only behavior. This must not be allowed to make always-on
|
|
||||||
workers consume engineering Tasks before their deterministic plan is registered.
|
|
||||||
|
|
||||||
The worker therefore distinguishes a real read-only source capture from an engineering placeholder:
|
|
||||||
|
|
||||||
```text
|
|
||||||
source_capture
|
|
||||||
+ capability_scope contains governed.write.branch or governed.write.file
|
|
||||||
= PENDING_DETERMINISTIC_PLAN
|
|
||||||
```
|
|
||||||
|
|
||||||
For such a placeholder the worker:
|
|
||||||
|
|
||||||
1. re-resolves through the canonical plan registry **without replaying the stale source-capture
|
|
||||||
envelope**;
|
|
||||||
2. if no deterministic plan exists, emits `TASK_SKIPPED_PENDING_DETERMINISTIC_PLAN` and does not
|
|
||||||
claim;
|
|
||||||
3. if the registered canonical envelope is deterministic and the governed binding still matches,
|
|
||||||
uses that deterministic envelope and claims normally.
|
|
||||||
|
|
||||||
A real read-only source-capture Task with only `git.read`/`git.test` still replays its exact persisted
|
|
||||||
payload and remains claimable.
|
|
||||||
|
|
||||||
This makes persistent workers safe with the currently deployed Bridge while preserving the same
|
|
||||||
Task/Job binding. No Task DB rewrite, cancellation, replacement Task, fake NEEDS_REWORK, or
|
|
||||||
pre-claim metadata mutation is required.
|
|
||||||
|
|
||||||
A later Bridge cleanup may implement true binding-only projection, but Worker Availability does not
|
|
||||||
depend on that cleanup.
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
# R4.2 P0-A TaskPlane Scope Isolation
|
|
||||||
|
|
||||||
Status: candidate implementation. This is not proof of protected-main promotion or production deployment.
|
|
||||||
|
|
||||||
Task scope is now represented by `scope_type`, `tenant_id`, `user_id`, `project_id`, `conversation_id`, and `run_id` in the model and SQLite persistence. Historical rows become `legacy`; their tenant is never guessed. Tenant tasks require `tenant_id`. System resources use `scope_type=system` without a fabricated tenant.
|
|
||||||
|
|
||||||
The HTTP adapter uses the authenticated identity. `PolicyEngine` resolves that identity through server-owned `identity_actor_contexts`; request payload fields do not grant scope. Public create binds the trusted scope, and public list/get apply parameterized scope predicates in `TaskPlaneStore`.
|
|
||||||
|
|
||||||
SQLite is the current runtime database. PostgreSQL remains a separate migration and runtime-acceptance gate.
|
|
||||||
@@ -1,38 +0,0 @@
|
|||||||
# R4.2 P0-B — Durable Message / Handoff Fabric
|
|
||||||
|
|
||||||
Status: candidate implementation; not evidence of protected-main merge or Production deployment.
|
|
||||||
|
|
||||||
`MessageEnvelope` is the single durable handoff identity and carries source/target/type, tenant/user/
|
|
||||||
project/conversation/task/run scope, causation, sequence, capability scope, confidentiality scope,
|
|
||||||
idempotency key, expiry and delivery state.
|
|
||||||
|
|
||||||
Identity includes **target + scope + sequence + idempotency key + payload hash**, fixing the existing
|
|
||||||
AgentEventBus target collision. Server ActorContext is authoritative; caller scope may narrow but may
|
|
||||||
not widen/forge trusted tenant/user/project/conversation/run fields.
|
|
||||||
|
|
||||||
`DurableHandoffStore` provides SQLite durability in the current runtime. Each logical stream has
|
|
||||||
unique `(stream_key, idempotency_key)` and `(stream_key, sequence)`. Exact retry returns `REPLAY`;
|
|
||||||
changed payload under the same key is rejected; sequence gaps are rejected; expired messages are not
|
|
||||||
delivered.
|
|
||||||
|
|
||||||
Confidentiality scopes:
|
|
||||||
|
|
||||||
- public: no tenant requirement
|
|
||||||
- tenant: same tenant
|
|
||||||
- project: same tenant + project
|
|
||||||
- private: same tenant + project + user
|
|
||||||
- run: same tenant + project + user + run
|
|
||||||
|
|
||||||
Required `capability_scope` must be present on the authenticated delivery actor unless the actor is
|
|
||||||
explicitly administrative.
|
|
||||||
|
|
||||||
Compatibility adapters convert AgentEventBus events, TaskPlane CloudEvents and legacy Agent OS
|
|
||||||
MessageBus messages into the durable envelope. Existing message_outbox remains intact as a migration
|
|
||||||
source; P0-B does not delete or silently rewrite it.
|
|
||||||
|
|
||||||
Required gates:
|
|
||||||
`MESSAGE_SCOPE_MISMATCH_DENY`, `MESSAGE_TARGET_COLLISION_DENY`, `MESSAGE_IDEMPOTENCY_REPLAY`,
|
|
||||||
`MESSAGE_SEQUENCE_GAP_DETECT`, `MESSAGE_CONFIDENTIALITY_SCOPE_ENFORCE`,
|
|
||||||
`HANDOFF_CROSS_TENANT_DENY`, plus existing P0-A TaskPlane scope/security regressions.
|
|
||||||
|
|
||||||
SQLite remains current runtime persistence. PostgreSQL remains a separate migration gate.
|
|
||||||
@@ -48,8 +48,3 @@ class MessageBus:
|
|||||||
messages = list(self.queue)
|
messages = list(self.queue)
|
||||||
self.queue.clear()
|
self.queue.clear()
|
||||||
return messages
|
return messages
|
||||||
|
|
||||||
# R4.2 P0-B compatibility adapter.
|
|
||||||
def to_handoff_envelope(message, *, target=None, sequence=1, idempotency_key=None, **scope):
|
|
||||||
from message_handoff_adapters import legacy_message_to_envelope
|
|
||||||
return legacy_message_to_envelope(message, target=target, sequence=sequence, idempotency_key=idempotency_key, **scope)
|
|
||||||
|
|||||||
@@ -0,0 +1,93 @@
|
|||||||
|
# Autonomous Engineering Runtime Contract v1
|
||||||
|
|
||||||
|
The Online Agent Worker and the Hermes/Codex engineering runtime are separate
|
||||||
|
security domains.
|
||||||
|
|
||||||
|
The worker owns:
|
||||||
|
- Agent Task polling, atomic claim, task lease and heartbeat;
|
||||||
|
- intent resolution and restart state;
|
||||||
|
- authoritative repository/base selection;
|
||||||
|
- independent diff inspection and secret checks;
|
||||||
|
- verification command allowlisting and re-execution;
|
||||||
|
- Governed GitOps task/lease/commit;
|
||||||
|
- final result submission.
|
||||||
|
|
||||||
|
The runtime broker and Hermes/Codex backend do **not** receive:
|
||||||
|
- Agent Task API token;
|
||||||
|
- task lease token;
|
||||||
|
- governed writer lease or Bridge credential;
|
||||||
|
- authority repository write access.
|
||||||
|
|
||||||
|
## stdin request
|
||||||
|
|
||||||
|
The backend configured behind the broker receives one JSON object:
|
||||||
|
|
||||||
|
`schema = aios.online-agent.runtime-request.v1`
|
||||||
|
|
||||||
|
It contains task goal/instructions/success criteria and a pinned disposable
|
||||||
|
worktree path. The backend may inspect and modify that worktree. It must not
|
||||||
|
commit or push.
|
||||||
|
|
||||||
|
## stdout response
|
||||||
|
|
||||||
|
The backend returns one JSON object:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"schema": "aios.online-agent.runtime-result.v1",
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"summary": "root cause and implementation summary",
|
||||||
|
"verification": [
|
||||||
|
{
|
||||||
|
"label": "focused test",
|
||||||
|
"argv": ["/usr/bin/python3", "-m", "pytest", "-q", "tests/test_x.py"],
|
||||||
|
"cwd": ".",
|
||||||
|
"timeout_seconds": 600
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"evidence": {
|
||||||
|
"root_cause": "..."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`status` may also be `BLOCKED` or `FAILED`.
|
||||||
|
|
||||||
|
The worker ignores any backend claim about changed files and independently reads
|
||||||
|
Git state. It also reruns verification requests through its local command
|
||||||
|
allowlist.
|
||||||
|
|
||||||
|
## Real runtime requirement
|
||||||
|
|
||||||
|
A chat/router/WeChat Hermes service is not an engineering backend. A compliant
|
||||||
|
backend must be able to read and mutate the disposable worktree and emit the
|
||||||
|
runtime result schema above.
|
||||||
|
|
||||||
|
The exact Hermes/Codex CLI/API must be discovered from the installed runtime.
|
||||||
|
Do not invent flags. `/etc/aios/agent-runtime-hermes.conf` contains only
|
||||||
|
non-secret backend command configuration. Any model/runtime credential must be
|
||||||
|
delivered to the broker/backend through its own systemd credential projection,
|
||||||
|
never through the Online Agent worker token.
|
||||||
|
|
||||||
|
|
||||||
|
## OpenClaw headless engineering backend
|
||||||
|
|
||||||
|
For hosts that have a current OpenClaw CLI, use the bundled
|
||||||
|
`openclaw_engineering_backend.py`.
|
||||||
|
|
||||||
|
The wrapper invokes the documented headless coding entry point:
|
||||||
|
|
||||||
|
`openclaw agent exec --message-file - --cwd <pinned-worktree> --model <model> --auth-env-only --timeout <seconds> --json`
|
||||||
|
|
||||||
|
It deliberately does **not** use the Gateway/WeChat/chat route. `agent exec`
|
||||||
|
runs an embedded, one-shot agent turn and scopes its workspace/tool working
|
||||||
|
directory to `--cwd`.
|
||||||
|
|
||||||
|
Provider credentials, if required, are projected into the broker service under
|
||||||
|
separate systemd credentials and mapped to provider environment variables only
|
||||||
|
for the OpenClaw child. The Online Agent Task token and writer lease remain
|
||||||
|
absent.
|
||||||
|
|
||||||
|
Before enabling the autonomous worker, run a disposable broker self-test that
|
||||||
|
proves OpenClaw changes the requested worktree while Git HEAD remains at the
|
||||||
|
base OID.
|
||||||
@@ -0,0 +1,297 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import array
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import pwd
|
||||||
|
import re
|
||||||
|
import socket
|
||||||
|
import stat
|
||||||
|
import subprocess
|
||||||
|
from typing import Any, Mapping
|
||||||
|
|
||||||
|
|
||||||
|
MAX_BYTES = 1024 * 1024
|
||||||
|
ADAPTER = os.environ.get("AIOS_AGENT_RUNTIME_ADAPTER", "").strip()
|
||||||
|
SOCKET_PATH = Path(os.environ.get("AIOS_AGENT_RUNTIME_SOCKET", ""))
|
||||||
|
WORKSPACE_ROOT = Path(os.environ.get("AIOS_AGENT_RUNTIME_WORKSPACE_ROOT", ""))
|
||||||
|
BACKEND_ARGV_JSON = os.environ.get("AIOS_AGENT_RUNTIME_BACKEND_ARGV_JSON", "")
|
||||||
|
ALLOWED_PEER_UID = int(os.environ.get("AIOS_AGENT_RUNTIME_ALLOWED_PEER_UID", "-1"))
|
||||||
|
ALLOWED_PEER_USER = os.environ.get("AIOS_AGENT_RUNTIME_ALLOWED_PEER_USER", "").strip()
|
||||||
|
CONNECTION_TIMEOUT = float(os.environ.get("AIOS_AGENT_RUNTIME_CONNECTION_TIMEOUT_SECONDS", "15"))
|
||||||
|
BACKEND_TIMEOUT = int(os.environ.get("AIOS_AGENT_RUNTIME_BACKEND_TIMEOUT_SECONDS", "3600"))
|
||||||
|
BACKEND_ENV_ALLOWLIST = (
|
||||||
|
"AIOS_OPENCLAW_BIN",
|
||||||
|
"AIOS_OPENCLAW_MODEL",
|
||||||
|
"AIOS_OPENCLAW_PROVIDER_CREDENTIAL_MAP_JSON",
|
||||||
|
"AIOS_OPENCLAW_EXEC_TIMEOUT_SECONDS",
|
||||||
|
)
|
||||||
|
SENSITIVE_RE = re.compile(r"(?i)(authorization\s*:\s*bearer\s+\S+|(?:password|passwd|secret|token|api[_-]?key)\s*[:=]\s*\S+)")
|
||||||
|
|
||||||
|
|
||||||
|
class BrokerError(RuntimeError):
|
||||||
|
def __init__(self, code: str, *, details: Mapping[str, Any] | None = None) -> None:
|
||||||
|
super().__init__(code)
|
||||||
|
self.code = code
|
||||||
|
self.details = dict(details or {})
|
||||||
|
|
||||||
|
|
||||||
|
def peer_uid(connection: socket.socket) -> int:
|
||||||
|
if not hasattr(socket, "SO_PEERCRED"):
|
||||||
|
raise BrokerError("SO_PEERCRED_UNAVAILABLE")
|
||||||
|
raw = connection.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12)
|
||||||
|
pid, uid, gid = array.array("i", raw[:12])
|
||||||
|
return int(uid)
|
||||||
|
|
||||||
|
|
||||||
|
def configured_peer_uid() -> int:
|
||||||
|
if ALLOWED_PEER_UID >= 0:
|
||||||
|
return ALLOWED_PEER_UID
|
||||||
|
if not ALLOWED_PEER_USER:
|
||||||
|
raise BrokerError("ALLOWED_PEER_UID_NOT_CONFIGURED")
|
||||||
|
try:
|
||||||
|
return int(pwd.getpwnam(ALLOWED_PEER_USER).pw_uid)
|
||||||
|
except KeyError as exc:
|
||||||
|
raise BrokerError("ALLOWED_PEER_USER_NOT_FOUND") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def authorize_peer(connection: socket.socket) -> None:
|
||||||
|
uid = peer_uid(connection)
|
||||||
|
expected = configured_peer_uid()
|
||||||
|
if uid != expected:
|
||||||
|
raise BrokerError("PEER_UID_REJECTED", details={"peer_uid": uid})
|
||||||
|
|
||||||
|
|
||||||
|
def read_request(connection: socket.socket) -> dict[str, Any]:
|
||||||
|
connection.settimeout(CONNECTION_TIMEOUT)
|
||||||
|
total = 0
|
||||||
|
chunks: list[bytes] = []
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
chunk = connection.recv(65536)
|
||||||
|
except socket.timeout as exc:
|
||||||
|
raise BrokerError("REQUEST_TIMEOUT") from exc
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
total += len(chunk)
|
||||||
|
if total > MAX_BYTES:
|
||||||
|
raise BrokerError("REQUEST_TOO_LARGE")
|
||||||
|
chunks.append(chunk)
|
||||||
|
try:
|
||||||
|
value = json.loads(b"".join(chunks).decode("utf-8"))
|
||||||
|
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||||
|
raise BrokerError("REQUEST_INVALID_JSON") from exc
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise BrokerError("REQUEST_OBJECT_REQUIRED")
|
||||||
|
return dict(value)
|
||||||
|
|
||||||
|
|
||||||
|
def backend_argv() -> list[str]:
|
||||||
|
try:
|
||||||
|
value = json.loads(BACKEND_ARGV_JSON)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise BrokerError("BACKEND_ARGV_JSON_INVALID") from exc
|
||||||
|
if not isinstance(value, list) or not value or not all(isinstance(item, str) and item for item in value):
|
||||||
|
raise BrokerError("BACKEND_ARGV_INVALID")
|
||||||
|
executable = Path(value[0])
|
||||||
|
if not executable.is_absolute():
|
||||||
|
raise BrokerError("BACKEND_EXECUTABLE_INVALID")
|
||||||
|
try:
|
||||||
|
resolved = executable.resolve(strict=True)
|
||||||
|
except OSError as exc:
|
||||||
|
raise BrokerError("BACKEND_EXECUTABLE_INVALID") from exc
|
||||||
|
if not resolved.is_file():
|
||||||
|
raise BrokerError("BACKEND_EXECUTABLE_INVALID")
|
||||||
|
return list(value)
|
||||||
|
|
||||||
|
|
||||||
|
def validate_request(value: Mapping[str, Any]) -> tuple[dict[str, Any], int]:
|
||||||
|
if value.get("schema") != "aios.online-agent.runtime-request.v1":
|
||||||
|
raise BrokerError("REQUEST_SCHEMA_INVALID")
|
||||||
|
if value.get("adapter") != ADAPTER or ADAPTER not in {"hermes", "codex", "openclaw"}:
|
||||||
|
raise BrokerError("REQUEST_ADAPTER_INVALID")
|
||||||
|
context = value.get("execution_context")
|
||||||
|
policy = value.get("policy")
|
||||||
|
if not isinstance(context, Mapping) or not isinstance(policy, Mapping):
|
||||||
|
raise BrokerError("REQUEST_CONTEXT_INVALID")
|
||||||
|
worktree_raw = context.get("worktree")
|
||||||
|
if not isinstance(worktree_raw, str):
|
||||||
|
raise BrokerError("REQUEST_WORKTREE_INVALID")
|
||||||
|
for field in (
|
||||||
|
"authoritative_repository_write",
|
||||||
|
"direct_git_ref_mutation",
|
||||||
|
"direct_git_push",
|
||||||
|
"direct_git_commit",
|
||||||
|
"main_write",
|
||||||
|
"force_push",
|
||||||
|
"production_write",
|
||||||
|
"tag_write",
|
||||||
|
"baseline_write",
|
||||||
|
"task_api_credential_visible",
|
||||||
|
"writer_lease_visible",
|
||||||
|
):
|
||||||
|
if policy.get(field) is not False:
|
||||||
|
raise BrokerError("REQUEST_POLICY_REJECTED", details={"field": field})
|
||||||
|
|
||||||
|
root = WORKSPACE_ROOT.resolve()
|
||||||
|
candidate = Path(worktree_raw)
|
||||||
|
if not candidate.is_absolute():
|
||||||
|
raise BrokerError("REQUEST_WORKTREE_INVALID")
|
||||||
|
parent = candidate.parent.resolve()
|
||||||
|
if root == parent or root not in parent.parents:
|
||||||
|
raise BrokerError("REQUEST_WORKTREE_OUT_OF_SCOPE")
|
||||||
|
|
||||||
|
flags = getattr(os, "O_PATH", os.O_RDONLY) | getattr(os, "O_NOFOLLOW", 0) | os.O_DIRECTORY
|
||||||
|
try:
|
||||||
|
fd = os.open(str(candidate), flags)
|
||||||
|
except OSError as exc:
|
||||||
|
raise BrokerError("REQUEST_WORKTREE_OPEN_FAILED") from exc
|
||||||
|
st = os.fstat(fd)
|
||||||
|
if not stat.S_ISDIR(st.st_mode):
|
||||||
|
os.close(fd)
|
||||||
|
raise BrokerError("REQUEST_WORKTREE_NOT_DIRECTORY")
|
||||||
|
pinned = Path(f"/proc/self/fd/{fd}")
|
||||||
|
try:
|
||||||
|
resolved = pinned.resolve(strict=True)
|
||||||
|
except OSError:
|
||||||
|
os.close(fd)
|
||||||
|
raise BrokerError("REQUEST_WORKTREE_PIN_FAILED")
|
||||||
|
if root not in resolved.parents:
|
||||||
|
os.close(fd)
|
||||||
|
raise BrokerError("REQUEST_WORKTREE_OUT_OF_SCOPE")
|
||||||
|
return dict(value), fd
|
||||||
|
|
||||||
|
|
||||||
|
def sanitize(value: Any) -> Any:
|
||||||
|
if isinstance(value, Mapping):
|
||||||
|
cleaned: dict[str, Any] = {}
|
||||||
|
for key, inner in value.items():
|
||||||
|
lowered = str(key).lower()
|
||||||
|
if any(marker in lowered for marker in ("token", "secret", "credential", "authorization", "password", "api_key", "apikey", "access_key")):
|
||||||
|
cleaned[str(key)] = "[REDACTED]"
|
||||||
|
else:
|
||||||
|
cleaned[str(key)] = sanitize(inner)
|
||||||
|
return cleaned
|
||||||
|
if isinstance(value, list):
|
||||||
|
return [sanitize(item) for item in value]
|
||||||
|
if isinstance(value, str):
|
||||||
|
return SENSITIVE_RE.sub("[REDACTED]", value)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def sanitized_failure(code: str, details: Mapping[str, Any] | None = None) -> dict[str, Any]:
|
||||||
|
safe: dict[str, Any] = {}
|
||||||
|
for key, value in dict(details or {}).items():
|
||||||
|
lowered = str(key).lower()
|
||||||
|
if any(marker in lowered for marker in ("token", "secret", "credential", "authorization", "password", "api_key", "apikey", "access_key")):
|
||||||
|
safe[str(key)] = "[REDACTED]"
|
||||||
|
else:
|
||||||
|
safe[str(key)] = value
|
||||||
|
return {
|
||||||
|
"schema": "aios.online-agent.runtime-result.v1",
|
||||||
|
"status": "FAILED",
|
||||||
|
"summary": code,
|
||||||
|
"verification": [],
|
||||||
|
"evidence": safe,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def execute_backend(request: Mapping[str, Any], worktree_fd: int) -> dict[str, Any]:
|
||||||
|
argv = backend_argv()
|
||||||
|
request = dict(request)
|
||||||
|
context = dict(request["execution_context"])
|
||||||
|
# Pass a pinned inode-backed path, not the original user-visible pathname.
|
||||||
|
context["worktree"] = f"/proc/self/fd/{worktree_fd}"
|
||||||
|
request["execution_context"] = context
|
||||||
|
payload = json.dumps(request, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
environment = {
|
||||||
|
"PATH": "/usr/local/bin:/usr/bin:/bin",
|
||||||
|
"LC_ALL": "C.UTF-8",
|
||||||
|
"AIOS_AGENT_RUNTIME_ADAPTER": ADAPTER,
|
||||||
|
}
|
||||||
|
for name in BACKEND_ENV_ALLOWLIST:
|
||||||
|
value = os.environ.get(name)
|
||||||
|
if value is not None:
|
||||||
|
environment[name] = value
|
||||||
|
credentials_dir = os.environ.get("CREDENTIALS_DIRECTORY")
|
||||||
|
if credentials_dir:
|
||||||
|
environment["CREDENTIALS_DIRECTORY"] = credentials_dir
|
||||||
|
|
||||||
|
try:
|
||||||
|
completed = subprocess.run(
|
||||||
|
argv,
|
||||||
|
input=payload,
|
||||||
|
cwd=f"/proc/self/fd/{worktree_fd}",
|
||||||
|
pass_fds=(worktree_fd,),
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
timeout=BACKEND_TIMEOUT,
|
||||||
|
env=environment,
|
||||||
|
)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
return sanitized_failure("AGENT_BACKEND_TIMEOUT")
|
||||||
|
except OSError as exc:
|
||||||
|
return sanitized_failure("AGENT_BACKEND_EXEC_FAILED", {"error": type(exc).__name__})
|
||||||
|
if completed.returncode != 0:
|
||||||
|
return sanitized_failure("AGENT_BACKEND_FAILED", {"exit_code": completed.returncode})
|
||||||
|
if len(completed.stdout) > MAX_BYTES:
|
||||||
|
return sanitized_failure("AGENT_BACKEND_RESULT_TOO_LARGE")
|
||||||
|
try:
|
||||||
|
value = json.loads(completed.stdout.decode("utf-8"))
|
||||||
|
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||||
|
return sanitized_failure("AGENT_BACKEND_RESULT_INVALID")
|
||||||
|
if not isinstance(value, Mapping) or value.get("schema") != "aios.online-agent.runtime-result.v1":
|
||||||
|
return sanitized_failure("AGENT_BACKEND_RESULT_SCHEMA_INVALID")
|
||||||
|
sanitized = sanitize(dict(value))
|
||||||
|
return sanitized if isinstance(sanitized, dict) else sanitized_failure("AGENT_BACKEND_RESULT_INVALID")
|
||||||
|
|
||||||
|
|
||||||
|
def serve_once(listener: socket.socket) -> None:
|
||||||
|
connection, _ = listener.accept()
|
||||||
|
worktree_fd: int | None = None
|
||||||
|
try:
|
||||||
|
authorize_peer(connection)
|
||||||
|
request = read_request(connection)
|
||||||
|
request, worktree_fd = validate_request(request)
|
||||||
|
response = execute_backend(request, worktree_fd)
|
||||||
|
except BrokerError as exc:
|
||||||
|
response = sanitized_failure(exc.code, exc.details)
|
||||||
|
except Exception as exc:
|
||||||
|
response = sanitized_failure("BROKER_INTERNAL_ERROR", {"error": type(exc).__name__})
|
||||||
|
finally:
|
||||||
|
if worktree_fd is not None:
|
||||||
|
os.close(worktree_fd)
|
||||||
|
raw = json.dumps(response, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + b"\n"
|
||||||
|
try:
|
||||||
|
connection.settimeout(CONNECTION_TIMEOUT)
|
||||||
|
connection.sendall(raw[:MAX_BYTES])
|
||||||
|
finally:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
if ADAPTER not in {"hermes", "codex", "openclaw"}:
|
||||||
|
raise BrokerError("BROKER_ADAPTER_INVALID")
|
||||||
|
if not SOCKET_PATH.is_absolute() or not WORKSPACE_ROOT.is_absolute():
|
||||||
|
raise BrokerError("BROKER_PATH_INVALID")
|
||||||
|
SOCKET_PATH.parent.mkdir(parents=True, mode=0o750, exist_ok=True)
|
||||||
|
if SOCKET_PATH.exists() or SOCKET_PATH.is_symlink():
|
||||||
|
SOCKET_PATH.unlink()
|
||||||
|
listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||||
|
listener.bind(str(SOCKET_PATH))
|
||||||
|
os.chmod(SOCKET_PATH, 0o660)
|
||||||
|
listener.listen(8)
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
serve_once(listener)
|
||||||
|
finally:
|
||||||
|
listener.close()
|
||||||
|
if SOCKET_PATH.exists():
|
||||||
|
SOCKET_PATH.unlink()
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,369 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""OpenClaw engineering backend for the AIOS autonomous runtime broker.
|
||||||
|
|
||||||
|
This process is intentionally *not* an Online Agent Task worker. It receives
|
||||||
|
one secret-free `aios.online-agent.runtime-request.v1` object on stdin, invokes
|
||||||
|
OpenClaw's headless `agent exec` against the pinned disposable worktree, and
|
||||||
|
returns one `aios.online-agent.runtime-result.v1` object on stdout.
|
||||||
|
|
||||||
|
Online Task claim/heartbeat/result and Governed GitOps remain owned by the
|
||||||
|
AIOS autonomous worker.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from typing import Any, Mapping
|
||||||
|
|
||||||
|
|
||||||
|
REQUEST_SCHEMA = "aios.online-agent.runtime-request.v1"
|
||||||
|
RESULT_SCHEMA = "aios.online-agent.runtime-result.v1"
|
||||||
|
MAX_INPUT_BYTES = 1024 * 1024
|
||||||
|
MAX_OUTPUT_BYTES = 1024 * 1024
|
||||||
|
DEFAULT_TIMEOUT = 1800
|
||||||
|
ALLOWED_ADAPTERS = frozenset({"hermes", "codex", "openclaw"})
|
||||||
|
SECRET_PATTERNS = (
|
||||||
|
re.compile(r"\bsk-[A-Za-z0-9_-]{20,}\b"),
|
||||||
|
re.compile(r"\bghp_[A-Za-z0-9]{20,}\b"),
|
||||||
|
re.compile(r"\bAKIA[0-9A-Z]{16}\b"),
|
||||||
|
re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class BackendError(RuntimeError):
|
||||||
|
def __init__(self, code: str, *, details: Mapping[str, Any] | None = None) -> None:
|
||||||
|
super().__init__(code)
|
||||||
|
self.code = code
|
||||||
|
self.details = dict(details or {})
|
||||||
|
|
||||||
|
|
||||||
|
def redact_text(value: str, limit: int = 8192) -> str:
|
||||||
|
value = value[:limit]
|
||||||
|
for pattern in SECRET_PATTERNS:
|
||||||
|
value = pattern.sub("[REDACTED]", value)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def result(
|
||||||
|
status: str,
|
||||||
|
summary: str,
|
||||||
|
*,
|
||||||
|
verification: list[dict[str, Any]] | None = None,
|
||||||
|
evidence: Mapping[str, Any] | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"schema": RESULT_SCHEMA,
|
||||||
|
"status": status,
|
||||||
|
"summary": redact_text(summary) or status,
|
||||||
|
"verification": verification or [],
|
||||||
|
"evidence": dict(evidence or {}),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def emit(value: Mapping[str, Any]) -> int:
|
||||||
|
raw = json.dumps(dict(value), ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||||
|
sys.stdout.write(raw)
|
||||||
|
sys.stdout.write("\n")
|
||||||
|
sys.stdout.flush()
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def read_request() -> dict[str, Any]:
|
||||||
|
raw = sys.stdin.buffer.read(MAX_INPUT_BYTES + 1)
|
||||||
|
if len(raw) > MAX_INPUT_BYTES:
|
||||||
|
raise BackendError("REQUEST_TOO_LARGE")
|
||||||
|
try:
|
||||||
|
value = json.loads(raw.decode("utf-8"))
|
||||||
|
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||||
|
raise BackendError("REQUEST_INVALID_JSON") from exc
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise BackendError("REQUEST_OBJECT_REQUIRED")
|
||||||
|
if value.get("schema") != REQUEST_SCHEMA:
|
||||||
|
raise BackendError("REQUEST_SCHEMA_INVALID")
|
||||||
|
return dict(value)
|
||||||
|
|
||||||
|
|
||||||
|
def nonempty(value: Any, field: str, limit: int) -> str:
|
||||||
|
if not isinstance(value, str):
|
||||||
|
raise BackendError(f"{field.upper()}_INVALID")
|
||||||
|
value = value.strip()
|
||||||
|
if not value or len(value) > limit:
|
||||||
|
raise BackendError(f"{field.upper()}_INVALID")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def string_list(value: Any, field: str) -> list[str]:
|
||||||
|
if not isinstance(value, list) or not value or len(value) > 64:
|
||||||
|
raise BackendError(f"{field.upper()}_INVALID")
|
||||||
|
result_value: list[str] = []
|
||||||
|
for item in value:
|
||||||
|
result_value.append(nonempty(item, field, 4096))
|
||||||
|
return result_value
|
||||||
|
|
||||||
|
|
||||||
|
def validate_request(value: Mapping[str, Any]) -> tuple[dict[str, Any], Path]:
|
||||||
|
adapter = str(value.get("adapter") or "")
|
||||||
|
if adapter not in ALLOWED_ADAPTERS:
|
||||||
|
raise BackendError("ADAPTER_INVALID")
|
||||||
|
|
||||||
|
task = value.get("task")
|
||||||
|
context = value.get("execution_context")
|
||||||
|
policy = value.get("policy")
|
||||||
|
if not isinstance(task, Mapping) or not isinstance(context, Mapping) or not isinstance(policy, Mapping):
|
||||||
|
raise BackendError("REQUEST_CONTEXT_INVALID")
|
||||||
|
|
||||||
|
task_id = nonempty(task.get("task_id"), "task_id", 180)
|
||||||
|
goal = nonempty(task.get("goal"), "goal", 16384)
|
||||||
|
instructions = nonempty(task.get("instructions"), "instructions", 65536)
|
||||||
|
success = string_list(task.get("success_criteria"), "success_criteria")
|
||||||
|
|
||||||
|
worktree_raw = context.get("worktree")
|
||||||
|
if not isinstance(worktree_raw, str) or not worktree_raw.startswith("/proc/self/fd/"):
|
||||||
|
raise BackendError("PINNED_WORKTREE_REQUIRED")
|
||||||
|
worktree = Path(worktree_raw)
|
||||||
|
try:
|
||||||
|
resolved = worktree.resolve(strict=True)
|
||||||
|
except OSError as exc:
|
||||||
|
raise BackendError("PINNED_WORKTREE_UNAVAILABLE") from exc
|
||||||
|
if not resolved.is_dir():
|
||||||
|
raise BackendError("PINNED_WORKTREE_INVALID")
|
||||||
|
|
||||||
|
for field in (
|
||||||
|
"authoritative_repository_write",
|
||||||
|
"direct_git_ref_mutation",
|
||||||
|
"direct_git_push",
|
||||||
|
"direct_git_commit",
|
||||||
|
"main_write",
|
||||||
|
"force_push",
|
||||||
|
"production_write",
|
||||||
|
"tag_write",
|
||||||
|
"baseline_write",
|
||||||
|
"task_api_credential_visible",
|
||||||
|
"writer_lease_visible",
|
||||||
|
):
|
||||||
|
if policy.get(field) is not False:
|
||||||
|
raise BackendError("REQUEST_POLICY_REJECTED", details={"field": field})
|
||||||
|
|
||||||
|
return {
|
||||||
|
"adapter": adapter,
|
||||||
|
"task_id": task_id,
|
||||||
|
"goal": goal,
|
||||||
|
"instructions": instructions,
|
||||||
|
"success_criteria": success,
|
||||||
|
"repository": str(context.get("repository") or ""),
|
||||||
|
"base_oid": str(context.get("base_oid") or ""),
|
||||||
|
"target_branch": str(context.get("target_branch") or ""),
|
||||||
|
}, worktree
|
||||||
|
|
||||||
|
|
||||||
|
def openclaw_binary() -> Path:
|
||||||
|
configured = os.environ.get("AIOS_OPENCLAW_BIN", "").strip()
|
||||||
|
if not configured:
|
||||||
|
raise BackendError("OPENCLAW_BINARY_NOT_CONFIGURED")
|
||||||
|
path = Path(configured)
|
||||||
|
if not path.is_absolute():
|
||||||
|
raise BackendError("OPENCLAW_BINARY_INVALID")
|
||||||
|
try:
|
||||||
|
resolved = path.resolve(strict=True)
|
||||||
|
except OSError as exc:
|
||||||
|
raise BackendError("OPENCLAW_BINARY_UNAVAILABLE") from exc
|
||||||
|
if not resolved.is_file() or not os.access(resolved, os.X_OK):
|
||||||
|
raise BackendError("OPENCLAW_BINARY_UNAVAILABLE")
|
||||||
|
return resolved
|
||||||
|
|
||||||
|
|
||||||
|
def provider_environment() -> dict[str, str]:
|
||||||
|
"""Build an environment containing only explicitly projected provider auth.
|
||||||
|
|
||||||
|
`--auth-env-only` tells OpenClaw not to load ambient config/auth stores.
|
||||||
|
The mapping is non-secret; values are read from systemd credential files.
|
||||||
|
"""
|
||||||
|
raw = os.environ.get("AIOS_OPENCLAW_PROVIDER_CREDENTIAL_MAP_JSON", "{}")
|
||||||
|
try:
|
||||||
|
mapping = json.loads(raw)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise BackendError("PROVIDER_CREDENTIAL_MAP_INVALID") from exc
|
||||||
|
if not isinstance(mapping, Mapping):
|
||||||
|
raise BackendError("PROVIDER_CREDENTIAL_MAP_INVALID")
|
||||||
|
|
||||||
|
credentials_directory = os.environ.get("CREDENTIALS_DIRECTORY", "").strip()
|
||||||
|
environment = {
|
||||||
|
"PATH": "/usr/local/bin:/usr/bin:/bin",
|
||||||
|
"LC_ALL": "C.UTF-8",
|
||||||
|
"NO_COLOR": "1",
|
||||||
|
# Prevent optional login-shell env snapshot import into exec runs.
|
||||||
|
"OPENCLAW_EXEC_SHELL_SNAPSHOT": "0",
|
||||||
|
}
|
||||||
|
if mapping:
|
||||||
|
if not credentials_directory:
|
||||||
|
raise BackendError("PROVIDER_CREDENTIAL_DIRECTORY_UNAVAILABLE")
|
||||||
|
credential_root = Path(credentials_directory)
|
||||||
|
for env_name, credential_name in mapping.items():
|
||||||
|
if (
|
||||||
|
not isinstance(env_name, str)
|
||||||
|
or not re.fullmatch(r"[A-Z][A-Z0-9_]{2,63}", env_name)
|
||||||
|
or not isinstance(credential_name, str)
|
||||||
|
or not re.fullmatch(r"[A-Za-z0-9._-]{1,128}", credential_name)
|
||||||
|
):
|
||||||
|
raise BackendError("PROVIDER_CREDENTIAL_MAP_INVALID")
|
||||||
|
path = credential_root / credential_name
|
||||||
|
if not path.is_file() or path.is_symlink():
|
||||||
|
raise BackendError("PROVIDER_CREDENTIAL_UNAVAILABLE", details={"name": credential_name})
|
||||||
|
secret = path.read_text(encoding="utf-8").strip()
|
||||||
|
if not secret:
|
||||||
|
raise BackendError("PROVIDER_CREDENTIAL_EMPTY", details={"name": credential_name})
|
||||||
|
environment[env_name] = secret
|
||||||
|
return environment
|
||||||
|
|
||||||
|
|
||||||
|
def build_prompt(task: Mapping[str, Any]) -> str:
|
||||||
|
criteria = "\n".join(f"- {item}" for item in task["success_criteria"])
|
||||||
|
return f"""You are the autonomous engineering runtime for AIOS task {task['task_id']}.
|
||||||
|
|
||||||
|
Repository: {task['repository']}
|
||||||
|
Base OID: {task['base_oid']}
|
||||||
|
Target branch (owned by AIOS Governed GitOps, not by you): {task['target_branch']}
|
||||||
|
|
||||||
|
Goal:
|
||||||
|
{task['goal']}
|
||||||
|
|
||||||
|
Instructions:
|
||||||
|
{task['instructions']}
|
||||||
|
|
||||||
|
Success criteria:
|
||||||
|
{criteria}
|
||||||
|
|
||||||
|
Execution boundary:
|
||||||
|
- Work only inside the current disposable worktree.
|
||||||
|
- Inspect the repository yourself and determine the root cause and files to change.
|
||||||
|
- Implement the smallest correct fix.
|
||||||
|
- Run focused tests and repair failures when feasible.
|
||||||
|
- Do NOT run `git commit`, `git push`, `git update-ref`, create branches/tags, or modify authoritative refs.
|
||||||
|
- Do NOT attempt to access AIOS Agent Task credentials, writer leases, or Governed GitOps credentials.
|
||||||
|
- Leave Git HEAD unchanged; AIOS independently inspects the diff, reruns verification, and performs the governed commit.
|
||||||
|
- In your final answer, concisely state root cause, files changed, and tests you ran.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def openclaw_timeout() -> int:
|
||||||
|
raw = os.environ.get("AIOS_OPENCLAW_EXEC_TIMEOUT_SECONDS", str(DEFAULT_TIMEOUT))
|
||||||
|
try:
|
||||||
|
value = int(raw)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise BackendError("OPENCLAW_TIMEOUT_INVALID") from exc
|
||||||
|
if not 60 <= value <= 7200:
|
||||||
|
raise BackendError("OPENCLAW_TIMEOUT_INVALID")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def invoke_openclaw(task: Mapping[str, Any], worktree: Path) -> dict[str, Any]:
|
||||||
|
binary = openclaw_binary()
|
||||||
|
model = nonempty(os.environ.get("AIOS_OPENCLAW_MODEL"), "openclaw_model", 256)
|
||||||
|
timeout_seconds = openclaw_timeout()
|
||||||
|
prompt = build_prompt(task)
|
||||||
|
|
||||||
|
argv = [
|
||||||
|
str(binary),
|
||||||
|
"agent",
|
||||||
|
"exec",
|
||||||
|
"--message-file",
|
||||||
|
"-",
|
||||||
|
"--cwd",
|
||||||
|
str(worktree),
|
||||||
|
"--model",
|
||||||
|
model,
|
||||||
|
"--auth-env-only",
|
||||||
|
"--timeout",
|
||||||
|
str(timeout_seconds),
|
||||||
|
"--json",
|
||||||
|
]
|
||||||
|
|
||||||
|
try:
|
||||||
|
pinned_fd = int(worktree.name)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise BackendError("PINNED_WORKTREE_FD_INVALID") from exc
|
||||||
|
|
||||||
|
try:
|
||||||
|
completed = subprocess.run(
|
||||||
|
argv,
|
||||||
|
input=prompt.encode("utf-8"),
|
||||||
|
cwd=str(worktree),
|
||||||
|
pass_fds=(pinned_fd,),
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
timeout=timeout_seconds + 90,
|
||||||
|
env=provider_environment(),
|
||||||
|
)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
return result(
|
||||||
|
"FAILED",
|
||||||
|
"OPENCLAW_HARD_TIMEOUT",
|
||||||
|
evidence={"backend": "openclaw-agent-exec", "timeout_seconds": timeout_seconds},
|
||||||
|
)
|
||||||
|
except OSError as exc:
|
||||||
|
return result(
|
||||||
|
"FAILED",
|
||||||
|
"OPENCLAW_EXEC_FAILED",
|
||||||
|
evidence={"backend": "openclaw-agent-exec", "error": type(exc).__name__},
|
||||||
|
)
|
||||||
|
|
||||||
|
stdout = completed.stdout[:MAX_OUTPUT_BYTES]
|
||||||
|
try:
|
||||||
|
envelope = json.loads(stdout.decode("utf-8"))
|
||||||
|
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||||
|
return result(
|
||||||
|
"FAILED",
|
||||||
|
"OPENCLAW_RESULT_INVALID",
|
||||||
|
evidence={
|
||||||
|
"backend": "openclaw-agent-exec",
|
||||||
|
"exit_code": completed.returncode,
|
||||||
|
"stderr_sha256": __import__("hashlib").sha256(completed.stderr).hexdigest(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
if not isinstance(envelope, Mapping):
|
||||||
|
return result("FAILED", "OPENCLAW_RESULT_INVALID", evidence={"backend": "openclaw-agent-exec"})
|
||||||
|
|
||||||
|
ok = envelope.get("ok") is True and envelope.get("status") == "ok" and completed.returncode == 0
|
||||||
|
final_text = str(envelope.get("final") or "").strip()
|
||||||
|
evidence = {
|
||||||
|
"backend": "openclaw-agent-exec",
|
||||||
|
"openclaw_status": envelope.get("status"),
|
||||||
|
"exit_code": completed.returncode,
|
||||||
|
"model": envelope.get("model"),
|
||||||
|
"provider": envelope.get("provider"),
|
||||||
|
"session_id_present": bool(envelope.get("sessionId")),
|
||||||
|
"code_mode_engaged": envelope.get("codeModeEngaged"),
|
||||||
|
"assistant_turns": envelope.get("assistantTurns"),
|
||||||
|
"tool_summary": envelope.get("toolSummary") if isinstance(envelope.get("toolSummary"), Mapping) else {},
|
||||||
|
}
|
||||||
|
|
||||||
|
if not ok:
|
||||||
|
error = envelope.get("error")
|
||||||
|
if isinstance(error, Mapping):
|
||||||
|
evidence["error_kind"] = error.get("kind")
|
||||||
|
summary = "OPENCLAW_" + str(envelope.get("status") or "ERROR").upper()
|
||||||
|
return result("FAILED", summary, evidence=evidence)
|
||||||
|
|
||||||
|
summary = final_text or "OpenClaw completed the engineering turn."
|
||||||
|
return result("COMPLETED", summary, evidence=evidence)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
try:
|
||||||
|
request = read_request()
|
||||||
|
task, worktree = validate_request(request)
|
||||||
|
return emit(invoke_openclaw(task, worktree))
|
||||||
|
except BackendError as exc:
|
||||||
|
return emit(result("BLOCKED", exc.code, evidence=exc.details))
|
||||||
|
except Exception as exc:
|
||||||
|
return emit(result("FAILED", "OPENCLAW_BACKEND_INTERNAL_ERROR", evidence={"error": type(exc).__name__}))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,822 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import grp
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from typing import Any, Mapping, Sequence
|
||||||
|
|
||||||
|
from intent_runtime import IntentError, Resolution, is_claimable_intent, resolve
|
||||||
|
from runtime_client import RuntimeErrorV2, collect_changes, invoke, request_payload, scan_secrets
|
||||||
|
|
||||||
|
|
||||||
|
API_BASE = os.environ.get("AIOS_AGENT_TASK_API_URL", "http://127.0.0.1:18940").rstrip("/")
|
||||||
|
BRIDGE_BASE = os.environ.get("AIOS_GOVERNED_WRITE_API_URL", "http://127.0.0.1:19998").rstrip("/")
|
||||||
|
AGENT_ID = os.environ.get("AIOS_AGENT_ID", "online-engineering-worker-cn-network-mkt-autonomous-20260815")
|
||||||
|
AGENT_VERSION = os.environ.get("AIOS_AGENT_VERSION", "2026-08-15.4-autonomous-intent")
|
||||||
|
REPOSITORY = os.environ.get("AIOS_AUTONOMOUS_REPOSITORY", "aios/aios")
|
||||||
|
BASE_REF = os.environ.get("AIOS_AUTONOMOUS_BASE_REF", "refs/heads/main")
|
||||||
|
AUTHORITY_GIT_DIR = Path(os.environ.get("AIOS_AUTHORITY_GIT_DIR", "/data/aios-gitops/repo.git"))
|
||||||
|
STATE_ROOT = Path(os.environ.get("AIOS_AUTONOMOUS_STATE_ROOT", "/var/lib/aios-autonomous-agent-worker"))
|
||||||
|
WORKSPACE_ROOT = Path(os.environ.get("AIOS_AUTONOMOUS_WORKSPACE_ROOT", str(STATE_ROOT / "workspaces")))
|
||||||
|
SHARED_GROUP = os.environ.get("AIOS_AGENT_RUNTIME_SHARED_GROUP", "aios-agent-runtime")
|
||||||
|
DEFAULT_ADAPTER = os.environ.get("AIOS_AUTONOMOUS_DEFAULT_ADAPTER", "hermes")
|
||||||
|
HERMES_SOCKET = Path(os.environ.get("AIOS_HERMES_BROKER_SOCKET", "/run/aios-agent-runtime/hermes.sock"))
|
||||||
|
CODEX_SOCKET = Path(os.environ.get("AIOS_CODEX_BROKER_SOCKET", "/run/aios-agent-runtime/codex.sock"))
|
||||||
|
OPENCLAW_SOCKET = Path(os.environ.get("AIOS_OPENCLAW_BROKER_SOCKET", "/run/aios-agent-runtime/openclaw.sock"))
|
||||||
|
POLL_SECONDS = int(os.environ.get("AIOS_AUTONOMOUS_POLL_SECONDS", "5"))
|
||||||
|
HEARTBEAT_SECONDS = int(os.environ.get("AIOS_AUTONOMOUS_HEARTBEAT_SECONDS", "10"))
|
||||||
|
LEASE_SECONDS = int(os.environ.get("AIOS_AUTONOMOUS_LEASE_SECONDS", "300"))
|
||||||
|
RUNTIME_TIMEOUT_SECONDS = int(os.environ.get("AIOS_AUTONOMOUS_RUNTIME_TIMEOUT_SECONDS", "3600"))
|
||||||
|
MAX_OUTPUT_BYTES = 64 * 1024
|
||||||
|
OID_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||||
|
SENSITIVE_RE = re.compile(
|
||||||
|
r"(?i)(authorization\s*:\s*bearer\s+\S+|(?:password|passwd|secret|token|api[_-]?key)\s*[:=]\s*\S+)"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class WorkerError(RuntimeError):
|
||||||
|
def __init__(self, code: str, *, phase: str, details: Mapping[str, Any] | None = None) -> None:
|
||||||
|
super().__init__(code)
|
||||||
|
self.code = code
|
||||||
|
self.phase = phase
|
||||||
|
self.details = dict(details or {})
|
||||||
|
|
||||||
|
|
||||||
|
def now() -> str:
|
||||||
|
return datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def sha256_bytes(value: bytes) -> str:
|
||||||
|
return hashlib.sha256(value).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def sha256_json(value: Any) -> str:
|
||||||
|
return sha256_bytes(json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
def sanitize(value: Any) -> Any:
|
||||||
|
if isinstance(value, Mapping):
|
||||||
|
result: dict[str, Any] = {}
|
||||||
|
for key, inner in value.items():
|
||||||
|
lowered = str(key).lower()
|
||||||
|
if any(word in lowered for word in ("token", "secret", "credential", "authorization", "password", "api_key", "apikey", "access_key")):
|
||||||
|
result[str(key)] = "[REDACTED]"
|
||||||
|
else:
|
||||||
|
result[str(key)] = sanitize(inner)
|
||||||
|
return result
|
||||||
|
if isinstance(value, list):
|
||||||
|
return [sanitize(item) for item in value]
|
||||||
|
if isinstance(value, tuple):
|
||||||
|
return [sanitize(item) for item in value]
|
||||||
|
if isinstance(value, str):
|
||||||
|
return SENSITIVE_RE.sub("[REDACTED]", value)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def audit(event_type: str, **fields: Any) -> None:
|
||||||
|
STATE_ROOT.mkdir(parents=True, mode=0o750, exist_ok=True)
|
||||||
|
path = STATE_ROOT / "events.jsonl"
|
||||||
|
record = {
|
||||||
|
"event_type": event_type,
|
||||||
|
"created_at": now(),
|
||||||
|
"agent_id": AGENT_ID,
|
||||||
|
"agent_version": AGENT_VERSION,
|
||||||
|
"credential_material_exposed": False,
|
||||||
|
"writer_lease_material_exposed": False,
|
||||||
|
**sanitize(fields),
|
||||||
|
}
|
||||||
|
with path.open("a", encoding="utf-8") as handle:
|
||||||
|
handle.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n")
|
||||||
|
os.chmod(path, 0o640)
|
||||||
|
print(json.dumps(record, ensure_ascii=False, sort_keys=True), flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
def atomic_json(path: Path, value: Mapping[str, Any]) -> None:
|
||||||
|
path.parent.mkdir(parents=True, mode=0o750, exist_ok=True)
|
||||||
|
if path.parent.is_symlink() or path.is_symlink():
|
||||||
|
raise WorkerError("STATE_PATH_SYMLINK_REJECTED", phase="state")
|
||||||
|
fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=str(path.parent))
|
||||||
|
try:
|
||||||
|
os.fchmod(fd, 0o640)
|
||||||
|
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||||
|
json.dump(sanitize(dict(value)), handle, ensure_ascii=False, sort_keys=True)
|
||||||
|
handle.write("\n")
|
||||||
|
handle.flush()
|
||||||
|
os.fsync(handle.fileno())
|
||||||
|
os.replace(temporary, path)
|
||||||
|
finally:
|
||||||
|
if os.path.exists(temporary):
|
||||||
|
os.unlink(temporary)
|
||||||
|
|
||||||
|
|
||||||
|
def state_path(task_id: str) -> Path:
|
||||||
|
safe = re.sub(r"[^A-Za-z0-9._:-]+", "-", task_id)[:180]
|
||||||
|
return STATE_ROOT / "tasks" / f"{safe}.json"
|
||||||
|
|
||||||
|
|
||||||
|
def load_state(task_id: str) -> dict[str, Any] | None:
|
||||||
|
path = state_path(task_id)
|
||||||
|
if not path.is_file() or path.is_symlink():
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
value = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
return value if isinstance(value, dict) else None
|
||||||
|
|
||||||
|
|
||||||
|
def save_state(task_id: str, value: Mapping[str, Any]) -> None:
|
||||||
|
atomic_json(state_path(task_id), value)
|
||||||
|
|
||||||
|
|
||||||
|
def api_call(method: str, route: str, token: str, body: Mapping[str, Any] | None = None) -> tuple[int, dict[str, Any]]:
|
||||||
|
payload = None if body is None else json.dumps(dict(body), ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
headers = {"Authorization": "Bearer " + token, "Accept": "application/json"}
|
||||||
|
if payload is not None:
|
||||||
|
headers["Content-Type"] = "application/json"
|
||||||
|
request = urllib.request.Request(API_BASE + route, data=payload, headers=headers, method=method)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(request, timeout=20) as response:
|
||||||
|
value = json.loads(response.read().decode("utf-8"))
|
||||||
|
return response.status, value if isinstance(value, dict) else {}
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
try:
|
||||||
|
value = json.loads(exc.read(4096).decode("utf-8", "replace"))
|
||||||
|
except Exception:
|
||||||
|
value = {"error": "HTTP_ERROR"}
|
||||||
|
return exc.code, value if isinstance(value, dict) else {"error": "HTTP_ERROR"}
|
||||||
|
except (urllib.error.URLError, TimeoutError):
|
||||||
|
return 599, {"error": "AGENT_TASK_API_UNAVAILABLE"}
|
||||||
|
|
||||||
|
|
||||||
|
def bridge_call(route: str, body: Mapping[str, Any]) -> tuple[int, dict[str, Any]]:
|
||||||
|
if BRIDGE_BASE != "http://127.0.0.1:19998":
|
||||||
|
raise WorkerError(
|
||||||
|
"NON_STAGING_BRIDGE_REJECTED",
|
||||||
|
phase="startup",
|
||||||
|
details={"bridge_base": BRIDGE_BASE},
|
||||||
|
)
|
||||||
|
payload = json.dumps(dict(body), ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
request = urllib.request.Request(
|
||||||
|
BRIDGE_BASE + route,
|
||||||
|
data=payload,
|
||||||
|
headers={"Accept": "application/json", "Content-Type": "application/json"},
|
||||||
|
method="POST",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(request, timeout=60) as response:
|
||||||
|
value = json.loads(response.read().decode("utf-8"))
|
||||||
|
return response.status, value if isinstance(value, dict) else {}
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
try:
|
||||||
|
value = json.loads(exc.read(4096).decode("utf-8", "replace"))
|
||||||
|
except Exception:
|
||||||
|
value = {"error": "HTTP_ERROR"}
|
||||||
|
return exc.code, value if isinstance(value, dict) else {"error": "HTTP_ERROR"}
|
||||||
|
except (urllib.error.URLError, TimeoutError):
|
||||||
|
return 599, {"error": "GOVERNED_WRITE_API_UNAVAILABLE"}
|
||||||
|
|
||||||
|
|
||||||
|
def load_agent_token() -> str:
|
||||||
|
directory = Path(os.environ.get("CREDENTIALS_DIRECTORY", ""))
|
||||||
|
path = directory / "agent_token"
|
||||||
|
if not path.is_file():
|
||||||
|
raise WorkerError("AGENT_RUNTIME_CREDENTIAL_UNAVAILABLE", phase="startup")
|
||||||
|
token = path.read_text(encoding="utf-8").strip()
|
||||||
|
if len(token) < 24:
|
||||||
|
raise WorkerError("AGENT_RUNTIME_CREDENTIAL_INVALID", phase="startup")
|
||||||
|
return token
|
||||||
|
|
||||||
|
|
||||||
|
def available_adapters() -> tuple[str, ...]:
|
||||||
|
values: list[str] = []
|
||||||
|
for name, path in (("hermes", HERMES_SOCKET), ("codex", CODEX_SOCKET), ("openclaw", OPENCLAW_SOCKET)):
|
||||||
|
try:
|
||||||
|
if path.is_socket():
|
||||||
|
values.append(name)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
return tuple(values)
|
||||||
|
|
||||||
|
|
||||||
|
def capabilities() -> tuple[str, ...]:
|
||||||
|
values = {
|
||||||
|
"online_agent_task_api", "task.poll", "task.claim", "task.heartbeat", "task.result",
|
||||||
|
"intent.task", "engineering.consumer", "engineering.execute", "engineering.autonomous",
|
||||||
|
"workspace.worktree", "git.read", "git.diff", "git.test",
|
||||||
|
"governed.write.branch", "governed.write.file",
|
||||||
|
}
|
||||||
|
values.update(f"agent.runtime.{name}" for name in available_adapters())
|
||||||
|
return tuple(sorted(values))
|
||||||
|
|
||||||
|
|
||||||
|
class LeaseHeartbeat:
|
||||||
|
def __init__(self, task_id: str, agent_token: str, lease_token: str) -> None:
|
||||||
|
self.task_id = task_id
|
||||||
|
self.agent_token = agent_token
|
||||||
|
self.lease_token = lease_token
|
||||||
|
self.stage = "claimed"
|
||||||
|
self.failed: str | None = None
|
||||||
|
self.stop = threading.Event()
|
||||||
|
self.thread = threading.Thread(target=self._run, daemon=True, name=f"heartbeat-{task_id}")
|
||||||
|
|
||||||
|
def set_stage(self, stage: str) -> None:
|
||||||
|
self.stage = stage
|
||||||
|
audit("TASK_PROGRESS_STAGE", task_id=self.task_id, stage=stage)
|
||||||
|
|
||||||
|
def send(self) -> None:
|
||||||
|
status, response = api_call(
|
||||||
|
"POST",
|
||||||
|
f"/v1/tasks/{self.task_id}/heartbeat",
|
||||||
|
self.agent_token,
|
||||||
|
{
|
||||||
|
"lease_token": self.lease_token,
|
||||||
|
"lease_seconds": LEASE_SECONDS,
|
||||||
|
"progress": {
|
||||||
|
"stage": self.stage,
|
||||||
|
"event_type": "TASK_STARTED" if self.stage == "started" else "TASK_PROGRESS",
|
||||||
|
"execution_mode": "autonomous-intent-v2",
|
||||||
|
"production_side_effects": False,
|
||||||
|
"main_write": False,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if 200 <= status < 300:
|
||||||
|
audit(
|
||||||
|
"TASK_HEARTBEAT",
|
||||||
|
task_id=self.task_id,
|
||||||
|
stage=self.stage,
|
||||||
|
api_status=status,
|
||||||
|
task_status=response.get("status"),
|
||||||
|
lease_expires_at=response.get("lease_expires_at"),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.failed = str(response.get("error") or status)
|
||||||
|
audit("TASK_HEARTBEAT_FAILED", task_id=self.task_id, stage=self.stage, error=self.failed)
|
||||||
|
|
||||||
|
def _run(self) -> None:
|
||||||
|
while not self.stop.wait(HEARTBEAT_SECONDS):
|
||||||
|
try:
|
||||||
|
self.send()
|
||||||
|
except Exception as exc:
|
||||||
|
self.failed = type(exc).__name__
|
||||||
|
audit("TASK_HEARTBEAT_FAILED", task_id=self.task_id, error=self.failed)
|
||||||
|
|
||||||
|
def start(self) -> None:
|
||||||
|
self.thread.start()
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
self.stop.set()
|
||||||
|
self.thread.join(timeout=3)
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_roots() -> None:
|
||||||
|
if WORKSPACE_ROOT.is_symlink() or STATE_ROOT.is_symlink() or AUTHORITY_GIT_DIR.is_symlink():
|
||||||
|
raise WorkerError("ROOT_SYMLINK_REJECTED", phase="startup")
|
||||||
|
if not AUTHORITY_GIT_DIR.is_dir():
|
||||||
|
raise WorkerError("AUTHORITY_GIT_DIR_UNAVAILABLE", phase="startup")
|
||||||
|
STATE_ROOT.mkdir(parents=True, mode=0o750, exist_ok=True)
|
||||||
|
WORKSPACE_ROOT.mkdir(parents=True, mode=0o750, exist_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
def run_git(args: Sequence[str], *, cwd: Path, timeout: int = 180) -> subprocess.CompletedProcess[bytes]:
|
||||||
|
completed = subprocess.run(
|
||||||
|
["/usr/bin/git", *args],
|
||||||
|
cwd=str(cwd),
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
timeout=timeout,
|
||||||
|
env={"PATH": "/usr/bin:/bin", "LC_ALL": "C.UTF-8"},
|
||||||
|
)
|
||||||
|
if completed.returncode != 0:
|
||||||
|
raise WorkerError(
|
||||||
|
"GIT_COMMAND_FAILED",
|
||||||
|
phase="workspace",
|
||||||
|
details={"operation": args[0] if args else "", "exit_code": completed.returncode},
|
||||||
|
)
|
||||||
|
return completed
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_worktree(task_id: str, base_oid: str) -> tuple[Path, Path]:
|
||||||
|
ensure_roots()
|
||||||
|
run_root = Path(tempfile.mkdtemp(prefix=f"{task_id[:48]}-", dir=str(WORKSPACE_ROOT)))
|
||||||
|
os.chmod(run_root, 0o750)
|
||||||
|
clone = run_root / "repository"
|
||||||
|
worktree = run_root / "worktree"
|
||||||
|
run_git(["clone", "--no-local", "--no-hardlinks", str(AUTHORITY_GIT_DIR), str(clone)], cwd=run_root, timeout=300)
|
||||||
|
run_git(["-C", str(clone), "rev-parse", "--verify", f"{base_oid}^{{commit}}"], cwd=run_root)
|
||||||
|
run_git(["-C", str(clone), "worktree", "add", "--detach", str(worktree), base_oid], cwd=run_root, timeout=300)
|
||||||
|
return run_root, worktree
|
||||||
|
|
||||||
|
|
||||||
|
def share_run_root(run_root: Path) -> None:
|
||||||
|
try:
|
||||||
|
gid = grp.getgrnam(SHARED_GROUP).gr_gid
|
||||||
|
except KeyError as exc:
|
||||||
|
raise WorkerError("RUNTIME_SHARED_GROUP_UNAVAILABLE", phase="workspace") from exc
|
||||||
|
root = run_root.resolve()
|
||||||
|
if not root.is_dir() or run_root.is_symlink():
|
||||||
|
raise WorkerError("RUNTIME_RUN_ROOT_INVALID", phase="workspace")
|
||||||
|
for current, dirs, files in os.walk(root, topdown=True, followlinks=False):
|
||||||
|
current_path = Path(current)
|
||||||
|
for name in dirs + files:
|
||||||
|
candidate = current_path / name
|
||||||
|
if candidate.is_symlink():
|
||||||
|
raise WorkerError(
|
||||||
|
"RUNTIME_WORKSPACE_SYMLINK_REJECTED",
|
||||||
|
phase="workspace",
|
||||||
|
details={"path": str(candidate.relative_to(root))},
|
||||||
|
)
|
||||||
|
for candidate in [current_path, *(current_path / name for name in dirs), *(current_path / name for name in files)]:
|
||||||
|
st = candidate.stat()
|
||||||
|
os.chown(candidate, -1, gid)
|
||||||
|
mode = st.st_mode & 0o777
|
||||||
|
os.chmod(candidate, mode | (0o070 if candidate.is_dir() else 0o060))
|
||||||
|
|
||||||
|
|
||||||
|
def cleanup_run_root(run_root: Path) -> None:
|
||||||
|
root = WORKSPACE_ROOT.resolve()
|
||||||
|
candidate = run_root.resolve(strict=False)
|
||||||
|
if candidate == root or root not in candidate.parents:
|
||||||
|
raise WorkerError("CLEANUP_SCOPE_INVALID", phase="cleanup")
|
||||||
|
if candidate.exists():
|
||||||
|
shutil.rmtree(candidate)
|
||||||
|
|
||||||
|
|
||||||
|
def allowed_programs() -> set[str]:
|
||||||
|
raw = os.environ.get("AIOS_AUTONOMOUS_ALLOWED_PROGRAMS_JSON", '["/usr/bin/git","/usr/bin/python3"]')
|
||||||
|
try:
|
||||||
|
value = json.loads(raw)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise WorkerError("ALLOWED_PROGRAMS_CONFIG_INVALID", phase="startup") from exc
|
||||||
|
if not isinstance(value, list) or not all(isinstance(item, str) and item.startswith("/") for item in value):
|
||||||
|
raise WorkerError("ALLOWED_PROGRAMS_CONFIG_INVALID", phase="startup")
|
||||||
|
return set(value)
|
||||||
|
|
||||||
|
|
||||||
|
def validate_verification_argv(value: Any) -> list[str]:
|
||||||
|
if not isinstance(value, list) or not value or not all(isinstance(item, str) and item for item in value):
|
||||||
|
raise WorkerError("VERIFICATION_ARGV_INVALID", phase="verification")
|
||||||
|
argv = list(value)
|
||||||
|
if argv[0] not in allowed_programs():
|
||||||
|
raise WorkerError("VERIFICATION_PROGRAM_NOT_ALLOWED", phase="verification", details={"program": Path(argv[0]).name})
|
||||||
|
if argv[0].endswith("/git") and any(item in {"push", "update-ref", "receive-pack", "send-pack", "commit", "reset", "clean"} for item in argv[1:]):
|
||||||
|
raise WorkerError("VERIFICATION_GIT_MUTATION_REJECTED", phase="verification")
|
||||||
|
if argv[0].endswith("/python3") and "-c" in argv:
|
||||||
|
raise WorkerError("VERIFICATION_INLINE_PYTHON_REJECTED", phase="verification")
|
||||||
|
return argv
|
||||||
|
|
||||||
|
|
||||||
|
def run_verification(label: str, argv: Sequence[str], *, cwd: Path, timeout: int) -> dict[str, Any]:
|
||||||
|
started = time.monotonic()
|
||||||
|
completed = subprocess.run(
|
||||||
|
list(argv),
|
||||||
|
cwd=str(cwd),
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
timeout=timeout,
|
||||||
|
env={"PATH": "/usr/local/bin:/usr/bin:/bin", "LC_ALL": "C.UTF-8"},
|
||||||
|
)
|
||||||
|
result = {
|
||||||
|
"label": label[:120],
|
||||||
|
"argv": [Path(item).name if idx == 0 else item for idx, item in enumerate(argv)],
|
||||||
|
"exit_code": completed.returncode,
|
||||||
|
"stdout_sha256": sha256_bytes(completed.stdout),
|
||||||
|
"stderr_sha256": sha256_bytes(completed.stderr),
|
||||||
|
"duration_ms": int((time.monotonic() - started) * 1000),
|
||||||
|
}
|
||||||
|
audit("VERIFICATION_RESULT", **result)
|
||||||
|
if completed.returncode != 0:
|
||||||
|
raise WorkerError(
|
||||||
|
"VERIFICATION_FAILED",
|
||||||
|
phase="verification",
|
||||||
|
details={"label": label[:120], "exit_code": completed.returncode},
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def independent_verification(
|
||||||
|
worktree: Path,
|
||||||
|
changed_files: Sequence[str],
|
||||||
|
runtime_verification: Sequence[Mapping[str, Any]],
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
results = [
|
||||||
|
run_verification(
|
||||||
|
"git_diff_check",
|
||||||
|
["/usr/bin/git", "-C", str(worktree), "diff", "--check"],
|
||||||
|
cwd=worktree,
|
||||||
|
timeout=120,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
py_files = [item for item in changed_files if item.endswith(".py")]
|
||||||
|
if py_files:
|
||||||
|
results.append(
|
||||||
|
run_verification(
|
||||||
|
"changed_python_compile",
|
||||||
|
["/usr/bin/python3", "-m", "py_compile", *py_files],
|
||||||
|
cwd=worktree,
|
||||||
|
timeout=300,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
test_files = [
|
||||||
|
item for item in py_files
|
||||||
|
if item.startswith("tests/") or Path(item).name.startswith("test_")
|
||||||
|
]
|
||||||
|
if test_files:
|
||||||
|
results.append(
|
||||||
|
run_verification(
|
||||||
|
"changed_pytest",
|
||||||
|
["/usr/bin/python3", "-m", "pytest", "-q", *test_files],
|
||||||
|
cwd=worktree,
|
||||||
|
timeout=900,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
seen: set[str] = set()
|
||||||
|
for item in runtime_verification:
|
||||||
|
argv = validate_verification_argv(item.get("argv"))
|
||||||
|
cwd_value = item.get("cwd", ".")
|
||||||
|
if cwd_value in {"", "."}:
|
||||||
|
cwd = worktree
|
||||||
|
else:
|
||||||
|
if not isinstance(cwd_value, str) or cwd_value.startswith(("/", "~")) or ".." in Path(cwd_value).parts:
|
||||||
|
raise WorkerError("VERIFICATION_CWD_INVALID", phase="verification")
|
||||||
|
cwd = (worktree / cwd_value).resolve()
|
||||||
|
if worktree.resolve() not in cwd.parents and cwd != worktree.resolve():
|
||||||
|
raise WorkerError("VERIFICATION_CWD_ESCAPE", phase="verification")
|
||||||
|
timeout = item.get("timeout_seconds", 300)
|
||||||
|
if not isinstance(timeout, int) or isinstance(timeout, bool) or not 1 <= timeout <= 1800:
|
||||||
|
raise WorkerError("VERIFICATION_TIMEOUT_INVALID", phase="verification")
|
||||||
|
key = sha256_json({"argv": argv, "cwd": str(cwd), "timeout": timeout})
|
||||||
|
if key in seen:
|
||||||
|
continue
|
||||||
|
seen.add(key)
|
||||||
|
results.append(
|
||||||
|
run_verification(str(item.get("label") or "agent_requested_verification"), argv, cwd=cwd, timeout=timeout)
|
||||||
|
)
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def governed_commit(
|
||||||
|
task_id: str,
|
||||||
|
resolution: Resolution,
|
||||||
|
worktree: Path,
|
||||||
|
changed_files: Sequence[str],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
files: dict[str, str] = {}
|
||||||
|
for relative in changed_files:
|
||||||
|
path = (worktree / relative).resolve()
|
||||||
|
if worktree.resolve() not in path.parents or not path.is_file():
|
||||||
|
raise WorkerError("COMMIT_FILE_INVALID", phase="committing", details={"path": relative})
|
||||||
|
files[relative] = base64.b64encode(path.read_bytes()).decode("ascii")
|
||||||
|
|
||||||
|
governed_task_id = f"{task_id}-writer"
|
||||||
|
create_key = f"{task_id}-governed-v2"
|
||||||
|
commit_key = f"{task_id}-commit-v2"
|
||||||
|
status, created = bridge_call(
|
||||||
|
"/v2/governed-task",
|
||||||
|
{
|
||||||
|
"task_id": governed_task_id,
|
||||||
|
"repository": resolution.repository,
|
||||||
|
"base_oid": resolution.base_oid,
|
||||||
|
"target_branch": resolution.target_branch,
|
||||||
|
"risk": "engineering",
|
||||||
|
"description": f"Autonomous intent task {task_id}",
|
||||||
|
"idempotency_key": create_key,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if not 200 <= status < 300:
|
||||||
|
raise WorkerError(
|
||||||
|
"GOVERNED_TASK_CREATE_FAILED",
|
||||||
|
phase="committing",
|
||||||
|
details={"api_status": status, "error": created.get("error")},
|
||||||
|
)
|
||||||
|
|
||||||
|
generation = int(created.get("generation") or 1)
|
||||||
|
expected = str(created.get("expected_branch_oid") or resolution.base_oid)
|
||||||
|
branch_mode = str(created.get("branch_mode") or "CREATE_NEW")
|
||||||
|
status, lease = bridge_call(
|
||||||
|
"/v2/writer-lease",
|
||||||
|
{
|
||||||
|
"schema": "aios.task-writer-lease.v2",
|
||||||
|
"task_id": governed_task_id,
|
||||||
|
"repository": resolution.repository,
|
||||||
|
"branch_mode": branch_mode,
|
||||||
|
"base_oid": resolution.base_oid,
|
||||||
|
"expected_branch_oid": expected,
|
||||||
|
"target_branch": resolution.target_branch,
|
||||||
|
"generation": generation,
|
||||||
|
"lease_seconds": 900,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if not 200 <= status < 300:
|
||||||
|
raise WorkerError(
|
||||||
|
"WRITER_LEASE_ACQUIRE_FAILED",
|
||||||
|
phase="committing",
|
||||||
|
details={"api_status": status, "error": lease.get("error")},
|
||||||
|
)
|
||||||
|
|
||||||
|
status, committed = bridge_call(
|
||||||
|
"/v2/commit-files",
|
||||||
|
{
|
||||||
|
"task_id": governed_task_id,
|
||||||
|
"repository": resolution.repository,
|
||||||
|
"expected_branch_oid": expected,
|
||||||
|
"target_branch": resolution.target_branch,
|
||||||
|
"generation": generation,
|
||||||
|
"idempotency_key": commit_key,
|
||||||
|
"files": files,
|
||||||
|
"message": f"chore(agent): execute online intent task {task_id}"[:240],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if not 200 <= status < 300:
|
||||||
|
raise WorkerError(
|
||||||
|
"GOVERNED_COMMIT_FAILED",
|
||||||
|
phase="committing",
|
||||||
|
details={"api_status": status, "error": committed.get("error")},
|
||||||
|
)
|
||||||
|
commit_oid = committed.get("commit_oid")
|
||||||
|
if not isinstance(commit_oid, str) or not OID_RE.fullmatch(commit_oid):
|
||||||
|
raise WorkerError("GOVERNED_COMMIT_OID_INVALID", phase="committing")
|
||||||
|
return {
|
||||||
|
"governed_task_id": governed_task_id,
|
||||||
|
"generation": generation,
|
||||||
|
"branch_mode": branch_mode,
|
||||||
|
"commit_oid": commit_oid,
|
||||||
|
"commit_visible": committed.get("commit_visible"),
|
||||||
|
"target_branch": resolution.target_branch,
|
||||||
|
"create_idempotency_key": create_key,
|
||||||
|
"commit_idempotency_key": commit_key,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def socket_for(adapter: str) -> Path:
|
||||||
|
if adapter == "hermes":
|
||||||
|
return HERMES_SOCKET
|
||||||
|
if adapter == "codex":
|
||||||
|
return CODEX_SOCKET
|
||||||
|
if adapter == "openclaw":
|
||||||
|
return OPENCLAW_SOCKET
|
||||||
|
raise WorkerError("ADAPTER_INVALID", phase="runtime")
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_task(task: Mapping[str, Any], prior: Mapping[str, Any] | None = None) -> Resolution:
|
||||||
|
try:
|
||||||
|
return resolve(
|
||||||
|
task,
|
||||||
|
authority_git_dir=AUTHORITY_GIT_DIR,
|
||||||
|
repository=REPOSITORY,
|
||||||
|
base_ref=BASE_REF,
|
||||||
|
available_adapters=available_adapters(),
|
||||||
|
default_adapter=DEFAULT_ADAPTER,
|
||||||
|
prior=prior,
|
||||||
|
)
|
||||||
|
except IntentError as exc:
|
||||||
|
raise WorkerError(exc.code, phase="intent_resolution", details=exc.details) from exc
|
||||||
|
|
||||||
|
|
||||||
|
def execute_claim(task: Mapping[str, Any], claim: Mapping[str, Any], token: str) -> int:
|
||||||
|
task_id = str(claim.get("task_id") or task.get("task_id") or "")
|
||||||
|
lease_token = claim.get("lease_token")
|
||||||
|
if not isinstance(lease_token, str) or len(lease_token) < 24:
|
||||||
|
raise WorkerError("LEASE_TOKEN_NOT_RETURNED", phase="claim")
|
||||||
|
attempt = int(claim.get("attempt") or 0)
|
||||||
|
state = load_state(task_id) or {"task_id": task_id, "phase": "claimed"}
|
||||||
|
prior = state.get("resolution") if isinstance(state.get("resolution"), Mapping) else None
|
||||||
|
resolution = resolve_task(task, prior=prior)
|
||||||
|
state["resolution"] = resolution.to_json()
|
||||||
|
state["attempt"] = attempt
|
||||||
|
save_state(task_id, state)
|
||||||
|
|
||||||
|
heartbeat = LeaseHeartbeat(task_id, token, lease_token)
|
||||||
|
heartbeat.start()
|
||||||
|
run_root: Path | None = None
|
||||||
|
try:
|
||||||
|
heartbeat.set_stage("started")
|
||||||
|
heartbeat.send()
|
||||||
|
audit("TASK_STARTED", task_id=task_id, attempt=attempt, adapter=resolution.adapter)
|
||||||
|
|
||||||
|
existing_commit = state.get("governed")
|
||||||
|
if isinstance(existing_commit, Mapping) and isinstance(existing_commit.get("commit_oid"), str):
|
||||||
|
result = {
|
||||||
|
"status": "SUCCEEDED",
|
||||||
|
"verification_state": "VERIFIED_BY_WORKER_READBACK_PENDING_SERVER",
|
||||||
|
"task_id": task_id,
|
||||||
|
"execution_mode": "autonomous-intent-v2",
|
||||||
|
"adapter": resolution.adapter,
|
||||||
|
"resolution_digest": resolution.resolution_digest,
|
||||||
|
"governed": dict(existing_commit),
|
||||||
|
"recovered_after_commit": True,
|
||||||
|
"secret_material_exposed": False,
|
||||||
|
"direct_main_write": False,
|
||||||
|
"force_push": False,
|
||||||
|
"production_write": False,
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
heartbeat.set_stage("workspace_preparing")
|
||||||
|
run_root, worktree = prepare_worktree(task_id, resolution.base_oid)
|
||||||
|
share_run_root(run_root)
|
||||||
|
state["phase"] = "workspace_prepared"
|
||||||
|
save_state(task_id, state)
|
||||||
|
|
||||||
|
heartbeat.set_stage("agent_executing")
|
||||||
|
payload = request_payload(
|
||||||
|
adapter=resolution.adapter,
|
||||||
|
task_id=task_id,
|
||||||
|
goal=resolution.goal,
|
||||||
|
instructions=resolution.instructions,
|
||||||
|
success_criteria=resolution.success_criteria,
|
||||||
|
repository=resolution.repository,
|
||||||
|
base_oid=resolution.base_oid,
|
||||||
|
target_branch=resolution.target_branch,
|
||||||
|
worktree=worktree,
|
||||||
|
)
|
||||||
|
runtime = invoke(
|
||||||
|
socket_for(resolution.adapter),
|
||||||
|
payload,
|
||||||
|
timeout_seconds=RUNTIME_TIMEOUT_SECONDS,
|
||||||
|
)
|
||||||
|
heartbeat.set_stage("validating")
|
||||||
|
changed_files = collect_changes(worktree, resolution.base_oid)
|
||||||
|
scan_secrets(worktree, changed_files)
|
||||||
|
verification = independent_verification(worktree, changed_files, runtime.verification)
|
||||||
|
|
||||||
|
heartbeat.set_stage("committing")
|
||||||
|
governed = governed_commit(task_id, resolution, worktree, changed_files)
|
||||||
|
state.update({"phase": "commit_done", "governed": governed, "changed_files": list(changed_files)})
|
||||||
|
save_state(task_id, state)
|
||||||
|
|
||||||
|
result = {
|
||||||
|
"status": "SUCCEEDED",
|
||||||
|
"verification_state": "VERIFIED_BY_WORKER_READBACK_PENDING_SERVER",
|
||||||
|
"task_id": task_id,
|
||||||
|
"execution_mode": "autonomous-intent-v2",
|
||||||
|
"adapter": resolution.adapter,
|
||||||
|
"resolution_digest": resolution.resolution_digest,
|
||||||
|
"base_oid": resolution.base_oid,
|
||||||
|
"target_branch": resolution.target_branch,
|
||||||
|
"changed_files": list(changed_files),
|
||||||
|
"tests": verification,
|
||||||
|
"agent_summary": runtime.summary,
|
||||||
|
"agent_evidence": sanitize(runtime.evidence),
|
||||||
|
"governed": governed,
|
||||||
|
"secret_material_exposed": False,
|
||||||
|
"direct_main_write": False,
|
||||||
|
"force_push": False,
|
||||||
|
"production_write": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
if heartbeat.failed:
|
||||||
|
raise WorkerError("HEARTBEAT_UNHEALTHY", phase="result_submitting", details={"error": heartbeat.failed})
|
||||||
|
heartbeat.set_stage("result_submitting")
|
||||||
|
result_key = f"{task_id}-result-v2"
|
||||||
|
status, response = api_call(
|
||||||
|
"POST",
|
||||||
|
f"/v1/tasks/{task_id}/result",
|
||||||
|
token,
|
||||||
|
{
|
||||||
|
"lease_token": lease_token,
|
||||||
|
"idempotency_key": result_key,
|
||||||
|
"result": result,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if not 200 <= status < 300:
|
||||||
|
raise WorkerError(
|
||||||
|
"TASK_RESULT_SUBMIT_FAILED",
|
||||||
|
phase="result_submitting",
|
||||||
|
details={"api_status": status, "error": response.get("error")},
|
||||||
|
)
|
||||||
|
audit("RESULT_SUBMITTED", task_id=task_id, task_status=response.get("status"), result_idempotency_key=result_key)
|
||||||
|
state["phase"] = "result_submitted"
|
||||||
|
save_state(task_id, state)
|
||||||
|
return 0
|
||||||
|
except (WorkerError, RuntimeErrorV2) as exc:
|
||||||
|
code = exc.code if hasattr(exc, "code") else type(exc).__name__
|
||||||
|
phase = exc.phase if isinstance(exc, WorkerError) else "runtime"
|
||||||
|
details = exc.details if hasattr(exc, "details") else {}
|
||||||
|
audit("AUTONOMOUS_EXECUTION_FAILED", task_id=task_id, error=code, phase=phase, details=details)
|
||||||
|
failure_status = "BLOCKED" if code in {
|
||||||
|
"AGENT_RUNTIME_UNAVAILABLE", "RUNTIME_UNAVAILABLE", "RUNTIME_BLOCKED",
|
||||||
|
"GOVERNED_WRITE_API_UNAVAILABLE", "NON_STAGING_BRIDGE_REJECTED",
|
||||||
|
} else "FAILED_RETRYABLE"
|
||||||
|
failure = {
|
||||||
|
"status": failure_status,
|
||||||
|
"verification_state": "UNVERIFIED",
|
||||||
|
"task_id": task_id,
|
||||||
|
"execution_mode": "autonomous-intent-v2",
|
||||||
|
"error": code,
|
||||||
|
"phase": phase,
|
||||||
|
"details": sanitize(details),
|
||||||
|
"secret_material_exposed": False,
|
||||||
|
"direct_main_write": False,
|
||||||
|
"force_push": False,
|
||||||
|
"production_write": False,
|
||||||
|
}
|
||||||
|
if not heartbeat.failed:
|
||||||
|
status, response = api_call(
|
||||||
|
"POST",
|
||||||
|
f"/v1/tasks/{task_id}/result",
|
||||||
|
token,
|
||||||
|
{
|
||||||
|
"lease_token": lease_token,
|
||||||
|
"idempotency_key": f"{task_id}-attempt-{attempt}-failure-v2",
|
||||||
|
"result": failure,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
audit("FAILURE_RESULT_SUBMITTED", task_id=task_id, api_status=status, task_status=response.get("status"))
|
||||||
|
return 20
|
||||||
|
finally:
|
||||||
|
heartbeat.close()
|
||||||
|
if run_root is not None:
|
||||||
|
try:
|
||||||
|
cleanup_run_root(run_root)
|
||||||
|
audit("WORKSPACE_CLEANED", task_id=task_id, cleanup="PASS")
|
||||||
|
except Exception as exc:
|
||||||
|
audit("WORKSPACE_CLEANUP_FAILED", task_id=task_id, error=type(exc).__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def worker_once(token: str) -> int:
|
||||||
|
status, poll = api_call("POST", "/v1/tasks/poll", token, {"limit": 100})
|
||||||
|
if status != 200:
|
||||||
|
audit("TASK_POLL_FAILED", api_status=status, error=poll.get("error"))
|
||||||
|
return 10
|
||||||
|
|
||||||
|
worker_caps = capabilities()
|
||||||
|
candidates: list[Mapping[str, Any]] = []
|
||||||
|
for item in poll.get("tasks", []):
|
||||||
|
if not isinstance(item, Mapping):
|
||||||
|
continue
|
||||||
|
if not is_claimable_intent(item, worker_caps):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
# Read-only preflight. Do not consume an attempt if runtime/base is unavailable.
|
||||||
|
resolve_task(item, prior=(load_state(str(item.get("task_id") or "")) or {}).get("resolution"))
|
||||||
|
except WorkerError as exc:
|
||||||
|
audit("TASK_SKIPPED_PREFLIGHT", task_id=item.get("task_id"), error=exc.code)
|
||||||
|
continue
|
||||||
|
candidates.append(item)
|
||||||
|
|
||||||
|
if not candidates:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
# Server priority/order remains authoritative; take the first visible compatible task.
|
||||||
|
task = candidates[0]
|
||||||
|
task_id = str(task.get("task_id") or "")
|
||||||
|
status, claim = api_call("POST", f"/v1/tasks/{task_id}/claim", token, {"lease_seconds": LEASE_SECONDS})
|
||||||
|
if status != 200:
|
||||||
|
audit("TASK_CLAIM_FAILED", task_id=task_id, api_status=status, error=claim.get("error"))
|
||||||
|
return 11
|
||||||
|
audit(
|
||||||
|
"TASK_CLAIMED",
|
||||||
|
task_id=task_id,
|
||||||
|
attempt=claim.get("attempt"),
|
||||||
|
revision=claim.get("revision"),
|
||||||
|
claim_mode="server_atomic",
|
||||||
|
execution_mode="autonomous-intent-v2",
|
||||||
|
)
|
||||||
|
merged = dict(task)
|
||||||
|
merged.update({key: value for key, value in claim.items() if key not in {"lease_token"}})
|
||||||
|
merged["task_id"] = task_id
|
||||||
|
return execute_claim(merged, claim, token)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
ensure_roots()
|
||||||
|
if BRIDGE_BASE != "http://127.0.0.1:19998":
|
||||||
|
raise WorkerError("NON_STAGING_BRIDGE_REJECTED", phase="startup", details={"bridge_base": BRIDGE_BASE})
|
||||||
|
token = load_agent_token()
|
||||||
|
audit(
|
||||||
|
"AUTONOMOUS_WORKER_READY",
|
||||||
|
poll_mode="continuous",
|
||||||
|
claim_mode="server_atomic",
|
||||||
|
task_scope="intent-only",
|
||||||
|
bridge_base=BRIDGE_BASE,
|
||||||
|
available_adapters=list(available_adapters()),
|
||||||
|
capabilities=list(capabilities()),
|
||||||
|
main_write=False,
|
||||||
|
production_write=False,
|
||||||
|
force_push=False,
|
||||||
|
)
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
result = worker_once(token)
|
||||||
|
except Exception as exc:
|
||||||
|
audit("WORKER_LOOP_ERROR", error=type(exc).__name__)
|
||||||
|
result = 20
|
||||||
|
time.sleep(POLL_SECONDS if result == 0 else max(POLL_SECONDS, 5))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,273 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import asdict, dataclass
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
from typing import Any, Mapping, Sequence
|
||||||
|
|
||||||
|
|
||||||
|
OID_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||||
|
TASK_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{2,179}$")
|
||||||
|
REPOSITORY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]{2,199}$")
|
||||||
|
ALLOWED_ADAPTERS = frozenset({"hermes", "codex", "openclaw"})
|
||||||
|
|
||||||
|
|
||||||
|
class IntentError(RuntimeError):
|
||||||
|
def __init__(self, code: str, *, details: Mapping[str, Any] | None = None) -> None:
|
||||||
|
super().__init__(code)
|
||||||
|
self.code = code
|
||||||
|
self.details = dict(details or {})
|
||||||
|
|
||||||
|
|
||||||
|
def stable_digest(value: Any) -> str:
|
||||||
|
raw = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _text(value: Any, field: str, limit: int) -> str:
|
||||||
|
if not isinstance(value, str):
|
||||||
|
raise IntentError(f"{field.upper()}_INVALID")
|
||||||
|
value = value.strip()
|
||||||
|
if not value or len(value) > limit:
|
||||||
|
raise IntentError(f"{field.upper()}_INVALID")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _string_list(value: Any, field: str, *, required: bool) -> tuple[str, ...]:
|
||||||
|
if value is None and not required:
|
||||||
|
return ()
|
||||||
|
if not isinstance(value, list) or len(value) > 64 or (required and not value):
|
||||||
|
raise IntentError(f"{field.upper()}_INVALID")
|
||||||
|
return tuple(_text(item, field, 4096) for item in value)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class IntentTask:
|
||||||
|
task_id: str
|
||||||
|
goal: str
|
||||||
|
instructions: str
|
||||||
|
success_criteria: tuple[str, ...]
|
||||||
|
required_capabilities: tuple[str, ...]
|
||||||
|
digest: str
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def parse(cls, task: Mapping[str, Any]) -> "IntentTask":
|
||||||
|
task_id = str(task.get("task_id") or "")
|
||||||
|
if not TASK_ID_RE.fullmatch(task_id):
|
||||||
|
raise IntentError("TASK_ID_INVALID")
|
||||||
|
|
||||||
|
metadata = task.get("metadata")
|
||||||
|
if isinstance(metadata, Mapping) and isinstance(metadata.get("execution"), Mapping):
|
||||||
|
raise IntentError("STRUCTURED_TASK_NOT_INTENT")
|
||||||
|
|
||||||
|
goal = _text(task.get("goal"), "goal", 16384)
|
||||||
|
instructions = _text(
|
||||||
|
task.get("instructions") or "Inspect the repository, implement the goal, and satisfy the success criteria.",
|
||||||
|
"instructions",
|
||||||
|
65536,
|
||||||
|
)
|
||||||
|
success = _string_list(task.get("success_criteria"), "success_criteria", required=True)
|
||||||
|
capabilities = _string_list(task.get("required_capabilities") or [], "required_capabilities", required=False)
|
||||||
|
|
||||||
|
required_markers = {"intent.task", "engineering.execute"}
|
||||||
|
missing = sorted(required_markers - set(capabilities))
|
||||||
|
if missing:
|
||||||
|
raise IntentError("INTENT_CAPABILITY_MARKER_REQUIRED", details={"missing": missing})
|
||||||
|
|
||||||
|
canonical = {
|
||||||
|
"task_id": task_id,
|
||||||
|
"goal": goal,
|
||||||
|
"instructions": instructions,
|
||||||
|
"success_criteria": success,
|
||||||
|
"required_capabilities": sorted(capabilities),
|
||||||
|
}
|
||||||
|
return cls(
|
||||||
|
task_id=task_id,
|
||||||
|
goal=goal,
|
||||||
|
instructions=instructions,
|
||||||
|
success_criteria=success,
|
||||||
|
required_capabilities=capabilities,
|
||||||
|
digest=stable_digest(canonical),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Resolution:
|
||||||
|
schema: str
|
||||||
|
task_id: str
|
||||||
|
repository: str
|
||||||
|
base_ref: str
|
||||||
|
base_oid: str
|
||||||
|
target_branch: str
|
||||||
|
adapter: str
|
||||||
|
intent_digest: str
|
||||||
|
goal: str
|
||||||
|
instructions: str
|
||||||
|
success_criteria: tuple[str, ...]
|
||||||
|
required_capabilities: tuple[str, ...]
|
||||||
|
resolution_digest: str
|
||||||
|
|
||||||
|
def to_json(self) -> dict[str, Any]:
|
||||||
|
value = asdict(self)
|
||||||
|
value["success_criteria"] = list(self.success_criteria)
|
||||||
|
value["required_capabilities"] = list(self.required_capabilities)
|
||||||
|
return value
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_json(cls, value: Mapping[str, Any], *, intent: IntentTask) -> "Resolution":
|
||||||
|
if value.get("schema") != "aios.autonomous-intent-resolution.v2":
|
||||||
|
raise IntentError("RESOLUTION_SCHEMA_INVALID")
|
||||||
|
if value.get("task_id") != intent.task_id or value.get("intent_digest") != intent.digest:
|
||||||
|
raise IntentError("RESOLUTION_TASK_DRIFT")
|
||||||
|
result = cls(
|
||||||
|
schema=str(value["schema"]),
|
||||||
|
task_id=str(value["task_id"]),
|
||||||
|
repository=str(value["repository"]),
|
||||||
|
base_ref=str(value["base_ref"]),
|
||||||
|
base_oid=str(value["base_oid"]),
|
||||||
|
target_branch=str(value["target_branch"]),
|
||||||
|
adapter=str(value["adapter"]),
|
||||||
|
intent_digest=str(value["intent_digest"]),
|
||||||
|
goal=str(value["goal"]),
|
||||||
|
instructions=str(value["instructions"]),
|
||||||
|
success_criteria=tuple(str(item) for item in value.get("success_criteria") or []),
|
||||||
|
required_capabilities=tuple(str(item) for item in value.get("required_capabilities") or []),
|
||||||
|
resolution_digest=str(value["resolution_digest"]),
|
||||||
|
)
|
||||||
|
validate_resolution(result)
|
||||||
|
if result.resolution_digest != resolution_digest(result):
|
||||||
|
raise IntentError("RESOLUTION_DIGEST_INVALID")
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def read_base_oid(authority_git_dir: Path, base_ref: str) -> str:
|
||||||
|
if not base_ref.startswith("refs/heads/"):
|
||||||
|
raise IntentError("BASE_REF_INVALID")
|
||||||
|
if authority_git_dir.is_symlink() or not authority_git_dir.is_dir():
|
||||||
|
raise IntentError("AUTHORITY_GIT_DIR_INVALID")
|
||||||
|
completed = subprocess.run(
|
||||||
|
["/usr/bin/git", "--git-dir", str(authority_git_dir), "rev-parse", "--verify", f"{base_ref}^{{commit}}"],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=False,
|
||||||
|
timeout=30,
|
||||||
|
env={"PATH": "/usr/bin:/bin", "LC_ALL": "C.UTF-8"},
|
||||||
|
)
|
||||||
|
oid = completed.stdout.strip()
|
||||||
|
if completed.returncode != 0 or not OID_RE.fullmatch(oid):
|
||||||
|
raise IntentError("BASE_OID_UNAVAILABLE")
|
||||||
|
return oid
|
||||||
|
|
||||||
|
|
||||||
|
def derive_branch(task_id: str, digest: str) -> str:
|
||||||
|
slug = re.sub(r"[^A-Za-z0-9._-]+", "-", task_id).strip(".-").lower() or "task"
|
||||||
|
slug = slug[:100]
|
||||||
|
branch = f"agent/intent/{slug}-{digest[:12]}"
|
||||||
|
if len(branch) > 180:
|
||||||
|
branch = branch[:167] + "-" + digest[:12]
|
||||||
|
if ".." in branch or branch.endswith(("/", ".", ".lock")):
|
||||||
|
raise IntentError("TARGET_BRANCH_INVALID")
|
||||||
|
return branch
|
||||||
|
|
||||||
|
|
||||||
|
def select_adapter(capabilities: Sequence[str], available: Sequence[str], default_adapter: str) -> str:
|
||||||
|
advertised = set(available)
|
||||||
|
if not advertised.issubset(ALLOWED_ADAPTERS):
|
||||||
|
raise IntentError("ADAPTER_CONFIGURATION_INVALID")
|
||||||
|
requested = {
|
||||||
|
item.split(".", 2)[-1]
|
||||||
|
for item in capabilities
|
||||||
|
if item.startswith("agent.runtime.")
|
||||||
|
}
|
||||||
|
requested.discard("")
|
||||||
|
if len(requested) > 1:
|
||||||
|
raise IntentError("MULTIPLE_AGENT_RUNTIMES_REQUIRED")
|
||||||
|
adapter = next(iter(requested)) if requested else default_adapter
|
||||||
|
if adapter not in ALLOWED_ADAPTERS:
|
||||||
|
raise IntentError("AGENT_RUNTIME_INVALID")
|
||||||
|
if adapter not in advertised:
|
||||||
|
raise IntentError("AGENT_RUNTIME_UNAVAILABLE", details={"adapter": adapter})
|
||||||
|
return adapter
|
||||||
|
|
||||||
|
|
||||||
|
def resolution_digest(value: Resolution) -> str:
|
||||||
|
return stable_digest({
|
||||||
|
"schema": value.schema,
|
||||||
|
"task_id": value.task_id,
|
||||||
|
"repository": value.repository,
|
||||||
|
"base_ref": value.base_ref,
|
||||||
|
"base_oid": value.base_oid,
|
||||||
|
"target_branch": value.target_branch,
|
||||||
|
"adapter": value.adapter,
|
||||||
|
"intent_digest": value.intent_digest,
|
||||||
|
"goal": value.goal,
|
||||||
|
"instructions": value.instructions,
|
||||||
|
"success_criteria": value.success_criteria,
|
||||||
|
"required_capabilities": sorted(value.required_capabilities),
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def validate_resolution(value: Resolution) -> None:
|
||||||
|
if not TASK_ID_RE.fullmatch(value.task_id):
|
||||||
|
raise IntentError("TASK_ID_INVALID")
|
||||||
|
if not REPOSITORY_RE.fullmatch(value.repository) or value.repository.startswith(("/", ".")):
|
||||||
|
raise IntentError("REPOSITORY_INVALID")
|
||||||
|
if not value.base_ref.startswith("refs/heads/"):
|
||||||
|
raise IntentError("BASE_REF_INVALID")
|
||||||
|
if not OID_RE.fullmatch(value.base_oid):
|
||||||
|
raise IntentError("BASE_OID_INVALID")
|
||||||
|
if not value.target_branch.startswith("agent/intent/"):
|
||||||
|
raise IntentError("TARGET_BRANCH_POLICY_INVALID")
|
||||||
|
if value.adapter not in ALLOWED_ADAPTERS:
|
||||||
|
raise IntentError("AGENT_RUNTIME_INVALID")
|
||||||
|
|
||||||
|
|
||||||
|
def resolve(
|
||||||
|
task: Mapping[str, Any],
|
||||||
|
*,
|
||||||
|
authority_git_dir: Path,
|
||||||
|
repository: str,
|
||||||
|
base_ref: str,
|
||||||
|
available_adapters: Sequence[str],
|
||||||
|
default_adapter: str,
|
||||||
|
prior: Mapping[str, Any] | None = None,
|
||||||
|
) -> Resolution:
|
||||||
|
intent = IntentTask.parse(task)
|
||||||
|
if prior is not None:
|
||||||
|
return Resolution.from_json(prior, intent=intent)
|
||||||
|
if not REPOSITORY_RE.fullmatch(repository) or repository.startswith(("/", ".")):
|
||||||
|
raise IntentError("REPOSITORY_INVALID")
|
||||||
|
base_oid = read_base_oid(authority_git_dir, base_ref)
|
||||||
|
adapter = select_adapter(intent.required_capabilities, available_adapters, default_adapter)
|
||||||
|
provisional = Resolution(
|
||||||
|
schema="aios.autonomous-intent-resolution.v2",
|
||||||
|
task_id=intent.task_id,
|
||||||
|
repository=repository,
|
||||||
|
base_ref=base_ref,
|
||||||
|
base_oid=base_oid,
|
||||||
|
target_branch=derive_branch(intent.task_id, intent.digest),
|
||||||
|
adapter=adapter,
|
||||||
|
intent_digest=intent.digest,
|
||||||
|
goal=intent.goal,
|
||||||
|
instructions=intent.instructions,
|
||||||
|
success_criteria=intent.success_criteria,
|
||||||
|
required_capabilities=intent.required_capabilities,
|
||||||
|
resolution_digest="",
|
||||||
|
)
|
||||||
|
result = Resolution(
|
||||||
|
**{**asdict(provisional), "resolution_digest": resolution_digest(provisional)}
|
||||||
|
)
|
||||||
|
validate_resolution(result)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def is_claimable_intent(task: Mapping[str, Any], worker_capabilities: Sequence[str]) -> bool:
|
||||||
|
try:
|
||||||
|
intent = IntentTask.parse(task)
|
||||||
|
except IntentError:
|
||||||
|
return False
|
||||||
|
required = set(intent.required_capabilities)
|
||||||
|
return required.issubset(set(worker_capabilities))
|
||||||
@@ -0,0 +1,197 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
BASE = os.environ.get("AIOS_AGENT_TASK_API_URL", "http://127.0.0.1:18940").rstrip("/")
|
||||||
|
AGENT_ID = os.environ.get("AIOS_AGENT_ID", "online-engineering-worker-cn-network-mkt-autonomous-20260815")
|
||||||
|
AGENT_TYPE = os.environ.get("AIOS_AGENT_TYPE", "trusted-online-engineering-worker")
|
||||||
|
AGENT_VERSION = os.environ.get("AIOS_AGENT_VERSION", "2026-08-15.4-autonomous-intent")
|
||||||
|
STATE_DIR = Path(os.environ.get("AIOS_WORKER_REGISTRATION_STATE", "/var/lib/aios-autonomous-agent-worker/registration"))
|
||||||
|
CREDENTIAL_PATH = Path(os.environ.get("AIOS_WORKER_ENCRYPTED_CREDENTIAL", "/etc/credstore.encrypted/aios-autonomous-agent-worker/agent_token"))
|
||||||
|
IDENTITY_PATH = STATE_DIR / "identity.json"
|
||||||
|
PROJECTION_PROVIDER = "BridgeV2RuntimeCredentialProjection"
|
||||||
|
|
||||||
|
|
||||||
|
def now() -> str:
|
||||||
|
return datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def runtime_capabilities() -> list[str]:
|
||||||
|
values = [
|
||||||
|
"online_agent_task_api", "task.poll", "task.claim", "task.heartbeat", "task.result",
|
||||||
|
"intent.task", "engineering.consumer", "engineering.execute", "engineering.autonomous",
|
||||||
|
"workspace.worktree", "git.read", "git.diff", "git.test",
|
||||||
|
"governed.write.branch", "governed.write.file",
|
||||||
|
]
|
||||||
|
hermes = os.environ.get("AIOS_HERMES_BROKER_SOCKET", "")
|
||||||
|
codex = os.environ.get("AIOS_CODEX_BROKER_SOCKET", "")
|
||||||
|
openclaw = os.environ.get("AIOS_OPENCLAW_BROKER_SOCKET", "")
|
||||||
|
if hermes and Path(hermes).is_socket():
|
||||||
|
values.append("agent.runtime.hermes")
|
||||||
|
if codex and Path(codex).is_socket():
|
||||||
|
values.append("agent.runtime.codex")
|
||||||
|
if openclaw and Path(openclaw).is_socket():
|
||||||
|
values.append("agent.runtime.openclaw")
|
||||||
|
return values
|
||||||
|
|
||||||
|
|
||||||
|
def api_call(method: str, route: str, token: str, body: dict[str, Any] | None = None) -> tuple[int, dict[str, Any]]:
|
||||||
|
payload = None if body is None else json.dumps(body, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
headers = {"Authorization": "Bearer " + token, "Accept": "application/json"}
|
||||||
|
if payload is not None:
|
||||||
|
headers["Content-Type"] = "application/json"
|
||||||
|
request = urllib.request.Request(BASE + route, data=payload, headers=headers, method=method)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(request, timeout=15) as response:
|
||||||
|
value = json.loads(response.read().decode("utf-8"))
|
||||||
|
return response.status, value if isinstance(value, dict) else {}
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
try:
|
||||||
|
value = json.loads(exc.read(2048).decode("utf-8"))
|
||||||
|
except Exception:
|
||||||
|
value = {"error": "HTTP_ERROR"}
|
||||||
|
return exc.code, value if isinstance(value, dict) else {"error": "HTTP_ERROR"}
|
||||||
|
except (urllib.error.URLError, TimeoutError):
|
||||||
|
return 599, {"error": "AGENT_TASK_API_UNAVAILABLE"}
|
||||||
|
|
||||||
|
|
||||||
|
def encrypted_credential_is_valid() -> bool:
|
||||||
|
if not CREDENTIAL_PATH.is_file() or CREDENTIAL_PATH.is_symlink():
|
||||||
|
return False
|
||||||
|
result = subprocess.run(
|
||||||
|
["systemd-creds", "decrypt", "--with-key=host", str(CREDENTIAL_PATH), "/dev/null"],
|
||||||
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, timeout=15,
|
||||||
|
)
|
||||||
|
return result.returncode == 0
|
||||||
|
|
||||||
|
|
||||||
|
def load_identity() -> dict[str, Any] | None:
|
||||||
|
if not IDENTITY_PATH.is_file() or IDENTITY_PATH.is_symlink():
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
value = json.loads(IDENTITY_PATH.read_text(encoding="utf-8"))
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
return value if isinstance(value, dict) else None
|
||||||
|
|
||||||
|
|
||||||
|
def identity_matches(value: dict[str, Any] | None) -> bool:
|
||||||
|
return bool(
|
||||||
|
value
|
||||||
|
and value.get("agent_id") == AGENT_ID
|
||||||
|
and value.get("agent_type") == AGENT_TYPE
|
||||||
|
and value.get("agent_version") == AGENT_VERSION
|
||||||
|
and value.get("task_scope") == "intent-only"
|
||||||
|
and value.get("credential_projection_provider") == PROJECTION_PROVIDER
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def encrypt_agent_token(token: str) -> None:
|
||||||
|
CREDENTIAL_PATH.parent.mkdir(parents=True, mode=0o700, exist_ok=True)
|
||||||
|
fd, temporary = tempfile.mkstemp(prefix=".agent-token.", dir=str(CREDENTIAL_PATH.parent))
|
||||||
|
os.close(fd)
|
||||||
|
os.unlink(temporary)
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
["systemd-creds", "encrypt", "--with-key=host", "--name=agent_token", "/dev/stdin", temporary],
|
||||||
|
input=(token + "\n").encode("utf-8"),
|
||||||
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, timeout=20,
|
||||||
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
raise RuntimeError("AGENT_CREDENTIAL_ENCRYPT_FAILED")
|
||||||
|
os.chmod(temporary, 0o600)
|
||||||
|
os.replace(temporary, CREDENTIAL_PATH)
|
||||||
|
finally:
|
||||||
|
if os.path.exists(temporary):
|
||||||
|
os.unlink(temporary)
|
||||||
|
|
||||||
|
|
||||||
|
def write_identity() -> None:
|
||||||
|
value = {
|
||||||
|
"agent_id": AGENT_ID,
|
||||||
|
"agent_type": AGENT_TYPE,
|
||||||
|
"agent_version": AGENT_VERSION,
|
||||||
|
"task_scope": "intent-only",
|
||||||
|
"credential_projection_provider": PROJECTION_PROVIDER,
|
||||||
|
"credential_delivery": "systemd-LoadCredentialEncrypted",
|
||||||
|
"task_api_url": BASE,
|
||||||
|
"task_api_schema": "aios.agent-task.v1",
|
||||||
|
"execution_backend": "autonomous-intent-worker-v2",
|
||||||
|
"runtime_only_secret_delivery": True,
|
||||||
|
"main_write": False,
|
||||||
|
"production_write": False,
|
||||||
|
"force_push": False,
|
||||||
|
"tag_write": False,
|
||||||
|
"baseline_write": False,
|
||||||
|
"capabilities": runtime_capabilities(),
|
||||||
|
"created_at": now(),
|
||||||
|
}
|
||||||
|
STATE_DIR.mkdir(parents=True, mode=0o750, exist_ok=True)
|
||||||
|
fd, temporary = tempfile.mkstemp(prefix="identity.", dir=str(STATE_DIR))
|
||||||
|
try:
|
||||||
|
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||||
|
json.dump(value, handle, ensure_ascii=False, sort_keys=True)
|
||||||
|
handle.write("\n")
|
||||||
|
os.chmod(temporary, 0o640)
|
||||||
|
os.replace(temporary, IDENTITY_PATH)
|
||||||
|
finally:
|
||||||
|
if os.path.exists(temporary):
|
||||||
|
os.unlink(temporary)
|
||||||
|
|
||||||
|
|
||||||
|
def register_or_recover() -> str:
|
||||||
|
STATE_DIR.mkdir(parents=True, mode=0o750, exist_ok=True)
|
||||||
|
existing = load_identity()
|
||||||
|
if CREDENTIAL_PATH.exists():
|
||||||
|
if not identity_matches(existing):
|
||||||
|
raise RuntimeError("AGENT_IDENTITY_BINDING_CONFLICT")
|
||||||
|
if not encrypted_credential_is_valid():
|
||||||
|
raise RuntimeError("AGENT_ENCRYPTED_CREDENTIAL_INVALID")
|
||||||
|
return "RECOVERED"
|
||||||
|
if existing is not None:
|
||||||
|
raise RuntimeError("AGENT_CREDENTIAL_MISSING_AFTER_PRIOR_REGISTRATION")
|
||||||
|
credential_directory = Path(os.environ.get("CREDENTIALS_DIRECTORY", ""))
|
||||||
|
control_path = credential_directory / "bridge_control_token"
|
||||||
|
if not control_path.is_file():
|
||||||
|
raise RuntimeError("BRIDGE_RUNTIME_CONTROL_CREDENTIAL_UNAVAILABLE")
|
||||||
|
control_token = control_path.read_text(encoding="utf-8").strip()
|
||||||
|
status, response = api_call(
|
||||||
|
"POST", "/v1/agents/register", control_token,
|
||||||
|
{
|
||||||
|
"agent_id": AGENT_ID,
|
||||||
|
"agent_type": AGENT_TYPE,
|
||||||
|
"agent_version": AGENT_VERSION,
|
||||||
|
"capabilities": runtime_capabilities(),
|
||||||
|
"metadata": {
|
||||||
|
"task_scope": "intent-only",
|
||||||
|
"credential_projection_provider": PROJECTION_PROVIDER,
|
||||||
|
"runtime_only_secret_delivery": True,
|
||||||
|
"execution_backend": "autonomous-intent-worker-v2",
|
||||||
|
"main_write": False,
|
||||||
|
"production_write": False,
|
||||||
|
"force_push": False,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if status != 201:
|
||||||
|
raise RuntimeError("AGENT_REGISTER_FAILED_" + str(response.get("error") or status))
|
||||||
|
token = response.get("agent_token")
|
||||||
|
if not isinstance(token, str) or len(token) < 24:
|
||||||
|
raise RuntimeError("AGENT_TOKEN_NOT_RETURNED")
|
||||||
|
encrypt_agent_token(token)
|
||||||
|
write_identity()
|
||||||
|
return "REGISTERED"
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(0 if register_or_recover() in {"REGISTERED", "RECOVERED"} else 20)
|
||||||
@@ -0,0 +1,224 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import socket
|
||||||
|
import subprocess
|
||||||
|
from typing import Any, Mapping, Sequence
|
||||||
|
|
||||||
|
|
||||||
|
OID_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||||
|
MAX_BYTES = 1024 * 1024
|
||||||
|
MAX_CHANGED_FILES = 256
|
||||||
|
MAX_FILE_BYTES = 4 * 1024 * 1024
|
||||||
|
SENSITIVE_NAMES = frozenset({
|
||||||
|
".env", ".env.local", ".env.production", "id_rsa", "id_ed25519",
|
||||||
|
"credentials", "credentials.json",
|
||||||
|
})
|
||||||
|
SENSITIVE_SUFFIXES = (".key", ".p12", ".pfx")
|
||||||
|
SECRET_PATTERNS = (
|
||||||
|
re.compile(rb"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"),
|
||||||
|
re.compile(rb"\bAKIA[0-9A-Z]{16}\b"),
|
||||||
|
re.compile(rb"\bghp_[A-Za-z0-9]{30,}\b"),
|
||||||
|
re.compile(rb"\bsk-[A-Za-z0-9_-]{24,}\b"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class RuntimeErrorV2(RuntimeError):
|
||||||
|
def __init__(self, code: str, *, details: Mapping[str, Any] | None = None) -> None:
|
||||||
|
super().__init__(code)
|
||||||
|
self.code = code
|
||||||
|
self.details = dict(details or {})
|
||||||
|
|
||||||
|
|
||||||
|
def safe_relative_path(value: str) -> str:
|
||||||
|
if (
|
||||||
|
not value
|
||||||
|
or value.startswith(("/", "~"))
|
||||||
|
or "\\" in value
|
||||||
|
or "\x00" in value
|
||||||
|
or any(part in {"", ".", ".."} for part in value.split("/"))
|
||||||
|
):
|
||||||
|
raise RuntimeErrorV2("CHANGED_PATH_INVALID", details={"path": value[:160]})
|
||||||
|
lowered = value.split("/")[-1].lower()
|
||||||
|
if value.startswith(".git/") or value in {".git", ".gitmodules"}:
|
||||||
|
raise RuntimeErrorV2("GIT_METADATA_CHANGE_REJECTED", details={"path": value})
|
||||||
|
if lowered in SENSITIVE_NAMES or lowered.endswith(SENSITIVE_SUFFIXES):
|
||||||
|
raise RuntimeErrorV2("SENSITIVE_PATH_REJECTED", details={"path": value})
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def git(worktree: Path, args: Sequence[str], timeout: int = 60) -> bytes:
|
||||||
|
completed = subprocess.run(
|
||||||
|
["/usr/bin/git", "-C", str(worktree), *args],
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
timeout=timeout,
|
||||||
|
env={"PATH": "/usr/bin:/bin", "LC_ALL": "C.UTF-8"},
|
||||||
|
)
|
||||||
|
if completed.returncode != 0:
|
||||||
|
raise RuntimeErrorV2("WORKTREE_GIT_FAILED", details={"operation": args[0] if args else ""})
|
||||||
|
return completed.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def _zpaths(raw: bytes) -> list[str]:
|
||||||
|
result: list[str] = []
|
||||||
|
for item in raw.split(b"\0"):
|
||||||
|
if not item:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
result.append(item.decode("utf-8"))
|
||||||
|
except UnicodeDecodeError as exc:
|
||||||
|
raise RuntimeErrorV2("WORKTREE_PATH_ENCODING_INVALID") from exc
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def collect_changes(worktree: Path, base_oid: str) -> list[str]:
|
||||||
|
if not OID_RE.fullmatch(base_oid):
|
||||||
|
raise RuntimeErrorV2("BASE_OID_INVALID")
|
||||||
|
head = git(worktree, ["rev-parse", "HEAD"]).decode("ascii", "strict").strip()
|
||||||
|
if head != base_oid:
|
||||||
|
raise RuntimeErrorV2("WORKTREE_HEAD_MUTATED", details={"expected": base_oid, "actual": head})
|
||||||
|
tracked = _zpaths(git(worktree, ["diff", "--name-only", "--no-renames", "-z", base_oid, "--"]))
|
||||||
|
untracked = _zpaths(git(worktree, ["ls-files", "--others", "--exclude-standard", "-z"]))
|
||||||
|
changed = sorted({safe_relative_path(item) for item in tracked + untracked})
|
||||||
|
if not changed:
|
||||||
|
raise RuntimeErrorV2("AGENT_PRODUCED_NO_CHANGES")
|
||||||
|
if len(changed) > MAX_CHANGED_FILES:
|
||||||
|
raise RuntimeErrorV2("CHANGED_FILE_LIMIT_EXCEEDED")
|
||||||
|
root = worktree.resolve()
|
||||||
|
for relative in changed:
|
||||||
|
path = (root / relative).resolve(strict=False)
|
||||||
|
if root not in path.parents or not path.is_file() or path.is_symlink():
|
||||||
|
raise RuntimeErrorV2("CHANGED_FILE_UNSUPPORTED", details={"path": relative})
|
||||||
|
if path.stat().st_size > MAX_FILE_BYTES:
|
||||||
|
raise RuntimeErrorV2("CHANGED_FILE_TOO_LARGE", details={"path": relative})
|
||||||
|
return changed
|
||||||
|
|
||||||
|
|
||||||
|
def scan_secrets(worktree: Path, changed: Sequence[str]) -> None:
|
||||||
|
root = worktree.resolve()
|
||||||
|
for relative in changed:
|
||||||
|
path = (root / safe_relative_path(relative)).resolve()
|
||||||
|
raw = path.read_bytes()
|
||||||
|
for pattern in SECRET_PATTERNS:
|
||||||
|
if pattern.search(raw):
|
||||||
|
raise RuntimeErrorV2("SECRET_MATERIAL_REJECTED", details={"path": relative})
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class RuntimeResult:
|
||||||
|
status: str
|
||||||
|
summary: str
|
||||||
|
verification: tuple[dict[str, Any], ...]
|
||||||
|
evidence: dict[str, Any]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def parse(cls, value: Mapping[str, Any]) -> "RuntimeResult":
|
||||||
|
if value.get("schema") != "aios.online-agent.runtime-result.v1":
|
||||||
|
raise RuntimeErrorV2("RUNTIME_RESULT_SCHEMA_INVALID")
|
||||||
|
status = str(value.get("status") or "")
|
||||||
|
if status not in {"COMPLETED", "BLOCKED", "FAILED"}:
|
||||||
|
raise RuntimeErrorV2("RUNTIME_RESULT_STATUS_INVALID")
|
||||||
|
summary = str(value.get("summary") or "").strip()
|
||||||
|
if not summary or len(summary) > 8192:
|
||||||
|
raise RuntimeErrorV2("RUNTIME_RESULT_SUMMARY_INVALID")
|
||||||
|
verification = value.get("verification") or []
|
||||||
|
evidence = value.get("evidence") or {}
|
||||||
|
if not isinstance(verification, list) or len(verification) > 64:
|
||||||
|
raise RuntimeErrorV2("RUNTIME_VERIFICATION_INVALID")
|
||||||
|
if not isinstance(evidence, Mapping):
|
||||||
|
raise RuntimeErrorV2("RUNTIME_EVIDENCE_INVALID")
|
||||||
|
return cls(status, summary, tuple(dict(item) for item in verification if isinstance(item, Mapping)), dict(evidence))
|
||||||
|
|
||||||
|
|
||||||
|
def request_payload(
|
||||||
|
*,
|
||||||
|
adapter: str,
|
||||||
|
task_id: str,
|
||||||
|
goal: str,
|
||||||
|
instructions: str,
|
||||||
|
success_criteria: Sequence[str],
|
||||||
|
repository: str,
|
||||||
|
base_oid: str,
|
||||||
|
target_branch: str,
|
||||||
|
worktree: Path,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if adapter not in {"hermes", "codex", "openclaw"}:
|
||||||
|
raise RuntimeErrorV2("ADAPTER_INVALID")
|
||||||
|
if not OID_RE.fullmatch(base_oid):
|
||||||
|
raise RuntimeErrorV2("BASE_OID_INVALID")
|
||||||
|
return {
|
||||||
|
"schema": "aios.online-agent.runtime-request.v1",
|
||||||
|
"adapter": adapter,
|
||||||
|
"task": {
|
||||||
|
"task_id": task_id,
|
||||||
|
"goal": goal,
|
||||||
|
"instructions": instructions,
|
||||||
|
"success_criteria": list(success_criteria),
|
||||||
|
},
|
||||||
|
"execution_context": {
|
||||||
|
"repository": repository,
|
||||||
|
"base_oid": base_oid,
|
||||||
|
"target_branch": target_branch,
|
||||||
|
"worktree": str(worktree),
|
||||||
|
},
|
||||||
|
"policy": {
|
||||||
|
"authoritative_repository_write": False,
|
||||||
|
"direct_git_ref_mutation": False,
|
||||||
|
"direct_git_push": False,
|
||||||
|
"direct_git_commit": False,
|
||||||
|
"main_write": False,
|
||||||
|
"force_push": False,
|
||||||
|
"production_write": False,
|
||||||
|
"tag_write": False,
|
||||||
|
"baseline_write": False,
|
||||||
|
"task_api_credential_visible": False,
|
||||||
|
"writer_lease_visible": False,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def invoke(socket_path: Path, payload: Mapping[str, Any], timeout_seconds: int) -> RuntimeResult:
|
||||||
|
if not socket_path.is_absolute():
|
||||||
|
raise RuntimeErrorV2("RUNTIME_SOCKET_NOT_ABSOLUTE")
|
||||||
|
if not 1 <= timeout_seconds <= 7200:
|
||||||
|
raise RuntimeErrorV2("RUNTIME_TIMEOUT_INVALID")
|
||||||
|
raw = json.dumps(dict(payload), ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + b"\n"
|
||||||
|
if len(raw) > MAX_BYTES:
|
||||||
|
raise RuntimeErrorV2("RUNTIME_REQUEST_TOO_LARGE")
|
||||||
|
|
||||||
|
client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||||
|
client.settimeout(timeout_seconds)
|
||||||
|
try:
|
||||||
|
client.connect(str(socket_path))
|
||||||
|
client.sendall(raw)
|
||||||
|
client.shutdown(socket.SHUT_WR)
|
||||||
|
chunks: list[bytes] = []
|
||||||
|
total = 0
|
||||||
|
while True:
|
||||||
|
chunk = client.recv(65536)
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
total += len(chunk)
|
||||||
|
if total > MAX_BYTES:
|
||||||
|
raise RuntimeErrorV2("RUNTIME_RESPONSE_TOO_LARGE")
|
||||||
|
chunks.append(chunk)
|
||||||
|
except (OSError, socket.timeout) as exc:
|
||||||
|
raise RuntimeErrorV2("RUNTIME_UNAVAILABLE", details={"socket": str(socket_path)}) from exc
|
||||||
|
finally:
|
||||||
|
client.close()
|
||||||
|
try:
|
||||||
|
value = json.loads(b"".join(chunks).decode("utf-8"))
|
||||||
|
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||||
|
raise RuntimeErrorV2("RUNTIME_RESPONSE_INVALID") from exc
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise RuntimeErrorV2("RUNTIME_RESPONSE_INVALID")
|
||||||
|
result = RuntimeResult.parse(value)
|
||||||
|
if result.status == "BLOCKED":
|
||||||
|
raise RuntimeErrorV2("RUNTIME_BLOCKED", details={"summary": result.summary})
|
||||||
|
if result.status == "FAILED":
|
||||||
|
raise RuntimeErrorV2("RUNTIME_FAILED", details={"summary": result.summary})
|
||||||
|
return result
|
||||||
@@ -0,0 +1,101 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
from runtime_client import collect_changes, invoke, request_payload, scan_secrets
|
||||||
|
|
||||||
|
|
||||||
|
WORKSPACE_ROOT = Path(os.environ.get(
|
||||||
|
"AIOS_AUTONOMOUS_WORKSPACE_ROOT",
|
||||||
|
"/var/lib/aios-autonomous-agent-worker/workspaces",
|
||||||
|
))
|
||||||
|
ADAPTER = os.environ.get("AIOS_AUTONOMOUS_SELFTEST_ADAPTER", "openclaw")
|
||||||
|
SOCKET = Path(os.environ.get(
|
||||||
|
"AIOS_AUTONOMOUS_SELFTEST_BROKER_SOCKET",
|
||||||
|
f"/run/aios-agent-runtime/{ADAPTER}.sock",
|
||||||
|
))
|
||||||
|
|
||||||
|
|
||||||
|
def run(argv: list[str], cwd: Path) -> str:
|
||||||
|
completed = subprocess.run(
|
||||||
|
argv,
|
||||||
|
cwd=str(cwd),
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
timeout=60,
|
||||||
|
env={"PATH": "/usr/bin:/bin", "LC_ALL": "C.UTF-8"},
|
||||||
|
)
|
||||||
|
if completed.returncode != 0:
|
||||||
|
raise RuntimeError(f"SELFTEST_COMMAND_FAILED:{argv[0]}:{completed.returncode}")
|
||||||
|
return completed.stdout.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
if not SOCKET.is_socket():
|
||||||
|
raise RuntimeError("AUTONOMOUS_ENGINEERING_BACKEND_UNAVAILABLE")
|
||||||
|
WORKSPACE_ROOT.mkdir(parents=True, mode=0o750, exist_ok=True)
|
||||||
|
run_root = Path(tempfile.mkdtemp(prefix="runtime-selftest-", dir=str(WORKSPACE_ROOT)))
|
||||||
|
repo = run_root / "worktree"
|
||||||
|
repo.mkdir()
|
||||||
|
try:
|
||||||
|
run(["/usr/bin/git", "init", str(repo)], run_root)
|
||||||
|
run(["/usr/bin/git", "-C", str(repo), "config", "user.email", "selftest@example.invalid"], run_root)
|
||||||
|
run(["/usr/bin/git", "-C", str(repo), "config", "user.name", "AIOS Runtime Selftest"], run_root)
|
||||||
|
marker = repo / "agent_runtime_selftest.txt"
|
||||||
|
marker.write_text("VALUE=1\n", encoding="utf-8")
|
||||||
|
run(["/usr/bin/git", "-C", str(repo), "add", "agent_runtime_selftest.txt"], run_root)
|
||||||
|
run(["/usr/bin/git", "-C", str(repo), "commit", "-m", "selftest base"], run_root)
|
||||||
|
base_oid = run(["/usr/bin/git", "-C", str(repo), "rev-parse", "HEAD"], run_root)
|
||||||
|
|
||||||
|
payload = request_payload(
|
||||||
|
adapter=ADAPTER,
|
||||||
|
task_id="autonomous-runtime-selftest",
|
||||||
|
goal="Change agent_runtime_selftest.txt so it contains exactly VALUE=2 followed by a newline.",
|
||||||
|
instructions=(
|
||||||
|
"Modify only agent_runtime_selftest.txt in the provided disposable worktree. "
|
||||||
|
"Do not commit, push, change Git refs, or access task/governed-write credentials."
|
||||||
|
),
|
||||||
|
success_criteria=[
|
||||||
|
"agent_runtime_selftest.txt contains VALUE=2",
|
||||||
|
"Git HEAD remains unchanged",
|
||||||
|
],
|
||||||
|
repository="selftest/disposable",
|
||||||
|
base_oid=base_oid,
|
||||||
|
target_branch="agent/intent/runtime-selftest",
|
||||||
|
worktree=repo,
|
||||||
|
)
|
||||||
|
result = invoke(SOCKET, payload, timeout_seconds=300)
|
||||||
|
changed = collect_changes(repo, base_oid)
|
||||||
|
scan_secrets(repo, changed)
|
||||||
|
content = marker.read_text(encoding="utf-8")
|
||||||
|
head = run(["/usr/bin/git", "-C", str(repo), "rev-parse", "HEAD"], run_root)
|
||||||
|
if content != "VALUE=2\n":
|
||||||
|
raise RuntimeError("RUNTIME_SELFTEST_CONTENT_MISMATCH")
|
||||||
|
if head != base_oid:
|
||||||
|
raise RuntimeError("RUNTIME_SELFTEST_HEAD_MUTATED")
|
||||||
|
if changed != ["agent_runtime_selftest.txt"]:
|
||||||
|
raise RuntimeError("RUNTIME_SELFTEST_UNEXPECTED_DIFF")
|
||||||
|
print(json.dumps({
|
||||||
|
"status": "PASS",
|
||||||
|
"adapter": ADAPTER,
|
||||||
|
"base_oid": base_oid,
|
||||||
|
"changed_files": changed,
|
||||||
|
"head_unchanged": True,
|
||||||
|
"runtime_summary": result.summary,
|
||||||
|
"task_api_token_visible_to_runtime": False,
|
||||||
|
"writer_lease_visible_to_runtime": False,
|
||||||
|
}, sort_keys=True))
|
||||||
|
return 0
|
||||||
|
finally:
|
||||||
|
shutil.rmtree(run_root, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
# Generic Governed Online Agent Worker
|
||||||
|
|
||||||
|
This worker is a pull-based, restartable consumer for `aios.agent-task.v1`.
|
||||||
|
The service configuration contains only stable runtime identity, capability
|
||||||
|
names, API endpoints, heartbeat timing, and the credential projection. It does
|
||||||
|
not contain a task id, repository base OID, target branch, or plan-file path.
|
||||||
|
|
||||||
|
## Dynamic task contract
|
||||||
|
|
||||||
|
Engineering tasks carry `metadata.execution` with schema
|
||||||
|
`aios.online-agent.execution.v2`. The payload supplies the repository, exact
|
||||||
|
base OID, target branch, detached-worktree policy, adapter name, explicit file
|
||||||
|
change set, verification commands, and governed commit idempotency keys.
|
||||||
|
Task prose is descriptive only and is never passed to a shell.
|
||||||
|
|
||||||
|
The worker accepts only structured operations:
|
||||||
|
|
||||||
|
- `deterministic`: allow-listed file changes and test commands;
|
||||||
|
- `codex`: fail-closed until a task supplies an explicit Codex file/command
|
||||||
|
policy;
|
||||||
|
- `hermes`: fail-closed because the current Hermes runtime is a user-chat
|
||||||
|
service, not an engineering executor.
|
||||||
|
|
||||||
|
All command execution uses `shell=False`, fixed program allowlists, relative
|
||||||
|
paths beneath an ephemeral detached worktree, and output redaction. Direct
|
||||||
|
`git branch`, `update-ref`, `push`, `commit`, `reset`, `clean`, main,
|
||||||
|
production, tag, and baseline mutations are rejected by the worker.
|
||||||
|
|
||||||
|
## Governed write path
|
||||||
|
|
||||||
|
The worker calls the Bridge in this order:
|
||||||
|
|
||||||
|
1. `POST /v2/governed-task`;
|
||||||
|
2. `POST /v2/writer-lease`;
|
||||||
|
3. `POST /v2/commit-files`;
|
||||||
|
4. read-only authority-ref visibility verification.
|
||||||
|
|
||||||
|
CREATE_NEW is represented by the task payload and is normalized by the Bridge
|
||||||
|
lease authority. The worker never creates an authoritative ref itself.
|
||||||
|
|
||||||
|
## Lifecycle and recovery
|
||||||
|
|
||||||
|
The runtime registers with `register_or_recover`, receives its agent token
|
||||||
|
once, and stores only the encrypted systemd credential. The consumer loops over
|
||||||
|
poll, atomically claims one task, sends the first heartbeat, emits
|
||||||
|
`TASK_STARTED`, and continues heartbeats while executing.
|
||||||
|
|
||||||
|
State is atomically stored per task without agent or lease token material. If
|
||||||
|
the process stops after a governed commit but before result submission, the
|
||||||
|
next lease attempt reuses the stable governed idempotency key and submits the
|
||||||
|
stored commit evidence without issuing a second commit.
|
||||||
|
|
||||||
|
The control plane must independently verify submitted results; worker-local
|
||||||
|
success is not production or mainline authorization.
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
"""Generic governed Online Agent Worker runtime."""
|
||||||
|
|
||||||
|
__all__ = ["worker"]
|
||||||
@@ -0,0 +1,937 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Generic, restartable Online Agent Worker.
|
||||||
|
|
||||||
|
The worker is deliberately task-agnostic. A task's signed/durable payload is
|
||||||
|
the only source of repository, branch, change, verification, and governance
|
||||||
|
parameters. There is no task id, base OID, branch name, or plan file in the
|
||||||
|
runtime configuration.
|
||||||
|
|
||||||
|
The worker performs local work only in an ephemeral detached worktree. All
|
||||||
|
authoritative ref updates go through the governed Bridge API; direct branch,
|
||||||
|
update-ref, push, main, production, tag, and baseline mutations are rejected.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from abc import ABC, abstractmethod
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from typing import Any, Mapping
|
||||||
|
from urllib.parse import unquote
|
||||||
|
|
||||||
|
|
||||||
|
API_BASE = os.environ.get("AIOS_AGENT_TASK_API_URL", "http://127.0.0.1:18940").rstrip("/")
|
||||||
|
BRIDGE_BASE = os.environ.get("AIOS_GOVERNED_WRITE_API_URL", "http://127.0.0.1:18892").rstrip("/")
|
||||||
|
AGENT_ID = os.environ.get(
|
||||||
|
"AIOS_AGENT_ID", "online-engineering-worker-cn-network-mkt-generic-20260815"
|
||||||
|
)
|
||||||
|
AGENT_TYPE = os.environ.get("AIOS_AGENT_TYPE", "trusted-online-engineering-worker")
|
||||||
|
AGENT_VERSION = os.environ.get("AIOS_AGENT_VERSION", "2026-08-15.3-generic")
|
||||||
|
AUTHORITY_GIT_DIR = Path(
|
||||||
|
os.environ.get("AIOS_AUTHORITY_GIT_DIR", "/data/aios-gitops/repo.git")
|
||||||
|
)
|
||||||
|
STATE_ROOT = Path(
|
||||||
|
os.environ.get("AIOS_WORKER_STATE_ROOT", "/var/lib/aios-online-agent-worker-20260815")
|
||||||
|
)
|
||||||
|
WORKSPACE_ROOT = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_WORKER_WORKSPACE_ROOT", "/var/lib/aios-online-agent-worker-20260815/workspaces"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
GIT_CONFIG_GLOBAL = STATE_ROOT / "gitconfig"
|
||||||
|
POLL_SECONDS = int(os.environ.get("AIOS_WORKER_POLL_SECONDS", "5"))
|
||||||
|
HEARTBEAT_SECONDS = int(os.environ.get("AIOS_WORKER_HEARTBEAT_SECONDS", "20"))
|
||||||
|
LEASE_SECONDS = int(os.environ.get("AIOS_WORKER_LEASE_SECONDS", "300"))
|
||||||
|
MAX_TASK_BYTES = 512 * 1024
|
||||||
|
MAX_FILE_BYTES = 2 * 1024 * 1024
|
||||||
|
OID_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||||
|
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
|
||||||
|
ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{2,179}$")
|
||||||
|
BRANCH_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]*$")
|
||||||
|
PATH_RE = re.compile(r"^[^\\\x00-\x1f\x7f]+$")
|
||||||
|
SENSITIVE_RE = re.compile(
|
||||||
|
r"(?i)(authorization\s*:\s*bearer\s+\S+|(?:password|passwd|secret|token|api[_-]?key)\s*[:=]\s*\S+)"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class WorkerError(RuntimeError):
|
||||||
|
def __init__(self, code: str, *, phase: str, details: Mapping[str, Any] | None = None) -> None:
|
||||||
|
super().__init__(code)
|
||||||
|
self.code = code
|
||||||
|
self.phase = phase
|
||||||
|
self.details = dict(details or {})
|
||||||
|
|
||||||
|
|
||||||
|
def now() -> str:
|
||||||
|
return datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def digest_bytes(value: bytes) -> str:
|
||||||
|
return hashlib.sha256(value).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def digest_json(value: Any) -> str:
|
||||||
|
return digest_bytes(
|
||||||
|
json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def sanitize(value: Any) -> Any:
|
||||||
|
if isinstance(value, Mapping):
|
||||||
|
cleaned: dict[str, Any] = {}
|
||||||
|
for key, inner in value.items():
|
||||||
|
lowered = str(key).lower()
|
||||||
|
if any(marker in lowered for marker in ("token", "secret", "credential", "password", "authorization")):
|
||||||
|
cleaned[str(key)] = "[REDACTED]"
|
||||||
|
else:
|
||||||
|
cleaned[str(key)] = sanitize(inner)
|
||||||
|
return cleaned
|
||||||
|
if isinstance(value, list):
|
||||||
|
return [sanitize(item) for item in value]
|
||||||
|
if isinstance(value, tuple):
|
||||||
|
return [sanitize(item) for item in value]
|
||||||
|
if isinstance(value, str):
|
||||||
|
return SENSITIVE_RE.sub("[REDACTED]", value)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def excerpt(value: str, limit: int = 800) -> str:
|
||||||
|
value = str(sanitize(value))
|
||||||
|
return value if len(value) <= limit else value[: limit // 2] + "...[TRUNCATED]..." + value[-limit // 2 :]
|
||||||
|
|
||||||
|
|
||||||
|
def audit(event_type: str, **fields: Any) -> None:
|
||||||
|
STATE_ROOT.mkdir(mode=0o750, parents=True, exist_ok=True)
|
||||||
|
path = STATE_ROOT / "events.jsonl"
|
||||||
|
record = {
|
||||||
|
"event_type": event_type,
|
||||||
|
"created_at": now(),
|
||||||
|
"agent_id": AGENT_ID,
|
||||||
|
"credential_material_exposed": False,
|
||||||
|
"lease_material_exposed": False,
|
||||||
|
**sanitize(fields),
|
||||||
|
}
|
||||||
|
with path.open("a", encoding="utf-8") as handle:
|
||||||
|
handle.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n")
|
||||||
|
os.chmod(path, 0o640)
|
||||||
|
print(json.dumps(record, ensure_ascii=False, sort_keys=True), flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
def atomic_json(path: Path, value: Mapping[str, Any], mode: int = 0o640) -> None:
|
||||||
|
path.parent.mkdir(mode=0o750, parents=True, exist_ok=True)
|
||||||
|
if path.parent.is_symlink() or path.is_symlink():
|
||||||
|
raise WorkerError("STATE_PATH_SYMLINK_REJECTED", phase="state")
|
||||||
|
fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=str(path.parent))
|
||||||
|
try:
|
||||||
|
os.fchmod(fd, mode)
|
||||||
|
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||||
|
json.dump(sanitize(dict(value)), handle, ensure_ascii=False, sort_keys=True)
|
||||||
|
handle.write("\n")
|
||||||
|
handle.flush()
|
||||||
|
os.fsync(handle.fileno())
|
||||||
|
os.replace(temporary, path)
|
||||||
|
os.chmod(path, mode)
|
||||||
|
finally:
|
||||||
|
if os.path.exists(temporary):
|
||||||
|
os.unlink(temporary)
|
||||||
|
|
||||||
|
|
||||||
|
def safe_relative_path(value: Any, *, field: str = "path") -> str:
|
||||||
|
if not isinstance(value, str) or not value or len(value) > 240:
|
||||||
|
raise WorkerError(f"{field.upper()}_INVALID", phase="validation")
|
||||||
|
if unquote(value) != value or not PATH_RE.fullmatch(value):
|
||||||
|
raise WorkerError(f"{field.upper()}_INVALID", phase="validation")
|
||||||
|
if value.startswith(("/", "~")) or re.match(r"^[A-Za-z]:", value):
|
||||||
|
raise WorkerError(f"{field.upper()}_ABSOLUTE", phase="validation")
|
||||||
|
parts = value.replace("\\", "/").split("/")
|
||||||
|
if any(part in {"", ".", ".."} for part in parts):
|
||||||
|
raise WorkerError(f"{field.upper()}_TRAVERSAL", phase="validation")
|
||||||
|
return "/".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_under(root: Path, relative: str, *, write: bool = False) -> Path:
|
||||||
|
relative = safe_relative_path(relative)
|
||||||
|
root = root.resolve()
|
||||||
|
candidate = (root / relative).resolve(strict=False)
|
||||||
|
if candidate != root and root not in candidate.parents:
|
||||||
|
raise WorkerError("WORKSPACE_ESCAPE", phase="validation")
|
||||||
|
current = root
|
||||||
|
for part in Path(relative).parts:
|
||||||
|
current /= part
|
||||||
|
if current.exists() and current.is_symlink():
|
||||||
|
raise WorkerError("WORKSPACE_SYMLINK_REJECTED", phase="validation")
|
||||||
|
if write and candidate.exists() and candidate.is_symlink():
|
||||||
|
raise WorkerError("WRITE_SYMLINK_REJECTED", phase="validation")
|
||||||
|
return candidate
|
||||||
|
|
||||||
|
|
||||||
|
def validate_oid(value: Any, field: str = "oid") -> str:
|
||||||
|
if not isinstance(value, str) or not OID_RE.fullmatch(value):
|
||||||
|
raise WorkerError(f"{field.upper()}_INVALID", phase="validation")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def validate_sha(value: Any, field: str = "sha256") -> str:
|
||||||
|
if not isinstance(value, str) or not SHA256_RE.fullmatch(value):
|
||||||
|
raise WorkerError(f"{field.upper()}_INVALID", phase="validation")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def validate_branch(value: Any) -> str:
|
||||||
|
if not isinstance(value, str) or not 3 <= len(value) <= 180:
|
||||||
|
raise WorkerError("TARGET_BRANCH_INVALID", phase="validation")
|
||||||
|
if value in {"main", "master", "HEAD"} or value.startswith(
|
||||||
|
("main/", "master/", "refs/", "tags/", "tag/", "release/", "releases/", "baseline/", "production/")
|
||||||
|
):
|
||||||
|
raise WorkerError("PROTECTED_BRANCH_REJECTED", phase="validation")
|
||||||
|
if (
|
||||||
|
value.startswith("/")
|
||||||
|
or value.endswith(("/", ".", ".lock"))
|
||||||
|
or ".." in value
|
||||||
|
or "//" in value
|
||||||
|
or "@{" in value
|
||||||
|
or any(ch in value for ch in " ~^:?*[\\")
|
||||||
|
or any(part in {"", ".", ".."} for part in value.split("/"))
|
||||||
|
or not BRANCH_RE.fullmatch(value)
|
||||||
|
):
|
||||||
|
raise WorkerError("TARGET_BRANCH_INVALID", phase="validation")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def validate_program(argv: Any, *, internal_git: bool = False) -> list[str]:
|
||||||
|
if not isinstance(argv, list) or not argv or not all(isinstance(item, str) and item for item in argv):
|
||||||
|
raise WorkerError("STRUCTURED_ARGV_REQUIRED", phase="validation")
|
||||||
|
command = argv[0]
|
||||||
|
if command not in {"/usr/bin/git", "/usr/bin/python3"}:
|
||||||
|
raise WorkerError("PROGRAM_NOT_ALLOWLISTED", phase="validation", details={"program": Path(command).name})
|
||||||
|
if command.endswith("python3"):
|
||||||
|
if any(item in {"-c", "-", "--command"} for item in argv):
|
||||||
|
raise WorkerError("PYTHON_INLINE_CODE_REJECTED", phase="validation")
|
||||||
|
if "-m" in argv:
|
||||||
|
index = argv.index("-m")
|
||||||
|
if index + 1 >= len(argv) or argv[index + 1] not in {"pytest", "py_compile"}:
|
||||||
|
raise WorkerError("PYTHON_MODULE_NOT_ALLOWLISTED", phase="validation")
|
||||||
|
if command.endswith("git") and not internal_git:
|
||||||
|
rejected = {"push", "update-ref", "receive-pack", "send-pack", "reset", "clean", "branch", "commit", "merge", "rebase"}
|
||||||
|
if any(item in rejected for item in argv[1:]):
|
||||||
|
raise WorkerError("GIT_MUTATION_NOT_ALLOWLISTED", phase="validation")
|
||||||
|
return list(argv)
|
||||||
|
|
||||||
|
|
||||||
|
def run_command(
|
||||||
|
label: str,
|
||||||
|
argv: list[str],
|
||||||
|
*,
|
||||||
|
cwd: Path,
|
||||||
|
timeout: int,
|
||||||
|
internal_git: bool = False,
|
||||||
|
allowed_returncodes: tuple[int, ...] = (0,),
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
validate_program(argv, internal_git=internal_git)
|
||||||
|
if not 1 <= timeout <= 3600:
|
||||||
|
raise WorkerError("COMMAND_TIMEOUT_INVALID", phase=label)
|
||||||
|
started = time.monotonic()
|
||||||
|
try:
|
||||||
|
completed = subprocess.run(
|
||||||
|
argv,
|
||||||
|
cwd=str(cwd),
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
shell=False,
|
||||||
|
timeout=timeout,
|
||||||
|
env={"PATH": "/usr/bin:/bin", "LC_ALL": "C.UTF-8", "GIT_CONFIG_GLOBAL": str(GIT_CONFIG_GLOBAL)},
|
||||||
|
)
|
||||||
|
except subprocess.TimeoutExpired as exc:
|
||||||
|
raise WorkerError("COMMAND_TIMEOUT", phase=label, details={"timeout_seconds": timeout}) from exc
|
||||||
|
result = {
|
||||||
|
"label": label,
|
||||||
|
"argv": [Path(item).name if index == 0 else item for index, item in enumerate(argv)],
|
||||||
|
"exit_code": completed.returncode,
|
||||||
|
"stdout_sha256": digest_bytes(completed.stdout.encode("utf-8", "replace")),
|
||||||
|
"stderr_sha256": digest_bytes(completed.stderr.encode("utf-8", "replace")),
|
||||||
|
"stdout_excerpt": excerpt(completed.stdout),
|
||||||
|
"stderr_excerpt": excerpt(completed.stderr),
|
||||||
|
"duration_ms": int((time.monotonic() - started) * 1000),
|
||||||
|
}
|
||||||
|
audit("COMMAND_RESULT", **result)
|
||||||
|
if completed.returncode not in allowed_returncodes:
|
||||||
|
raise WorkerError("COMMAND_FAILED", phase=label, details={"exit_code": completed.returncode, "stderr": excerpt(completed.stderr, 400)})
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ExecutionEnvelope:
|
||||||
|
task_id: str
|
||||||
|
repository: str
|
||||||
|
base_oid: str
|
||||||
|
target_branch: str
|
||||||
|
adapter: str
|
||||||
|
change_set: tuple[dict[str, Any], ...]
|
||||||
|
verification: tuple[dict[str, Any], ...]
|
||||||
|
governance: dict[str, Any]
|
||||||
|
workspace: dict[str, Any]
|
||||||
|
digest: str
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_task(cls, task: Mapping[str, Any]) -> "ExecutionEnvelope":
|
||||||
|
task_id = str(task.get("task_id") or "")
|
||||||
|
if not ID_RE.fullmatch(task_id):
|
||||||
|
raise WorkerError("TASK_ID_INVALID", phase="validation")
|
||||||
|
metadata = task.get("metadata")
|
||||||
|
if not isinstance(metadata, Mapping):
|
||||||
|
raise WorkerError("TASK_METADATA_REQUIRED", phase="validation")
|
||||||
|
execution = metadata.get("execution")
|
||||||
|
if not isinstance(execution, Mapping):
|
||||||
|
raise WorkerError("DYNAMIC_EXECUTION_PAYLOAD_REQUIRED", phase="validation")
|
||||||
|
if execution.get("schema") != "aios.online-agent.execution.v2":
|
||||||
|
raise WorkerError("EXECUTION_SCHEMA_INVALID", phase="validation")
|
||||||
|
repository = execution.get("repository")
|
||||||
|
if not isinstance(repository, str) or not repository:
|
||||||
|
raise WorkerError("REPOSITORY_INVALID", phase="validation")
|
||||||
|
base_oid = validate_oid(execution.get("base_oid"), "base_oid")
|
||||||
|
target_branch = validate_branch(execution.get("target_branch"))
|
||||||
|
workspace = execution.get("workspace")
|
||||||
|
if not isinstance(workspace, Mapping):
|
||||||
|
raise WorkerError("WORKSPACE_POLICY_REQUIRED", phase="validation")
|
||||||
|
workspace_value = dict(workspace)
|
||||||
|
if workspace_value.get("authority_git_dir") != str(AUTHORITY_GIT_DIR):
|
||||||
|
raise WorkerError("AUTHORITY_ROOT_INVALID", phase="validation")
|
||||||
|
if workspace_value.get("read_only_repository") is not True:
|
||||||
|
raise WorkerError("REPOSITORY_WRITE_POLICY_INVALID", phase="validation")
|
||||||
|
if workspace_value.get("cleanup_policy") not in {"always", "cleanup_on_success"}:
|
||||||
|
raise WorkerError("WORKSPACE_CLEANUP_POLICY_INVALID", phase="validation")
|
||||||
|
adapter_value = execution.get("adapter")
|
||||||
|
if isinstance(adapter_value, Mapping):
|
||||||
|
adapter = str(adapter_value.get("name") or "")
|
||||||
|
else:
|
||||||
|
adapter = str(adapter_value or "")
|
||||||
|
if adapter not in {"deterministic", "codex", "hermes"}:
|
||||||
|
raise WorkerError("RUNTIME_ADAPTER_UNSUPPORTED", phase="validation")
|
||||||
|
change_set = execution.get("change_set")
|
||||||
|
if not isinstance(change_set, list) or not change_set or len(change_set) > 128:
|
||||||
|
raise WorkerError("CHANGE_SET_INVALID", phase="validation")
|
||||||
|
normalized_changes: list[dict[str, Any]] = []
|
||||||
|
for item in change_set:
|
||||||
|
if not isinstance(item, Mapping):
|
||||||
|
raise WorkerError("CHANGE_OPERATION_INVALID", phase="validation")
|
||||||
|
operation = str(item.get("operation") or "")
|
||||||
|
relative = safe_relative_path(item.get("path"))
|
||||||
|
normalized = dict(item)
|
||||||
|
normalized["operation"] = operation
|
||||||
|
normalized["path"] = relative
|
||||||
|
if operation == "write_file":
|
||||||
|
content = item.get("content")
|
||||||
|
if not isinstance(content, str) or len(content.encode("utf-8")) > MAX_FILE_BYTES:
|
||||||
|
raise WorkerError("WRITE_CONTENT_INVALID", phase="validation")
|
||||||
|
elif operation == "replace_text":
|
||||||
|
if not isinstance(item.get("old"), str) or not item.get("old") or not isinstance(item.get("new"), str):
|
||||||
|
raise WorkerError("REPLACE_TEXT_INVALID", phase="validation")
|
||||||
|
if len(str(item["old"]).encode("utf-8")) > MAX_FILE_BYTES:
|
||||||
|
raise WorkerError("REPLACE_TEXT_INVALID", phase="validation")
|
||||||
|
elif operation == "delete_file":
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
raise WorkerError("CHANGE_OPERATION_UNSUPPORTED", phase="validation", details={"operation": operation})
|
||||||
|
expected = item.get("expected_sha256")
|
||||||
|
if expected is not None:
|
||||||
|
validate_sha(expected, "expected_sha256")
|
||||||
|
normalized_changes.append(normalized)
|
||||||
|
verification = execution.get("verification") or []
|
||||||
|
if not isinstance(verification, list) or len(verification) > 128:
|
||||||
|
raise WorkerError("VERIFICATION_INVALID", phase="validation")
|
||||||
|
normalized_verification: list[dict[str, Any]] = []
|
||||||
|
for item in verification:
|
||||||
|
if not isinstance(item, Mapping):
|
||||||
|
raise WorkerError("VERIFICATION_COMMAND_INVALID", phase="validation")
|
||||||
|
label = str(item.get("label") or "verification")
|
||||||
|
argv = validate_program(item.get("argv"))
|
||||||
|
cwd_value = item.get("cwd")
|
||||||
|
cwd = "." if cwd_value in {None, "", "."} else safe_relative_path(cwd_value, field="cwd")
|
||||||
|
timeout = item.get("timeout_seconds", 120)
|
||||||
|
if not isinstance(timeout, int) or isinstance(timeout, bool) or not 1 <= timeout <= 3600:
|
||||||
|
raise WorkerError("VERIFICATION_TIMEOUT_INVALID", phase="validation")
|
||||||
|
normalized_verification.append({"label": label, "argv": argv, "cwd": cwd, "timeout_seconds": timeout})
|
||||||
|
governance = execution.get("governance")
|
||||||
|
if not isinstance(governance, Mapping):
|
||||||
|
raise WorkerError("GOVERNANCE_POLICY_REQUIRED", phase="validation")
|
||||||
|
governance_value = dict(governance)
|
||||||
|
governed_task_id = str(governance_value.get("task_id") or f"{task_id}-writer")
|
||||||
|
if not ID_RE.fullmatch(governed_task_id):
|
||||||
|
raise WorkerError("GOVERNED_TASK_ID_INVALID", phase="validation")
|
||||||
|
idempotency_key = str(governance_value.get("idempotency_key") or f"{task_id}-governed-v1")
|
||||||
|
commit_key = str(governance_value.get("commit_idempotency_key") or f"{task_id}-commit-v1")
|
||||||
|
for field, value in (("idempotency_key", idempotency_key), ("commit_idempotency_key", commit_key)):
|
||||||
|
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:-]{7,159}", value):
|
||||||
|
raise WorkerError(f"{field.upper()}_INVALID", phase="validation")
|
||||||
|
governance_value.update(
|
||||||
|
{
|
||||||
|
"task_id": governed_task_id,
|
||||||
|
"idempotency_key": idempotency_key,
|
||||||
|
"commit_idempotency_key": commit_key,
|
||||||
|
"message": str(governance_value.get("message") or f"chore(gitops): execute task {task_id}"),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
for field in ("main_write", "force_push", "production_write", "tag_write", "baseline_write", "protected_ref_bypass"):
|
||||||
|
if governance_value.get(field, False) is not False:
|
||||||
|
raise WorkerError("GOVERNANCE_POLICY_REJECTED", phase="validation", details={"field": field})
|
||||||
|
allowed_files = governance_value.get("allowed_files")
|
||||||
|
if allowed_files is not None:
|
||||||
|
if not isinstance(allowed_files, list):
|
||||||
|
raise WorkerError("ALLOWED_FILES_INVALID", phase="validation")
|
||||||
|
allowed = {safe_relative_path(item, field="allowed_file") for item in allowed_files}
|
||||||
|
if not {item["path"] for item in normalized_changes}.issubset(allowed):
|
||||||
|
raise WorkerError("CHANGE_OUTSIDE_ALLOWED_FILES", phase="validation")
|
||||||
|
governance_value["allowed_files"] = sorted(allowed)
|
||||||
|
return cls(
|
||||||
|
task_id=task_id,
|
||||||
|
repository=repository,
|
||||||
|
base_oid=base_oid,
|
||||||
|
target_branch=target_branch,
|
||||||
|
adapter=adapter,
|
||||||
|
change_set=tuple(normalized_changes),
|
||||||
|
verification=tuple(normalized_verification),
|
||||||
|
governance=governance_value,
|
||||||
|
workspace=workspace_value,
|
||||||
|
digest=digest_json(execution),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class RuntimeAdapter(ABC):
|
||||||
|
name: str
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def execute(self, envelope: ExecutionEnvelope, worktree: Path) -> dict[str, Any]:
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
|
||||||
|
class DeterministicExecutor(RuntimeAdapter):
|
||||||
|
name = "deterministic"
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _write_atomic(path: Path, content: str) -> None:
|
||||||
|
path.parent.mkdir(mode=0o750, parents=True, exist_ok=True)
|
||||||
|
fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=str(path.parent))
|
||||||
|
try:
|
||||||
|
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||||
|
handle.write(content)
|
||||||
|
handle.flush()
|
||||||
|
os.fsync(handle.fileno())
|
||||||
|
os.replace(temporary, path)
|
||||||
|
finally:
|
||||||
|
if os.path.exists(temporary):
|
||||||
|
os.unlink(temporary)
|
||||||
|
|
||||||
|
def execute(self, envelope: ExecutionEnvelope, worktree: Path) -> dict[str, Any]:
|
||||||
|
changed: set[str] = set()
|
||||||
|
for change in envelope.change_set:
|
||||||
|
relative = str(change["path"])
|
||||||
|
path = resolve_under(worktree, relative, write=True)
|
||||||
|
expected = change.get("expected_sha256")
|
||||||
|
current = path.read_bytes() if path.is_file() else None
|
||||||
|
if expected is not None and (current is None or digest_bytes(current) != expected):
|
||||||
|
raise WorkerError("WORKTREE_SOURCE_DRIFT", phase="implementing", details={"path": relative})
|
||||||
|
operation = change["operation"]
|
||||||
|
if operation == "write_file":
|
||||||
|
self._write_atomic(path, str(change["content"]))
|
||||||
|
elif operation == "replace_text":
|
||||||
|
if current is None:
|
||||||
|
raise WorkerError("REPLACE_SOURCE_MISSING", phase="implementing", details={"path": relative})
|
||||||
|
original = current.decode("utf-8")
|
||||||
|
old = str(change["old"])
|
||||||
|
count = original.count(old)
|
||||||
|
expected_count = int(change.get("count", 1))
|
||||||
|
if count != expected_count:
|
||||||
|
raise WorkerError("REPLACE_TEXT_MATCH_COUNT_INVALID", phase="implementing", details={"path": relative, "count": count})
|
||||||
|
self._write_atomic(path, original.replace(old, str(change["new"])))
|
||||||
|
elif operation == "delete_file":
|
||||||
|
if path.exists():
|
||||||
|
path.unlink()
|
||||||
|
changed.add(relative)
|
||||||
|
audit("REPOSITORY_CHANGE_APPLIED", operation=operation, path=relative, sha256=digest_bytes(path.read_bytes()) if path.exists() else None)
|
||||||
|
commands: list[dict[str, Any]] = []
|
||||||
|
if changed:
|
||||||
|
commands.append(run_command("git_intent_to_add", ["/usr/bin/git", "-C", str(worktree), "add", "-N", "--", *sorted(changed)], cwd=worktree, timeout=120, internal_git=True))
|
||||||
|
commands.append(run_command("git_diff_check", ["/usr/bin/git", "-C", str(worktree), "diff", "--check"], cwd=worktree, timeout=120))
|
||||||
|
commands.append(run_command("git_diff_stat", ["/usr/bin/git", "-C", str(worktree), "diff", "--no-ext-diff", "--stat"], cwd=worktree, timeout=120))
|
||||||
|
tests: list[dict[str, Any]] = []
|
||||||
|
for item in envelope.verification:
|
||||||
|
cwd = worktree if item["cwd"] == "." else resolve_under(worktree, item["cwd"])
|
||||||
|
tests.append(run_command(item["label"], item["argv"], cwd=cwd, timeout=item["timeout_seconds"]))
|
||||||
|
return {
|
||||||
|
"changed_files": sorted(changed),
|
||||||
|
"commands": commands,
|
||||||
|
"tests": tests,
|
||||||
|
"adapter": self.name,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class CodexExecutor(RuntimeAdapter):
|
||||||
|
name = "codex"
|
||||||
|
|
||||||
|
def execute(self, envelope: ExecutionEnvelope, worktree: Path) -> dict[str, Any]:
|
||||||
|
raise WorkerError(
|
||||||
|
"CODEX_ADAPTER_REQUIRES_EXPLICIT_POLICY",
|
||||||
|
phase="adapter",
|
||||||
|
details={"reason": "Codex is not allowed to mutate a governed worktree without an explicit file/command policy"},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class HermesExecutor(RuntimeAdapter):
|
||||||
|
name = "hermes"
|
||||||
|
|
||||||
|
def execute(self, envelope: ExecutionEnvelope, worktree: Path) -> dict[str, Any]:
|
||||||
|
raise WorkerError(
|
||||||
|
"HERMES_ENGINEERING_ADAPTER_UNAVAILABLE",
|
||||||
|
phase="adapter",
|
||||||
|
details={"reason": "The live Hermes service is a user-chat runtime, not an engineering executor"},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def adapter_for(name: str) -> RuntimeAdapter:
|
||||||
|
return {"deterministic": DeterministicExecutor, "codex": CodexExecutor, "hermes": HermesExecutor}[name]()
|
||||||
|
|
||||||
|
|
||||||
|
class HttpClient:
|
||||||
|
@staticmethod
|
||||||
|
def call(base: str, method: str, route: str, token: str | None, body: Mapping[str, Any] | None = None, timeout: int = 30) -> tuple[int, dict[str, Any]]:
|
||||||
|
payload = None if body is None else json.dumps(body, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
headers = {"Accept": "application/json"}
|
||||||
|
if token:
|
||||||
|
headers["Authorization"] = "Bearer " + token
|
||||||
|
if payload is not None:
|
||||||
|
headers["Content-Type"] = "application/json"
|
||||||
|
request = urllib.request.Request(base + route, data=payload, headers=headers, method=method)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||||
|
value = json.loads(response.read().decode("utf-8"))
|
||||||
|
return response.status, value if isinstance(value, dict) else {}
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
raw = exc.read(8192).decode("utf-8", "replace")
|
||||||
|
try:
|
||||||
|
value = json.loads(raw)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
value = {"error": "HTTP_ERROR"}
|
||||||
|
return exc.code, value if isinstance(value, dict) else {"error": "HTTP_ERROR"}
|
||||||
|
except (urllib.error.URLError, TimeoutError):
|
||||||
|
return 599, {"error": "SERVICE_UNAVAILABLE"}
|
||||||
|
|
||||||
|
|
||||||
|
class AgentTaskClient:
|
||||||
|
def __init__(self, token: str) -> None:
|
||||||
|
self.token = token
|
||||||
|
|
||||||
|
def poll(self) -> list[dict[str, Any]]:
|
||||||
|
status, response = HttpClient.call(API_BASE, "POST", "/v1/tasks/poll", self.token, {"limit": 100})
|
||||||
|
if status != 200:
|
||||||
|
raise WorkerError("TASK_POLL_FAILED", phase="poll", details={"api_status": status, "error": response.get("error")})
|
||||||
|
tasks = response.get("tasks")
|
||||||
|
return [dict(item) for item in tasks if isinstance(item, Mapping)] if isinstance(tasks, list) else []
|
||||||
|
|
||||||
|
def claim(self, task_id: str) -> tuple[int, dict[str, Any]]:
|
||||||
|
return HttpClient.call(API_BASE, "POST", f"/v1/tasks/{task_id}/claim", self.token, {"lease_seconds": LEASE_SECONDS})
|
||||||
|
|
||||||
|
def heartbeat(self, task_id: str, lease_token: str, stage: str) -> tuple[int, dict[str, Any]]:
|
||||||
|
return HttpClient.call(
|
||||||
|
API_BASE,
|
||||||
|
"POST",
|
||||||
|
f"/v1/tasks/{task_id}/heartbeat",
|
||||||
|
self.token,
|
||||||
|
{"lease_token": lease_token, "lease_seconds": LEASE_SECONDS, "progress": {"stage": stage, "worker_version": AGENT_VERSION}},
|
||||||
|
)
|
||||||
|
|
||||||
|
def result(self, task_id: str, lease_token: str, idempotency_key: str, result: Mapping[str, Any]) -> tuple[int, dict[str, Any]]:
|
||||||
|
return HttpClient.call(
|
||||||
|
API_BASE,
|
||||||
|
"POST",
|
||||||
|
f"/v1/tasks/{task_id}/result",
|
||||||
|
self.token,
|
||||||
|
{"lease_token": lease_token, "idempotency_key": idempotency_key, "result": dict(result)},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class BridgeClient:
|
||||||
|
def call(self, route: str, body: Mapping[str, Any]) -> tuple[int, dict[str, Any]]:
|
||||||
|
return HttpClient.call(BRIDGE_BASE, "POST", route, None, body, timeout=60)
|
||||||
|
|
||||||
|
|
||||||
|
class LeaseHeartbeat:
|
||||||
|
def __init__(self, client: AgentTaskClient, task_id: str, lease_token: str) -> None:
|
||||||
|
self.client = client
|
||||||
|
self.task_id = task_id
|
||||||
|
self.lease_token = lease_token
|
||||||
|
self.stage = "claimed"
|
||||||
|
self.failed: str | None = None
|
||||||
|
self.stop_event = threading.Event()
|
||||||
|
self.lock = threading.Lock()
|
||||||
|
self.thread = threading.Thread(target=self._run, name="online-agent-heartbeat", daemon=True)
|
||||||
|
|
||||||
|
def first(self) -> None:
|
||||||
|
status, response = self.client.heartbeat(self.task_id, self.lease_token, self.stage)
|
||||||
|
if not 200 <= status < 300:
|
||||||
|
self.failed = str(response.get("error") or status)
|
||||||
|
raise WorkerError("TASK_HEARTBEAT_FAILED", phase="heartbeat", details={"api_status": status, "error": self.failed})
|
||||||
|
audit("TASK_HEARTBEAT", task_id=self.task_id, stage=self.stage, api_status=status)
|
||||||
|
|
||||||
|
def start(self) -> None:
|
||||||
|
self.thread.start()
|
||||||
|
|
||||||
|
def set_stage(self, stage: str) -> None:
|
||||||
|
with self.lock:
|
||||||
|
self.stage = stage
|
||||||
|
|
||||||
|
def _run(self) -> None:
|
||||||
|
while not self.stop_event.wait(HEARTBEAT_SECONDS):
|
||||||
|
with self.lock:
|
||||||
|
stage = self.stage
|
||||||
|
try:
|
||||||
|
status, response = self.client.heartbeat(self.task_id, self.lease_token, stage)
|
||||||
|
if not 200 <= status < 300:
|
||||||
|
self.failed = str(response.get("error") or status)
|
||||||
|
audit("TASK_HEARTBEAT_FAILED", task_id=self.task_id, stage=stage, api_status=status, error=self.failed)
|
||||||
|
else:
|
||||||
|
audit("TASK_HEARTBEAT", task_id=self.task_id, stage=stage, api_status=status)
|
||||||
|
except Exception as exc: # pragma: no cover - defensive runtime guard
|
||||||
|
self.failed = type(exc).__name__
|
||||||
|
audit("TASK_HEARTBEAT_FAILED", task_id=self.task_id, stage=stage, error=self.failed)
|
||||||
|
|
||||||
|
def ensure(self) -> None:
|
||||||
|
if self.failed:
|
||||||
|
raise WorkerError("TASK_LEASE_LOST", phase="heartbeat", details={"error": self.failed})
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
self.stop_event.set()
|
||||||
|
if self.thread.ident is not None:
|
||||||
|
self.thread.join(timeout=2)
|
||||||
|
|
||||||
|
|
||||||
|
def load_agent_token() -> str:
|
||||||
|
directory = Path(os.environ.get("CREDENTIALS_DIRECTORY", ""))
|
||||||
|
path = directory / "agent_token"
|
||||||
|
if not path.is_file() or path.is_symlink():
|
||||||
|
raise WorkerError("AGENT_RUNTIME_CREDENTIAL_UNAVAILABLE", phase="startup")
|
||||||
|
value = path.read_text(encoding="utf-8").strip()
|
||||||
|
if len(value) < 24:
|
||||||
|
raise WorkerError("AGENT_RUNTIME_CREDENTIAL_INVALID", phase="startup")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_roots() -> None:
|
||||||
|
if AUTHORITY_GIT_DIR.is_symlink() or STATE_ROOT.is_symlink() or WORKSPACE_ROOT.is_symlink():
|
||||||
|
raise WorkerError("ROOT_SYMLINK_REJECTED", phase="startup")
|
||||||
|
if not AUTHORITY_GIT_DIR.is_dir():
|
||||||
|
raise WorkerError("AUTHORITY_GIT_DIR_UNAVAILABLE", phase="startup")
|
||||||
|
STATE_ROOT.mkdir(mode=0o750, parents=True, exist_ok=True)
|
||||||
|
WORKSPACE_ROOT.mkdir(mode=0o750, parents=True, exist_ok=True)
|
||||||
|
if GIT_CONFIG_GLOBAL.is_symlink():
|
||||||
|
raise WorkerError("GIT_CONFIG_SYMLINK_REJECTED", phase="startup")
|
||||||
|
expected = f"[safe]\n\tdirectory = {AUTHORITY_GIT_DIR}\n"
|
||||||
|
if GIT_CONFIG_GLOBAL.exists() and GIT_CONFIG_GLOBAL.read_text(encoding="utf-8") != expected:
|
||||||
|
raise WorkerError("GIT_CONFIG_BINDING_CONFLICT", phase="startup")
|
||||||
|
if not GIT_CONFIG_GLOBAL.exists():
|
||||||
|
GIT_CONFIG_GLOBAL.write_text(expected, encoding="utf-8")
|
||||||
|
os.chmod(GIT_CONFIG_GLOBAL, 0o640)
|
||||||
|
|
||||||
|
|
||||||
|
def task_state_path(task_id: str) -> Path:
|
||||||
|
if not ID_RE.fullmatch(task_id):
|
||||||
|
raise WorkerError("TASK_ID_INVALID", phase="state")
|
||||||
|
return STATE_ROOT / "tasks" / f"{task_id}.json"
|
||||||
|
|
||||||
|
|
||||||
|
def load_task_state(task_id: str, execution_digest: str) -> dict[str, Any] | None:
|
||||||
|
path = task_state_path(task_id)
|
||||||
|
if not path.is_file() or path.is_symlink():
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
value = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
except Exception:
|
||||||
|
raise WorkerError("TASK_STATE_INVALID", phase="state")
|
||||||
|
if not isinstance(value, dict):
|
||||||
|
raise WorkerError("TASK_STATE_INVALID", phase="state")
|
||||||
|
if value.get("execution_digest") not in {None, execution_digest}:
|
||||||
|
raise WorkerError("TASK_PAYLOAD_DRIFT", phase="recovery")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def save_task_state(task_id: str, value: Mapping[str, Any]) -> None:
|
||||||
|
atomic_json(task_state_path(task_id), value)
|
||||||
|
|
||||||
|
|
||||||
|
def new_run_root() -> Path:
|
||||||
|
ensure_roots()
|
||||||
|
root = Path(tempfile.mkdtemp(prefix="task-", dir=str(WORKSPACE_ROOT)))
|
||||||
|
os.chmod(root, 0o750)
|
||||||
|
return root
|
||||||
|
|
||||||
|
|
||||||
|
def cleanup_run_root(path: Path) -> None:
|
||||||
|
root = WORKSPACE_ROOT.resolve()
|
||||||
|
candidate = path.resolve(strict=False)
|
||||||
|
if candidate == root or root not in candidate.parents:
|
||||||
|
raise WorkerError("CLEANUP_SCOPE_INVALID", phase="cleanup")
|
||||||
|
if candidate.exists() or candidate.is_symlink():
|
||||||
|
shutil.rmtree(candidate)
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_worktree(run_root: Path, base_oid: str) -> Path:
|
||||||
|
clone = run_root / "repository"
|
||||||
|
worktree = run_root / "worktree"
|
||||||
|
run_command("workspace_clone", ["/usr/bin/git", "clone", "--no-local", "--no-hardlinks", str(AUTHORITY_GIT_DIR), str(clone)], cwd=run_root, timeout=300, internal_git=True)
|
||||||
|
run_command("workspace_base_probe", ["/usr/bin/git", "-C", str(clone), "rev-parse", "--verify", base_oid + "^{commit}"], cwd=run_root, timeout=60)
|
||||||
|
run_command("workspace_worktree_create", ["/usr/bin/git", "-C", str(clone), "worktree", "add", "--detach", str(worktree), base_oid], cwd=run_root, timeout=300, internal_git=True)
|
||||||
|
return worktree
|
||||||
|
|
||||||
|
|
||||||
|
def governed_commit(envelope: ExecutionEnvelope, worktree: Path, bridge: BridgeClient, heartbeat: LeaseHeartbeat) -> dict[str, Any]:
|
||||||
|
governance = envelope.governance
|
||||||
|
create_body = {
|
||||||
|
"task_id": governance["task_id"],
|
||||||
|
"repository": envelope.repository,
|
||||||
|
"base_oid": envelope.base_oid,
|
||||||
|
"target_branch": envelope.target_branch,
|
||||||
|
"risk": str(governance.get("risk") or "engineering"),
|
||||||
|
"description": str(governance.get("description") or f"Governed execution for {envelope.task_id}"),
|
||||||
|
"idempotency_key": governance["idempotency_key"],
|
||||||
|
}
|
||||||
|
status, response = bridge.call("/v2/governed-task", create_body)
|
||||||
|
if not 200 <= status < 300:
|
||||||
|
raise WorkerError("GOVERNED_TASK_CREATE_FAILED", phase="committing", details={"api_status": status, "error": response.get("error")})
|
||||||
|
generation = int(response.get("generation") or 0)
|
||||||
|
expected_branch_oid = str(response.get("expected_branch_oid") or envelope.base_oid)
|
||||||
|
branch_mode = str(response.get("branch_mode") or "CREATE_NEW")
|
||||||
|
lease_body = {
|
||||||
|
"schema": "aios.task-writer-lease.v2",
|
||||||
|
"task_id": governance["task_id"],
|
||||||
|
"repository": envelope.repository,
|
||||||
|
"branch_mode": branch_mode,
|
||||||
|
"base_oid": envelope.base_oid,
|
||||||
|
"expected_branch_oid": expected_branch_oid,
|
||||||
|
"target_branch": envelope.target_branch,
|
||||||
|
"generation": generation,
|
||||||
|
"lease_seconds": int(governance.get("lease_seconds") or 900),
|
||||||
|
}
|
||||||
|
status, lease_response = bridge.call("/v2/writer-lease", lease_body)
|
||||||
|
if not 200 <= status < 300:
|
||||||
|
raise WorkerError("WRITER_LEASE_ACQUIRE_FAILED", phase="committing", details={"api_status": status, "error": lease_response.get("error")})
|
||||||
|
heartbeat.ensure()
|
||||||
|
files: dict[str, str] = {}
|
||||||
|
paths = governance.get("allowed_files") or [item["path"] for item in envelope.change_set]
|
||||||
|
for relative in sorted({safe_relative_path(item, field="allowed_file") for item in paths}):
|
||||||
|
path = resolve_under(worktree, relative)
|
||||||
|
if not path.is_file():
|
||||||
|
raise WorkerError("COMMIT_FILE_MISSING", phase="committing", details={"path": relative})
|
||||||
|
files[relative] = base64.b64encode(path.read_bytes()).decode("ascii")
|
||||||
|
commit_body = {
|
||||||
|
"task_id": governance["task_id"],
|
||||||
|
"repository": envelope.repository,
|
||||||
|
"expected_branch_oid": expected_branch_oid,
|
||||||
|
"target_branch": envelope.target_branch,
|
||||||
|
"generation": generation,
|
||||||
|
"idempotency_key": governance["commit_idempotency_key"],
|
||||||
|
"files": files,
|
||||||
|
"message": governance["message"],
|
||||||
|
}
|
||||||
|
status, commit_response = bridge.call("/v2/commit-files", commit_body)
|
||||||
|
if not 200 <= status < 300:
|
||||||
|
raise WorkerError("GOVERNED_COMMIT_FAILED", phase="committing", details={"api_status": status, "error": commit_response.get("error")})
|
||||||
|
commit_oid = commit_response.get("commit_oid")
|
||||||
|
if not isinstance(commit_oid, str) or not OID_RE.fullmatch(commit_oid):
|
||||||
|
raise WorkerError("GOVERNED_COMMIT_OID_INVALID", phase="committing")
|
||||||
|
visible_oid: str | None = None
|
||||||
|
for _ in range(5):
|
||||||
|
probe = subprocess.run(
|
||||||
|
["/usr/bin/git", "--git-dir", str(AUTHORITY_GIT_DIR), "rev-parse", "--verify", f"refs/heads/{envelope.target_branch}"],
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
env={"PATH": "/usr/bin:/bin", "LC_ALL": "C.UTF-8", "GIT_CONFIG_GLOBAL": str(GIT_CONFIG_GLOBAL)},
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
if probe.returncode == 0:
|
||||||
|
visible_oid = probe.stdout.strip()
|
||||||
|
if visible_oid == commit_oid:
|
||||||
|
break
|
||||||
|
time.sleep(1)
|
||||||
|
if visible_oid != commit_oid:
|
||||||
|
raise WorkerError("GIT_VISIBILITY_MISMATCH", phase="verification", details={"commit_oid": commit_oid, "visible_oid": visible_oid})
|
||||||
|
return {
|
||||||
|
"governed_task_id": governance["task_id"],
|
||||||
|
"branch_mode": branch_mode,
|
||||||
|
"generation": generation,
|
||||||
|
"target_branch": envelope.target_branch,
|
||||||
|
"commit_oid": commit_oid,
|
||||||
|
"visible_oid": visible_oid,
|
||||||
|
"commit_idempotency_key": governance["commit_idempotency_key"],
|
||||||
|
"api_commit_visible": commit_response.get("commit_visible"),
|
||||||
|
"direct_main_write": False,
|
||||||
|
"force_push": False,
|
||||||
|
"production_write": False,
|
||||||
|
"protected_ref_bypass": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def build_result(task_id: str, envelope: ExecutionEnvelope, execution: Mapping[str, Any], governed: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"status": "SUCCEEDED",
|
||||||
|
"verification_state": "VERIFIED_BY_WORKER_PENDING_SERVER_REVIEW",
|
||||||
|
"task_id": task_id,
|
||||||
|
"execution_backend": "generic-online-agent-worker-v2",
|
||||||
|
"adapter": envelope.adapter,
|
||||||
|
"execution_digest": envelope.digest,
|
||||||
|
"repository": envelope.repository,
|
||||||
|
"base_oid": envelope.base_oid,
|
||||||
|
"target_branch": envelope.target_branch,
|
||||||
|
"changed_files": execution.get("changed_files", []),
|
||||||
|
"commands": [{k: item.get(k) for k in ("label", "exit_code", "stdout_sha256", "stderr_sha256", "duration_ms")} for item in execution.get("commands", [])],
|
||||||
|
"tests": [{k: item.get(k) for k in ("label", "exit_code", "stdout_sha256", "stderr_sha256", "duration_ms")} for item in execution.get("tests", [])],
|
||||||
|
"governed": dict(governed),
|
||||||
|
"workspace_cleanup": "PASS",
|
||||||
|
"secret_material_exposed": False,
|
||||||
|
"direct_main_write": False,
|
||||||
|
"force_push": False,
|
||||||
|
"production_write": False,
|
||||||
|
"protected_ref_bypass": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class WorkerSupervisor:
|
||||||
|
def __init__(self, token: str) -> None:
|
||||||
|
self.agent = AgentTaskClient(token)
|
||||||
|
self.bridge = BridgeClient()
|
||||||
|
|
||||||
|
def process_claim(self, task: Mapping[str, Any], claim: Mapping[str, Any]) -> None:
|
||||||
|
task_id = str(claim.get("task_id") or task.get("task_id") or "")
|
||||||
|
lease_token = claim.get("lease_token")
|
||||||
|
if not isinstance(lease_token, str) or len(lease_token) < 24:
|
||||||
|
raise WorkerError("LEASE_TOKEN_NOT_RETURNED", phase="claim")
|
||||||
|
audit("TASK_CLAIMED", task_id=task_id, attempt=claim.get("attempt"), revision=claim.get("revision"))
|
||||||
|
heartbeat = LeaseHeartbeat(self.agent, task_id, lease_token)
|
||||||
|
try:
|
||||||
|
heartbeat.first()
|
||||||
|
heartbeat.set_stage("started")
|
||||||
|
heartbeat.start()
|
||||||
|
audit("TASK_STARTED", task_id=task_id, attempt=claim.get("attempt"))
|
||||||
|
envelope = ExecutionEnvelope.from_task(task)
|
||||||
|
state = load_task_state(task_id, envelope.digest) or {"task_id": task_id, "execution_digest": envelope.digest, "phase": "claimed"}
|
||||||
|
governed = state.get("governed") if isinstance(state.get("governed"), Mapping) else None
|
||||||
|
if isinstance(governed, Mapping) and isinstance(governed.get("commit_oid"), str) and OID_RE.fullmatch(governed["commit_oid"]):
|
||||||
|
result = state.get("result")
|
||||||
|
if not isinstance(result, Mapping):
|
||||||
|
result = {"status": "SUCCEEDED", "verification_state": "VERIFIED_BY_WORKER_PENDING_SERVER_REVIEW", "task_id": task_id, "execution_digest": envelope.digest, "governed": dict(governed), "recovered_without_duplicate_commit": True, "direct_main_write": False, "production_write": False, "force_push": False, "protected_ref_bypass": False}
|
||||||
|
audit("TASK_RECOVERED_COMMIT", task_id=task_id, commit_oid=governed["commit_oid"], duplicate_commit=False)
|
||||||
|
else:
|
||||||
|
run_root = new_run_root()
|
||||||
|
try:
|
||||||
|
worktree = prepare_worktree(run_root, envelope.base_oid)
|
||||||
|
state.update({"phase": "workspace_prepared", "execution_digest": envelope.digest, "attempt": claim.get("attempt")})
|
||||||
|
save_task_state(task_id, state)
|
||||||
|
adapter = adapter_for(envelope.adapter)
|
||||||
|
execution = adapter.execute(envelope, worktree)
|
||||||
|
state.update({"phase": "verification_passed", "execution": execution})
|
||||||
|
save_task_state(task_id, state)
|
||||||
|
heartbeat.ensure()
|
||||||
|
heartbeat.set_stage("committing")
|
||||||
|
governed = governed_commit(envelope, worktree, self.bridge, heartbeat)
|
||||||
|
result = build_result(task_id, envelope, execution, governed)
|
||||||
|
state.update({"phase": "commit_done", "governed": governed, "result": result})
|
||||||
|
save_task_state(task_id, state)
|
||||||
|
finally:
|
||||||
|
cleanup_run_root(run_root)
|
||||||
|
audit("WORKSPACE_CLEANED", task_id=task_id, cleanup="PASS")
|
||||||
|
heartbeat.ensure()
|
||||||
|
heartbeat.set_stage("result")
|
||||||
|
result_key = str(result.get("governed", {}).get("commit_idempotency_key") or f"{task_id}-result-{envelope.digest[:16]}")
|
||||||
|
status, response = self.agent.result(task_id, lease_token, result_key, result)
|
||||||
|
if not 200 <= status < 300:
|
||||||
|
raise WorkerError("TASK_RESULT_SUBMIT_FAILED", phase="result", details={"api_status": status, "error": response.get("error")})
|
||||||
|
audit("TASK_RESULT_SUBMITTED", task_id=task_id, api_status=status, result_idempotency_key=result_key, commit_oid=result.get("governed", {}).get("commit_oid"))
|
||||||
|
except WorkerError as exc:
|
||||||
|
audit("TASK_FAILED", task_id=task_id, error=exc.code, phase=exc.phase, details=exc.details)
|
||||||
|
failure_key = f"{task_id}-attempt-{claim.get('attempt', 0)}-failure"
|
||||||
|
failure = {
|
||||||
|
"status": "FAILED",
|
||||||
|
"verification_state": "UNVERIFIED",
|
||||||
|
"task_id": task_id,
|
||||||
|
"execution_backend": "generic-online-agent-worker-v2",
|
||||||
|
"error": exc.code,
|
||||||
|
"phase": exc.phase,
|
||||||
|
"details": exc.details,
|
||||||
|
"direct_main_write": False,
|
||||||
|
"force_push": False,
|
||||||
|
"production_write": False,
|
||||||
|
"protected_ref_bypass": False,
|
||||||
|
"secret_material_exposed": False,
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
status, response = self.agent.result(task_id, lease_token, failure_key, failure)
|
||||||
|
audit("TASK_FAILURE_RESULT_SUBMITTED", task_id=task_id, api_status=status, error=response.get("error"))
|
||||||
|
except Exception as submit_exc: # pragma: no cover - defensive runtime guard
|
||||||
|
audit("TASK_FAILURE_RESULT_SUBMIT_FAILED", task_id=task_id, error=type(submit_exc).__name__)
|
||||||
|
finally:
|
||||||
|
heartbeat.close()
|
||||||
|
|
||||||
|
def once(self) -> int:
|
||||||
|
try:
|
||||||
|
tasks = self.agent.poll()
|
||||||
|
audit("TASK_POLL", visible_count=len(tasks))
|
||||||
|
except WorkerError as exc:
|
||||||
|
audit("TASK_POLL_FAILED", error=exc.code, phase=exc.phase, details=exc.details)
|
||||||
|
return 10
|
||||||
|
dynamic_tasks: list[dict[str, Any]] = []
|
||||||
|
for task in tasks:
|
||||||
|
metadata = task.get("metadata")
|
||||||
|
execution = metadata.get("execution") if isinstance(metadata, Mapping) else None
|
||||||
|
if not isinstance(execution, Mapping) or execution.get("schema") != "aios.online-agent.execution.v2":
|
||||||
|
audit("TASK_SKIPPED_NON_DYNAMIC", task_id=task.get("task_id"), reason="DYNAMIC_EXECUTION_PAYLOAD_REQUIRED")
|
||||||
|
continue
|
||||||
|
dynamic_tasks.append(task)
|
||||||
|
for task in dynamic_tasks:
|
||||||
|
task_id = str(task.get("task_id") or "")
|
||||||
|
status, claim = self.agent.claim(task_id)
|
||||||
|
if status == 409:
|
||||||
|
audit("TASK_CLAIM_SKIPPED", task_id=task_id, api_status=status, error=claim.get("error"))
|
||||||
|
continue
|
||||||
|
if status != 200:
|
||||||
|
audit("TASK_CLAIM_FAILED", task_id=task_id, api_status=status, error=claim.get("error"))
|
||||||
|
continue
|
||||||
|
self.process_claim(task, claim)
|
||||||
|
return 0
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def run_forever(self) -> int:
|
||||||
|
ensure_roots()
|
||||||
|
audit("WORKER_STARTED", task_agnostic=True, api_base=API_BASE, bridge_base=BRIDGE_BASE, adapter_names=["deterministic", "codex", "hermes"], runtime_only_credentials=True)
|
||||||
|
while True:
|
||||||
|
self.once()
|
||||||
|
time.sleep(POLL_SECONDS)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
try:
|
||||||
|
token = load_agent_token()
|
||||||
|
supervisor = WorkerSupervisor(token)
|
||||||
|
if os.environ.get("AIOS_WORKER_ONCE") == "1":
|
||||||
|
ensure_roots()
|
||||||
|
return supervisor.once()
|
||||||
|
return supervisor.run_forever()
|
||||||
|
except WorkerError as exc:
|
||||||
|
audit("WORKER_START_FAILED", error=exc.code, phase=exc.phase, details=exc.details)
|
||||||
|
return 20
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,157 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Register or recover the generic Online Agent using runtime-only secrets."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
BASE = os.environ.get("AIOS_AGENT_TASK_API_URL", "http://127.0.0.1:18940").rstrip("/")
|
||||||
|
AGENT_ID = os.environ.get("AIOS_AGENT_ID", "online-engineering-worker-cn-network-mkt-generic-20260815")
|
||||||
|
AGENT_TYPE = os.environ.get("AIOS_AGENT_TYPE", "trusted-online-engineering-worker")
|
||||||
|
AGENT_VERSION = os.environ.get("AIOS_AGENT_VERSION", "2026-08-15.3-generic")
|
||||||
|
PROJECTION_PROVIDER = "BridgeV2RuntimeCredentialProjection"
|
||||||
|
STATE_DIR = Path(os.environ.get("AIOS_WORKER_REGISTRATION_STATE", "/var/lib/aios-online-agent-worker-20260815/registration"))
|
||||||
|
CREDENTIAL_PATH = Path(os.environ.get("AIOS_WORKER_ENCRYPTED_CREDENTIAL", "/etc/credstore.encrypted/aios-online-agent-worker-20260815/agent_token"))
|
||||||
|
IDENTITY_PATH = STATE_DIR / "identity.json"
|
||||||
|
AUDIT_PATH = STATE_DIR / "events.jsonl"
|
||||||
|
CAPABILITIES = [
|
||||||
|
"online_agent_task_api", "task.poll", "task.claim", "task.heartbeat", "task.result",
|
||||||
|
"engineering.consumer", "engineering.execute", "workspace.worktree", "git.read", "git.diff",
|
||||||
|
"git.test", "governed.write.branch", "governed.write.file",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def now() -> str:
|
||||||
|
return datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def audit(event_type: str, **fields: Any) -> None:
|
||||||
|
STATE_DIR.mkdir(mode=0o750, parents=True, exist_ok=True)
|
||||||
|
record = {"event_type": event_type, "created_at": now(), "agent_id": AGENT_ID, "credential_material_exposed": False, **fields}
|
||||||
|
with AUDIT_PATH.open("a", encoding="utf-8") as handle:
|
||||||
|
handle.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n")
|
||||||
|
os.chmod(AUDIT_PATH, 0o640)
|
||||||
|
print(json.dumps(record, ensure_ascii=False, sort_keys=True), flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
def api_call(method: str, route: str, token: str, body: dict[str, Any] | None = None) -> tuple[int, dict[str, Any]]:
|
||||||
|
payload = None if body is None else json.dumps(body, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
headers = {"Authorization": "Bearer " + token, "Accept": "application/json"}
|
||||||
|
if payload is not None:
|
||||||
|
headers["Content-Type"] = "application/json"
|
||||||
|
request = urllib.request.Request(BASE + route, data=payload, headers=headers, method=method)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(request, timeout=15) as response:
|
||||||
|
value = json.loads(response.read().decode("utf-8"))
|
||||||
|
return response.status, value if isinstance(value, dict) else {}
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
try:
|
||||||
|
value = json.loads(exc.read(2048).decode("utf-8"))
|
||||||
|
except Exception:
|
||||||
|
value = {"error": "HTTP_ERROR"}
|
||||||
|
return exc.code, value if isinstance(value, dict) else {"error": "HTTP_ERROR"}
|
||||||
|
except (urllib.error.URLError, TimeoutError):
|
||||||
|
return 599, {"error": "AGENT_TASK_API_UNAVAILABLE"}
|
||||||
|
|
||||||
|
|
||||||
|
def read_identity() -> dict[str, Any] | None:
|
||||||
|
if not IDENTITY_PATH.is_file() or IDENTITY_PATH.is_symlink():
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
value = json.loads(IDENTITY_PATH.read_text(encoding="utf-8"))
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
return value if isinstance(value, dict) else None
|
||||||
|
|
||||||
|
|
||||||
|
def identity_matches(value: dict[str, Any] | None) -> bool:
|
||||||
|
return bool(value and value.get("agent_id") == AGENT_ID and value.get("agent_type") == AGENT_TYPE and value.get("agent_version") == AGENT_VERSION and value.get("credential_projection_provider") == PROJECTION_PROVIDER and value.get("task_scope") == "dynamic")
|
||||||
|
|
||||||
|
|
||||||
|
def encrypted_credential_is_valid() -> bool:
|
||||||
|
if not CREDENTIAL_PATH.is_file() or CREDENTIAL_PATH.is_symlink():
|
||||||
|
return False
|
||||||
|
result = subprocess.run(["systemd-creds", "decrypt", "--with-key=host", str(CREDENTIAL_PATH), "/dev/null"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, timeout=15)
|
||||||
|
return result.returncode == 0
|
||||||
|
|
||||||
|
|
||||||
|
def write_identity() -> None:
|
||||||
|
value = {"agent_id": AGENT_ID, "agent_type": AGENT_TYPE, "agent_version": AGENT_VERSION, "task_scope": "dynamic", "credential_projection_provider": PROJECTION_PROVIDER, "credential_delivery": "systemd-LoadCredentialEncrypted", "task_api_url": BASE, "task_api_schema": "aios.agent-task.v1", "execution_backend": "generic-online-agent-worker-v2", "runtime_only_secret_delivery": True, "production_write": False, "main_write": False, "force_push": False, "tag_write": False, "baseline_write": False, "writer_lease": True, "created_at": now()}
|
||||||
|
STATE_DIR.mkdir(mode=0o750, parents=True, exist_ok=True)
|
||||||
|
fd, temporary = tempfile.mkstemp(prefix="identity.", dir=str(STATE_DIR))
|
||||||
|
try:
|
||||||
|
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||||
|
json.dump(value, handle, ensure_ascii=False, sort_keys=True)
|
||||||
|
handle.write("\n")
|
||||||
|
os.chmod(temporary, 0o640)
|
||||||
|
os.replace(temporary, IDENTITY_PATH)
|
||||||
|
finally:
|
||||||
|
if os.path.exists(temporary):
|
||||||
|
os.unlink(temporary)
|
||||||
|
|
||||||
|
|
||||||
|
def encrypt_agent_token(agent_token: str) -> None:
|
||||||
|
CREDENTIAL_PATH.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||||
|
fd, temporary = tempfile.mkstemp(prefix=".online-agent-token.", dir=str(CREDENTIAL_PATH.parent))
|
||||||
|
os.close(fd)
|
||||||
|
os.unlink(temporary)
|
||||||
|
try:
|
||||||
|
result = subprocess.run(["systemd-creds", "encrypt", "--with-key=host", "--name=agent_token", "/dev/stdin", temporary], input=(agent_token + "\n").encode("utf-8"), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, timeout=20)
|
||||||
|
if result.returncode != 0:
|
||||||
|
raise RuntimeError("AGENT_CREDENTIAL_ENCRYPT_FAILED")
|
||||||
|
os.chmod(temporary, 0o600)
|
||||||
|
os.replace(temporary, CREDENTIAL_PATH)
|
||||||
|
os.chmod(CREDENTIAL_PATH, 0o600)
|
||||||
|
finally:
|
||||||
|
if os.path.exists(temporary):
|
||||||
|
os.unlink(temporary)
|
||||||
|
|
||||||
|
|
||||||
|
def register_or_recover() -> str:
|
||||||
|
STATE_DIR.mkdir(mode=0o750, parents=True, exist_ok=True)
|
||||||
|
existing = read_identity()
|
||||||
|
if CREDENTIAL_PATH.exists():
|
||||||
|
if not identity_matches(existing):
|
||||||
|
raise RuntimeError("AGENT_IDENTITY_BINDING_CONFLICT")
|
||||||
|
if not encrypted_credential_is_valid():
|
||||||
|
raise RuntimeError("AGENT_ENCRYPTED_CREDENTIAL_INVALID")
|
||||||
|
audit("AGENT_RECOVERED", credential_delivery="systemd-LoadCredentialEncrypted", credential_projection_provider=PROJECTION_PROVIDER, register_or_recover=True)
|
||||||
|
return "RECOVERED"
|
||||||
|
if existing is not None:
|
||||||
|
raise RuntimeError("AGENT_CREDENTIAL_MISSING_AFTER_PRIOR_REGISTRATION")
|
||||||
|
credential_directory = Path(os.environ.get("CREDENTIALS_DIRECTORY", ""))
|
||||||
|
control_path = credential_directory / "bridge_control_token"
|
||||||
|
if not control_path.is_file():
|
||||||
|
raise RuntimeError("BRIDGE_RUNTIME_CONTROL_CREDENTIAL_UNAVAILABLE")
|
||||||
|
control_token = control_path.read_text(encoding="utf-8").strip()
|
||||||
|
if len(control_token) < 32:
|
||||||
|
raise RuntimeError("BRIDGE_RUNTIME_CONTROL_CREDENTIAL_INVALID")
|
||||||
|
status, response = api_call("POST", "/v1/agents/register", control_token, {"agent_id": AGENT_ID, "agent_type": AGENT_TYPE, "agent_version": AGENT_VERSION, "capabilities": CAPABILITIES, "metadata": {"task_scope": "dynamic", "credential_projection_provider": PROJECTION_PROVIDER, "runtime_only_secret_delivery": True, "task_api_schema": "aios.agent-task.v1", "execution_backend": "generic-online-agent-worker-v2", "production_write": False, "main_write": False, "force_push": False, "tag_write": False, "baseline_write": False, "writer_lease": True}})
|
||||||
|
if status == 409 and response.get("error") == "AGENT_ALREADY_EXISTS":
|
||||||
|
raise RuntimeError("AGENT_ALREADY_EXISTS_RECOVERY_CREDENTIAL_UNAVAILABLE")
|
||||||
|
if status != 201:
|
||||||
|
raise RuntimeError("AGENT_REGISTER_FAILED_" + str(response.get("error") or status))
|
||||||
|
agent_token = response.get("agent_token")
|
||||||
|
if not isinstance(agent_token, str) or len(agent_token) < 24:
|
||||||
|
raise RuntimeError("AGENT_TOKEN_NOT_RETURNED")
|
||||||
|
encrypt_agent_token(agent_token)
|
||||||
|
write_identity()
|
||||||
|
audit("AGENT_REGISTERED", credential_delivery="systemd-LoadCredentialEncrypted", credential_projection_provider=PROJECTION_PROVIDER, register_or_recover=True)
|
||||||
|
return "REGISTERED"
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
try:
|
||||||
|
raise SystemExit(0 if register_or_recover() in {"REGISTERED", "RECOVERED"} else 20)
|
||||||
|
except Exception as exc:
|
||||||
|
audit("AGENT_REGISTRATION_FAILED", error=type(exc).__name__)
|
||||||
|
raise SystemExit(20)
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from aios_online_agent_worker.worker import ExecutionEnvelope, WorkerError, sanitize # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
BASE = "a" * 40
|
||||||
|
|
||||||
|
|
||||||
|
def payload(task_id: str, branch: str, *, base_oid: str = BASE) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"task_id": task_id,
|
||||||
|
"metadata": {
|
||||||
|
"execution": {
|
||||||
|
"schema": "aios.online-agent.execution.v2",
|
||||||
|
"repository": "aios/aios",
|
||||||
|
"base_oid": base_oid,
|
||||||
|
"target_branch": branch,
|
||||||
|
"adapter": "deterministic",
|
||||||
|
"workspace": {
|
||||||
|
"authority_git_dir": str(Path(os.environ.get("AIOS_AUTHORITY_GIT_DIR", "/data/aios-gitops/repo.git"))),
|
||||||
|
"read_only_repository": True,
|
||||||
|
"cleanup_policy": "always",
|
||||||
|
},
|
||||||
|
"change_set": [
|
||||||
|
{
|
||||||
|
"operation": "write_file",
|
||||||
|
"path": "tests/generated_marker.py",
|
||||||
|
"content": "def test_marker():\n assert True\n",
|
||||||
|
"expected_sha256": None,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"verification": [
|
||||||
|
{"label": "compile", "argv": ["/usr/bin/python3", "-m", "py_compile", "tests/generated_marker.py"], "cwd": ".", "timeout_seconds": 60}
|
||||||
|
],
|
||||||
|
"governance": {
|
||||||
|
"task_id": f"{task_id}-writer",
|
||||||
|
"idempotency_key": f"{task_id}-governed-v1",
|
||||||
|
"commit_idempotency_key": f"{task_id}-commit-v1",
|
||||||
|
"allowed_files": ["tests/generated_marker.py"],
|
||||||
|
"main_write": False,
|
||||||
|
"force_push": False,
|
||||||
|
"production_write": False,
|
||||||
|
"tag_write": False,
|
||||||
|
"baseline_write": False,
|
||||||
|
"protected_ref_bypass": False,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_dynamic_payload_is_not_bound_to_task_or_base() -> None:
|
||||||
|
first = ExecutionEnvelope.from_task(payload("task-alpha-20260815", "agent/alpha-20260815"))
|
||||||
|
second = ExecutionEnvelope.from_task(payload("task-beta-20260815", "agent/beta-20260815", base_oid="b" * 40))
|
||||||
|
assert first.task_id != second.task_id
|
||||||
|
assert first.base_oid != second.base_oid
|
||||||
|
assert first.target_branch != second.target_branch
|
||||||
|
|
||||||
|
|
||||||
|
def test_protected_branch_is_rejected() -> None:
|
||||||
|
try:
|
||||||
|
ExecutionEnvelope.from_task(payload("task-main-20260815", "main"))
|
||||||
|
except WorkerError as exc:
|
||||||
|
assert exc.code == "PROTECTED_BRANCH_REJECTED"
|
||||||
|
else:
|
||||||
|
raise AssertionError("protected branch accepted")
|
||||||
|
|
||||||
|
|
||||||
|
def test_task_payload_does_not_persist_secret_values() -> None:
|
||||||
|
value = sanitize({"agent_token": "agt-secret", "nested": {"authorization": "Bearer secret"}})
|
||||||
|
assert value == {"agent_token": "[REDACTED]", "nested": {"authorization": "[REDACTED]"}}
|
||||||
|
|
||||||
|
|
||||||
|
def test_execution_digest_is_stable() -> None:
|
||||||
|
value = payload("task-digest-20260815", "agent/digest-20260815")
|
||||||
|
assert ExecutionEnvelope.from_task(value).digest == ExecutionEnvelope.from_task(value).digest
|
||||||
@@ -45,9 +45,6 @@ class ShadowClientRBACPublicInterfaceTests(unittest.TestCase):
|
|||||||
identity_verifier=self.IdentityVerifier({"shadow-client"}),
|
identity_verifier=self.IdentityVerifier({"shadow-client"}),
|
||||||
execution_enabled=False,
|
execution_enabled=False,
|
||||||
identity_capabilities={"shadow-client": set(capabilities)},
|
identity_capabilities={"shadow-client": set(capabilities)},
|
||||||
identity_actor_contexts={
|
|
||||||
"shadow-client": {"scope_type": "system"}
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
service = self.TaskPlaneService(store, policy=policy)
|
service = self.TaskPlaneService(store, policy=policy)
|
||||||
|
|
||||||
|
|||||||
@@ -1,206 +0,0 @@
|
|||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from dataclasses import dataclass, field
|
|
||||||
from pathlib import Path
|
|
||||||
import sys
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[3]
|
|
||||||
ACTIVE_RUNTIME = ROOT / "active-runtime"
|
|
||||||
if str(ACTIVE_RUNTIME) not in sys.path:
|
|
||||||
sys.path.insert(0, str(ACTIVE_RUNTIME))
|
|
||||||
|
|
||||||
import sqlite3
|
|
||||||
import threading
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from app.task_plane.fleet import FleetConflict, FleetScheduler
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class Plan:
|
|
||||||
task_id: str
|
|
||||||
required_capabilities: list[str] = field(default_factory=lambda:["code.read"])
|
|
||||||
metadata: dict = field(default_factory=dict)
|
|
||||||
scope_type: str = "tenant"
|
|
||||||
tenant_id: str = "tenant-a"
|
|
||||||
project_id: str | None = "project-a"
|
|
||||||
user_id: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class Task:
|
|
||||||
task_id: str
|
|
||||||
revision: int = 1
|
|
||||||
status: str = "QUEUED"
|
|
||||||
created_at: float = 1.0
|
|
||||||
task_plan: Plan | None = None
|
|
||||||
|
|
||||||
def __post_init__(self):
|
|
||||||
if self.task_plan is None:
|
|
||||||
self.task_plan=Plan(self.task_id)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class Worker:
|
|
||||||
worker_id: str
|
|
||||||
capabilities: list[str]=field(default_factory=lambda:["code.read"])
|
|
||||||
|
|
||||||
|
|
||||||
class Store:
|
|
||||||
def __init__(self, path=":memory:"):
|
|
||||||
self.conn=sqlite3.connect(path,check_same_thread=False)
|
|
||||||
self._lock=threading.RLock()
|
|
||||||
self.conn.executescript("""
|
|
||||||
CREATE TABLE IF NOT EXISTS claims(
|
|
||||||
claim_id TEXT PRIMARY KEY,task_id TEXT,worker_id TEXT,status TEXT,lease_expires_at REAL
|
|
||||||
);
|
|
||||||
""")
|
|
||||||
self.tasks=[]
|
|
||||||
self.workers={}
|
|
||||||
|
|
||||||
def list_tasks(self,_ctx=None):
|
|
||||||
return list(self.tasks)
|
|
||||||
|
|
||||||
def get_worker(self,worker_id):
|
|
||||||
return self.workers[worker_id]
|
|
||||||
|
|
||||||
|
|
||||||
class Registry:
|
|
||||||
def __init__(self):
|
|
||||||
self.online=set()
|
|
||||||
def is_online(self,worker_id,now=None):
|
|
||||||
return worker_id in self.online
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def fleet():
|
|
||||||
store=Store(); registry=Registry()
|
|
||||||
for wid in ("w-a","w-b","w-c"):
|
|
||||||
store.workers[wid]=Worker(wid)
|
|
||||||
registry.online.add(wid)
|
|
||||||
scheduler=FleetScheduler(store,registry,now_fn=lambda:1000.0)
|
|
||||||
scheduler.configure_worker("w-a",max_slots=1,failure_domain="zone-a",weight=1)
|
|
||||||
scheduler.configure_worker("w-b",max_slots=2,failure_domain="zone-b",weight=1)
|
|
||||||
scheduler.configure_worker("w-c",max_slots=1,failure_domain="zone-c",weight=2)
|
|
||||||
return store,registry,scheduler
|
|
||||||
|
|
||||||
|
|
||||||
def test_FLEET_AGENT_REGISTRATION_capacity_policy_is_durable(fleet):
|
|
||||||
store,_,scheduler=fleet
|
|
||||||
policy=scheduler.state.worker_policy("w-b")
|
|
||||||
assert policy.max_slots==2 and policy.failure_domain=="zone-b"
|
|
||||||
reloaded=type(scheduler)(store, scheduler.workers, now_fn=lambda:1000.0)
|
|
||||||
assert reloaded.state.worker_policy("w-b").max_slots==2
|
|
||||||
|
|
||||||
|
|
||||||
def test_FLEET_CAPACITY_SCHEDULING_does_not_oversubscribe_worker(fleet):
|
|
||||||
store,_,scheduler=fleet
|
|
||||||
store.tasks=[Task("task-1"),Task("task-2"),Task("task-3")]
|
|
||||||
decisions=scheduler.select(["w-a"],limit=3,now=1000.0)
|
|
||||||
assert [d.worker_id for d in decisions]==["w-a"]
|
|
||||||
|
|
||||||
|
|
||||||
def test_FLEET_CAPACITY_SCHEDULING_accounts_for_active_claims(fleet):
|
|
||||||
store,_,scheduler=fleet
|
|
||||||
store.conn.execute(
|
|
||||||
"INSERT INTO claims VALUES ('claim-1','other','w-a','ACTIVE',2000)"
|
|
||||||
)
|
|
||||||
store.tasks=[Task("task-1")]
|
|
||||||
assert scheduler.select(["w-a"],limit=1,now=1000.0)==[]
|
|
||||||
|
|
||||||
|
|
||||||
def test_FLEET_FAIRNESS_scope_virtual_finish_prevents_hot_scope_monopoly(fleet):
|
|
||||||
store,_,scheduler=fleet
|
|
||||||
store.tasks=[
|
|
||||||
Task("a1",created_at=1,task_plan=Plan("a1",tenant_id="tenant-a",project_id="p-a")),
|
|
||||||
Task("a2",created_at=2,task_plan=Plan("a2",tenant_id="tenant-a",project_id="p-a")),
|
|
||||||
Task("b1",created_at=3,task_plan=Plan("b1",tenant_id="tenant-b",project_id="p-b")),
|
|
||||||
]
|
|
||||||
first=scheduler.select(["w-b"],limit=1,now=1000.0)[0]
|
|
||||||
assert first.task_id=="a1"
|
|
||||||
scheduler.state.release("a1",state="COMPLETED",now=1000.1)
|
|
||||||
store.tasks=[task for task in store.tasks if task.task_id!="a1"]
|
|
||||||
second=scheduler.select(["w-b"],limit=1,now=1000.2)[0]
|
|
||||||
assert second.task_id=="b1"
|
|
||||||
|
|
||||||
|
|
||||||
def test_FLEET_FAULT_ISOLATION_opens_worker_circuit_after_threshold(fleet):
|
|
||||||
store,_,scheduler=fleet
|
|
||||||
store.tasks=[Task("task-1")]
|
|
||||||
for _ in range(3):
|
|
||||||
scheduler.state.record_worker_failure("w-a","runtime_crash",now=1000.0)
|
|
||||||
assert scheduler.state.worker_circuit_open("w-a",1001.0)
|
|
||||||
assert scheduler.select(["w-a"],limit=1,now=1001.0)==[]
|
|
||||||
|
|
||||||
|
|
||||||
def test_FLEET_WORKER_REASSIGNMENT_avoids_quarantined_worker(fleet):
|
|
||||||
store,_,scheduler=fleet
|
|
||||||
store.tasks=[Task("task-1")]
|
|
||||||
scheduler.state.record_task_worker_failure(
|
|
||||||
"task-1","w-a","lease_expired",quarantine_seconds=60,now=1000.0
|
|
||||||
)
|
|
||||||
decision=scheduler.select(["w-a","w-b"],limit=1,now=1001.0)[0]
|
|
||||||
assert decision.worker_id=="w-b"
|
|
||||||
|
|
||||||
|
|
||||||
def test_FLEET_SCOPE_ISOLATION_uses_tenant_project_fairness_keys(fleet):
|
|
||||||
store,_,scheduler=fleet
|
|
||||||
store.tasks=[
|
|
||||||
Task("a",task_plan=Plan("a",tenant_id="tenant-a",project_id="p")),
|
|
||||||
Task("b",task_plan=Plan("b",tenant_id="tenant-b",project_id="p")),
|
|
||||||
]
|
|
||||||
decisions=scheduler.select(["w-b"],limit=2,now=1000.0)
|
|
||||||
assert {d.scope_key for d in decisions}=={
|
|
||||||
"tenant:tenant-a|project:p","tenant:tenant-b|project:p"
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def test_FLEET_IDEMPOTENT_DISPATCH_replay_same_task_worker(fleet):
|
|
||||||
store,_,scheduler=fleet
|
|
||||||
store.tasks=[Task("task-1")]
|
|
||||||
one=scheduler.select(["w-b"],limit=1,now=1000.0)[0]
|
|
||||||
# Reservation remains authoritative; a conflicting second worker cannot steal it.
|
|
||||||
with pytest.raises(FleetConflict):
|
|
||||||
scheduler.state.reserve(
|
|
||||||
task_id="task-1",worker_id="w-c",task_revision=1,
|
|
||||||
scope_key=one.scope_key,failure_domain="zone-c",score=one.score,
|
|
||||||
reservation_seconds=30,now=1001.0,scope_weight=1.0,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_FLEET_FAILURE_RECOVERY_worker_success_closes_circuit(fleet):
|
|
||||||
store,_,scheduler=fleet
|
|
||||||
for _ in range(3):
|
|
||||||
scheduler.state.record_worker_failure("w-a","runtime_crash",now=1000.0)
|
|
||||||
scheduler.state.record_worker_success("w-a",now=1001.0)
|
|
||||||
assert not scheduler.state.worker_circuit_open("w-a",1001.1)
|
|
||||||
|
|
||||||
|
|
||||||
def test_affinity_and_failure_domain_anti_affinity(fleet):
|
|
||||||
store,_,scheduler=fleet
|
|
||||||
scheduler.configure_worker("w-b",max_slots=2,failure_domain="zone-b",labels={"gpu":"false"})
|
|
||||||
scheduler.configure_worker("w-c",max_slots=1,failure_domain="zone-c",labels={"gpu":"true"})
|
|
||||||
task=Task("gpu-task")
|
|
||||||
task.task_plan.metadata={
|
|
||||||
"fleet_affinity_labels":{"gpu":"true"},
|
|
||||||
"fleet_avoid_failure_domains":["zone-b"],
|
|
||||||
}
|
|
||||||
store.tasks=[task]
|
|
||||||
decision=scheduler.select(["w-b","w-c"],limit=1,now=1000.0)[0]
|
|
||||||
assert decision.worker_id=="w-c"
|
|
||||||
|
|
||||||
|
|
||||||
def test_FLEET_FAILURE_RECOVERY_state_survives_sqlite_restart(tmp_path):
|
|
||||||
path=tmp_path/"task-plane.sqlite3"
|
|
||||||
first=Store(str(path)); registry=Registry()
|
|
||||||
first.workers["w-a"]=Worker("w-a"); registry.online.add("w-a")
|
|
||||||
scheduler=FleetScheduler(first,registry,now_fn=lambda:1000.0)
|
|
||||||
scheduler.configure_worker("w-a",max_slots=3,failure_domain="zone-a")
|
|
||||||
scheduler.state.record_worker_failure("w-a","runtime_crash",threshold=1,now=1000.0)
|
|
||||||
first.conn.close()
|
|
||||||
|
|
||||||
second=Store(str(path)); second.workers["w-a"]=Worker("w-a")
|
|
||||||
resumed=FleetScheduler(second,registry,now_fn=lambda:1001.0)
|
|
||||||
assert resumed.state.worker_policy("w-a").max_slots==3
|
|
||||||
assert resumed.state.worker_circuit_open("w-a",1001.0)
|
|
||||||
@@ -1,70 +0,0 @@
|
|||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import importlib.util
|
|
||||||
from pathlib import Path
|
|
||||||
import sys
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[3]
|
|
||||||
ACTIVE_RUNTIME = ROOT / "active-runtime"
|
|
||||||
if str(ACTIVE_RUNTIME) not in sys.path:
|
|
||||||
sys.path.insert(0, str(ACTIVE_RUNTIME))
|
|
||||||
|
|
||||||
from app.task_plane.scheduler import TaskScheduler
|
|
||||||
|
|
||||||
|
|
||||||
def _security_contract():
|
|
||||||
path = Path(__file__).with_name("test_task_plane_security_contract.py")
|
|
||||||
spec = importlib.util.spec_from_file_location("p2b_order_security_contract", path)
|
|
||||||
module = importlib.util.module_from_spec(spec)
|
|
||||||
sys.modules[spec.name] = module
|
|
||||||
spec.loader.exec_module(module)
|
|
||||||
return module
|
|
||||||
|
|
||||||
|
|
||||||
def _create_task(service, contract, task_id, tenant, project):
|
|
||||||
return service.create_task(
|
|
||||||
{
|
|
||||||
"task_id": task_id,
|
|
||||||
"issuer": {"type": "agent", "id": contract.ADMIN_IDENTITY},
|
|
||||||
"required_capabilities": ["code.read"],
|
|
||||||
"scope_type": "tenant",
|
|
||||||
"tenant_id": tenant,
|
|
||||||
"project_id": project,
|
|
||||||
"metadata": {"fleet_priority": 0},
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _task_id(claim):
|
|
||||||
return getattr(
|
|
||||||
claim,
|
|
||||||
"task_id",
|
|
||||||
claim.get("task_id") if isinstance(claim, dict) else None,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_P2B_FLEET_ORDER_is_preserved_through_legacy_claim_oracle():
|
|
||||||
contract = _security_contract()
|
|
||||||
service = contract._service()
|
|
||||||
service.register_worker(contract._worker_payload(max_concurrency=3))
|
|
||||||
worker_id = contract.WORKER_ID
|
|
||||||
|
|
||||||
scheduler = TaskScheduler(service.store)
|
|
||||||
scheduler.fleet_scheduler.configure_worker(
|
|
||||||
worker_id, max_slots=3, failure_domain="zone-a"
|
|
||||||
)
|
|
||||||
|
|
||||||
_create_task(service, contract, "p2b-order-a1", "tenant-a", "project-a")
|
|
||||||
_create_task(service, contract, "p2b-order-a2", "tenant-a", "project-a")
|
|
||||||
_create_task(service, contract, "p2b-order-b1", "tenant-b", "project-b")
|
|
||||||
|
|
||||||
first = scheduler.schedule_for_worker(worker_id)
|
|
||||||
second = scheduler.schedule_for_worker(worker_id)
|
|
||||||
|
|
||||||
assert _task_id(first) == "p2b-order-a1"
|
|
||||||
assert _task_id(second) == "p2b-order-b1"
|
|
||||||
|
|
||||||
rows = service.store.conn.execute(
|
|
||||||
"SELECT scope_key,dispatch_count FROM fleet_scope_state ORDER BY scope_key"
|
|
||||||
).fetchall()
|
|
||||||
assert len(rows) >= 2
|
|
||||||
@@ -1,181 +0,0 @@
|
|||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from dataclasses import dataclass, field
|
|
||||||
import importlib.util
|
|
||||||
from pathlib import Path
|
|
||||||
import sqlite3
|
|
||||||
import sys
|
|
||||||
import threading
|
|
||||||
import time
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[3]
|
|
||||||
ACTIVE_RUNTIME = ROOT / "active-runtime"
|
|
||||||
if str(ACTIVE_RUNTIME) not in sys.path:
|
|
||||||
sys.path.insert(0, str(ACTIVE_RUNTIME))
|
|
||||||
|
|
||||||
from app.task_plane.fleet import FleetConflict, FleetScheduler
|
|
||||||
from app.task_plane.errors import NotFoundError
|
|
||||||
from app.task_plane.scheduler import TaskScheduler
|
|
||||||
|
|
||||||
|
|
||||||
def _security_contract():
|
|
||||||
path = Path(__file__).with_name("test_task_plane_security_contract.py")
|
|
||||||
spec = importlib.util.spec_from_file_location("p2b_security_contract", path)
|
|
||||||
module = importlib.util.module_from_spec(spec)
|
|
||||||
sys.modules[spec.name] = module
|
|
||||||
spec.loader.exec_module(module)
|
|
||||||
return module
|
|
||||||
|
|
||||||
|
|
||||||
def test_P2B_REAL_DISPATCH_claim_binds_fleet_reservation_and_circuit():
|
|
||||||
contract = _security_contract()
|
|
||||||
service = contract._service()
|
|
||||||
service.register_worker(contract._worker_payload(max_concurrency=2))
|
|
||||||
worker_id = contract.WORKER_ID
|
|
||||||
|
|
||||||
service.create_task(
|
|
||||||
{
|
|
||||||
"task_id": "task-p2b-real-1",
|
|
||||||
"issuer": {"type": "agent", "id": contract.ADMIN_IDENTITY},
|
|
||||||
"required_capabilities": ["code.read"],
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
scheduler = TaskScheduler(service.store)
|
|
||||||
scheduler.fleet_scheduler.configure_worker(
|
|
||||||
worker_id,
|
|
||||||
max_slots=2,
|
|
||||||
failure_domain="zone-real",
|
|
||||||
labels={"runtime": "task-plane"},
|
|
||||||
)
|
|
||||||
first = scheduler.schedule_for_worker(worker_id)
|
|
||||||
assert first is not None
|
|
||||||
assert first.task_id == "task-p2b-real-1"
|
|
||||||
|
|
||||||
row = service.store.conn.execute(
|
|
||||||
"SELECT state,claim_id,worker_id FROM fleet_dispatches WHERE task_id=?",
|
|
||||||
("task-p2b-real-1",),
|
|
||||||
).fetchone()
|
|
||||||
assert tuple(row) == ("CLAIMED", first.claim_id, worker_id)
|
|
||||||
|
|
||||||
# A real circuit blocks the actual scheduler even though one capacity slot remains.
|
|
||||||
failure_now = time.time()
|
|
||||||
scheduler.record_worker_failure(
|
|
||||||
worker_id, "runtime_crash", threshold=1, now=failure_now
|
|
||||||
)
|
|
||||||
service.create_task(
|
|
||||||
{
|
|
||||||
"task_id": "task-p2b-real-2",
|
|
||||||
"issuer": {"type": "agent", "id": contract.ADMIN_IDENTITY},
|
|
||||||
"required_capabilities": ["code.read"],
|
|
||||||
}
|
|
||||||
)
|
|
||||||
with pytest.raises(NotFoundError, match="no queued tasks available"):
|
|
||||||
scheduler.schedule_for_worker(worker_id)
|
|
||||||
|
|
||||||
scheduler.record_worker_success(worker_id, now=failure_now + 1.0)
|
|
||||||
second = scheduler.schedule_for_worker(worker_id)
|
|
||||||
assert second is not None
|
|
||||||
assert second.task_id == "task-p2b-real-2"
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class Plan:
|
|
||||||
task_id: str
|
|
||||||
required_capabilities: list[str] = field(default_factory=lambda: ["code.read"])
|
|
||||||
metadata: dict = field(default_factory=dict)
|
|
||||||
scope_type: str = "tenant"
|
|
||||||
tenant_id: str = "tenant-a"
|
|
||||||
project_id: str | None = "project-a"
|
|
||||||
user_id: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class Task:
|
|
||||||
task_id: str
|
|
||||||
revision: int = 1
|
|
||||||
status: str = "QUEUED"
|
|
||||||
created_at: float = 1.0
|
|
||||||
task_plan: Plan | None = None
|
|
||||||
|
|
||||||
def __post_init__(self):
|
|
||||||
if self.task_plan is None:
|
|
||||||
self.task_plan = Plan(self.task_id)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class Worker:
|
|
||||||
worker_id: str
|
|
||||||
capabilities: list[str] = field(default_factory=lambda: ["code.read"])
|
|
||||||
|
|
||||||
|
|
||||||
class Store:
|
|
||||||
def __init__(self):
|
|
||||||
self.conn = sqlite3.connect(":memory:", check_same_thread=False)
|
|
||||||
self._lock = threading.RLock()
|
|
||||||
self.conn.execute(
|
|
||||||
"CREATE TABLE claims("
|
|
||||||
"claim_id TEXT PRIMARY KEY,task_id TEXT,worker_id TEXT,status TEXT,"
|
|
||||||
"lease_expires_at REAL)"
|
|
||||||
)
|
|
||||||
self.tasks = []
|
|
||||||
self.workers = {}
|
|
||||||
|
|
||||||
def list_tasks(self, _ctx=None):
|
|
||||||
return list(self.tasks)
|
|
||||||
|
|
||||||
def get_worker(self, worker_id):
|
|
||||||
return self.workers[worker_id]
|
|
||||||
|
|
||||||
|
|
||||||
class Registry:
|
|
||||||
def __init__(self):
|
|
||||||
self.online = set()
|
|
||||||
|
|
||||||
def is_online(self, worker_id, now=None):
|
|
||||||
return worker_id in self.online
|
|
||||||
|
|
||||||
|
|
||||||
def _fleet():
|
|
||||||
store = Store()
|
|
||||||
registry = Registry()
|
|
||||||
store.workers["w-a"] = Worker("w-a")
|
|
||||||
registry.online.add("w-a")
|
|
||||||
fleet = FleetScheduler(store, registry, now_fn=lambda: 1000.0)
|
|
||||||
fleet.configure_worker("w-a", max_slots=1, failure_domain="zone-a")
|
|
||||||
return store, fleet
|
|
||||||
|
|
||||||
|
|
||||||
def test_P2B_CAPACITY_RESERVATION_is_atomic_under_concurrent_pullers():
|
|
||||||
store, fleet = _fleet()
|
|
||||||
tasks = [Task("task-a"), Task("task-b")]
|
|
||||||
store.tasks = tasks
|
|
||||||
barrier = threading.Barrier(2)
|
|
||||||
outcomes = []
|
|
||||||
|
|
||||||
def reserve(task):
|
|
||||||
barrier.wait()
|
|
||||||
try:
|
|
||||||
fleet.reserve_for_worker("w-a", task, now=1000.0)
|
|
||||||
outcomes.append("reserved")
|
|
||||||
except FleetConflict:
|
|
||||||
outcomes.append("conflict")
|
|
||||||
|
|
||||||
threads = [threading.Thread(target=reserve, args=(task,)) for task in tasks]
|
|
||||||
for thread in threads:
|
|
||||||
thread.start()
|
|
||||||
for thread in threads:
|
|
||||||
thread.join()
|
|
||||||
|
|
||||||
assert sorted(outcomes) == ["conflict", "reserved"]
|
|
||||||
assert fleet.state.reserved_slots("w-a", 1000.0) == 1
|
|
||||||
|
|
||||||
|
|
||||||
def test_P2B_RANK_is_read_only_until_legacy_security_accepts_candidate():
|
|
||||||
store, fleet = _fleet()
|
|
||||||
store.tasks = [Task("task-a"), Task("task-b")]
|
|
||||||
ranked = fleet.rank_for_worker("w-a", now=1000.0)
|
|
||||||
assert [task.task_id for task in ranked] == ["task-a", "task-b"]
|
|
||||||
assert fleet.state.reserved_slots("w-a", 1000.0) == 0
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user