197 lines
8.5 KiB
Python
197 lines
8.5 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sqlite3
|
|
import tempfile
|
|
import unittest
|
|
|
|
from services.aios_codex_watcher.watcher import CommandWatcher, Config, WatcherError, validate_command
|
|
|
|
|
|
CONTRACT = "cbe33ecab8e9794ba5f318bea43aeaf447a56fbc"
|
|
|
|
|
|
class Clock:
|
|
def __init__(self) -> None:
|
|
self.value = 1000.0
|
|
|
|
def __call__(self) -> float:
|
|
return self.value
|
|
|
|
|
|
class WatcherTests(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.root = Path(self.temp.name)
|
|
for name in ("active", "completed", "dead", "state", "log"):
|
|
(self.root / name).mkdir(mode=0o700)
|
|
self.token = self.root / "token"
|
|
self.token.write_text("not-a-real-secret\n", encoding="utf-8")
|
|
self.token.chmod(0o600)
|
|
self.clock = Clock()
|
|
self.events: list[dict] = []
|
|
self.exec_count = 0
|
|
self.config = Config(
|
|
active_dir=self.root / "active", completed_dir=self.root / "completed",
|
|
dead_letter_dir=self.root / "dead", database_path=self.root / "state/state.sqlite3",
|
|
audit_log_path=self.root / "log/audit.jsonl",
|
|
gateway_endpoint="http://127.0.0.1:18920/v1/orchestrator/events",
|
|
gateway_token_file=self.token, contract_commit=CONTRACT,
|
|
lease_seconds=10, retry_seconds=2, max_attempts=3, poll_seconds=1,
|
|
executor_path=None, executor_args=(), executor_timeout_seconds=10,
|
|
)
|
|
|
|
def tearDown(self) -> None:
|
|
self.temp.cleanup()
|
|
|
|
def command(self, command_id: str = "cmd-1", *, contract: str = CONTRACT, kind: str = "acceptance_probe", text: str = "no-op") -> dict:
|
|
return {
|
|
"schema_version": 1, "command_id": command_id,
|
|
"project_key": "a0-codex-watcher-nonproduction",
|
|
"work_unit_id": "A0-CODEX-WATCHER-ACCEPTANCE",
|
|
"contract_commit": contract, "state_version": 0,
|
|
"execution": {"kind": kind, "input": text},
|
|
}
|
|
|
|
def write(self, value: dict, name: str = "command.json") -> Path:
|
|
path = self.config.active_dir / name
|
|
path.write_text(json.dumps(value), encoding="utf-8")
|
|
path.chmod(0o600)
|
|
return path
|
|
|
|
def watcher(self, **kwargs) -> CommandWatcher:
|
|
def execute(command: dict) -> dict:
|
|
self.exec_count += 1
|
|
return {"kind": command["execution"]["kind"], "exit_code": 0}
|
|
return CommandWatcher(
|
|
self.config, clock=self.clock,
|
|
event_poster=kwargs.get("event_poster", self.events.append),
|
|
executor=kwargs.get("executor", execute), worker_id=kwargs.get("worker_id"),
|
|
)
|
|
|
|
def row(self, command_id: str = "cmd-1") -> sqlite3.Row:
|
|
db = sqlite3.connect(self.config.database_path)
|
|
db.row_factory = sqlite3.Row
|
|
try:
|
|
return db.execute("SELECT * FROM commands WHERE command_id=?", (command_id,)).fetchone()
|
|
finally:
|
|
db.close()
|
|
|
|
def test_happy_path_posts_lifecycle_once_and_archives(self) -> None:
|
|
path = self.write(self.command())
|
|
self.assertTrue(self.watcher().process_path(path))
|
|
self.assertEqual([e["event_type"] for e in self.events], ["work_unit_started", "work_unit_ended"])
|
|
self.assertEqual(self.exec_count, 1)
|
|
self.assertEqual(self.row()["status"], "COMPLETED")
|
|
self.assertTrue((self.config.completed_dir / path.name).is_file())
|
|
markers = [json.loads(line)["marker"] for line in self.config.audit_log_path.read_text().splitlines()]
|
|
self.assertEqual(markers, ["CODEX_AUTO_CLAIM", "TASK_STARTED", "TASK_EXECUTED", "TASK_ENDED", "EVENT_POSTED"])
|
|
|
|
def test_contract_mismatch_is_rejected_without_registration(self) -> None:
|
|
path = self.write(self.command(contract="0" * 40))
|
|
self.assertFalse(self.watcher().process_path(path))
|
|
self.assertTrue((self.config.dead_letter_dir / path.name).is_file())
|
|
db = sqlite3.connect(self.config.database_path)
|
|
self.assertEqual(db.execute("SELECT count(*) FROM commands").fetchone()[0], 0)
|
|
|
|
def test_duplicate_id_with_different_payload_is_dead_lettered(self) -> None:
|
|
first = self.write(self.command(), "first.json")
|
|
self.assertTrue(self.watcher().process_path(first))
|
|
second = self.write(self.command(text="different"), "second.json")
|
|
self.assertFalse(self.watcher().process_path(second))
|
|
self.assertEqual(self.row()["status"], "COMPLETED")
|
|
self.assertTrue((self.config.dead_letter_dir / second.name).is_file())
|
|
|
|
def test_same_work_unit_with_different_command_id_is_not_executed(self) -> None:
|
|
first = self.write(self.command(), "first.json")
|
|
self.assertTrue(self.watcher().process_path(first))
|
|
second_command = self.command(command_id="cmd-2")
|
|
second = self.write(second_command, "second.json")
|
|
self.assertFalse(self.watcher().process_path(second))
|
|
self.assertEqual(self.exec_count, 1)
|
|
self.assertTrue((self.config.dead_letter_dir / second.name).is_file())
|
|
|
|
def test_two_workers_cannot_hold_same_lease(self) -> None:
|
|
path = self.write(self.command())
|
|
one = self.watcher(worker_id="worker-one")
|
|
command, digest = one._read_command(path)
|
|
one._register(command, digest, path)
|
|
self.assertIsNotNone(one._claim("cmd-1"))
|
|
two = self.watcher(worker_id="worker-two")
|
|
self.assertIsNone(two._claim("cmd-1"))
|
|
|
|
def test_stale_lease_is_recovered(self) -> None:
|
|
path = self.write(self.command())
|
|
one = self.watcher(worker_id="worker-one")
|
|
command, digest = one._read_command(path)
|
|
one._register(command, digest, path)
|
|
one._claim("cmd-1")
|
|
self.clock.value += 11
|
|
self.assertIsNotNone(self.watcher(worker_id="worker-two")._claim("cmd-1"))
|
|
|
|
def test_end_post_retry_does_not_execute_twice(self) -> None:
|
|
calls = 0
|
|
attempted_end_events: list[dict] = []
|
|
def poster(event: dict) -> None:
|
|
nonlocal calls
|
|
calls += 1
|
|
if event["event_type"] == "work_unit_ended":
|
|
attempted_end_events.append(event)
|
|
if calls == 2:
|
|
raise WatcherError("temporary")
|
|
self.events.append(event)
|
|
path = self.write(self.command())
|
|
watcher = self.watcher(event_poster=poster)
|
|
self.assertFalse(watcher.process_path(path))
|
|
self.assertEqual(self.row()["phase"], "END")
|
|
self.assertEqual(self.exec_count, 1)
|
|
self.clock.value += 3
|
|
self.assertTrue(watcher.process_path(path))
|
|
self.assertEqual(self.exec_count, 1)
|
|
self.assertEqual(len(attempted_end_events), 2)
|
|
self.assertEqual(attempted_end_events[0], attempted_end_events[1])
|
|
self.assertEqual(self.row()["status"], "COMPLETED")
|
|
|
|
def test_failure_exhaustion_moves_to_dead_letter(self) -> None:
|
|
executions = 0
|
|
def fail(_command: dict) -> dict:
|
|
nonlocal executions
|
|
executions += 1
|
|
raise WatcherError("executor failure")
|
|
path = self.write(self.command())
|
|
watcher = self.watcher(executor=fail)
|
|
for _ in range(3):
|
|
self.assertFalse(watcher.process_path(path))
|
|
self.clock.value += 3
|
|
self.assertEqual(self.row()["status"], "DEAD_LETTER")
|
|
self.assertEqual(executions, 1)
|
|
self.assertTrue((self.config.dead_letter_dir / path.name).is_file())
|
|
|
|
def test_symlink_command_is_rejected_without_following(self) -> None:
|
|
outside = self.root / "outside.json"
|
|
outside.write_text(json.dumps(self.command()), encoding="utf-8")
|
|
link = self.config.active_dir / "link.json"
|
|
os.symlink(outside, link)
|
|
self.assertFalse(self.watcher().process_path(link))
|
|
self.assertTrue(outside.is_file())
|
|
self.assertTrue((self.config.dead_letter_dir / link.name).is_symlink())
|
|
|
|
def test_codex_mode_fails_closed_when_executor_absent(self) -> None:
|
|
path = self.write(self.command(kind="codex"))
|
|
watcher = CommandWatcher(self.config, clock=self.clock, event_poster=self.events.append)
|
|
self.assertFalse(watcher.process_path(path))
|
|
self.assertEqual(self.row()["phase"], "EXECUTE")
|
|
self.assertIn("codex_executor_unavailable", self.row()["last_error"])
|
|
|
|
def test_validation_rejects_unknown_fields(self) -> None:
|
|
command = self.command()
|
|
command["secret"] = "must-not-be-accepted"
|
|
with self.assertRaisesRegex(WatcherError, "invalid_command_fields"):
|
|
validate_command(command, CONTRACT)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|