aios/tests/test_completed_replay_reconciliation_v7.py

from __future__ import annotations

import json
import sqlite3
from datetime import datetime, timezone
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from types import SimpleNamespace

import pytest

from aios_gitops_job_controller import BindingConflict, GitOpsJobController, JobError, JobState, JobStore, sha256_json


TASK = "TASK-CONV-V2R2C2-RECONCILE-SNAPSHOT-20260826"
EXTERNAL = "job-external-v7"
BRANCH = "agent/v2r2c2-convergence-integration-20260825"
BASE = "e" * 40
COMMIT = "1" * 40


def make_store(tmp_path: Path) -> tuple[GitOpsJobController, str, dict]:
    store = JobStore(tmp_path / "job-root")
    controller = GitOpsJobController(
        store, clock=lambda: datetime(2026, 9, 8, tzinfo=timezone.utc)
    )
    request = {
        "idempotency_key": "v7-replay",
        "campaign_id": TASK,
        "task_id": TASK,
        "external_job_id": EXTERNAL,
        "repository": "aios/aios",
        "operation": "CONTINUATION_COMMIT_FILES",
        "branch": BRANCH,
        "target_branch": BRANCH,
        "branch_mode": "CONTINUE_EXISTING",
        "base_oid": BASE,
        "lineage_anchor_oid": BASE,
        "expected_branch_oid": BASE,
        "generation": 2,
        "payload_digest": "a" * 64,
        "files_digest": "b" * 64,
        "message_digest": "c" * 64,
        "worker_principal": "aios-autonomy-agent",
        "worker_lineage": "governed-continuation",
    }
    receipt = controller.submit(request)
    controller.advance(receipt.job_id, JobState.AUTHORIZED, side_effect_state="AUTHORIZED")
    row = store.read(receipt.job_id)
    controller.lease_job(
        receipt.job_id,
        lease_id="lease-v7-epoch-3",
        lineage_anchor_oid=BASE,
        current_head_oid=BASE,
        generation=2,
        lease_expires_at="2026-09-08T00:01:00Z",
        expected_revision=row["revision"],
    )
    with store._connect() as connection:
        connection.execute("ALTER TABLE branch_leases ADD COLUMN epoch INTEGER")
        connection.execute("UPDATE branch_leases SET epoch=3, state='USED' WHERE job_id=?", (receipt.job_id,))
        connection.execute("CREATE TABLE branch_lease_events (lease_id TEXT NOT NULL, state TEXT NOT NULL, current_head_oid TEXT NOT NULL, generation INTEGER NOT NULL)")
        connection.execute("INSERT INTO branch_lease_events VALUES (?, 'USED', ?, 2)", ("lease-v7-epoch-3", BASE))
    controller.advance(receipt.job_id, JobState.WORKTREE_READY, side_effect_state="WORKTREE_READY")
    controller.advance(receipt.job_id, JobState.EXECUTING, side_effect_state="IN_PROGRESS")
    completed = {
        "commit_oid": COMMIT,
        "authority_oid": COMMIT,
        "remote_verified": True,
        "worker_principal": request["worker_principal"],
        "worker_lineage": request["worker_lineage"],
    }
    controller.advance(receipt.job_id, JobState.REMOTE_VERIFIED, side_effect_state="REMOTE_VERIFIED", receipt=completed)
    final = controller.advance(receipt.job_id, JobState.COMPLETED, side_effect_state="REMOTE_VERIFIED", receipt=completed)
    row = store.read(receipt.job_id)
    envelope = {
        "task_id": TASK,
        "external_job_id": EXTERNAL,
        "internal_job_id": receipt.job_id,
        "repository": "aios/aios",
        "target_branch": BRANCH,
        "generation": 2,
        "branch_mode": "CONTINUE_EXISTING",
        "payload_digest": request["payload_digest"],
        "files_digest": request["files_digest"],
        "message_digest": request["message_digest"],
        "commit_oid": COMMIT,
        "authority_head_oid": COMMIT,
        "worker_principal": request["worker_principal"],
        "worker_lineage": request["worker_lineage"],
        "source_lease_epoch": 3,
        "historical_lease_head_oid": BASE,
        "durable_revision": row["revision"],
        "completion_identity": sha256_json({"job_id": receipt.job_id, "revision": row["revision"], "receipt": completed}),
        "reason": "COMPLETED_REPLAY_LEASE_EVIDENCE_RECONCILIATION",
        "provenance": {"source": "v7-test", "authority_readback": True},
    }
    assert final.state is JobState.COMPLETED
    return controller, receipt.job_id, envelope


def test_completed_replay_is_append_only_and_exactly_idempotent(tmp_path: Path):
    controller, job_id, envelope = make_store(tmp_path)
    with controller.store._connect() as connection:
        before = dict(connection.execute("SELECT * FROM branch_leases WHERE job_id=?", (job_id,)).fetchone())
        before_events = [dict(row) for row in connection.execute("SELECT * FROM branch_lease_events WHERE lease_id=?", ("lease-v7-epoch-3",)).fetchall()]
        old_receipt = connection.execute("SELECT receipt_json FROM jobs WHERE job_id=?", (job_id,)).fetchone()[0]
    first = controller.store.append_completed_replay_reconciliation(job_id, envelope, authority_head_oid=COMMIT)
    second = controller.store.append_completed_replay_reconciliation(job_id, envelope, authority_head_oid=COMMIT)
    restarted = GitOpsJobController(JobStore(controller.store.root))
    third = restarted.store.append_completed_replay_reconciliation(job_id, envelope, authority_head_oid=COMMIT)
    assert first["reconciliation_id"] == second["reconciliation_id"] == third["reconciliation_id"]
    assert first["current_head_oid"] == COMMIT
    with controller.store._connect() as connection:
        after = dict(connection.execute("SELECT * FROM branch_leases WHERE job_id=?", (job_id,)).fetchone())
        after_events = [dict(row) for row in connection.execute("SELECT * FROM branch_lease_events WHERE lease_id=?", ("lease-v7-epoch-3",)).fetchall()]
        new_receipt = connection.execute("SELECT receipt_json FROM jobs WHERE job_id=?", (job_id,)).fetchone()[0]
        count = connection.execute("SELECT COUNT(*) FROM completed_replay_reconciliations WHERE job_id=?", (job_id,)).fetchone()[0]
    assert before == after
    assert before_events == after_events
    assert old_receipt == new_receipt
    assert count == 1
    with pytest.raises(sqlite3.IntegrityError, match="COMPLETED_REPLAY_APPEND_ONLY"):
        with controller.store._connect() as connection:
            connection.execute(
                "UPDATE completed_replay_reconciliations SET reason='tampered' WHERE job_id=?",
                (job_id,),
            )
    with pytest.raises(sqlite3.IntegrityError, match="COMPLETED_REPLAY_APPEND_ONLY"):
        with controller.store._connect() as connection:
            connection.execute(
                "DELETE FROM completed_replay_reconciliations WHERE job_id=?",
                (job_id,),
            )


def test_completed_replay_authority_conflict_fails_closed(tmp_path: Path):
    controller, job_id, envelope = make_store(tmp_path)
    with pytest.raises(BindingConflict, match="AUTHORITY_CONFLICT"):
        controller.store.append_completed_replay_reconciliation(job_id, envelope, authority_head_oid="2" * 40)


def test_completed_replay_binding_conflict_fails_closed(tmp_path: Path):
    controller, job_id, envelope = make_store(tmp_path)
    conflicting = {**envelope, "files_digest": "d" * 64}
    with pytest.raises(BindingConflict, match="BINDING_CONFLICT"):
        controller.store.append_completed_replay_reconciliation(job_id, conflicting, authority_head_oid=COMMIT)


def test_identical_reconciliation_converges_under_contention(tmp_path: Path):
    controller, job_id, envelope = make_store(tmp_path)

    def run(_: int) -> str:
        return controller.store.append_completed_replay_reconciliation(
            job_id, envelope, authority_head_oid=COMMIT
        )["reconciliation_id"]

    with ThreadPoolExecutor(max_workers=8) as pool:
        values = list(pool.map(run, range(16)))
    assert set(values) == {values[0]}
    assert controller.store.completed_replay_evidence(job_id)["commit_oid"] == COMMIT



def test_public_job_projection_exposes_external_reconciled_head(tmp_path: Path):
    from aios_chatgpt_bridge_v2_server import GovernedRuntime

    controller, job_id, envelope = make_store(tmp_path)
    reconciliation = controller.store.append_completed_replay_reconciliation(
        job_id, envelope, authority_head_oid=COMMIT
    )

    class Authority:
        def read_ref(self, ref: str):
            assert ref == "refs/heads/" + BRANCH
            return COMMIT

    runtime = object.__new__(GovernedRuntime)
    runtime.authority = Authority()
    runtime._controller_for_job = lambda _job_id: controller
    result = GovernedRuntime.job(runtime, {"job_id": job_id}, "get")
    assert result["job_id"] == EXTERNAL
    assert result["current_head_oid"] == COMMIT
    assert result["completed_replay_reconciliation_id"] == reconciliation["reconciliation_id"]
    assert result["receipt"]["commit_oid"] == result["receipt"]["authority_oid"] == COMMIT


def test_writer_lease_admits_exact_completed_replay_without_new_lease(tmp_path: Path):
    from aios_chatgpt_bridge_v2_server import GovernedRuntime

    controller, job_id, envelope = make_store(tmp_path)
    controller.store.append_completed_replay_reconciliation(
        job_id, envelope, authority_head_oid=COMMIT
    )

    class Authority:
        def read_ref(self, ref: str):
            assert ref == "refs/heads/" + BRANCH
            return COMMIT

    runtime = object.__new__(GovernedRuntime)
    runtime.config = SimpleNamespace(state_root=tmp_path)
    runtime.controllers = {"job": controller}
    runtime.authority = Authority()
    body = {
        "task_id": TASK,
        "job_id": EXTERNAL,
        "repository": "aios/aios",
        "branch_mode": "CONTINUE_EXISTING",
        "base_oid": BASE,
        "expected_branch_oid": BASE,
        "target_branch": BRANCH,
        "generation": 2,
        "lease_seconds": 900,
    }
    result = GovernedRuntime.acquire_lease(runtime, body)
    assert result["completed_replay"] is True
    assert result["lease_created"] is False
    assert result["expected_branch_oid"] == BASE
    assert result["current_head_oid"] == COMMIT
    assert result["completed_replay_reconciliation_id"].startswith("reconcile-")
    with controller.store._connect() as connection:
        assert connection.execute(
            "SELECT COUNT(*) FROM branch_leases WHERE job_id=?", (job_id,)
        ).fetchone()[0] == 1


def test_conflicting_reconciliation_has_no_ambiguous_winner(tmp_path: Path):
    controller, job_id, envelope = make_store(tmp_path)
    conflicting = {**envelope, "reason": "CONFLICTING_PROVENANCE"}
    values = [envelope, conflicting] * 8

    def run(candidate: dict) -> str:
        try:
            result = controller.store.append_completed_replay_reconciliation(
                job_id, candidate, authority_head_oid=COMMIT
            )
            return "ok:" + result["identity_digest"]
        except BindingConflict:
            return "conflict"

    with ThreadPoolExecutor(max_workers=8) as pool:
        outcomes = list(pool.map(run, values))
    winners = {item for item in outcomes if item.startswith("ok:")}
    assert len(winners) == 1
    assert "conflict" in outcomes
    with controller.store._connect() as connection:
        assert connection.execute(
            "SELECT COUNT(*) FROM completed_replay_reconciliations WHERE job_id=?", (job_id,)
        ).fetchone()[0] == 1


def test_reconciliation_rejects_head_alias_before_append(tmp_path: Path):
    controller, job_id, envelope = make_store(tmp_path)
    conflicting = {**envelope, "historical_lease_head_oid": COMMIT}
    with pytest.raises(BindingConflict, match="HISTORICAL_LEASE_CONFLICT"):
        controller.store.append_completed_replay_reconciliation(
            job_id, conflicting, authority_head_oid=COMMIT
        )
    with controller.store._connect() as connection:
        assert connection.execute(
            "SELECT COUNT(*) FROM completed_replay_reconciliations WHERE job_id=?",
            (job_id,),
        ).fetchone()[0] == 0


def test_http_commit_allowlist_carries_replay_identity(tmp_path: Path):
    from aios_chatgpt_bridge_v2_server import BridgeHandler

    handler = object.__new__(BridgeHandler)
    body = {
        "task_id": TASK,
        "job_id": EXTERNAL,
        "source_lease_epoch": 3,
        "repository": "aios/aios",
        "expected_branch_oid": BASE,
        "target_branch": BRANCH,
        "generation": 2,
        "idempotency_key": "v7-replay",
        "files": [{"path": "x.txt", "content_base64": "eA=="}],
        "message": "replay",
    }
    handler._allowed("commitFilesGoverned", body)


def test_cold_append_requires_existing_database(tmp_path: Path):
    missing = tmp_path / "missing-job-root"
    with pytest.raises(JobError, match="JOB_ROOT_NOT_FOUND"):
        JobStore.for_existing_append(missing)
    assert not missing.exists()


def test_real_http_post_carries_external_replay_identity(tmp_path: Path):
    from http.client import HTTPConnection
    from http.server import HTTPServer
    from threading import Thread
    from aios_chatgpt_bridge_v2_server import BridgeHandler

    observed = {}
    class Adapter:
        def getRuntimeStatus(self):
            return {
                "governed_writes_enabled": True,
                "legacy_global_writes_enabled": False,
                "task_principal_writes_enabled": True,
            }

        def commitFilesGoverned(self, body):
            observed.update(body)
            return {"job_id": body["job_id"], "current_head_oid": COMMIT}

    def verify_principal(transport, request):
        assert transport["operation"] == "commitFilesGoverned"
        assert transport["headers"]["authorization"] == "Bearer fixture-principal"
        return {"task_id": request["task_id"], "tenant_id": "fixture"}

    server = HTTPServer(("127.0.0.1", 0), BridgeHandler)
    BridgeHandler.adapter = Adapter()
    BridgeHandler.principal_verifier = verify_principal
    thread = Thread(target=server.serve_forever, daemon=True)
    thread.start()
    body = {
        "task_id": TASK,
        "job_id": EXTERNAL,
        "source_lease_epoch": 3,
        "repository": "aios/aios",
        "expected_branch_oid": BASE,
        "target_branch": BRANCH,
        "generation": 2,
        "idempotency_key": "v7-replay",
        "files": [{"path": "x.txt", "content_base64": "eA=="}],
        "message": "replay",
    }
    try:
        connection = HTTPConnection("127.0.0.1", server.server_port, timeout=5)
        connection.request(
            "POST",
            "/v2/commit-files",
            body=json.dumps(body),
            headers={
                "Content-Type": "application/json",
                "Authorization": "Bearer fixture-principal",
            },
        )
        response = connection.getresponse()
        payload = json.loads(response.read().decode())
        connection.close()
    finally:
        server.shutdown()
        server.server_close()
        thread.join(timeout=5)
        BridgeHandler.adapter = None
        BridgeHandler.principal_verifier = None
    assert response.status == 200
    assert observed["job_id"] == EXTERNAL
    assert observed["source_lease_epoch"] == 3
    assert payload["job_id"] == EXTERNAL
    assert payload["current_head_oid"] == COMMIT