Files
aios/tests/test_repository_discovery.py
T

515 lines
20 KiB
Python

from __future__ import annotations
import json
from pathlib import Path
import subprocess
import sys
import pytest
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "active-runtime" / "app"))
from aios_repository_discovery import ( # noqa: E402
RepositoryDiscovery,
RepositoryDiscoveryError,
RepositoryDiscoveryNotFound,
get_diff_index,
list_repository_tree,
read_repository_file,
search_repository_source,
)
REPOSITORY = "aios/aios"
SECRET = b"repository-discovery-test-secret"
def git(*args: str, cwd: Path | None = None) -> str:
return subprocess.run(
["git", *( ["-C", str(cwd)] if cwd is not None else []), *args],
check=True,
capture_output=True,
text=True,
).stdout.strip()
@pytest.fixture()
def repository(tmp_path: Path):
bare = tmp_path / "authoritative.git"
seed = tmp_path / "seed"
git("init", "--bare", str(bare))
git("init", "-b", "main", str(seed))
git("config", "user.name", "Discovery Test", cwd=seed)
git("config", "user.email", "discovery@example.invalid", cwd=seed)
(seed / "README.md").write_text("first immutable snapshot\n", encoding="utf-8")
(seed / "00-large.txt").write_text("x" * 512, encoding="utf-8")
(seed / "mystery" / "jobstore").mkdir(parents=True)
(seed / "mystery" / "jobstore" / "recovery.py").write_text(
"def locate():\n return 'JobStore.for_existing_readonly recovery marker'\n",
encoding="utf-8",
)
for index in range(3):
(seed / "search" / f"match-{index}.py").parent.mkdir(exist_ok=True)
(seed / "search" / f"match-{index}.py").write_text(
f"P0_SEARCH_MARKER line {index}\n", encoding="utf-8"
)
(seed / "binary.bin").write_bytes(b"binary-marker\x00not-source\n")
(seed / "nested" / "a" / "b").mkdir(parents=True)
(seed / "nested" / "a" / "b" / "value.txt").write_text("deep value\n", encoding="utf-8")
git("add", ".", cwd=seed)
git("commit", "-m", "initial", cwd=seed)
main_oid = git("rev-parse", "HEAD", cwd=seed)
git("tag", "v1.0.0", cwd=seed)
git("remote", "add", "origin", str(bare), cwd=seed)
git("push", "origin", "main", "--tags", cwd=seed)
git("--git-dir", str(bare), "symbolic-ref", "HEAD", "refs/heads/main")
git("checkout", "-b", "feature/discovery", cwd=seed)
(seed / "docs").mkdir()
git("mv", "README.md", "docs/README.md", cwd=seed)
for index in range(18):
changed = seed / "zz-diff-index" / f"changed-{index:02}.txt"
changed.parent.mkdir(exist_ok=True)
changed.write_text(f"change {index}\n", encoding="utf-8")
git("add", ".", cwd=seed)
git("commit", "-m", "large indexed diff", cwd=seed)
feature_oid = git("rev-parse", "HEAD", cwd=seed)
git("push", "origin", "feature/discovery", cwd=seed)
git("--git-dir", str(bare), "config", "remote.gitea.url", "https://g.example.invalid/git/aios/aios.git")
git("branch", "ambiguous", feature_oid, cwd=seed)
git("tag", "ambiguous", main_oid, cwd=seed)
git("push", "origin", "refs/heads/ambiguous:refs/heads/ambiguous", cwd=seed)
git("push", "origin", "refs/tags/ambiguous", cwd=seed)
discovery = RepositoryDiscovery(
allowed_repositories=(REPOSITORY,), git_dir=bare, cursor_secret=SECRET,
)
return {"bare": bare, "seed": seed, "main_oid": main_oid, "feature_oid": feature_oid, "discovery": discovery}
def test_exact_allowlist_and_verified_identity_reject_other_repositories(repository):
discovery = repository["discovery"]
with pytest.raises(RepositoryDiscoveryError, match="REPOSITORY_NOT_ALLOWED"):
discovery.list_tree(repository="other/repo", revision="HEAD")
multiple = RepositoryDiscovery(
allowed_repositories=(REPOSITORY, "other/repo"), git_dir=repository["bare"], cursor_secret=SECRET,
)
with pytest.raises(RepositoryDiscoveryError, match="REPOSITORY_IDENTITY_MISMATCH"):
multiple.list_tree(repository="other/repo", revision="HEAD")
def test_head_branch_tag_and_full_oid_resolve_to_one_immutable_commit(repository):
discovery = repository["discovery"]
expected = repository["main_oid"]
assert discovery.resolve_revision(REPOSITORY, "HEAD") == expected
assert discovery.resolve_revision(REPOSITORY, "main") == expected
assert discovery.resolve_revision(REPOSITORY, "v1.0.0") == expected
assert discovery.resolve_revision(REPOSITORY, expected) == expected
for dangerous in ("HEAD~1", "main^{tree}", "--help", "../main", "refs/remotes/origin/main"):
with pytest.raises(RepositoryDiscoveryError, match="INVALID_REVISION"):
discovery.resolve_revision(REPOSITORY, dangerous)
with pytest.raises(RepositoryDiscoveryError, match="AMBIGUOUS_REVISION"):
discovery.resolve_revision(REPOSITORY, "ambiguous")
def test_resolved_oid_remains_immutable_after_floating_head_moves(repository):
discovery = repository["discovery"]
original = discovery.read_file(
repository=REPOSITORY,
revision=repository["main_oid"],
file_path="README.md",
)
assert original["resolved_oid"] == repository["main_oid"]
git(
"--git-dir",
str(repository["bare"]),
"update-ref",
"refs/heads/main",
repository["feature_oid"],
)
moved = discovery.list_tree(
repository=REPOSITORY, revision="main", prefix="docs", limit=10
)
assert moved["resolved_oid"] == repository["feature_oid"]
with pytest.raises(RepositoryDiscoveryNotFound, match="FILE_NOT_FOUND"):
discovery.read_file(
repository=REPOSITORY, revision="main", file_path="README.md"
)
still_original = discovery.read_file(
repository=REPOSITORY,
revision=original["resolved_oid"],
file_path="README.md",
)
pinned_tree = discovery.list_tree(
repository=REPOSITORY, revision=original["resolved_oid"], limit=100
)
pinned_search = discovery.search_source(
repository=REPOSITORY,
revision=original["resolved_oid"],
query="JobStore.for_existing_readonly",
path_globs=("mystery/**/*.py",),
max_matches=5,
max_bytes=4096,
)
pinned_diff = discovery.diff_index(
repository=REPOSITORY,
from_revision=original["resolved_oid"],
to_revision=repository["feature_oid"],
limit=1,
)
assert pinned_tree["resolved_oid"] == repository["main_oid"]
assert any(item["path"] == "README.md" for item in pinned_tree["entries"])
assert pinned_search["resolved_oid"] == repository["main_oid"]
assert pinned_search["matches"]
assert pinned_diff["resolved_from_oid"] == repository["main_oid"]
assert pinned_diff["resolved_to_oid"] == repository["feature_oid"]
assert still_original["resolved_oid"] == repository["main_oid"]
assert still_original["blob_oid"] == original["blob_oid"]
assert still_original["content"] == original["content"]
def test_tree_pagination_cursor_is_signed_and_query_bound(repository):
discovery = repository["discovery"]
first = discovery.list_tree(repository=REPOSITORY, revision="main", limit=2)
assert first["resolved_oid"] == repository["main_oid"]
assert len(first["entries"]) == 2
assert first["truncated"] is True
cursor = first["next_cursor"]
assert isinstance(cursor, str)
second = discovery.list_tree(repository=REPOSITORY, revision=repository["main_oid"], limit=2, cursor=cursor)
assert {item["path"] for item in first["entries"]}.isdisjoint({item["path"] for item in second["entries"]})
with pytest.raises(RepositoryDiscoveryError, match="CURSOR_INVALID"):
discovery.list_tree(repository=REPOSITORY, revision="main", limit=3, cursor=cursor)
tampered_cursor = cursor[:-1] + ("0" if cursor[-1] != "0" else "1")
with pytest.raises(RepositoryDiscoveryError, match="CURSOR_INVALID"):
discovery.list_tree(repository=REPOSITORY, revision="main", limit=2, cursor=tampered_cursor)
@pytest.mark.parametrize("value", ("../secret", "/etc/passwd", "nested/../a", "--output", "nested\\a", ":(glob)**", "nested/*.txt"))
def test_path_traversal_and_git_option_like_paths_are_rejected(repository, value):
discovery = repository["discovery"]
with pytest.raises(RepositoryDiscoveryError, match="PATH_INVALID"):
discovery.list_tree(repository=REPOSITORY, revision="main", prefix=value)
with pytest.raises(RepositoryDiscoveryError, match="PATH_INVALID"):
discovery.read_file(repository=REPOSITORY, revision="main", file_path=value)
def test_source_search_finds_unknown_jobstore_path_with_literal_globs_and_bounds(repository):
discovery = repository["discovery"]
result = discovery.search_source(
repository=REPOSITORY,
revision="main",
query="JobStore.for_existing_readonly",
path_globs=("mystery/**/*.py",),
max_matches=5,
max_bytes=4096,
)
assert result["resolved_oid"] == repository["main_oid"]
assert result["matches"]
match = result["matches"][0]
assert match["path"] == "mystery/jobstore/recovery.py"
assert match["line_start"] == 2
assert match["line_end"] == 2
assert len(match["blob_oid"]) == 40
assert "recovery marker" in match["snippet"]
bounded = discovery.search_source(
repository=REPOSITORY, revision="main", query="x", max_matches=1, max_bytes=8,
)
assert bounded["scanned_bytes"] <= 8
assert bounded["truncated"] is True
def test_source_search_cursor_is_signed_and_binary_blobs_are_skipped(repository):
discovery = repository["discovery"]
first = discovery.search_source(
repository=REPOSITORY,
revision="main",
query="P0_SEARCH_MARKER",
path_globs=("search/*.py",),
max_matches=1,
max_bytes=4096,
)
assert len(first["matches"]) == 1
assert first["truncated"] is True
assert isinstance(first["next_cursor"], str)
second = discovery.search_source(
repository=REPOSITORY,
revision="main",
query="P0_SEARCH_MARKER",
path_globs=("search/*.py",),
max_matches=1,
max_bytes=4096,
cursor=first["next_cursor"],
)
assert second["matches"]
assert first["matches"][0]["path"] != second["matches"][0]["path"]
with pytest.raises(RepositoryDiscoveryError, match="CURSOR_INVALID"):
discovery.search_source(
repository=REPOSITORY,
revision="main",
query="different",
path_globs=("search/*.py",),
max_matches=1,
max_bytes=4096,
cursor=first["next_cursor"],
)
binary = discovery.search_source(
repository=REPOSITORY,
revision="main",
query="binary-marker",
path_globs=("binary.bin",),
max_matches=5,
max_bytes=4096,
)
assert binary["matches"] == []
assert binary["binary_blobs_skipped"] == 1
def test_diff_index_is_paginated_rename_aware_and_never_returns_patch(repository):
discovery = repository["discovery"]
first = discovery.diff_index(
repository=REPOSITORY, from_revision="main", to_revision="feature/discovery", limit=5,
)
assert first["resolved_from_oid"] == repository["main_oid"]
assert first["resolved_to_oid"] == repository["feature_oid"]
assert len(first["entries"]) == 5
assert first["truncated"] is True
assert "patch" not in first
assert all("additions" in item and "deletions" in item for item in first["entries"])
assert any(item["old_path"] == "README.md" and item["path"] == "docs/README.md" for item in first["entries"])
second = discovery.diff_index(
repository=REPOSITORY, from_revision="main", to_revision="feature/discovery", limit=5, cursor=first["next_cursor"],
)
assert {item["path"] for item in first["entries"]}.isdisjoint({item["path"] for item in second["entries"]})
scoped = discovery.diff_index(
repository=REPOSITORY,
from_revision="main",
to_revision="feature/discovery",
prefix="zz-diff-index",
limit=100,
)
assert scoped["entries"]
assert all(item["path"].startswith("zz-diff-index/") for item in scoped["entries"])
assert "resolved_from_oid" in scoped and "resolved_to_oid" in scoped
def test_read_file_returns_resolved_and_blob_identity_with_bounded_content(repository):
discovery = repository["discovery"]
result = discovery.read_file(
repository=REPOSITORY, revision="main", file_path="README.md", max_bytes=5,
)
assert result["resolved_oid"] == repository["main_oid"]
assert len(result["blob_oid"]) == 40
assert result["content"] == "first"
assert result["content_bytes"] == 5
assert result["truncated"] is True
with pytest.raises(RepositoryDiscoveryNotFound, match="FILE_NOT_FOUND"):
discovery.read_file(repository=REPOSITORY, revision="main", file_path="missing.txt")
def test_mapping_adapters_and_module_helpers_return_dicts_with_status_codes(repository):
discovery = repository["discovery"]
tree = discovery.list_repository_tree({"repository": REPOSITORY, "revision": "main", "limit": 2})
assert isinstance(tree, dict)
source = search_repository_source(
{"repository": REPOSITORY, "revision": "main", "query": "recovery marker"}, discovery=discovery,
)
assert source["matches"]
index = get_diff_index(
{"repository": REPOSITORY, "from": "main", "to": "feature/discovery", "limit": 2}, discovery=discovery,
)
assert index["entries"]
blob = read_repository_file(
{"repository": REPOSITORY, "revision": "main", "path": "README.md", "max_bytes": 8}, discovery=discovery,
)
assert blob["path"] == "README.md"
assert list_repository_tree({"repository": REPOSITORY, "revision": "main", "limit": 1}, discovery=discovery)["entries"]
with pytest.raises(RepositoryDiscoveryError) as failure:
discovery.read_repository_file({"repository": REPOSITORY, "revision": "main", "path": "../secret"})
assert str(failure.value) == "REPOSITORY_DISCOVERY_PATH_INVALID"
assert failure.value.status_code == 400
def test_public_request_names_pin_ref_and_return_required_identity_fields(repository):
discovery = repository["discovery"]
tree = discovery.list_repository_tree(
{"repository": REPOSITORY, "ref": "feature/discovery", "limit": 1}
)
assert tree["resolved_oid"] == repository["feature_oid"]
assert {"path", "type", "object_oid"} <= set(tree["entries"][0])
source = discovery.search_repository_source(
{
"repository": REPOSITORY,
"ref": "main",
"query": "P0_SEARCH_MARKER",
"path_globs": ["search/*.py"],
"max_matches": 2,
"max_bytes": 4096,
}
)
assert source["matches"][0]["line_start"] == source["matches"][0]["line_end"] == 1
blob = discovery.read_repository_file(
{"repository": REPOSITORY, "ref": "main", "path": "README.md", "max_bytes": 8}
)
assert blob["content_base64"]
assert len(blob["blob_oid"]) == 40
@pytest.fixture()
def large_diff_repository(tmp_path: Path):
bare = tmp_path / "large.git"
seed = tmp_path / "seed"
git("init", "--bare", str(bare))
git("init", "-b", "main", str(seed))
git("config", "user.name", "Discovery Test", cwd=seed)
git("config", "user.email", "discovery@example.invalid", cwd=seed)
(seed / "README.md").write_text("base\n", encoding="utf-8")
git("add", ".", cwd=seed)
git("commit", "-m", "base", cwd=seed)
main_oid = git("rev-parse", "HEAD", cwd=seed)
git("remote", "add", "origin", str(bare), cwd=seed)
git("push", "origin", "main", cwd=seed)
git("--git-dir", str(bare), "symbolic-ref", "HEAD", "refs/heads/main")
for index in range(18_000):
filename = "long-prefix-" + ("x" * 42) + f"-{index:05}.txt"
(seed / "bulk" / filename).parent.mkdir(exist_ok=True)
(seed / "bulk" / filename).write_text("large diff\n", encoding="utf-8")
git("add", "bulk", cwd=seed)
git("commit", "-m", "large diff", cwd=seed)
feature_oid = git("rev-parse", "HEAD", cwd=seed)
git("push", "origin", "main", cwd=seed)
git("--git-dir", str(bare), "config", "remote.gitea.url", "https://g.example.invalid/git/aios/aios.git")
return {
"bare": bare,
"main_oid": main_oid,
"feature_oid": feature_oid,
"discovery": RepositoryDiscovery(
allowed_repositories=(REPOSITORY,), git_dir=bare, cursor_secret=SECRET
),
}
def test_large_diff_index_is_streamed_and_page_bounded(large_diff_repository):
result = large_diff_repository["discovery"].diff_index(
repository=REPOSITORY,
from_revision=large_diff_repository["main_oid"],
to_revision=large_diff_repository["feature_oid"],
limit=1,
)
assert len(result["entries"]) == 1
assert result["entries"][0]["path"].startswith("bulk/")
assert result["entries"][0]["additions"] == 1
assert result["entries"][0]["deletions"] == 0
assert result["truncated"] is True
def test_bridge_http_discovery_routes_are_get_only_and_do_not_enter_write_handlers(repository):
pytest.importorskip(
"fcntl",
reason="Bridge imports POSIX-only fcntl; Windows integration is unverified",
)
from http.client import HTTPConnection
from http.server import ThreadingHTTPServer
from threading import Thread
from urllib.parse import urlencode
from aios_chatgpt_bridge_v2 import BridgeV2Adapter
from aios_chatgpt_bridge_v2_server import BridgeHandler
discovery = repository["discovery"]
calls: list[str] = []
def invoke(operation: str, method: str):
def handler(request):
calls.append(operation)
return getattr(discovery, method)(request)
return handler
adapter = BridgeV2Adapter(
runtime_status={"deployment_state": "STAGING"},
handlers={
"listRepositoryTree": invoke("listRepositoryTree", "list_repository_tree"),
"searchRepositorySource": invoke(
"searchRepositorySource", "search_repository_source"
),
"getDiffIndex": invoke("getDiffIndex", "get_diff_index"),
"readRepositoryFile": invoke("readRepositoryFile", "read_repository_file"),
},
)
previous_adapter = BridgeHandler.adapter
server = ThreadingHTTPServer(("127.0.0.1", 0), BridgeHandler)
BridgeHandler.adapter = adapter
thread = Thread(target=server.serve_forever, daemon=True)
thread.start()
def request(method: str, path: str, body: str | None = None):
connection = HTTPConnection("127.0.0.1", server.server_port, timeout=10)
headers = {"Content-Type": "application/json"} if body is not None else {}
connection.request(method, path, body=body, headers=headers)
response = connection.getresponse()
raw = response.read()
status = response.status
connection.close()
return status, raw
try:
cases = [
(
"/v2/repository/tree",
{"repository": REPOSITORY, "ref": "main", "limit": 2},
),
(
"/v2/repository/search",
{
"repository": REPOSITORY,
"ref": "main",
"query": "recovery marker",
"max_matches": 2,
"max_bytes": 4096,
},
),
(
"/v2/repository/diff-index",
{
"repository": REPOSITORY,
"from": "main",
"to": "feature/discovery",
"limit": 2,
},
),
(
"/v2/repository/file",
{"repository": REPOSITORY, "ref": "main", "path": "README.md"},
),
]
for path, values in cases:
status, raw = request("GET", f"{path}?{urlencode(values)}")
assert status == 200, raw.decode("utf-8", errors="replace")
payload = json.loads(raw.decode("utf-8"))
assert payload["repository"] == REPOSITORY
assert "resolved_oid" in payload or "resolved_to_oid" in payload
assert len(calls) == 4
status, _ = request("POST", "/v2/repository/tree", "{}")
assert status == 405
assert len(calls) == 4
duplicate = (
"/v2/repository/tree?repository=aios%2Faios&ref=main&ref=main&limit=1"
)
status, _ = request("GET", duplicate)
assert status == 400
assert len(calls) == 4
finally:
server.shutdown()
server.server_close()
thread.join(timeout=5)
BridgeHandler.adapter = previous_adapter