471 lines
14 KiB
Python
471 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Canonical AIOS all-modified-files evidence packager.
|
|
|
|
The bundle contract is intentionally compatible with the historical
|
|
aios_package_all_modified_files_v1.sh output while keeping secret scanning in
|
|
version-controlled, tested code.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
import tempfile
|
|
import zipfile
|
|
from typing import Iterable
|
|
|
|
OID_RE = re.compile(r"^[0-9a-f]{40}$")
|
|
TASK_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{2,179}$")
|
|
|
|
# A PEM private-key finding requires the complete PEM header. Source code,
|
|
# tests, docs, or error text containing only "BEGIN PRIVATE KEY" are not
|
|
# credentials and must not be rejected.
|
|
PEM_PRIVATE_KEY_RE = re.compile(
|
|
rb"-----BEGIN (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----",
|
|
re.IGNORECASE,
|
|
)
|
|
BEARER_RE = re.compile(
|
|
rb"authorization\s*:\s*bearer\s+[A-Za-z0-9._~+/=-]{20,}",
|
|
re.IGNORECASE,
|
|
)
|
|
KEY_VALUE_SECRET_RE = re.compile(
|
|
rb"(?:password|passwd|secret|api[_-]?key)\s*[:=]\s*(?!//)[^\s,;]{12,}",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
DOCUMENT_SUFFIXES = {".md", ".rst", ".adoc", ".txt"}
|
|
DOCUMENT_PREFIXES = (
|
|
"README",
|
|
"CHANGELOG",
|
|
"RUNBOOK",
|
|
"ADR",
|
|
"ARCHITECTURE",
|
|
"DEPLOYMENT",
|
|
"TROUBLESHOOTING",
|
|
)
|
|
|
|
|
|
class PackagerError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def sha256(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def secret_finding(data: bytes) -> str | None:
|
|
if PEM_PRIVATE_KEY_RE.search(data):
|
|
return "PEM_PRIVATE_KEY"
|
|
if BEARER_RE.search(data):
|
|
return "AUTHORIZATION_BEARER"
|
|
if KEY_VALUE_SECRET_RE.search(data):
|
|
return "KEY_VALUE_SECRET"
|
|
return None
|
|
|
|
|
|
def scan_secret(data: bytes, label: str) -> None:
|
|
finding = secret_finding(data)
|
|
if finding is not None:
|
|
raise PackagerError(
|
|
f"MODIFIED_FILES_SECRET_SCAN_MATCH:{finding}:{label}"
|
|
)
|
|
|
|
|
|
def git_bytes(git_dir: Path, oid: str, path: str) -> bytes:
|
|
try:
|
|
return subprocess.check_output(
|
|
[
|
|
"/usr/bin/git",
|
|
f"--git-dir={git_dir}",
|
|
"show",
|
|
f"{oid}:{path}",
|
|
],
|
|
timeout=30,
|
|
)
|
|
except subprocess.CalledProcessError as exc:
|
|
raise PackagerError(
|
|
"MODIFIED_FILES_GIT_SHOW_FAILED:" + path
|
|
) from exc
|
|
|
|
|
|
def git_text(git_dir: Path, *args: str) -> str:
|
|
try:
|
|
return subprocess.check_output(
|
|
["/usr/bin/git", f"--git-dir={git_dir}", *args],
|
|
text=True,
|
|
timeout=30,
|
|
)
|
|
except subprocess.CalledProcessError as exc:
|
|
raise PackagerError(
|
|
"MODIFIED_FILES_GIT_COMMAND_FAILED:" + " ".join(args)
|
|
) from exc
|
|
|
|
|
|
def validate_lineage(git_dir: Path, base: str, candidate: str) -> None:
|
|
if OID_RE.fullmatch(base) is None or OID_RE.fullmatch(candidate) is None:
|
|
raise PackagerError("MODIFIED_FILES_OID_INVALID")
|
|
for oid in (base, candidate):
|
|
try:
|
|
subprocess.run(
|
|
[
|
|
"/usr/bin/git",
|
|
f"--git-dir={git_dir}",
|
|
"cat-file",
|
|
"-e",
|
|
f"{oid}^{{commit}}",
|
|
],
|
|
check=True,
|
|
timeout=30,
|
|
)
|
|
except subprocess.CalledProcessError as exc:
|
|
raise PackagerError("MODIFIED_FILES_COMMIT_MISSING:" + oid) from exc
|
|
if subprocess.run(
|
|
[
|
|
"/usr/bin/git",
|
|
f"--git-dir={git_dir}",
|
|
"merge-base",
|
|
"--is-ancestor",
|
|
base,
|
|
candidate,
|
|
],
|
|
check=False,
|
|
timeout=30,
|
|
).returncode != 0:
|
|
raise PackagerError("MODIFIED_FILES_LINEAGE_INVALID")
|
|
|
|
|
|
def safe_destination(root: Path, relative: str) -> Path:
|
|
path = Path(relative)
|
|
if path.is_absolute() or ".." in path.parts:
|
|
raise PackagerError("MODIFIED_FILE_PATH_INVALID:" + relative)
|
|
destination = root / path
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
return destination
|
|
|
|
|
|
def is_document_path(path: str) -> bool:
|
|
value = Path(path)
|
|
if path.startswith("docs/"):
|
|
return True
|
|
if value.suffix.lower() in DOCUMENT_SUFFIXES:
|
|
return True
|
|
upper = value.name.upper()
|
|
return any(upper.startswith(prefix) for prefix in DOCUMENT_PREFIXES)
|
|
|
|
|
|
def diff_entries(
|
|
git_dir: Path,
|
|
base: str,
|
|
candidate: str,
|
|
before_root: Path,
|
|
after_root: Path,
|
|
) -> list[dict[str, object]]:
|
|
raw = git_text(
|
|
git_dir,
|
|
"diff",
|
|
"--name-status",
|
|
"-M",
|
|
"-C",
|
|
base,
|
|
candidate,
|
|
)
|
|
entries: list[dict[str, object]] = []
|
|
for line in raw.splitlines():
|
|
if not line.strip():
|
|
continue
|
|
parts = line.split("\t")
|
|
status = parts[0]
|
|
code = status[0]
|
|
old_path: str | None = None
|
|
new_path: str | None = None
|
|
if code in {"R", "C"}:
|
|
if len(parts) != 3:
|
|
raise PackagerError(
|
|
"MODIFIED_FILES_DIFF_PARSE_INVALID:" + line
|
|
)
|
|
old_path, new_path = parts[1], parts[2]
|
|
else:
|
|
if len(parts) != 2:
|
|
raise PackagerError(
|
|
"MODIFIED_FILES_DIFF_PARSE_INVALID:" + line
|
|
)
|
|
new_path = parts[1] if code != "D" else None
|
|
old_path = parts[1] if code != "A" else None
|
|
|
|
entry: dict[str, object] = {"status": status}
|
|
if old_path is not None:
|
|
entry["previous_path"] = old_path
|
|
before = git_bytes(git_dir, base, old_path)
|
|
scan_secret(before, "before:" + old_path)
|
|
safe_destination(before_root, old_path).write_bytes(before)
|
|
entry["before_sha256"] = sha256(before)
|
|
entry["before_bytes"] = len(before)
|
|
if new_path is not None:
|
|
entry["path"] = new_path
|
|
after = git_bytes(git_dir, candidate, new_path)
|
|
scan_secret(after, "after:" + new_path)
|
|
safe_destination(after_root, new_path).write_bytes(after)
|
|
entry["after_sha256"] = sha256(after)
|
|
entry["after_bytes"] = len(after)
|
|
entry["deleted"] = code == "D"
|
|
entry["added"] = code == "A"
|
|
entry["renamed"] = code == "R"
|
|
entry["copied"] = code == "C"
|
|
entries.append(entry)
|
|
return entries
|
|
|
|
|
|
def execution_entries(
|
|
extra_root: Path | None,
|
|
execution_root: Path,
|
|
) -> list[dict[str, object]]:
|
|
if extra_root is None or not extra_root.exists():
|
|
return []
|
|
if extra_root.is_symlink() or not extra_root.is_dir():
|
|
raise PackagerError("EXTRA_FILES_DIR_INVALID")
|
|
|
|
entries: list[dict[str, object]] = []
|
|
for path in sorted(extra_root.rglob("*")):
|
|
if path.is_symlink():
|
|
raise PackagerError(
|
|
"EXTRA_FILE_SYMLINK_REJECTED:" + str(path)
|
|
)
|
|
if not path.is_file():
|
|
continue
|
|
relative = path.relative_to(extra_root).as_posix()
|
|
data = path.read_bytes()
|
|
scan_secret(data, "execution:" + relative)
|
|
safe_destination(execution_root, relative).write_bytes(data)
|
|
entries.append(
|
|
{
|
|
"path": relative,
|
|
"sha256": sha256(data),
|
|
"bytes": len(data),
|
|
}
|
|
)
|
|
return entries
|
|
|
|
|
|
def write_checksums(bundle: Path) -> None:
|
|
checks: list[tuple[str, str]] = []
|
|
for path in sorted(bundle.rglob("*")):
|
|
if path.is_file() and path.name != "checksums.sha256":
|
|
checks.append(
|
|
(
|
|
sha256(path.read_bytes()),
|
|
path.relative_to(bundle).as_posix(),
|
|
)
|
|
)
|
|
(bundle / "checksums.sha256").write_text(
|
|
"".join(f"{digest} {name}\n" for digest, name in checks),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
|
|
def verify_checksums(bundle: Path) -> None:
|
|
checksum_path = bundle / "checksums.sha256"
|
|
for line in checksum_path.read_text(encoding="utf-8").splitlines():
|
|
digest, relative = line.split(" ", 1)
|
|
path = bundle / relative
|
|
if not path.is_file() or sha256(path.read_bytes()) != digest:
|
|
raise PackagerError(
|
|
"MODIFIED_FILES_CHECKSUM_CLOSURE_INVALID:" + relative
|
|
)
|
|
|
|
|
|
def build_bundle(
|
|
*,
|
|
git_dir: Path,
|
|
base: str,
|
|
candidate: str,
|
|
task_id: str,
|
|
extra_dir: Path | None = None,
|
|
output_root: Path | None = None,
|
|
now: datetime | None = None,
|
|
) -> dict[str, object]:
|
|
if TASK_RE.fullmatch(task_id) is None:
|
|
raise PackagerError("TASK_ID_INVALID")
|
|
validate_lineage(git_dir, base, candidate)
|
|
|
|
timestamp = (now or datetime.now(timezone.utc)).strftime(
|
|
"%Y%m%dT%H%M%SZ"
|
|
)
|
|
safe_task = re.sub(r"[^A-Za-z0-9._-]+", "-", task_id).strip("-")
|
|
if output_root is None:
|
|
output_root = Path(
|
|
tempfile.mkdtemp(
|
|
prefix=f"aios-modified-files-bundle-{safe_task}-{timestamp}-",
|
|
dir="/var/tmp",
|
|
)
|
|
)
|
|
else:
|
|
output_root.mkdir(parents=True, exist_ok=True)
|
|
|
|
bundle = output_root / "bundle"
|
|
before_root = bundle / "repo" / "before"
|
|
after_root = bundle / "repo" / "after"
|
|
execution_root = bundle / "execution"
|
|
before_root.mkdir(parents=True)
|
|
after_root.mkdir(parents=True)
|
|
execution_root.mkdir(parents=True)
|
|
|
|
changes = diff_entries(
|
|
git_dir,
|
|
base,
|
|
candidate,
|
|
before_root,
|
|
after_root,
|
|
)
|
|
execution_files = execution_entries(extra_dir, execution_root)
|
|
|
|
document_changes = [
|
|
entry
|
|
for entry in changes
|
|
if is_document_path(
|
|
str(entry.get("path") or entry.get("previous_path") or "")
|
|
)
|
|
]
|
|
execution_documents = [
|
|
entry
|
|
for entry in execution_files
|
|
if is_document_path(str(entry["path"]))
|
|
]
|
|
|
|
documents_index = {
|
|
"schema": "aios.modified-documents.index.v1",
|
|
"task_id": task_id,
|
|
"base_oid": base,
|
|
"candidate_oid": candidate,
|
|
"repository_documents": document_changes,
|
|
"execution_documents": execution_documents,
|
|
"repository_document_count": len(document_changes),
|
|
"execution_document_count": len(execution_documents),
|
|
}
|
|
(bundle / "documents-index.json").write_text(
|
|
json.dumps(documents_index, indent=2, sort_keys=True) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
manifest = {
|
|
"schema": "aios.all-modified-files.bundle.v1",
|
|
"task_id": task_id,
|
|
"base_oid": base,
|
|
"candidate_oid": candidate,
|
|
"generated_at": (
|
|
now or datetime.now(timezone.utc)
|
|
).isoformat(),
|
|
"changed_file_count": len(changes),
|
|
"execution_file_count": len(execution_files),
|
|
"repository_document_count": len(document_changes),
|
|
"execution_document_count": len(execution_documents),
|
|
"changes": changes,
|
|
"execution_files": execution_files,
|
|
}
|
|
(bundle / "all-modified-files-manifest.json").write_text(
|
|
json.dumps(manifest, indent=2, sort_keys=True) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
write_checksums(bundle)
|
|
verify_checksums(bundle)
|
|
|
|
zip_name = (
|
|
f"AIOS_{task_id}_ALL_MODIFIED_FILES_{candidate[:12]}.zip"
|
|
.replace("/", "-")
|
|
)
|
|
zip_path = output_root / zip_name
|
|
with zipfile.ZipFile(
|
|
zip_path,
|
|
"w",
|
|
compression=zipfile.ZIP_DEFLATED,
|
|
) as archive:
|
|
for path in sorted(bundle.rglob("*")):
|
|
if path.is_symlink():
|
|
raise PackagerError(
|
|
"BUNDLE_SYMLINK_REJECTED:" + str(path)
|
|
)
|
|
if path.is_file():
|
|
archive.write(
|
|
path,
|
|
arcname=path.relative_to(bundle).as_posix(),
|
|
)
|
|
|
|
zip_sha = sha256(zip_path.read_bytes())
|
|
summary = {
|
|
"schema": "aios.all-modified-files.bundle-summary.v1",
|
|
"task_id": task_id,
|
|
"base_oid": base,
|
|
"candidate_oid": candidate,
|
|
"bundle_path": str(zip_path),
|
|
"bundle_sha256": zip_sha,
|
|
"changed_file_count": len(changes),
|
|
"execution_file_count": len(execution_files),
|
|
"repository_document_count": len(document_changes),
|
|
"execution_document_count": len(execution_documents),
|
|
}
|
|
(output_root / "ALL_MODIFIED_FILES_BUNDLE.json").write_text(
|
|
json.dumps(summary, indent=2, sort_keys=True) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
return summary
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("base")
|
|
parser.add_argument("candidate")
|
|
parser.add_argument("task_id")
|
|
parser.add_argument("extra_dir", nargs="?")
|
|
parser.add_argument(
|
|
"--authority-git-dir",
|
|
default=os.environ.get(
|
|
"AIOS_AUTHORITY_GIT_DIR",
|
|
"/data/aios-gitops/repo.git",
|
|
),
|
|
)
|
|
parser.add_argument("--output-root")
|
|
args = parser.parse_args()
|
|
|
|
try:
|
|
summary = build_bundle(
|
|
git_dir=Path(args.authority_git_dir),
|
|
base=args.base,
|
|
candidate=args.candidate,
|
|
task_id=args.task_id,
|
|
extra_dir=Path(args.extra_dir) if args.extra_dir else None,
|
|
output_root=(
|
|
Path(args.output_root) if args.output_root else None
|
|
),
|
|
)
|
|
except PackagerError as exc:
|
|
print("FATAL:" + str(exc))
|
|
return 20
|
|
|
|
print("ALL_MODIFIED_FILES_BUNDLE=PASS")
|
|
print("ALL_MODIFIED_FILES_BUNDLE_PATH=" + str(summary["bundle_path"]))
|
|
print("ALL_MODIFIED_FILES_BUNDLE_SHA256=" + str(summary["bundle_sha256"]))
|
|
print("CHANGED_FILE_COUNT=" + str(summary["changed_file_count"]))
|
|
print("EXECUTION_FILE_COUNT=" + str(summary["execution_file_count"]))
|
|
print(
|
|
"REPOSITORY_DOCUMENT_COUNT="
|
|
+ str(summary["repository_document_count"])
|
|
)
|
|
print(
|
|
"EXECUTION_DOCUMENT_COUNT="
|
|
+ str(summary["execution_document_count"])
|
|
)
|
|
print(
|
|
"ALL_MODIFIED_FILES_EVIDENCE="
|
|
+ str(Path(str(summary["bundle_path"])).parent)
|
|
)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|