hermes-agent/tests/scripts/test_archive_inputs.py

"""CI archives exact pinned inputs and supplies the real staging consumers."""
import hashlib
import importlib
import json
from pathlib import Path
import threading
from concurrent.futures import ThreadPoolExecutor

import pytest

from pm.artifact_mirror import object_key
from pm.downloader import HashError
from scripts.ci import archive_inputs as inputs
from scripts.releases import r2
from tests.scripts.test_release_r2 import r2_server  # noqa: F401
from tests.test_termux_runtime_libs import _Server, _build_deb


def write_pins(repo, packages, libs=None):
    (repo / "pm").mkdir(parents=True, exist_ok=True)
    (repo / "pm/lock.json").write_text(json.dumps({"schema": 1, "packages": packages}), encoding="utf-8")
    (repo / "scripts/termux").mkdir(parents=True, exist_ok=True)
    (repo / "scripts/termux/runtime_libs.json").write_text(json.dumps(libs or {"libs": {}}), encoding="utf-8")


@pytest.fixture
def upstream(tmp_path):
    root = tmp_path / "upstream"
    root.mkdir()
    _build_deb(root / "lib.deb", "libarchive-proof.so", b"pinned library")
    server = _Server(root)
    try:
        yield server, root
    finally:
        server.stop()


def test_target_selection_preserves_multi_archive_and_any_fallback(tmp_path):
    def row(name):
        return {"url": f"https://upstream.test/{name}.zip", "sha256": hashlib.sha256(name.encode()).hexdigest()}
    packages = {
        "engine": {"version": "1", "artifacts": {
            "win32-x64": [row("engine"), row("cudart")], "linux-x64": row("linux"),
        }},
        "portable": {"version": "1", "artifacts": {"any": row("portable")}},
        "container": {"version": "1", "artifacts": {"linux-arm64-bionic": {
            "url": "docker://termux/termux-docker@sha256:" + "a" * 64,
        }}},
    }
    lib = row("lib")
    write_pins(tmp_path, packages, {"libs": {"lib": lib}, "licenses": row("license")})
    win = inputs.pinned_inputs(tmp_path, target="win32-x64")
    assert {p.sha256 for p in win} == {row(n)["sha256"] for n in ("engine", "cudart", "portable")}
    bionic = inputs.pinned_inputs(tmp_path, target="linux-arm64-bionic")
    assert {p.sha256 for p in bionic} == {row(n)["sha256"] for n in ("portable", "lib", "license")}
    all_pins = inputs.pinned_inputs(tmp_path)
    assert {p.sha256 for p in all_pins} == {row(n)["sha256"] for n in ("engine", "cudart", "portable", "linux", "lib", "license")}


@pytest.mark.parametrize("bad", [{}, {"url": "https://u/file.zip"}, {"url": "http://u/file.zip", "sha256": "a" * 64}, {"url": "https://u/file.zip", "sha256": " A "}])
def test_invalid_pins_fail_without_rewriting_authority(tmp_path, bad):
    write_pins(tmp_path, {"bad": {"version": "1", "artifacts": {"any": bad}}})
    before = (tmp_path / "pm/lock.json").read_bytes()
    with pytest.raises(ValueError):
        inputs.pinned_inputs(tmp_path)
    assert (tmp_path / "pm/lock.json").read_bytes() == before


def test_real_cli_miss_hit_and_staging_use_the_same_archived_bytes(tmp_path, upstream, r2_server, monkeypatch):
    from pm import paths
    from pm.store import Store
    from scripts.termux.stage_runtime_libs import stage
    from tests.pm.test_stage_only import _FakePackage

    server, root = upstream
    body = (root / "lib.deb").read_bytes()
    digest = hashlib.sha256(body).hexdigest()
    row = {"url": server.url + "/lib.deb", "sha256": digest, "version": "1"}
    repo = tmp_path / "repo"
    write_pins(repo, {"stage-test": {"version": "1", "artifacts": {"linux-arm64-bionic": row}}}, {"libs": {"lib": row}})
    monkeypatch.setattr(paths, "repo_root", lambda: repo)
    monkeypatch.setattr(paths, "lockfile_path", lambda: repo / "pm/lock.json")
    assert inputs.main([]) == 0
    assert r2_server.store[object_key(digest)][0] == body
    puts = [r for r in r2_server.requests if r[0] == "PUT"]
    assert len(puts) == 1 and puts[0][2]["If-None-Match"] == "*"
    assert r2_server.requests[-1][0] == "GET"
    assert not (repo / ".archive-inputs").exists()
    (root / "lib.deb").unlink()
    r2_server.requests.clear()
    store = Store(tmp_path / "tools")
    payload = tmp_path / "payload"
    assert inputs.main(["--target", "linux-arm64-bionic", "--store", str(store.root), "--payload", str(payload)]) == 0
    assert not any(r[0] == "PUT" for r in r2_server.requests)
    assert (stage(payload, {"lib": row}) / "libarchive-proof.so").read_bytes().endswith(b"pinned library")
    engine = importlib.import_module("pm.ensure")
    monkeypatch.setattr(engine, "_store", lambda: store)
    monkeypatch.setattr(engine, "get_package", lambda _: _FakePackage())
    entry = engine.stage_only("stage-test", "linux-arm64-bionic")
    assert (entry / "bin/tool").read_bytes() == body
    assert not store.entry(f"fetch-{digest}").exists()
    assert r2.canary_doomed_keys([object_key(digest)], "99999999") == []


@pytest.mark.parametrize("failure", ["upstream-hash", "r2-hash", "r2-permission", "readback"])
def test_corrupt_or_denied_archive_never_publishes_a_destination(tmp_path, upstream, r2_server, monkeypatch, failure):
    server, root = upstream
    body = (root / "lib.deb").read_bytes()
    digest = hashlib.sha256(body).hexdigest()
    pin = inputs.InputPin("lib", server.url + "/lib.deb", digest, "library")
    key = object_key(digest)
    if failure == "upstream-hash":
        (root / "lib.deb").write_bytes(b"corrupt")
    elif failure == "r2-hash":
        r2_server.store[key] = (b"corrupt", '"etag"')
    elif failure == "readback":
        original = r2.put_object
        def corrupt(*args, **kwargs):
            original(*args, **kwargs)
            r2_server.store[key] = (b"x" * len(body), '"etag"')
        monkeypatch.setattr(r2, "put_object", corrupt)
    else:
        original = r2.signed_request
        def deny(method, url, **kwargs):
            if method == "HEAD":
                raise r2.R2RequestError(method, url, 403)
            return original(method, url, **kwargs)
        monkeypatch.setattr(r2, "signed_request", deny)
    dest = tmp_path / "preserved"
    dest.write_bytes(b"old")
    with pytest.raises((HashError, ValueError, r2.R2RequestError)):
        inputs.Archive(*r2.credentials()).fetch(pin, dest)
    assert dest.read_bytes() == b"old"
    assert any(r[0] == "PUT" for r in r2_server.requests) == (failure == "readback")


def test_racing_misses_verify_the_immutable_winner(tmp_path, upstream, r2_server, monkeypatch):
    server, root = upstream
    body = (root / "lib.deb").read_bytes()
    pin = inputs.InputPin("lib", server.url + "/lib.deb", hashlib.sha256(body).hexdigest(), "library")
    barrier = threading.Barrier(2)
    original = r2.signed_request
    def race(method, url, **kwargs):
        try:
            return original(method, url, **kwargs)
        except r2.R2RequestError as exc:
            if method == "HEAD" and exc.status == 404:
                barrier.wait(timeout=10)
            raise
    monkeypatch.setattr(r2, "signed_request", race)
    archive = inputs.Archive(*r2.credentials())
    paths = [tmp_path / f"race-{i}" for i in range(2)]
    with ThreadPoolExecutor(max_workers=2) as pool:
        assert list(pool.map(lambda p: archive.fetch(pin, p), paths)) == ["upstream", "upstream"]
    assert all(p.read_bytes() == body for p in paths)


def test_historical_recovery_keeps_the_original_digest(tmp_path, upstream, r2_server, monkeypatch):
    server, root = upstream
    body = (root / "lib.deb").read_bytes()
    pin = inputs.InputPin("lib", server.url + "/gone.deb", hashlib.sha256(body).hexdigest(), "library")
    monkeypatch.setattr(inputs, "historical_url", lambda _: server.url + "/lib.deb")
    dest = tmp_path / "recovered"
    assert inputs.Archive(*r2.credentials()).fetch(pin, dest) == "historical archive"
    assert dest.read_bytes() == body


def test_committed_inventory_matches_every_http_pin():
    repo = Path(__file__).resolve().parents[2]
    lock = json.loads((repo / "pm/lock.json").read_text(encoding="utf-8"))
    expected = set()
    for package in lock["packages"].values():
        for artifact in package.get("artifacts", {}).values():
            for row in artifact if isinstance(artifact, list) else [artifact]:
                if row["url"].startswith("https://"):
                    expected.add(row["sha256"])
    table = json.loads((repo / "scripts/termux/runtime_libs.json").read_text(encoding="utf-8"))
    expected.update(row["sha256"] for row in table["libs"].values())
    expected.add(table["licenses"]["sha256"])
    assert {p.sha256 for p in inputs.pinned_inputs(repo)} == expected


def test_ci_toolchain_seed_runs_before_the_tool_installer(tmp_path, upstream, r2_server, monkeypatch):
    from types import SimpleNamespace
    from scripts.ci import setup_toolchain
    from pm import paths
    from pm.store import Store, current_target

    server, root = upstream
    body = (root / "lib.deb").read_bytes()
    digest = hashlib.sha256(body).hexdigest()
    target = current_target()
    row = {"url": server.url + "/lib.deb", "sha256": digest}
    packages = {name: {"version": "1", "artifacts": {target: row}} for name in ("python", "uv")}
    repo = tmp_path / "repo"
    write_pins(repo, packages)
    monkeypatch.setattr(paths, "repo_root", lambda: repo)
    home = tmp_path / "ci-home"
    setup_toolchain.archive_inputs(SimpleNamespace(home=home, toolchain="python"))
    (root / "lib.deb").unlink()
    store = Store(home / "tools")
    with store.scratch() as scratch:
        assert store.fetch(row["url"], digest, scratch).read_bytes() == body