9f2ba1b74d
Branch semantics kept where main and PM disagree: update_cmd_deps.py, constraints-termux.txt, the Electron update-api-check module and the post-swap hand-off test stay deleted; the pending-fleet-restart catch-up and the local_runtime tag/download ladder stay retired (PM owns engines). Ported from main onto the branch's shape: profile_scoped_chore for the auto-archive and plugin-update housekeeping chores, the local-runtime cross-process boot lock and residency cap, the checkpoint tmp_pack sweep, the cua daemon-liveness status probe, the remote-served Desktop update flag (posix.sh / windows.ps1), sign-in for env-pinned remote gateways (urlDisabled on RemoteSetupFields), the uvloop extra split (uvicorn without [standard]), and the umask-scoping spawn test. uv.lock regenerated with pm.build_env --lock-only; new utf-8 reads from main switched to utf-8-sig (check-windows-footguns).
308 lines
11 KiB
Python
308 lines
11 KiB
Python
"""Behavior tests for scripts/validate_plugin_catalog.py.
|
|
|
|
The script is the no-install structural validator used by the plugin-catalog
|
|
admission CI: it must run with only stdlib + ruamel.yaml, take file paths or a
|
|
directory, exit 0/1, and support --json machine output. These tests exercise
|
|
the CLI contract via subprocess (the same way CI invokes it).
|
|
"""
|
|
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import hermes_yaml as yaml
|
|
import pytest
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
SCRIPT = REPO_ROOT / "scripts" / "validate_plugin_catalog.py"
|
|
|
|
VALID_ENTRY = {
|
|
"name": "example-plugin",
|
|
"repo": "https://github.com/NousResearch/hermes-example-plugins",
|
|
"sha": "38fe0fb53eff98d477f807432e965429e665ca33",
|
|
"subdir": "",
|
|
"description": "One-line description.",
|
|
"maintainer": "NousResearch",
|
|
"tier": "official",
|
|
"requires_hermes": ">=0.19",
|
|
"docs_url": "",
|
|
"platforms": [],
|
|
"capabilities": {
|
|
"provides_tools": ["example_tool"],
|
|
"provides_hooks": [],
|
|
"provides_middleware": [],
|
|
"requires_env": [],
|
|
},
|
|
}
|
|
|
|
|
|
def write_entry(tmp_path: Path, data: dict, filename: str | None = None) -> Path:
|
|
name = filename or f"{data.get('name', 'entry')}.yaml"
|
|
path = tmp_path / name
|
|
path.write_text(yaml.safe_dump(data), encoding="utf-8")
|
|
return path
|
|
|
|
|
|
def run_validator(*args: str) -> subprocess.CompletedProcess:
|
|
return subprocess.run(
|
|
[sys.executable, str(SCRIPT), *args],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
|
|
# ── valid input ────────────────────────────────────────────────────────
|
|
|
|
|
|
@pytest.mark.parametrize("bom", [b"", b"\xef\xbb\xbf"])
|
|
def test_valid_entry_passes(tmp_path, bom):
|
|
path = write_entry(tmp_path, {**VALID_ENTRY, "description": "café 東京"})
|
|
path.write_bytes(bom + path.read_bytes())
|
|
result = run_validator(str(path))
|
|
assert result.returncode == 0, result.stdout + result.stderr
|
|
|
|
|
|
def test_valid_entry_without_optional_fields_passes(tmp_path):
|
|
entry = {
|
|
"name": "minimal-plugin",
|
|
"repo": "https://github.com/example/minimal",
|
|
"sha": "a" * 40,
|
|
"description": "Minimal.",
|
|
"maintainer": "someone",
|
|
}
|
|
path = write_entry(tmp_path, entry)
|
|
result = run_validator(str(path))
|
|
assert result.returncode == 0, result.stdout + result.stderr
|
|
|
|
|
|
# ── each malformed field fails with a pointed error ────────────────────
|
|
|
|
|
|
def _expect_error(tmp_path, mutation: dict, expected_substring: str, drop: str = ""):
|
|
entry = {**VALID_ENTRY, **mutation}
|
|
if drop:
|
|
entry.pop(drop, None)
|
|
path = write_entry(tmp_path, entry, filename="entry.yaml")
|
|
result = run_validator(str(path))
|
|
combined = result.stdout + result.stderr
|
|
assert result.returncode == 1, combined
|
|
assert expected_substring in combined, combined
|
|
assert "entry.yaml" in combined, combined
|
|
|
|
|
|
def test_unknown_category_fails(tmp_path):
|
|
path = write_entry(tmp_path, {**VALID_ENTRY, "category": "memmory"})
|
|
result = run_validator(str(path))
|
|
assert result.returncode != 0
|
|
assert "category" in result.stdout + result.stderr
|
|
path = write_entry(tmp_path, {**VALID_ENTRY, "category": "memory"}, "ok.yaml")
|
|
assert run_validator(str(path)).returncode == 0
|
|
|
|
|
|
def test_version_and_image_are_validated_when_present(tmp_path):
|
|
"""Admission rejects a malformed label or an off-GitHub image so the site and CLI never have to
|
|
coerce one; a well-formed pair passes."""
|
|
_expect_error(tmp_path, {"image": "https://cdn.example.com/banner.png"}, "image")
|
|
_expect_error(tmp_path, {"version": "1.4.0 beta"}, "version")
|
|
path = write_entry(tmp_path, {**VALID_ENTRY, "version": "1.4.0", "image": "https://raw.githubusercontent.com/owner/repo/38fe0fb53eff98d477f807432e965429e665ca33/banner.png"}, "ok.yaml")
|
|
assert run_validator(str(path)).returncode == 0
|
|
|
|
|
|
def test_screenshots_and_readme_are_validated_when_present(tmp_path):
|
|
"""Page fields: screenshots follow the image host rule and are capped; readme is a bool and needs a
|
|
forge the site can fetch raw files from at the pinned commit."""
|
|
shot = "https://raw.githubusercontent.com/owner/repo/38fe0fb53eff98d477f807432e965429e665ca33/docs/1.png"
|
|
_expect_error(tmp_path, {"screenshots": [shot, "https://cdn.example.com/2.png"]}, "screenshots")
|
|
_expect_error(tmp_path, {"screenshots": shot}, "screenshots")
|
|
_expect_error(tmp_path, {"screenshots": [shot] * 7}, "at most 6")
|
|
_expect_error(tmp_path, {"readme": "yes"}, "readme")
|
|
_expect_error(tmp_path, {"readme": True, "repo": "https://codeberg.org/owner/repo"}, "readme: true needs")
|
|
path = write_entry(tmp_path, {**VALID_ENTRY, "screenshots": [shot, shot], "readme": True}, "ok.yaml")
|
|
assert run_validator(str(path)).returncode == 0
|
|
|
|
|
|
def test_bad_name_fails(tmp_path):
|
|
_expect_error(tmp_path, {"name": "Bad Name!"}, "name")
|
|
|
|
|
|
def test_name_too_long_fails(tmp_path):
|
|
_expect_error(tmp_path, {"name": "x" * 65}, "name")
|
|
|
|
|
|
def test_non_https_repo_fails(tmp_path):
|
|
_expect_error(tmp_path, {"repo": "git@github.com:evil/x.git"}, "repo")
|
|
|
|
|
|
def test_short_sha_fails(tmp_path):
|
|
_expect_error(tmp_path, {"sha": "abc123"}, "sha")
|
|
|
|
|
|
def test_non_hex_sha_fails(tmp_path):
|
|
_expect_error(tmp_path, {"sha": "z" * 40}, "sha")
|
|
|
|
|
|
def test_bad_tier_fails(tmp_path):
|
|
_expect_error(tmp_path, {"tier": "platinum"}, "tier")
|
|
|
|
|
|
def test_empty_description_fails(tmp_path):
|
|
_expect_error(tmp_path, {"description": ""}, "description")
|
|
|
|
|
|
def test_empty_maintainer_fails(tmp_path):
|
|
_expect_error(tmp_path, {"maintainer": ""}, "maintainer")
|
|
|
|
|
|
def test_missing_required_field_fails(tmp_path):
|
|
_expect_error(tmp_path, {}, "sha", drop="sha")
|
|
|
|
|
|
def test_capabilities_value_not_a_list_fails(tmp_path):
|
|
_expect_error(
|
|
tmp_path,
|
|
{"capabilities": {"provides_tools": "not-a-list"}},
|
|
"provides_tools",
|
|
)
|
|
|
|
|
|
def test_capabilities_list_of_non_strings_fails(tmp_path):
|
|
_expect_error(
|
|
tmp_path,
|
|
{"capabilities": {"requires_env": [1, 2]}},
|
|
"requires_env",
|
|
)
|
|
|
|
|
|
def test_bad_requires_hermes_spec_fails(tmp_path):
|
|
_expect_error(tmp_path, {"requires_hermes": "banana"}, "requires_hermes")
|
|
|
|
|
|
def test_comma_separated_requires_hermes_passes(tmp_path):
|
|
entry = {**VALID_ENTRY, "requires_hermes": ">=0.19, <2.0"}
|
|
path = write_entry(tmp_path, entry)
|
|
result = run_validator(str(path))
|
|
assert result.returncode == 0, result.stdout + result.stderr
|
|
|
|
|
|
def test_unknown_platform_fails(tmp_path):
|
|
_expect_error(tmp_path, {"platforms": ["linux", "amiga"]}, "platforms")
|
|
|
|
|
|
def test_entry_not_a_mapping_fails(tmp_path):
|
|
path = tmp_path / "entry.yaml"
|
|
path.write_text("- just\n- a\n- list\n", encoding="utf-8")
|
|
result = run_validator(str(path))
|
|
assert result.returncode == 1
|
|
assert "mapping" in (result.stdout + result.stderr)
|
|
|
|
|
|
# ── unknown top-level keys warn but do not fail ────────────────────────
|
|
|
|
|
|
def test_unknown_key_warns_but_passes(tmp_path):
|
|
entry = {**VALID_ENTRY, "future_field": "hello"}
|
|
path = write_entry(tmp_path, entry)
|
|
result = run_validator(str(path))
|
|
combined = result.stdout + result.stderr
|
|
assert result.returncode == 0, combined
|
|
assert "future_field" in combined
|
|
assert "warning" in combined.lower()
|
|
|
|
|
|
# ── removed.yaml shape ─────────────────────────────────────────────────
|
|
|
|
|
|
def test_valid_removed_yaml_passes(tmp_path):
|
|
path = tmp_path / "removed.yaml"
|
|
path.write_text(
|
|
yaml.safe_dump(
|
|
{
|
|
"removed": [
|
|
{
|
|
"name": "some-plugin",
|
|
"repo": "https://github.com/evil/some-plugin",
|
|
"reason": "Exfiltrated env vars",
|
|
"date": "2026-07-02",
|
|
}
|
|
]
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
result = run_validator(str(path))
|
|
assert result.returncode == 0, result.stdout + result.stderr
|
|
|
|
|
|
def test_removed_yaml_not_a_list_fails(tmp_path):
|
|
path = tmp_path / "removed.yaml"
|
|
path.write_text(yaml.safe_dump({"removed": "nope"}), encoding="utf-8")
|
|
result = run_validator(str(path))
|
|
assert result.returncode == 1
|
|
assert "removed" in (result.stdout + result.stderr)
|
|
|
|
|
|
def test_removed_item_missing_name_fails(tmp_path):
|
|
path = tmp_path / "removed.yaml"
|
|
path.write_text(
|
|
yaml.safe_dump({"removed": [{"reason": "bad", "date": "2026-01-01"}]}),
|
|
encoding="utf-8",
|
|
)
|
|
result = run_validator(str(path))
|
|
assert result.returncode == 1
|
|
assert "name" in (result.stdout + result.stderr)
|
|
|
|
|
|
# ── --json machine output ──────────────────────────────────────────────
|
|
|
|
|
|
def test_json_output_shape_on_failure(tmp_path):
|
|
bad = write_entry(tmp_path, {**VALID_ENTRY, "sha": "short"}, filename="bad.yaml")
|
|
result = run_validator("--json", str(bad))
|
|
assert result.returncode == 1
|
|
payload = json.loads(result.stdout)
|
|
assert payload["ok"] is False
|
|
assert isinstance(payload["files"], list)
|
|
entry = next(f for f in payload["files"] if f["path"].endswith("bad.yaml"))
|
|
assert entry["ok"] is False
|
|
assert any("sha" in e for e in entry["errors"])
|
|
|
|
|
|
def test_json_output_shape_on_success_with_warning(tmp_path):
|
|
good = write_entry(tmp_path, {**VALID_ENTRY, "future_field": 1})
|
|
result = run_validator("--json", str(good))
|
|
assert result.returncode == 0
|
|
payload = json.loads(result.stdout)
|
|
assert payload["ok"] is True
|
|
(entry,) = payload["files"]
|
|
assert entry["ok"] is True
|
|
assert entry["errors"] == []
|
|
assert any("future_field" in w for w in entry["warnings"])
|
|
|
|
|
|
# ── directory mode ─────────────────────────────────────────────────────
|
|
|
|
|
|
def test_directory_mode_validates_all_entries_and_removed(tmp_path):
|
|
write_entry(tmp_path, VALID_ENTRY)
|
|
write_entry(tmp_path, {**VALID_ENTRY, "name": "bad-one", "sha": "nope"})
|
|
(tmp_path / "removed.yaml").write_text(
|
|
yaml.safe_dump({"removed": [{"name": "gone", "reason": "test"}]}),
|
|
encoding="utf-8",
|
|
)
|
|
result = run_validator(str(tmp_path))
|
|
combined = result.stdout + result.stderr
|
|
assert result.returncode == 1
|
|
assert "bad-one.yaml" in combined
|
|
# the valid entry and removed.yaml must not produce errors
|
|
assert combined.count("ERROR") == combined.count("bad-one.yaml: ERROR")
|
|
|
|
|
|
def test_directory_mode_all_valid_exits_zero(tmp_path):
|
|
write_entry(tmp_path, VALID_ENTRY)
|
|
(tmp_path / "removed.yaml").write_text(
|
|
yaml.safe_dump({"removed": []}), encoding="utf-8"
|
|
)
|
|
result = run_validator(str(tmp_path))
|
|
assert result.returncode == 0, result.stdout + result.stderr
|