test(auth): OpenRouter PKCE invariants, fake-authority A/B harness, docs, contributor map
Two invariant tests (red on main): the PKCE key lands as an api_key pool row that
resolve_provider("auto") picks up while the bare --api-key path keeps its default, and a forged
callback path is a 404 while the genuine nonce path yields the code. evals/openrouter_pkce_ab
drives the real auth_add_command against a local fake /api/v1/auth/keys (verifier check,
single-use codes) for legit / wrong-state / replayed-code / malformed-response / api-key-path.
This commit is contained in:
Teknium
2
contributors/emails/aakash.j.abraham@gmail.com
Normal file
2
contributors/emails/aakash.j.abraham@gmail.com
Normal file
@@ -0,0 +1,2 @@
|
||||
nyx573
|
||||
# PR #102639 salvage (OpenRouter OAuth PKCE)
|
||||
194
evals/openrouter_pkce_ab/harness.py
Normal file
194
evals/openrouter_pkce_ab/harness.py
Normal file
@@ -0,0 +1,194 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Local A/B harness for `hermes auth add openrouter --type oauth` against a FAKE OpenRouter.
|
||||
|
||||
Runs the REAL entry point (``hermes_cli.auth_commands.auth_add_command``) with a temp HERMES_HOME.
|
||||
Only the network authority is replaced: ``webbrowser.open`` is swapped for a scripted "browser"
|
||||
that follows the auth URL's ``callback_url`` the way openrouter.ai would (redirecting the loopback
|
||||
listener with ``?code=``), and ``OPENROUTER_AUTH_KEYS_URL`` points at a local fake code-exchange
|
||||
server that enforces OpenRouter's documented contract (S256 verifier check, single-use code, 403).
|
||||
|
||||
Scenarios (each prints PASS/FAIL, exit code = number of failures):
|
||||
legit full flow → pool entry written, auth_type=api_key, source=manual:openrouter_pkce
|
||||
wrong_state browser redirects to a different callback path (forged nonce) → 404, no exchange
|
||||
replayed_code code already consumed at the fake server → 403 → AuthError, nothing persisted
|
||||
malformed exchange returns JSON without "key" → AuthError, nothing persisted
|
||||
api_key_path `hermes auth add openrouter --api-key` still works with no --type (regression)
|
||||
|
||||
Usage: HERMES_PYTHON=<venv python> python3 evals/openrouter_pkce_ab/harness.py [--json OUT]
|
||||
Run against origin/main to see the BEFORE state (every oauth scenario fails with SystemExit
|
||||
"not implemented"), then against the salvage branch for AFTER.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from types import SimpleNamespace
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.path.insert(0, REPO)
|
||||
|
||||
|
||||
class FakeOpenRouter(ThreadingHTTPServer):
|
||||
"""Fake ``POST /api/v1/auth/keys`` implementing the published contract."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(("127.0.0.1", 0), _Handler)
|
||||
self.issued: dict[str, str] = {} # code -> code_challenge
|
||||
self.consumed: set[str] = set()
|
||||
self.mode = "ok" # ok | malformed
|
||||
self.exchanges: list[dict] = []
|
||||
|
||||
@property
|
||||
def url(self):
|
||||
return f"http://127.0.0.1:{self.server_address[1]}/api/v1/auth/keys"
|
||||
|
||||
|
||||
class _Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, *a): # noqa: A003
|
||||
return
|
||||
|
||||
def do_POST(self): # noqa: N802
|
||||
srv: FakeOpenRouter = self.server # type: ignore[assignment]
|
||||
body = json.loads(self.rfile.read(int(self.headers.get("Content-Length", "0")) or 0) or b"{}")
|
||||
srv.exchanges.append(body)
|
||||
code, verifier, method = body.get("code"), body.get("code_verifier", ""), body.get("code_challenge_method")
|
||||
if method not in ("S256", "plain", None):
|
||||
return self._json(400, {"error": {"code": 400, "message": "Invalid code_challenge_method"}})
|
||||
if code not in srv.issued or code in srv.consumed:
|
||||
return self._json(403, {"error": {"code": 403, "message": "Invalid code or code_verifier"}})
|
||||
expected = base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest()).decode().rstrip("=")
|
||||
if expected != srv.issued[code]:
|
||||
return self._json(403, {"error": {"code": 403, "message": "Invalid code or code_verifier"}})
|
||||
srv.consumed.add(code)
|
||||
if srv.mode == "malformed":
|
||||
return self._json(200, {"user_id": "user_x"})
|
||||
return self._json(200, {"key": "sk-or-v1-" + hashlib.sha256(code.encode()).hexdigest(), "user_id": "user_x"})
|
||||
|
||||
def _json(self, status, payload):
|
||||
raw = json.dumps(payload).encode()
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(raw)))
|
||||
self.end_headers()
|
||||
self.wfile.write(raw)
|
||||
|
||||
|
||||
def scripted_browser(fake: FakeOpenRouter, *, tamper_path=False, pre_consume=False):
|
||||
"""Return a ``webbrowser.open`` stand-in that behaves like openrouter.ai/auth after user consent."""
|
||||
def _open(url):
|
||||
q = parse_qs(urlparse(url).query)
|
||||
callback = q["callback_url"][0]
|
||||
code = "auth_code_" + hashlib.sha1(callback.encode()).hexdigest()[:12]
|
||||
fake.issued[code] = q["code_challenge"][0]
|
||||
if pre_consume:
|
||||
fake.consumed.add(code)
|
||||
if tamper_path: # attacker guesses the port but not the nonce path
|
||||
p = urlparse(callback)
|
||||
callback = f"{p.scheme}://{p.netloc}/callback/forged-nonce"
|
||||
def _redirect():
|
||||
try:
|
||||
with urllib.request.urlopen(f"{callback}?code={code}", timeout=5) as r:
|
||||
_open.last_status = r.status
|
||||
except urllib.error.HTTPError as e:
|
||||
_open.last_status = e.code
|
||||
except Exception as e: # listener already closed
|
||||
_open.last_status = repr(e)
|
||||
threading.Thread(target=_redirect, daemon=True).start()
|
||||
return True
|
||||
_open.last_status = None
|
||||
return _open
|
||||
|
||||
|
||||
def run(scenario: str, fake: FakeOpenRouter, home: str) -> dict:
|
||||
os.environ["HERMES_HOME"] = home
|
||||
for k in ("OPENROUTER_API_KEY", "OPENAI_API_KEY", "SSH_CLIENT", "SSH_TTY"):
|
||||
os.environ.pop(k, None)
|
||||
for m in [m for m in sys.modules if m.startswith(("hermes_cli", "agent", "hermes_constants"))]:
|
||||
del sys.modules[m]
|
||||
fake.mode = "malformed" if scenario == "malformed" else "ok"
|
||||
browser = scripted_browser(fake, tamper_path=(scenario == "wrong_state"), pre_consume=(scenario == "replayed_code"))
|
||||
outcome = {"scenario": scenario, "exchanges_before": len(fake.exchanges)}
|
||||
try:
|
||||
from hermes_cli.auth_commands import auth_add_command
|
||||
except Exception as e: # e.g. the original PR branch's auth.py fails at import time
|
||||
outcome.update(result=f"IMPORT FAILURE {type(e).__name__}: {e}", browser_redirect_status=None,
|
||||
exchange_calls=0, pool_entries=[])
|
||||
outcome.pop("exchanges_before")
|
||||
return outcome
|
||||
try: # BEFORE (origin/main) has no auth_openrouter sibling; the flow itself must then fail.
|
||||
import hermes_cli.auth_openrouter as orm
|
||||
import hermes_cli.auth_device_flow as dfl
|
||||
orm.OPENROUTER_AUTH_KEYS_URL = fake.url
|
||||
dfl._can_open_graphical_browser = lambda: True
|
||||
orm.webbrowser.open = browser
|
||||
except ImportError:
|
||||
pass
|
||||
args = SimpleNamespace(provider="openrouter", auth_type="oauth", label="pkce-test", api_key=None,
|
||||
no_browser=False, timeout=6)
|
||||
if scenario == "api_key_path":
|
||||
args = SimpleNamespace(provider="openrouter", auth_type=None, label="plain", api_key="sk-or-v1-manual")
|
||||
try:
|
||||
auth_add_command(args)
|
||||
outcome["result"] = "ok"
|
||||
except SystemExit as e:
|
||||
outcome["result"] = f"SystemExit: {e}"
|
||||
except Exception as e: # AuthError etc.
|
||||
outcome["result"] = f"{type(e).__name__}({getattr(e, 'code', '')}): {e}"
|
||||
outcome["browser_redirect_status"] = browser.last_status
|
||||
outcome["exchange_calls"] = len(fake.exchanges) - outcome.pop("exchanges_before")
|
||||
auth_json = os.path.join(home, "auth.json")
|
||||
entries = []
|
||||
if os.path.exists(auth_json):
|
||||
entries = json.load(open(auth_json, encoding="utf-8")).get("credential_pool", {}).get("openrouter", [])
|
||||
outcome["pool_entries"] = [{k: e.get(k) for k in ("auth_type", "source", "label", "base_url")}
|
||||
| {"key_prefix": str(e.get("access_token", ""))[:9]} for e in entries]
|
||||
return outcome
|
||||
|
||||
|
||||
EXPECT = {
|
||||
"legit": lambda o: o["result"] == "ok" and o["exchange_calls"] == 1 and any(
|
||||
e["auth_type"] == "api_key" and e["source"] == "manual:openrouter_pkce" and e["key_prefix"] == "sk-or-v1-"
|
||||
for e in o["pool_entries"]),
|
||||
"wrong_state": lambda o: o["browser_redirect_status"] == 404 and o["exchange_calls"] == 0
|
||||
and "openrouter_callback_timeout" in o["result"] and not o["pool_entries"],
|
||||
"replayed_code": lambda o: "openrouter_token_exchange_denied" in o["result"] and not o["pool_entries"],
|
||||
"malformed": lambda o: "openrouter_token_exchange_invalid" in o["result"] and not o["pool_entries"],
|
||||
"api_key_path": lambda o: o["result"] == "ok" and o["exchange_calls"] == 0 and any(
|
||||
e["auth_type"] == "api_key" and e["source"] == "manual" for e in o["pool_entries"]),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--json")
|
||||
ap.add_argument("--only", nargs="*")
|
||||
a = ap.parse_args()
|
||||
fake = FakeOpenRouter()
|
||||
threading.Thread(target=fake.serve_forever, daemon=True).start()
|
||||
results, fails = [], 0
|
||||
for scenario in a.only or EXPECT:
|
||||
home = tempfile.mkdtemp(prefix=f"or-pkce-{scenario}-")
|
||||
o = run(scenario, fake, home)
|
||||
o["pass"] = bool(EXPECT[scenario](o))
|
||||
fails += not o["pass"]
|
||||
print(("PASS" if o["pass"] else "FAIL"), json.dumps(o))
|
||||
results.append(o)
|
||||
fake.shutdown()
|
||||
if a.json:
|
||||
with open(a.json, "w", encoding="utf-8") as f:
|
||||
json.dump(results, f, indent=1)
|
||||
sys.exit(fails)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1166,3 +1166,84 @@ def test_qwen_oauth_login_marks_active_through_moved_owner(monkeypatch):
|
||||
|
||||
assert auth_commands._qwen_oauth_login(None) is creds
|
||||
assert marked == [creds]
|
||||
|
||||
|
||||
def test_auth_add_openrouter_oauth_persists_pkce_key_without_touching_api_key_default(tmp_path, monkeypatch):
|
||||
"""`hermes auth add openrouter --type oauth` stores the PKCE-minted key as an ``api_key`` pool row
|
||||
(OpenRouter returns a plain key, no refresh pair) that ``resolve_provider("auto")`` picks up with no
|
||||
env var — same as a pasted key; the bare `--api-key` path keeps its API-key default."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes"))
|
||||
monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
||||
_write_auth_store(tmp_path, {"version": 1, "providers": {}})
|
||||
monkeypatch.setattr("hermes_cli.auth._openrouter_pkce_login", lambda **kw: {"api_key": "sk-or-v1-from-pkce"})
|
||||
|
||||
from hermes_cli.auth import resolve_provider
|
||||
from hermes_cli.auth_commands import auth_add_command
|
||||
|
||||
class _Oauth:
|
||||
provider = "openrouter"
|
||||
auth_type = "oauth"
|
||||
api_key = None
|
||||
label = "browser-login"
|
||||
timeout = None
|
||||
no_browser = True
|
||||
|
||||
class _Plain:
|
||||
provider = "openrouter"
|
||||
auth_type = None # no --type: must NOT fall into the OAuth flow
|
||||
api_key = "sk-or-v1-pasted"
|
||||
label = "pasted"
|
||||
|
||||
auth_add_command(_Oauth())
|
||||
# No env var, no config.yaml provider: the pooled PKCE key alone must make openrouter resolvable.
|
||||
assert resolve_provider("auto") == "openrouter"
|
||||
auth_add_command(_Plain())
|
||||
|
||||
payload = json.loads((tmp_path / "hermes" / "auth.json").read_text())
|
||||
by_source = {e["source"]: e for e in payload["credential_pool"]["openrouter"]}
|
||||
assert by_source["manual:openrouter_pkce"]["auth_type"] == "api_key"
|
||||
assert by_source["manual:openrouter_pkce"]["access_token"] == "sk-or-v1-from-pkce"
|
||||
assert by_source["manual:openrouter_pkce"]["base_url"] == "https://openrouter.ai/api/v1"
|
||||
assert by_source["manual"]["access_token"] == "sk-or-v1-pasted"
|
||||
|
||||
|
||||
def test_openrouter_loopback_callback_binds_nonce_path_and_rejects_forged_redirect(monkeypatch):
|
||||
"""The CSRF nonce lives in the callback PATH (OpenRouter echoes no ``state``): a redirect that
|
||||
knows the port but not the nonce is a 404 and never yields a code; the genuine path does."""
|
||||
import threading
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
import hermes_cli.auth_openrouter as orm
|
||||
|
||||
seen: dict = {}
|
||||
|
||||
def _browser(url):
|
||||
callback = urllib.parse.parse_qs(urllib.parse.urlparse(url).query)["callback_url"][0]
|
||||
seen["callback"] = callback
|
||||
forged = callback.rsplit("/", 1)[0] + "/forged-nonce?code=evil"
|
||||
|
||||
def _redirects():
|
||||
try:
|
||||
urllib.request.urlopen(forged, timeout=5)
|
||||
except urllib.error.HTTPError as exc:
|
||||
seen["forged_status"] = exc.code
|
||||
with urllib.request.urlopen(f"{callback}?code=good-code", timeout=5) as resp:
|
||||
seen["genuine_status"] = resp.status
|
||||
|
||||
threading.Thread(target=_redirects, daemon=True).start()
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(orm, "_can_open_graphical_browser", lambda: True)
|
||||
monkeypatch.setattr(orm.webbrowser, "open", _browser)
|
||||
|
||||
code = orm._openrouter_loopback_code(
|
||||
{"code_challenge": "c", "code_challenge_method": "S256"}, open_browser=True, timeout_seconds=10)
|
||||
|
||||
parsed = urllib.parse.urlparse(seen["callback"])
|
||||
assert parsed.hostname == "127.0.0.1" and parsed.path.startswith("/callback/") and len(parsed.path) > 20
|
||||
assert seen["forged_status"] == 404
|
||||
assert seen["genuine_status"] == 200
|
||||
assert code == "good-code"
|
||||
|
||||
@@ -39,6 +39,7 @@ Hermes prints the exact port it bound to on the `Waiting for callback on ...` li
|
||||
| `anthropic` (Claude Pro/Max) | n/a | No — paste-the-code flow |
|
||||
| `openai-codex` (ChatGPT Plus/Pro) | n/a | No — device code flow |
|
||||
| `minimax`, `nous-portal` | n/a | No — device code flow |
|
||||
| `openrouter` (`hermes auth add openrouter --type oauth`) | OS-assigned, local only | No — over SSH Hermes switches to OpenRouter's headless flow and asks you to paste the code shown in the browser |
|
||||
|
||||
If your provider isn't in the table, you don't need a tunnel.
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ You need at least one way to connect to an LLM. Use `hermes model` to switch pro
|
||||
| **GitHub Copilot** | `hermes model` (OAuth device code flow, `COPILOT_GITHUB_TOKEN`, `GH_TOKEN`, or `gh auth token`) |
|
||||
| **GitHub Copilot ACP** | `hermes model` (spawns local `copilot --acp --stdio`) |
|
||||
| **Anthropic** | `hermes model` (Claude Max + extra usage credits via OAuth; also supports Anthropic API key or manual setup-token — see note below) |
|
||||
| **OpenRouter** | `OPENROUTER_API_KEY` in `~/.hermes/.env` |
|
||||
| **OpenRouter** | `OPENROUTER_API_KEY` in `~/.hermes/.env`, or `hermes auth add openrouter --type oauth` (browser login via OpenRouter's PKCE flow; stores a key in the credential pool) |
|
||||
| **Ramp Router** | `RAMP_ROUTER_API_KEY` in `~/.hermes/.env` (provider: `router`; aliases: `ramp-router`, `ramp`, `router.com`; Responses-native gateway, live account-scoped catalog) |
|
||||
| **Fireworks AI** | `FIREWORKS_API_KEY` in `~/.hermes/.env` (provider: `fireworks`; aliases: `fireworks-ai`, `fw`) |
|
||||
| **NovitaAI** | `NOVITA_API_KEY` in `~/.hermes/.env` (provider: `novita`, 200+ models, Model API, Agent Sandbox, GPU Cloud) |
|
||||
|
||||
@@ -602,6 +602,7 @@ hermes auth # Interactive wizard
|
||||
hermes auth list # Show all pools
|
||||
hermes auth list openrouter # Show specific provider
|
||||
hermes auth add openrouter --api-key sk-or-v1-xxx # Add API key
|
||||
hermes auth add openrouter --type oauth # Browser login (OpenRouter PKCE) mints a key for you
|
||||
hermes auth add anthropic --type oauth # Add OAuth credential
|
||||
hermes auth add openai-codex --type oauth --priority 0 # Add an account and try it first
|
||||
hermes auth remove openrouter 2 # Remove by index
|
||||
|
||||
@@ -48,6 +48,9 @@ If you already have an API key set in `.env`, Hermes auto-discovers it as a 1-ke
|
||||
# Add a second OpenRouter key
|
||||
hermes auth add openrouter --api-key sk-or-v1-your-second-key
|
||||
|
||||
# ...or let a browser login mint one (OpenRouter OAuth PKCE; stored as a plain API key)
|
||||
hermes auth add openrouter --type oauth
|
||||
|
||||
# Add a second Anthropic key
|
||||
hermes auth add anthropic --type api-key --api-key sk-ant-api03-your-second-key
|
||||
|
||||
|
||||
Reference in New Issue
Block a user