fix(doctor): detect a dead IPv6 route and name network.force_ipv4

#114265 secondary finding 1: ``network.force_ipv4`` was undiscoverable (default
off, mentioned only by a rotating tip), so an advertised-but-blackholed IPv6
prefix cost the reporter weeks. ``hermes doctor`` now runs an ``IPv6 route``
probe in the API Connectivity section: one 2 s IPv6 TCP connect to a known
dual-stack host. A timeout is the dead-route signature and is reported as a
warning plus a summary issue naming ``network.force_ipv4: true``; no AAAA /
no IPv6 route at all is healthy (fails fast, no stall) and ``force_ipv4``
already set skips the probe. Two invariant tests over a mocked connect seam.

Docs: doctor reference and the network config section describe the check.
This commit is contained in:
teknium1
2026-09-18 03:40:18 -07:00
committed by Teknium
parent d029cddfd7
commit 72360ae1d2
4 changed files with 100 additions and 1 deletions

View File

@@ -7,10 +7,13 @@ print and issue strings to append. No printing inside workers — the caller pri
from __future__ import annotations
import concurrent.futures
import errno
import functools
import os
import socket
import sys
from typing import NamedTuple
from urllib.parse import urlsplit
from hermes_cli.colors import Colors, color
from hermes_cli.models import _HERMES_USER_AGENT
@@ -279,12 +282,58 @@ def _probe_azure_entra() -> ProbeResult:
return _row(name, "warn", f"({err})", [f"Azure Foundry Entra: {err}. {hint}"], label=label)
def _load_network_config() -> dict:
try:
from hermes_cli.config import load_config_readonly
net = (load_config_readonly() or {}).get("network")
except Exception:
return {}
return net if isinstance(net, dict) else {}
def _tcp_connect(sockaddr, timeout: float) -> None:
"""Open + close one IPv6 TCP connection; raises OSError (TimeoutError on a dead route)."""
with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as sock:
sock.settimeout(timeout)
sock.connect(sockaddr)
_IPV6_PROBE_TIMEOUT = 2.0
def _probe_ipv6_path() -> ProbeResult:
"""Dead-IPv6-route detector (#114265): an advertised AAAA path that only times out makes every
serial connect burn its full timeout before IPv4 answers. Name the remedy instead of stalling."""
name = "IPv6 route"
if _load_network_config().get("force_ipv4"):
return _skip(name) # IPv6 is never dialled
host = urlsplit(OPENROUTER_MODELS_URL).hostname
try:
infos = socket.getaddrinfo(host, 443, socket.AF_INET6, socket.SOCK_STREAM)
except OSError:
infos = []
if not infos:
return _skip(name) # no AAAA record / no IPv6 resolver: nothing to test
try:
_tcp_connect(infos[0][4], _IPV6_PROBE_TIMEOUT)
except TimeoutError:
remedy = "set `network.force_ipv4: true` in config.yaml (or fix the IPv6 route)"
return _row(name, "warn", f"(IPv6 route to {host} advertised but dead: connect timed out after "
f"{_IPV6_PROBE_TIMEOUT:g}s — {remedy})",
[f"Dead IPv6 route: every IPv6-first connect stalls before IPv4 answers. Fix: {remedy}"])
except OSError as e:
if e.errno in (errno.ENETUNREACH, errno.EHOSTUNREACH, errno.EADDRNOTAVAIL):
return _row(name, "ok", "(no IPv6 route — IPv4 only)") # fails fast, so no stall
return _row(name, "ok", f"(IPv6 path to {host} reachable)") # refused/reset also prove a live path
def build_probes() -> list:
"""(label, callable) pairs in display order."""
global _APIKEY_PROVIDERS_CACHE
if _APIKEY_PROVIDERS_CACHE is None:
_APIKEY_PROVIDERS_CACHE = _build_apikey_providers_list()
return [
("IPv6 route", _probe_ipv6_path),
("OpenRouter API", _probe_openrouter), ("Anthropic API", _probe_anthropic),
# functools.partial binds each row's args so every callable keeps its own provider.
*((row[0], functools.partial(_probe_apikey_provider, *row)) for row in _APIKEY_PROVIDERS_CACHE),

View File

@@ -0,0 +1,48 @@
"""``hermes doctor`` dead-IPv6 probe (#114265 secondary finding 1).
An advertised-but-blackholed IPv6 route stalls every serial connect for the full timeout;
the racer hides most of it, but nothing told the user that ``network.force_ipv4`` exists.
The probe connects to one known AAAA host over IPv6 with a short timeout and, on a
timeout, names the remedy. No IPv6 route at all is healthy (skip), as is an explicit
``force_ipv4: true``.
"""
from __future__ import annotations
import socket
from hermes_cli import doctor_connectivity as dc
_AAAA = [(socket.AF_INET6, socket.SOCK_STREAM, 6, "", ("2001:db8::1", 443, 0, 0))]
def _run(monkeypatch, connect, *, config=None, addrinfo=_AAAA):
monkeypatch.setattr(dc, "_load_network_config", lambda: config or {})
monkeypatch.setattr(dc.socket, "getaddrinfo", lambda *_a, **_k: addrinfo)
monkeypatch.setattr(dc, "_tcp_connect", connect)
return dc._probe_ipv6_path()
def test_dead_ipv6_route_warns_and_names_force_ipv4(monkeypatch):
def timed_out(_sockaddr, _timeout):
raise TimeoutError("timed out")
result = _run(monkeypatch, timed_out)
(glyph, _label, detail), = result.lines
assert "⚠" in glyph and "network.force_ipv4" in detail
assert result.issues and "network.force_ipv4" in result.issues[0]
def test_healthy_or_absent_ipv6_never_warns(monkeypatch):
def reachable(_sockaddr, _timeout):
return None
def no_route(_sockaddr, _timeout):
raise OSError(101, "Network is unreachable")
assert _run(monkeypatch, reachable).issues == []
assert _run(monkeypatch, no_route).issues == []
assert "✓" in _run(monkeypatch, reachable).lines[0][0]
# force_ipv4 already set, or no AAAA record: nothing to probe, nothing to say.
assert _run(monkeypatch, reachable, config={"force_ipv4": True}).lines == []
assert _run(monkeypatch, reachable, addrinfo=[]).lines == []

View File

@@ -909,6 +909,8 @@ hermes doctor [--fix]
|--------|-------------|
| `--fix` | Attempt automatic repairs where possible. |
The **API Connectivity** section includes an `IPv6 route` check: it opens one short (2 s) IPv6 TCP connection to a known dual-stack host. A route that is advertised but only times out (a blackholed IPv6 prefix) is reported as a warning naming the remedy, `network.force_ipv4: true`. Having no IPv6 route at all is healthy and reported as OK; the check is skipped when `force_ipv4` is already set.
## `hermes dump`
```bash

View File

@@ -2874,7 +2874,7 @@ network:
force_ipv4: false # Force IPv4 for outbound connections (default: false)
```
`force_ipv4` — on servers with broken or unreachable IPv6, Python resolves AAAA records first and can hang for the full TCP timeout before falling back to IPv4. Hermes already races IPv6 and IPv4 for every outbound connection it makes (Happy Eyeballs, RFC 8305: the IPv4 attempt starts 250 ms after IPv6 and whichever connects first wins), so an advertised-but-blackholed IPv6 route costs about a quarter second per connection instead of the full timeout. Set this to `true` only when you want to skip IPv6 entirely and connect over IPv4 directly.
`force_ipv4` — on servers with broken or unreachable IPv6, Python resolves AAAA records first and can hang for the full TCP timeout before falling back to IPv4. Hermes already races IPv6 and IPv4 for every outbound connection it makes (Happy Eyeballs, RFC 8305: the IPv4 attempt starts 250 ms after IPv6 and whichever connects first wins), so an advertised-but-blackholed IPv6 route costs about a quarter second per connection instead of the full timeout. This covers the gateway's WebSocket dials (relay connector, platform adapters) as well as HTTP. Set this to `true` only when you want to skip IPv6 entirely and connect over IPv4 directly. `hermes doctor` runs an `IPv6 route` check that detects a dead IPv6 path and points at this setting.
## Onboarding