fix(doctor): detect a dead IPv6 route and name network.force_ipv4
#114265 secondary finding 1: ``network.force_ipv4`` was undiscoverable (default off, mentioned only by a rotating tip), so an advertised-but-blackholed IPv6 prefix cost the reporter weeks. ``hermes doctor`` now runs an ``IPv6 route`` probe in the API Connectivity section: one 2 s IPv6 TCP connect to a known dual-stack host. A timeout is the dead-route signature and is reported as a warning plus a summary issue naming ``network.force_ipv4: true``; no AAAA / no IPv6 route at all is healthy (fails fast, no stall) and ``force_ipv4`` already set skips the probe. Two invariant tests over a mocked connect seam. Docs: doctor reference and the network config section describe the check.
This commit is contained in:
@@ -7,10 +7,13 @@ print and issue strings to append. No printing inside workers — the caller pri
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures
|
||||
import errno
|
||||
import functools
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
from typing import NamedTuple
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from hermes_cli.colors import Colors, color
|
||||
from hermes_cli.models import _HERMES_USER_AGENT
|
||||
@@ -279,12 +282,58 @@ def _probe_azure_entra() -> ProbeResult:
|
||||
return _row(name, "warn", f"({err})", [f"Azure Foundry Entra: {err}. {hint}"], label=label)
|
||||
|
||||
|
||||
def _load_network_config() -> dict:
|
||||
try:
|
||||
from hermes_cli.config import load_config_readonly
|
||||
net = (load_config_readonly() or {}).get("network")
|
||||
except Exception:
|
||||
return {}
|
||||
return net if isinstance(net, dict) else {}
|
||||
|
||||
|
||||
def _tcp_connect(sockaddr, timeout: float) -> None:
|
||||
"""Open + close one IPv6 TCP connection; raises OSError (TimeoutError on a dead route)."""
|
||||
with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as sock:
|
||||
sock.settimeout(timeout)
|
||||
sock.connect(sockaddr)
|
||||
|
||||
|
||||
_IPV6_PROBE_TIMEOUT = 2.0
|
||||
|
||||
|
||||
def _probe_ipv6_path() -> ProbeResult:
|
||||
"""Dead-IPv6-route detector (#114265): an advertised AAAA path that only times out makes every
|
||||
serial connect burn its full timeout before IPv4 answers. Name the remedy instead of stalling."""
|
||||
name = "IPv6 route"
|
||||
if _load_network_config().get("force_ipv4"):
|
||||
return _skip(name) # IPv6 is never dialled
|
||||
host = urlsplit(OPENROUTER_MODELS_URL).hostname
|
||||
try:
|
||||
infos = socket.getaddrinfo(host, 443, socket.AF_INET6, socket.SOCK_STREAM)
|
||||
except OSError:
|
||||
infos = []
|
||||
if not infos:
|
||||
return _skip(name) # no AAAA record / no IPv6 resolver: nothing to test
|
||||
try:
|
||||
_tcp_connect(infos[0][4], _IPV6_PROBE_TIMEOUT)
|
||||
except TimeoutError:
|
||||
remedy = "set `network.force_ipv4: true` in config.yaml (or fix the IPv6 route)"
|
||||
return _row(name, "warn", f"(IPv6 route to {host} advertised but dead: connect timed out after "
|
||||
f"{_IPV6_PROBE_TIMEOUT:g}s — {remedy})",
|
||||
[f"Dead IPv6 route: every IPv6-first connect stalls before IPv4 answers. Fix: {remedy}"])
|
||||
except OSError as e:
|
||||
if e.errno in (errno.ENETUNREACH, errno.EHOSTUNREACH, errno.EADDRNOTAVAIL):
|
||||
return _row(name, "ok", "(no IPv6 route — IPv4 only)") # fails fast, so no stall
|
||||
return _row(name, "ok", f"(IPv6 path to {host} reachable)") # refused/reset also prove a live path
|
||||
|
||||
|
||||
def build_probes() -> list:
|
||||
"""(label, callable) pairs in display order."""
|
||||
global _APIKEY_PROVIDERS_CACHE
|
||||
if _APIKEY_PROVIDERS_CACHE is None:
|
||||
_APIKEY_PROVIDERS_CACHE = _build_apikey_providers_list()
|
||||
return [
|
||||
("IPv6 route", _probe_ipv6_path),
|
||||
("OpenRouter API", _probe_openrouter), ("Anthropic API", _probe_anthropic),
|
||||
# functools.partial binds each row's args so every callable keeps its own provider.
|
||||
*((row[0], functools.partial(_probe_apikey_provider, *row)) for row in _APIKEY_PROVIDERS_CACHE),
|
||||
|
||||
48
tests/hermes_cli/test_doctor_ipv6_probe.py
Normal file
48
tests/hermes_cli/test_doctor_ipv6_probe.py
Normal file
@@ -0,0 +1,48 @@
|
||||
"""``hermes doctor`` dead-IPv6 probe (#114265 secondary finding 1).
|
||||
|
||||
An advertised-but-blackholed IPv6 route stalls every serial connect for the full timeout;
|
||||
the racer hides most of it, but nothing told the user that ``network.force_ipv4`` exists.
|
||||
The probe connects to one known AAAA host over IPv6 with a short timeout and, on a
|
||||
timeout, names the remedy. No IPv6 route at all is healthy (skip), as is an explicit
|
||||
``force_ipv4: true``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
|
||||
from hermes_cli import doctor_connectivity as dc
|
||||
|
||||
_AAAA = [(socket.AF_INET6, socket.SOCK_STREAM, 6, "", ("2001:db8::1", 443, 0, 0))]
|
||||
|
||||
|
||||
def _run(monkeypatch, connect, *, config=None, addrinfo=_AAAA):
|
||||
monkeypatch.setattr(dc, "_load_network_config", lambda: config or {})
|
||||
monkeypatch.setattr(dc.socket, "getaddrinfo", lambda *_a, **_k: addrinfo)
|
||||
monkeypatch.setattr(dc, "_tcp_connect", connect)
|
||||
return dc._probe_ipv6_path()
|
||||
|
||||
|
||||
def test_dead_ipv6_route_warns_and_names_force_ipv4(monkeypatch):
|
||||
def timed_out(_sockaddr, _timeout):
|
||||
raise TimeoutError("timed out")
|
||||
|
||||
result = _run(monkeypatch, timed_out)
|
||||
(glyph, _label, detail), = result.lines
|
||||
assert "⚠" in glyph and "network.force_ipv4" in detail
|
||||
assert result.issues and "network.force_ipv4" in result.issues[0]
|
||||
|
||||
|
||||
def test_healthy_or_absent_ipv6_never_warns(monkeypatch):
|
||||
def reachable(_sockaddr, _timeout):
|
||||
return None
|
||||
|
||||
def no_route(_sockaddr, _timeout):
|
||||
raise OSError(101, "Network is unreachable")
|
||||
|
||||
assert _run(monkeypatch, reachable).issues == []
|
||||
assert _run(monkeypatch, no_route).issues == []
|
||||
assert "✓" in _run(monkeypatch, reachable).lines[0][0]
|
||||
# force_ipv4 already set, or no AAAA record: nothing to probe, nothing to say.
|
||||
assert _run(monkeypatch, reachable, config={"force_ipv4": True}).lines == []
|
||||
assert _run(monkeypatch, reachable, addrinfo=[]).lines == []
|
||||
@@ -909,6 +909,8 @@ hermes doctor [--fix]
|
||||
|--------|-------------|
|
||||
| `--fix` | Attempt automatic repairs where possible. |
|
||||
|
||||
The **API Connectivity** section includes an `IPv6 route` check: it opens one short (2 s) IPv6 TCP connection to a known dual-stack host. A route that is advertised but only times out (a blackholed IPv6 prefix) is reported as a warning naming the remedy, `network.force_ipv4: true`. Having no IPv6 route at all is healthy and reported as OK; the check is skipped when `force_ipv4` is already set.
|
||||
|
||||
## `hermes dump`
|
||||
|
||||
```bash
|
||||
|
||||
@@ -2874,7 +2874,7 @@ network:
|
||||
force_ipv4: false # Force IPv4 for outbound connections (default: false)
|
||||
```
|
||||
|
||||
`force_ipv4` — on servers with broken or unreachable IPv6, Python resolves AAAA records first and can hang for the full TCP timeout before falling back to IPv4. Hermes already races IPv6 and IPv4 for every outbound connection it makes (Happy Eyeballs, RFC 8305: the IPv4 attempt starts 250 ms after IPv6 and whichever connects first wins), so an advertised-but-blackholed IPv6 route costs about a quarter second per connection instead of the full timeout. Set this to `true` only when you want to skip IPv6 entirely and connect over IPv4 directly.
|
||||
`force_ipv4` — on servers with broken or unreachable IPv6, Python resolves AAAA records first and can hang for the full TCP timeout before falling back to IPv4. Hermes already races IPv6 and IPv4 for every outbound connection it makes (Happy Eyeballs, RFC 8305: the IPv4 attempt starts 250 ms after IPv6 and whichever connects first wins), so an advertised-but-blackholed IPv6 route costs about a quarter second per connection instead of the full timeout. This covers the gateway's WebSocket dials (relay connector, platform adapters) as well as HTTP. Set this to `true` only when you want to skip IPv6 entirely and connect over IPv4 directly. `hermes doctor` runs an `IPv6 route` check that detects a dead IPv6 path and points at this setting.
|
||||
|
||||
## Onboarding
|
||||
|
||||
|
||||
Reference in New Issue
Block a user