diff --git a/agent/agent_init.py b/agent/agent_init.py index bce03a1d9a..800f216782 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -30,7 +30,7 @@ from agent.session_activity import ActivityProvenance from agent.model_metadata import ( MINIMUM_CONTEXT_LENGTH, fetch_model_metadata, is_local_endpoint, query_ollama_num_ctx ) -from agent.process_bootstrap import _install_safe_stdio, install_happy_eyeballs_socket_connect +from agent.process_bootstrap import _install_safe_stdio from agent.subdirectory_hints import SubdirectoryHintTracker from agent.think_scrubber import StreamingThinkScrubber from agent.tool_guardrails import ( @@ -2253,10 +2253,6 @@ def init_agent( load_soul_identity keeps ~/.hermes/SOUL.md as identity regardless. """ _install_safe_stdio() - # Every startup-path sync connect (catalog fetch via urllib, provider warm via - # requests, sync LLM clients via httpcore) races IPv6/IPv4 instead of stalling - # serially on a blackholed AAAA route (#114265). Idempotent process-level install. - install_happy_eyeballs_socket_connect() _params = locals() for _name in _PASSTHROUGH_PARAMS: diff --git a/agent/process_bootstrap.py b/agent/process_bootstrap.py index 03de4fc931..0e58dc852c 100644 --- a/agent/process_bootstrap.py +++ b/agent/process_bootstrap.py @@ -2,28 +2,24 @@ Lazy OpenAI SDK import (``_OpenAIProxy`` keeps ``isinstance`` and ``patch("agent.process_bootstrap.OpenAI")`` working), crash-resistant stdio -(``_SafeWriter``), env-only HTTP proxy resolution, and Codex dual-stack -(Happy Eyeballs) connection racing. +(``_SafeWriter``), env-only HTTP proxy resolution, and the httpcore backend that +runs sync httpx connects through the process-wide Happy Eyeballs racer +(``hermes_bootstrap``). """ from __future__ import annotations -import errno -import os -import selectors import socket import sys import threading -import time from typing import Any, Optional +from hermes_bootstrap import _happy_eyeballs_create_connection from utils import base_url_hostname, normalize_proxy_url from agent.proxy_bypass import first_proxy_env_value, should_bypass_proxy _OPENAI_CLS_CACHE = None -_HAPPY_EYEBALLS_DELAY_SECONDS = 0.25 -_SOCKET_CONNECT_RACER_INSTALLED = False # Process-wide pool of sync ``httpx.HTTPTransport`` objects shared by every # keepalive client with the same (verify, proxy, happy-eyeballs) identity. @@ -39,122 +35,6 @@ _SHARED_TRANSPORTS_MAX = 32 HERMES_TRANSPORT_OWNER_EXT = "hermes_transport_owner" -def _interleave_addrinfos(addrinfos: list[tuple]) -> list[tuple]: - """Round-robin the resolved address families (deduped), preserving resolver order within each.""" - queues: dict[int, list[tuple]] = {} - seen: set[tuple] = set() - for addrinfo in addrinfos: - family, socktype, proto, _canonname, sockaddr = addrinfo - if (family, socktype, proto, sockaddr) not in seen: - seen.add((family, socktype, proto, sockaddr)) - queues.setdefault(family, []).append(addrinfo) - interleaved: list[tuple] = [] - while any(queues.values()): - interleaved.extend(queue.pop(0) for queue in queues.values() if queue) - return interleaved - - -def _quiet_unregister(selector, sock) -> None: - try: - selector.unregister(sock) - except Exception: - pass - - -def _happy_eyeballs_create_connection(address: tuple[str, int], timeout: Optional[float], - source_address: Optional[tuple[str, int]] = None, socket_options=()): - """RFC 8305-style connect: staggered non-blocking attempts across families. - - ``socket.create_connection`` tries addresses serially, so broken-but- - advertised IPv6 can burn the whole timeout per AAAA record before IPv4. - """ - host, port = address - addrinfos = _interleave_addrinfos(socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)) - if not addrinfos: - raise OSError(f"getaddrinfo returned no addresses for {host}") - - selector = selectors.DefaultSelector() - active: set[socket.socket] = set() - winner = None - last_error: Optional[OSError] = None - deadline = None if timeout is None else time.monotonic() + max(timeout, 0.0) - next_launch = time.monotonic() - pending = list(addrinfos) - in_progress = {0, errno.EINPROGRESS, errno.EWOULDBLOCK, errno.EALREADY, errno.EINTR, getattr(errno, "WSAEWOULDBLOCK", 10035)} - - def start_attempt(addrinfo): - family, socktype, proto, _canonname, sockaddr = addrinfo - candidate = socket.socket(family, socktype, proto) - try: - if source_address is not None: - local_infos = socket.getaddrinfo(source_address[0], source_address[1], family=family, type=socktype) - if not local_infos: - raise OSError(f"getaddrinfo returned no local {family} address for {source_address[0]}") - candidate.bind(local_infos[0][4]) - candidate.setblocking(False) - result = candidate.connect_ex(sockaddr) - if result in (0, errno.EISCONN): - return candidate - if result not in in_progress: - raise OSError(result, os.strerror(result)) - selector.register(candidate, selectors.EVENT_WRITE) - active.add(candidate) - return None - except Exception: - candidate.close() - raise - - try: - while pending or active: - now = time.monotonic() - if deadline is not None and now >= deadline: - raise socket.timeout("timed out") - if pending and now >= next_launch: - try: - winner = start_attempt(pending.pop(0)) - except OSError as exc: - last_error = exc - if not active: - next_launch = now - continue - if winner is not None: - break - next_launch = now + _HAPPY_EYEBALLS_DELAY_SECONDS - wait_timeout = None if deadline is None else max(0.0, deadline - now) - if pending: - until_launch = max(0.0, next_launch - now) - wait_timeout = until_launch if wait_timeout is None else min(wait_timeout, until_launch) - for key, _mask in selector.select(wait_timeout): - candidate = key.fileobj - error_code = candidate.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) - selector.unregister(candidate) - active.discard(candidate) - if error_code == 0: - winner = candidate - break - candidate.close() - last_error = OSError(error_code, os.strerror(error_code)) - if winner is not None: - break - if not active and pending: - next_launch = time.monotonic() - - if winner is None: - raise last_error if last_error is not None else OSError(f"Could not connect to {host}:{port}") - _quiet_unregister(selector, winner) - active.discard(winner) - winner.settimeout(timeout) - for option in socket_options or (): - winner.setsockopt(*option) - winner.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - return winner - finally: - for candidate in active: - _quiet_unregister(selector, candidate) - candidate.close() - selector.close() - - class _HappyEyeballsSyncBackend: """httpcore sync backend with concurrent IPv6/IPv4 connection fallback.""" @@ -226,62 +106,6 @@ def enable_happy_eyeballs_on_client(client) -> None: _enable_happy_eyeballs(transport, proxy_pool_types) -def install_happy_eyeballs_socket_connect() -> None: - """Race IPv6/IPv4 for every sync TCP connect in the process (RFC 8305, #114265). - - The startup path does not build its HTTP clients in one place: the model catalog - fetch goes through ``urllib``/``http.client``, provider warm through - ``requests``/``urllib3``, and sync LLM clients through httpcore. All three funnel - their TCP connect into ``socket.create_connection`` (``http.client`` re-reads it - per connection; httpcore looks it up at call time; urllib3 re-exports its own - serial copy in ``urllib3.util.connection``), and the stock implementation walks the - ``getaddrinfo`` results serially — on a network whose advertised IPv6 route is - blackholed, each AAAA record burns the full connect timeout before IPv4 answers. - Patches both entry points with the racer from this module; idempotent, best-effort. - """ - global _SOCKET_CONNECT_RACER_INSTALLED - if _SOCKET_CONNECT_RACER_INSTALLED: - return - _SOCKET_CONNECT_RACER_INSTALLED = True - - _socket_original = socket.create_connection - - def _socket_racer(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None, *, all_errors=False): - # Stock create_connection leaves the sentinel alone, so the socket keeps the - # process default from socket.setdefaulttimeout(); the racer re-applies the - # timeout on the winner, so it must resolve the sentinel the same way. - effective = socket.getdefaulttimeout() if timeout is socket._GLOBAL_DEFAULT_TIMEOUT else timeout - try: - return _happy_eyeballs_create_connection(address, effective, source_address=source_address) - except OSError: - raise # every candidate failed — identical semantics to the serial original - except Exception: - return _socket_original(address, timeout, source_address=source_address, all_errors=all_errors) - - socket.create_connection = _socket_racer - - try: - from urllib3.util import connection as _urllib3_connection - from urllib3.util.timeout import _DEFAULT_TIMEOUT as _urllib3_sentinel - _urllib3_original = _urllib3_connection.create_connection - - def _urllib3_racer(address, timeout=_urllib3_sentinel, source_address=None, socket_options=None): - effective = socket.getdefaulttimeout() if timeout is _urllib3_sentinel else timeout - try: - return _happy_eyeballs_create_connection( - address, effective, source_address=source_address, - socket_options=tuple(socket_options or ())) - except OSError: - raise - except Exception: - return _urllib3_original( - address, timeout, source_address=source_address, socket_options=socket_options) - - _urllib3_connection.create_connection = _urllib3_racer - except Exception: - pass # requests warm keeps its serial connect; the socket/http.client/httpcore paths still race - - def _load_openai_cls() -> type: """Import and cache ``openai.OpenAI``.""" global _OPENAI_CLS_CACHE @@ -515,5 +339,5 @@ OpenAI = _OpenAIProxy() __all__ = [ "OpenAI", "_OpenAIProxy", "_load_openai_cls", "_SafeWriter", "_install_safe_stdio", "_get_proxy_from_env", "_get_proxy_for_base_url", "build_keepalive_http_client", "close_shared_transports", - "enable_happy_eyeballs_on_client", "install_happy_eyeballs_socket_connect", + "enable_happy_eyeballs_on_client", ] diff --git a/hermes_bootstrap.py b/hermes_bootstrap.py index 38913cf241..405a328dcb 100644 --- a/hermes_bootstrap.py +++ b/hermes_bootstrap.py @@ -1,4 +1,5 @@ -"""Windows UTF-8 bootstrap for Hermes entry points (no-op on POSIX). +"""Process bootstrap for Hermes entry points: Windows UTF-8 stdio, import-path +hardening, durable lazy-install target, and dual-stack (Happy Eyeballs) connects. Windows binds stdio to the console code page (cp1252), so ``print("café")`` raises ``UnicodeEncodeError``, and Python children inherit the same default unless @@ -7,15 +8,228 @@ point (``hermes``, ``hermes-agent``, ``hermes-acp``, ``gateway.run``, ``batch_ru ``cron/scheduler``). It does NOT re-exec with ``-X utf8``: ``open()`` in the current process still needs an explicit ``encoding="utf-8"`` (ruff ``PLW1514``). POSIX is left alone deliberately — users' ``LANG``/``LC_*`` choices are respected. + +Stdlib only: entry points import this before ``harden_import_path()`` runs, so nothing +here may pull in a Hermes package that a project-local directory could shadow. """ from __future__ import annotations +import errno +import importlib.abc +import importlib.util import os +import selectors +import socket import sys +import time _IS_WINDOWS = sys.platform == "win32" _bootstrap_applied = False +_HAPPY_EYEBALLS_DELAY_SECONDS = 0.25 +_URLLIB3_CONNECTION_MODULE = "urllib3.util.connection" + + +def _interleave_addrinfos(addrinfos: list[tuple]) -> list[tuple]: + """Round-robin the resolved address families (deduped), preserving resolver order within each.""" + queues: dict[int, list[tuple]] = {} + seen: set[tuple] = set() + for addrinfo in addrinfos: + family, socktype, proto, _canonname, sockaddr = addrinfo + if (family, socktype, proto, sockaddr) not in seen: + seen.add((family, socktype, proto, sockaddr)) + queues.setdefault(family, []).append(addrinfo) + interleaved: list[tuple] = [] + while any(queues.values()): + interleaved.extend(queue.pop(0) for queue in queues.values() if queue) + return interleaved + + +def _quiet_unregister(selector, sock) -> None: + try: + selector.unregister(sock) + except Exception: + pass + + +def _happy_eyeballs_create_connection(address: tuple[str, int], timeout: float | None, + source_address: tuple[str, int] | None = None, socket_options=()): + """RFC 8305-style connect: staggered non-blocking attempts across families. + + ``socket.create_connection`` tries addresses serially, so broken-but- + advertised IPv6 can burn the whole timeout per AAAA record before IPv4. + """ + host, port = address + addrinfos = _interleave_addrinfos(socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)) + if not addrinfos: + raise OSError(f"getaddrinfo returned no addresses for {host}") + + selector = selectors.DefaultSelector() + active: set[socket.socket] = set() + winner = None + last_error: OSError | None = None + deadline = None if timeout is None else time.monotonic() + max(timeout, 0.0) + next_launch = time.monotonic() + pending = list(addrinfos) + in_progress = {0, errno.EINPROGRESS, errno.EWOULDBLOCK, errno.EALREADY, errno.EINTR, getattr(errno, "WSAEWOULDBLOCK", 10035)} + + def start_attempt(addrinfo): + family, socktype, proto, _canonname, sockaddr = addrinfo + candidate = socket.socket(family, socktype, proto) + try: + if source_address is not None: + local_infos = socket.getaddrinfo(source_address[0], source_address[1], family=family, type=socktype) + if not local_infos: + raise OSError(f"getaddrinfo returned no local {family} address for {source_address[0]}") + candidate.bind(local_infos[0][4]) + candidate.setblocking(False) + result = candidate.connect_ex(sockaddr) + if result in (0, errno.EISCONN): + return candidate + if result not in in_progress: + raise OSError(result, os.strerror(result)) + selector.register(candidate, selectors.EVENT_WRITE) + active.add(candidate) + return None + except Exception: + candidate.close() + raise + + try: + while pending or active: + now = time.monotonic() + if deadline is not None and now >= deadline: + raise socket.timeout("timed out") + if pending and now >= next_launch: + try: + winner = start_attempt(pending.pop(0)) + except OSError as exc: + last_error = exc + if not active: + next_launch = now + continue + if winner is not None: + break + next_launch = now + _HAPPY_EYEBALLS_DELAY_SECONDS + wait_timeout = None if deadline is None else max(0.0, deadline - now) + if pending: + until_launch = max(0.0, next_launch - now) + wait_timeout = until_launch if wait_timeout is None else min(wait_timeout, until_launch) + for key, _mask in selector.select(wait_timeout): + candidate = key.fileobj + error_code = candidate.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) + selector.unregister(candidate) + active.discard(candidate) + if error_code == 0: + winner = candidate + break + candidate.close() + last_error = OSError(error_code, os.strerror(error_code)) + if winner is not None: + break + if not active and pending: + next_launch = time.monotonic() + + if winner is None: + raise last_error if last_error is not None else OSError(f"Could not connect to {host}:{port}") + _quiet_unregister(selector, winner) + active.discard(winner) + winner.settimeout(timeout) + for option in socket_options or (): + winner.setsockopt(*option) + winner.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + return winner + finally: + for candidate in active: + _quiet_unregister(selector, candidate) + candidate.close() + selector.close() + + +def _patch_urllib3_create_connection(module) -> None: + """Point ``urllib3.util.connection.create_connection`` (its own serial walker) at the racer.""" + if getattr(module.create_connection, "_hermes_happy_eyeballs", False): + return + urllib3_original = module.create_connection + urllib3_sentinel = module._DEFAULT_TIMEOUT + + def _urllib3_racer(address, timeout=urllib3_sentinel, source_address=None, socket_options=None): + effective = socket.getdefaulttimeout() if timeout is urllib3_sentinel else timeout + try: + return _happy_eyeballs_create_connection( + address, effective, source_address=source_address, socket_options=tuple(socket_options or ())) + except OSError: + raise + except Exception: + return urllib3_original(address, timeout, source_address=source_address, socket_options=socket_options) + + _urllib3_racer._hermes_happy_eyeballs = True # type: ignore[attr-defined] + module.create_connection = _urllib3_racer + + +class _Urllib3ConnectionPatcher(importlib.abc.MetaPathFinder, importlib.abc.Loader): + """One-shot import hook: patch urllib3's connect walker the moment the module loads. + + Importing urllib3 eagerly costs ~50 ms on every CLI start, and ``hermes`` / the TUI + gateway never load it unless something actually calls ``requests``. + """ + + def find_spec(self, fullname, path, target=None): + if fullname != _URLLIB3_CONNECTION_MODULE: + return None + if self in sys.meta_path: + sys.meta_path.remove(self) + spec = importlib.util.find_spec(fullname) + if spec is None or spec.loader is None: + return None + self._inner = spec.loader + spec.loader = self + return spec + + def create_module(self, spec): + return self._inner.create_module(spec) + + def exec_module(self, module): + self._inner.exec_module(module) + _patch_urllib3_create_connection(module) + + +def install_happy_eyeballs_socket_connect() -> None: + """Race IPv6/IPv4 for every sync TCP connect in the process (RFC 8305, #114265). + + The startup path does not build its HTTP clients in one place: the model catalog + fetch goes through ``requests``/``urllib3``, sync LLM and OAuth clients through + httpcore, plugins through ``urllib``/``http.client``. All of them funnel their TCP + connect into ``socket.create_connection`` (``http.client`` re-reads it per connection; + httpcore looks it up at call time) or into urllib3's own serial copy in + ``urllib3.util.connection``. The stock implementations walk the ``getaddrinfo`` + results serially — on a network whose advertised IPv6 route is blackholed, each AAAA + record burns the full connect timeout before IPv4 answers. Idempotent, best-effort. + """ + if getattr(socket.create_connection, "_hermes_happy_eyeballs", False): + return + socket_original = socket.create_connection + + def _socket_racer(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None, *, all_errors=False): + # Stock create_connection leaves the sentinel alone, so the socket keeps the + # process default from socket.setdefaulttimeout(); the racer re-applies the + # timeout on the winner, so it must resolve the sentinel the same way. + effective = socket.getdefaulttimeout() if timeout is socket._GLOBAL_DEFAULT_TIMEOUT else timeout + try: + return _happy_eyeballs_create_connection(address, effective, source_address=source_address) + except OSError: + raise # every candidate failed — identical semantics to the serial original + except Exception: + return socket_original(address, timeout, source_address=source_address, all_errors=all_errors) + + _socket_racer._hermes_happy_eyeballs = True # type: ignore[attr-defined] + socket.create_connection = _socket_racer + + urllib3_connection = sys.modules.get(_URLLIB3_CONNECTION_MODULE) + if urllib3_connection is not None: + _patch_urllib3_create_connection(urllib3_connection) + elif not any(isinstance(finder, _Urllib3ConnectionPatcher) for finder in sys.meta_path): + sys.meta_path.insert(0, _Urllib3ConnectionPatcher()) def apply_windows_utf8_bootstrap() -> bool: @@ -117,3 +331,4 @@ def activate_durable_lazy_target() -> None: apply_windows_utf8_bootstrap() suppress_platform_ver_console() activate_durable_lazy_target() +install_happy_eyeballs_socket_connect() diff --git a/tests/agent/test_codex_happy_eyeballs.py b/tests/agent/test_codex_happy_eyeballs.py index 0196f77fb9..3c30d10fa5 100644 --- a/tests/agent/test_codex_happy_eyeballs.py +++ b/tests/agent/test_codex_happy_eyeballs.py @@ -5,16 +5,9 @@ import socket import httpcore import pytest +import hermes_bootstrap from agent import process_bootstrap -# Captured at import time, before any test can construct an AIAgent (whose -# init_agent now installs the process-level racer) — the restore fixture below -# needs the pristine originals regardless of test ordering. -from urllib3.util import connection as _urllib3_connection_util - -_STOCK_SOCKET_CREATE_CONNECTION = socket.create_connection -_STOCK_URLLIB3_CREATE_CONNECTION = _urllib3_connection_util.create_connection - @pytest.fixture def no_proxy_env(monkeypatch): @@ -117,7 +110,7 @@ def test_connection_staggers_past_blackholed_ipv6(monkeypatch): pass monkeypatch.setattr( - process_bootstrap.socket, + hermes_bootstrap.socket, "getaddrinfo", lambda *_args, **_kwargs: [ ( @@ -136,22 +129,22 @@ def test_connection_staggers_past_blackholed_ipv6(monkeypatch): ), ], ) - monkeypatch.setattr(process_bootstrap.socket, "socket", FakeSocket) + monkeypatch.setattr(hermes_bootstrap.socket, "socket", FakeSocket) monkeypatch.setattr( - process_bootstrap.selectors, "DefaultSelector", FakeSelector + hermes_bootstrap.selectors, "DefaultSelector", FakeSelector ) monkeypatch.setattr( - process_bootstrap.time, "monotonic", lambda: clock[0] + hermes_bootstrap.time, "monotonic", lambda: clock[0] ) - winner = process_bootstrap._happy_eyeballs_create_connection( + winner = hermes_bootstrap._happy_eyeballs_create_connection( ("chatgpt.com", 443), timeout=10.0, ) assert winner.family == socket.AF_INET assert winner.timeout == 10.0 - assert clock[0] == process_bootstrap._HAPPY_EYEBALLS_DELAY_SECONDS + assert clock[0] == hermes_bootstrap._HAPPY_EYEBALLS_DELAY_SECONDS assert sockets[0].closed is True assert sockets[1].closed is False @@ -331,172 +324,3 @@ def test_codex_auth_http_client_uses_happy_eyeballs_backend(no_proxy_env): ) finally: client.close() - - -@pytest.fixture -def restored_socket_connect(): - yield - socket.create_connection = _STOCK_SOCKET_CREATE_CONNECTION - _urllib3_connection_util.create_connection = _STOCK_URLLIB3_CREATE_CONNECTION - process_bootstrap._SOCKET_CONNECT_RACER_INSTALLED = False - - -def test_install_happy_eyeballs_socket_connect_patches_both_stacks(restored_socket_connect): - process_bootstrap.install_happy_eyeballs_socket_connect() - - assert socket.create_connection is not _STOCK_SOCKET_CREATE_CONNECTION - assert _urllib3_connection_util.create_connection is not _STOCK_URLLIB3_CREATE_CONNECTION - - # Idempotent: a second install must not wrap the racer again. - first_socket_racer = socket.create_connection - first_urllib3_racer = _urllib3_connection_util.create_connection - process_bootstrap.install_happy_eyeballs_socket_connect() - assert socket.create_connection is first_socket_racer - assert _urllib3_connection_util.create_connection is first_urllib3_racer - - -def test_installed_socket_connect_races_past_blackholed_ipv6( - monkeypatch, restored_socket_connect): - clock = [0.0] - sockets = [] - - class FakeSocket: - def __init__(self, family, socktype, proto): - self.family = family - self.closed = False - self.timeout = None - sockets.append(self) - - def setsockopt(self, *_args): - pass - - def setblocking(self, _blocking): - pass - - def settimeout(self, timeout): - self.timeout = timeout - - def bind(self, _address): - pass - - def connect_ex(self, _address): - if self.family == socket.AF_INET6: - return errno.EINPROGRESS - return 0 - - def close(self): - self.closed = True - - class FakeSelector: - def __init__(self): - self.registered = set() - - def register(self, fileobj, _events): - self.registered.add(fileobj) - - def unregister(self, fileobj): - self.registered.discard(fileobj) - - def select(self, timeout): - clock[0] += timeout or 0.0 - return [] - - def close(self): - pass - - monkeypatch.setattr( - process_bootstrap.socket, - "getaddrinfo", - lambda *_args, **_kwargs: [ - ( - socket.AF_INET6, - socket.SOCK_STREAM, - socket.IPPROTO_TCP, - "", - ("2001:db8::1", 443, 0, 0), - ), - ( - socket.AF_INET, - socket.SOCK_STREAM, - socket.IPPROTO_TCP, - "", - ("192.0.2.1", 443), - ), - ], - ) - monkeypatch.setattr(process_bootstrap.socket, "socket", FakeSocket) - monkeypatch.setattr( - process_bootstrap.selectors, "DefaultSelector", FakeSelector - ) - monkeypatch.setattr( - process_bootstrap.time, "monotonic", lambda: clock[0] - ) - - process_bootstrap.install_happy_eyeballs_socket_connect() - - # http.client passes the module timeout sentinel through positionally. - winner = socket.create_connection( - ("example.com", 443), socket._GLOBAL_DEFAULT_TIMEOUT, None - ) - - assert winner.family == socket.AF_INET - assert winner.timeout is None # sentinel resolves to the process default (None here), like stock - assert clock[0] == process_bootstrap._HAPPY_EYEBALLS_DELAY_SECONDS - assert sockets[0].closed is True - assert sockets[1] is winner - - -def test_installed_racer_serves_http_client_and_urllib3_connects(restored_socket_connect): - import http.client - - import urllib3 - - listener = socket.socket() - listener.bind(("127.0.0.1", 0)) - listener.listen(4) - port = listener.getsockname()[1] - - process_bootstrap.install_happy_eyeballs_socket_connect() - http_conn = http.client.HTTPConnection("127.0.0.1", port, timeout=5) - urllib3_conn = urllib3.connection.HTTPConnection("127.0.0.1", port, timeout=5) - try: - http_conn.connect() - urllib3_conn.connect() # exercises the socket_options kwarg of the urllib3 racer - assert http_conn.sock is not None - assert urllib3_conn.sock is not None - finally: - http_conn.close() - urllib3_conn.close() - listener.close() - - -def test_installed_racer_honours_process_default_timeout_on_sentinel(restored_socket_connect): - # Stock create_connection leaves the sentinel alone, so the socket keeps the - # process default set by socket.setdefaulttimeout(); the racer re-applies the - # timeout on the winner and must resolve the sentinel to that same default - # instead of forcing a blocking socket. - listener = socket.socket() - listener.bind(("127.0.0.1", 0)) - listener.listen(4) - port = listener.getsockname()[1] - - process_bootstrap.install_happy_eyeballs_socket_connect() - socket.setdefaulttimeout(5.0) - try: - winner = socket.create_connection(("127.0.0.1", port), socket._GLOBAL_DEFAULT_TIMEOUT) - try: - assert winner.gettimeout() == 5.0 - finally: - winner.close() - finally: - socket.setdefaulttimeout(None) - listener.close() - - -def test_installed_racer_accepts_all_errors_keyword(restored_socket_connect): - # socket.create_connection gained the keyword-only all_errors parameter in - # Python 3.11 (the repo floor); forwarding it through the installed racer must - # not fail with a TypeError before the connect is even attempted. - process_bootstrap.install_happy_eyeballs_socket_connect() - with pytest.raises(OSError): - socket.create_connection(("127.0.0.1", 1), 1.0, None, all_errors=True) diff --git a/tests/test_hermes_bootstrap.py b/tests/test_hermes_bootstrap.py index a97b05d128..341b278653 100644 --- a/tests/test_hermes_bootstrap.py +++ b/tests/test_hermes_bootstrap.py @@ -19,8 +19,10 @@ Key invariants covered by these tests: from __future__ import annotations +import errno import io import os +import socket import subprocess import sys import textwrap @@ -367,3 +369,102 @@ class TestSuppressPlatformVerConsole: if original is not None: platform._syscmd_ver = original + +class TestHappyEyeballsSocketConnect: + """Importing the bootstrap races IPv6/IPv4 for every sync connect in the process (#114265).""" + + def test_import_routes_http_client_and_urllib3_connects_through_the_racer(self): + import http.client + + import urllib3 + import urllib3.util.connection as urllib3_connection + + hb = _fresh_import() + assert socket.create_connection.__module__ == hb.__name__ + # urllib3 keeps its own serial connect walker; it is patched once imported (lazily). + assert getattr(urllib3_connection.create_connection, "_hermes_happy_eyeballs", False) + # Re-importing the bootstrap (or importing it after urllib3) never wraps the racer twice. + racer = socket.create_connection + _fresh_import() + assert socket.create_connection is racer + # The bootstrap must not pay urllib3's import (~50 ms) on every process start: a fresh + # interpreter gets the patch the moment urllib3 loads, not before. + subprocess.run([sys.executable, "-c", textwrap.dedent(""" + import sys, hermes_bootstrap + assert "urllib3" not in sys.modules, "bootstrap imported urllib3 eagerly" + import urllib3.util.connection as c + assert c.create_connection._hermes_happy_eyeballs + """)], check=True, cwd=str(Path(hb.__file__).parent), timeout=60) + + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(4) + port = listener.getsockname()[1] + http_conn = http.client.HTTPConnection("127.0.0.1", port, timeout=5) + urllib3_conn = urllib3.connection.HTTPConnection("127.0.0.1", port, timeout=5) + try: + http_conn.connect() + urllib3_conn.connect() # exercises the socket_options kwarg of the urllib3 racer + assert http_conn.sock.getpeername()[1] == port + assert urllib3_conn.sock.getpeername()[1] == port + finally: + http_conn.close() + urllib3_conn.close() + listener.close() + + def test_installed_racer_wins_ipv4_while_ipv6_hangs(self, monkeypatch): + hb = _fresh_import() + clock = [0.0] + sockets = [] + + class FakeSocket: + def __init__(self, family, socktype, proto): + self.family = family + self.closed = False + self.timeout = None + sockets.append(self) + + def setsockopt(self, *_args): + pass + + def setblocking(self, _blocking): + pass + + def settimeout(self, timeout): + self.timeout = timeout + + def connect_ex(self, _address): + return errno.EINPROGRESS if self.family == socket.AF_INET6 else 0 + + def close(self): + self.closed = True + + class FakeSelector: + def register(self, *_args): + pass + + def unregister(self, *_args): + pass + + def select(self, timeout): + clock[0] += timeout or 0.0 # the v6 attempt never completes + return [] + + def close(self): + pass + + monkeypatch.setattr(hb.socket, "getaddrinfo", lambda *_a, **_k: [ + (socket.AF_INET6, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("2001:db8::1", 443, 0, 0)), + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("192.0.2.1", 443)), + ]) + monkeypatch.setattr(hb.socket, "socket", FakeSocket) + monkeypatch.setattr(hb.selectors, "DefaultSelector", FakeSelector) + monkeypatch.setattr(hb.time, "monotonic", lambda: clock[0]) + + # http.client passes the module timeout sentinel through positionally. + winner = socket.create_connection(("example.com", 443), socket._GLOBAL_DEFAULT_TIMEOUT, None) + + assert winner.family == socket.AF_INET + assert winner.timeout is None # sentinel resolves to the process default, like stock + assert clock[0] == hb._HAPPY_EYEBALLS_DELAY_SECONDS + assert sockets[0].closed is True and sockets[1] is winner diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index 11aa5062b9..34f29260fd 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -2874,7 +2874,7 @@ network: force_ipv4: false # Force IPv4 for outbound connections (default: false) ``` -`force_ipv4` — on servers with broken or unreachable IPv6, Python resolves AAAA records first and can hang for the full TCP timeout before falling back to IPv4. Set this to `true` to skip IPv6 entirely and connect over IPv4 directly. +`force_ipv4` — on servers with broken or unreachable IPv6, Python resolves AAAA records first and can hang for the full TCP timeout before falling back to IPv4. Hermes already races IPv6 and IPv4 for every outbound connection it makes (Happy Eyeballs, RFC 8305: the IPv4 attempt starts 250 ms after IPv6 and whichever connects first wins), so an advertised-but-blackholed IPv6 route costs about a quarter second per connection instead of the full timeout. Set this to `true` only when you want to skip IPv6 entirely and connect over IPv4 directly. ## Onboarding