8e4a804f53
A transient HTTP 500 during the range probe aborted bundle staging. Retry PM-owned requests and body transfers with one bounded policy, instead of retrying complete installs or stacking caller retry loops. Keep partial ranges across attempts and interrupt backoff on pause. Keep the CDN serial fallback active until its source finishes. Do not retry permanent HTTP errors, certificate failures, bad hashes or disk errors. Use the same policy for metadata reads and artifact hashing. Verified: 217 focused tests passed, 1 skipped, and Ruff passed on the changed Python files. The pinned Windows ARM64 uv archive also survived an injected HTTP 500 through the PM CLI and its installed binary ran. The full suite and desktop release build were not run.
86 lines
2.9 KiB
Python
86 lines
2.9 KiB
Python
"""Bounded retries for PM-owned, idempotent network operations."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import errno
|
|
import http.client
|
|
import logging
|
|
import socket
|
|
import ssl
|
|
import time
|
|
import urllib.error
|
|
from datetime import timezone
|
|
from email.utils import parsedate_to_datetime
|
|
from typing import Callable, TypeVar
|
|
|
|
_T = TypeVar("_T")
|
|
_ATTEMPTS = 4
|
|
_MAX_DELAY = 30.0
|
|
_HTTP_RETRY = frozenset({408, 429, 500, 502, 503, 504})
|
|
_NETWORK_ERRNOS = frozenset({
|
|
errno.ECONNABORTED, errno.ECONNREFUSED, errno.ECONNRESET,
|
|
errno.ENETDOWN, errno.ENETRESET, errno.ENETUNREACH,
|
|
errno.EHOSTUNREACH, errno.EPIPE, errno.ETIMEDOUT,
|
|
})
|
|
|
|
|
|
def is_transient(exc: Exception) -> bool:
|
|
if isinstance(exc, urllib.error.HTTPError):
|
|
return exc.code in _HTTP_RETRY
|
|
if isinstance(exc, urllib.error.URLError):
|
|
return isinstance(exc.reason, Exception) and is_transient(exc.reason)
|
|
if isinstance(exc, (ssl.SSLEOFError, ssl.SSLZeroReturnError)):
|
|
return True
|
|
if isinstance(exc, ssl.SSLError):
|
|
return False
|
|
if isinstance(exc, socket.gaierror):
|
|
return exc.errno == socket.EAI_AGAIN
|
|
if isinstance(exc, (TimeoutError, ConnectionError, http.client.IncompleteRead)):
|
|
return True
|
|
return isinstance(exc, OSError) and exc.errno in _NETWORK_ERRNOS
|
|
|
|
|
|
def _delay(exc: Exception, attempt: int) -> float:
|
|
delay = float(2 ** attempt)
|
|
if isinstance(exc, urllib.error.HTTPError):
|
|
value = exc.headers.get("Retry-After", "") if exc.headers else ""
|
|
if value:
|
|
try:
|
|
retry_after = int(value)
|
|
except ValueError:
|
|
try:
|
|
when = parsedate_to_datetime(value)
|
|
if when.tzinfo is None:
|
|
when = when.replace(tzinfo=timezone.utc)
|
|
retry_after = when.timestamp() - time.time()
|
|
except (TypeError, ValueError, OverflowError):
|
|
retry_after = 0.0
|
|
delay = max(delay, retry_after)
|
|
return min(delay, _MAX_DELAY)
|
|
|
|
|
|
def retry_network(
|
|
operation: Callable[[], _T], *, wait: Callable[[float], None] | None = None,
|
|
) -> _T:
|
|
"""Retry the complete request and body read, never an install or publish.
|
|
|
|
The caller resets attempt-local buffers and keeps any resumable bytes.
|
|
A download supplies an interruptible wait so pause does not wait for backoff.
|
|
"""
|
|
attempt = 1
|
|
while True:
|
|
try:
|
|
return operation()
|
|
except Exception as exc:
|
|
if isinstance(exc, urllib.error.HTTPError):
|
|
exc.close()
|
|
if not is_transient(exc) or attempt >= _ATTEMPTS:
|
|
raise
|
|
delay = _delay(exc, attempt - 1)
|
|
attempt += 1
|
|
logging.getLogger(__name__).warning(
|
|
"network request failed (%s); retrying in %gs (attempt %d/%d)",
|
|
exc, delay, attempt, _ATTEMPTS,
|
|
)
|
|
(wait or time.sleep)(delay)
|