718 lines
27 KiB
Python
718 lines
27 KiB
Python
"""Shared utility functions for hermes-agent."""
|
|
|
|
import errno
|
|
import json
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import stat
|
|
import tempfile
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Any, Union
|
|
from urllib.parse import urlparse
|
|
|
|
import yaml
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
TRUTHY_STRINGS = frozenset({"1", "true", "yes", "on"})
|
|
|
|
|
|
def is_truthy_value(value: Any, default: bool = False) -> bool:
|
|
"""Coerce bool-ish values using the project's shared truthy string set."""
|
|
if value is None:
|
|
return default
|
|
if isinstance(value, str):
|
|
return value.strip().lower() in TRUTHY_STRINGS
|
|
return bool(value)
|
|
|
|
|
|
def env_var_enabled(name: str, default: str = "") -> bool:
|
|
"""Return True when an environment variable is set to a truthy value."""
|
|
return is_truthy_value(os.getenv(name, default), default=False)
|
|
|
|
|
|
def _preserve_file_mode(path: Path) -> "int | None":
|
|
"""Capture the permission bits of *path* if it exists, else ``None``."""
|
|
try:
|
|
return stat.S_IMODE(path.stat().st_mode) if path.exists() else None
|
|
except OSError:
|
|
return None
|
|
|
|
|
|
def _preserve_file_owner(path: Path) -> "tuple[int, int] | None":
|
|
"""Capture the owning uid/gid of *path* if the platform supports it."""
|
|
try:
|
|
st = path.stat() if os.name == "posix" else None
|
|
except OSError:
|
|
return None
|
|
return (st.st_uid, st.st_gid) if st else None
|
|
|
|
|
|
def _restore_file_metadata(path: Path, owner: "tuple[int, int] | None", mode: "int | None") -> None:
|
|
"""Best-effort re-apply of uid/gid and permission bits after an atomic replace.
|
|
|
|
Docker/NAS installs often run some commands as root while the volume is owned by the runtime
|
|
user; ``os.replace`` swaps in the temp file's owner, leaving ``config.yaml`` root-owned, so
|
|
privileged callers chown it back (harmless otherwise). ``tempfile.mkstemp`` creates files 0o600;
|
|
without re-applying *mode* the target would inherit that and break volume mounts relying on
|
|
broader permissions.
|
|
"""
|
|
if owner is not None and hasattr(os, "chown"):
|
|
try:
|
|
os.chown(path, owner[0], owner[1])
|
|
except OSError:
|
|
pass
|
|
if mode is not None:
|
|
try:
|
|
os.chmod(path, mode)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def _restore_file_owner(path: Path, owner: "tuple[int, int] | None") -> None:
|
|
_restore_file_metadata(path, owner, None)
|
|
|
|
|
|
def _restore_file_mode(path: Path, mode: "int | None") -> None:
|
|
_restore_file_metadata(path, None, mode)
|
|
|
|
|
|
_IS_WINDOWS = os.name == "nt"
|
|
|
|
# Windows rename failures that can be caused by another handle on the target
|
|
# rather than by a permission problem. ``os.replace`` onto a file that any
|
|
# other handle has open is denied because CPython opens files without
|
|
# ``FILE_SHARE_DELETE``:
|
|
#
|
|
# 5 ERROR_ACCESS_DENIED — what a held *target* handle actually reports
|
|
# 32 ERROR_SHARING_VIOLATION — reported when the *source* temp file is held
|
|
# 33 ERROR_LOCK_VIOLATION — byte-range lock on the target
|
|
#
|
|
# Measured on Windows 11 (build 26200, CPython 3.11): a plain reader on the
|
|
# target — in-process or cross-process — yields winerror 5, NOT 32. Keying
|
|
# recovery on 32 alone therefore misses every real occurrence of this bug.
|
|
# These codes are ambiguous (a genuine ACL denial is also 5), which is why
|
|
# recovery is bounded and any still-failing write is re-raised unchanged
|
|
# rather than being classified up front.
|
|
_WINDOWS_CONTENDED_REPLACE_ERRORS = frozenset({5, 32, 33})
|
|
|
|
# Retry budget for the atomic rename. A rename that wins here keeps the write
|
|
# fully atomic, so the budget is sized to cover a realistic contended hold: an
|
|
# observed desktop auth-init holds auth.json past 100 ms, while an ordinary
|
|
# status read is ~0.05 ms. Measured on Windows 11 build 26200, this recovers
|
|
# holds up to ~200 ms atomically (~310 ms worst case).
|
|
#
|
|
# The cap matters as much as the attempt count. gateway_state.json is
|
|
# rewritten at every turn boundary, so a permanently-held target pays the full
|
|
# budget on every write: a longer 6 x 20..400 ms budget cost ~1.3 s per write
|
|
# under a persistent reader, versus ~0.3 s here for the same atomic coverage.
|
|
# Jittered so concurrent writers don't retry in lockstep.
|
|
_REPLACE_RETRY_ATTEMPTS = 4
|
|
_REPLACE_RETRY_BASE_DELAY_S = 0.02
|
|
_REPLACE_RETRY_MAX_DELAY_S = 0.1
|
|
|
|
|
|
def _is_contended_windows_replace_error(exc: OSError) -> bool:
|
|
"""Return True for Windows rename failures a retry might clear.
|
|
|
|
Only a *candidate* classification: ``ERROR_ACCESS_DENIED`` covers both a concurrent handle and a
|
|
real ACL denial, and the two are not reliably distinguishable up front.
|
|
"""
|
|
return _IS_WINDOWS and getattr(exc, "winerror", None) in _WINDOWS_CONTENDED_REPLACE_ERRORS
|
|
|
|
|
|
def _rewrite_in_place(tmp_str: str, real_path: str) -> None:
|
|
"""Overwrite *real_path* with the contents of *tmp_str*, in place.
|
|
|
|
Last-resort path for a target whose handle is still held after the retry budget: writing through
|
|
the existing file works where renaming onto it does not.
|
|
|
|
This is still not atomic — it is a strictly smaller window than a copy, not the absence of one —
|
|
so it runs only after the rename has genuinely failed. Writing through the target also preserves
|
|
its ACL, which ``os.replace`` does not (the temp file's inherited ACL wins there).
|
|
"""
|
|
with open(tmp_str, "rb") as src:
|
|
data = src.read()
|
|
flags = os.O_WRONLY | getattr(os, "O_BINARY", 0)
|
|
fd = os.open(real_path, flags)
|
|
try:
|
|
written = 0
|
|
while written < len(data):
|
|
written += os.write(fd, data[written:])
|
|
os.ftruncate(fd, len(data))
|
|
try:
|
|
os.fsync(fd)
|
|
except OSError:
|
|
pass
|
|
finally:
|
|
os.close(fd)
|
|
os.unlink(tmp_str)
|
|
|
|
|
|
def _copy_fallback(tmp_str: str, real_path: str) -> None:
|
|
"""Copy/fsync/unlink fallback for cross-device and bind-mount renames."""
|
|
shutil.copyfile(tmp_str, real_path)
|
|
try:
|
|
shutil.copystat(tmp_str, real_path)
|
|
except OSError:
|
|
pass
|
|
try:
|
|
with open(real_path, "rb") as f:
|
|
os.fsync(f.fileno())
|
|
except OSError:
|
|
pass
|
|
os.unlink(tmp_str)
|
|
|
|
|
|
def atomic_replace(tmp_path: Union[str, Path], target: Union[str, Path]) -> str:
|
|
"""Atomically move *tmp_path* onto *target*, preserving symlinks.
|
|
|
|
This helper resolves the symlink first so ``os.replace`` writes to the real file in-place while
|
|
the symlink survives. For non-symlink and non-existent paths the behavior is identical to a
|
|
plain ``os.replace`` call unless the rename fails with:
|
|
|
|
* ``EXDEV`` / ``EBUSY`` (any platform) — cross-device, bind-mount, and busy-file deployments
|
|
fall back to copy/fsync/unlink immediately. These never clear on retry. * A Windows rename
|
|
contended by another open handle (winerror 5/32/33).
|
|
"""
|
|
target_str = str(target)
|
|
real_path = os.path.realpath(target_str) if os.path.islink(target_str) else target_str
|
|
tmp_str = str(tmp_path)
|
|
try:
|
|
os.replace(tmp_str, real_path)
|
|
return real_path
|
|
except OSError as exc:
|
|
contended = _is_contended_windows_replace_error(exc)
|
|
if exc.errno not in (errno.EXDEV, errno.EBUSY) and not contended:
|
|
raise
|
|
if contended:
|
|
# Lazy import: keeps ``utils`` free of a package-level dependency
|
|
# on ``agent`` for every consumer that never hits this path.
|
|
from agent.retry_utils import jittered_backoff
|
|
|
|
for attempt in range(1, _REPLACE_RETRY_ATTEMPTS + 1):
|
|
time.sleep(jittered_backoff(
|
|
attempt, base_delay=_REPLACE_RETRY_BASE_DELAY_S, max_delay=_REPLACE_RETRY_MAX_DELAY_S
|
|
))
|
|
try:
|
|
os.replace(tmp_str, real_path)
|
|
return real_path
|
|
except OSError as retry_exc:
|
|
if retry_exc.errno in (errno.EXDEV, errno.EBUSY):
|
|
# Not contention after all — stop burning the budget.
|
|
exc = retry_exc
|
|
contended = False
|
|
break
|
|
if not _is_contended_windows_replace_error(retry_exc):
|
|
raise
|
|
exc = retry_exc
|
|
logger.debug(
|
|
"atomic_replace: %s -> %s failed with %s; falling back to %s",
|
|
tmp_str, real_path,
|
|
getattr(exc, "winerror", None) or errno.errorcode.get(exc.errno or 0, exc.errno),
|
|
"in-place rewrite" if contended else "copy",
|
|
)
|
|
if contended:
|
|
# Re-raises the rewrite's own error (not the rename's) when the
|
|
# target is genuinely unwritable — an ACL denial stays an ACL
|
|
# denial rather than being reported as contention.
|
|
_rewrite_in_place(tmp_str, real_path)
|
|
else:
|
|
_copy_fallback(tmp_str, real_path)
|
|
return real_path
|
|
|
|
|
|
def _atomic_write(
|
|
path: Path,
|
|
write,
|
|
*,
|
|
prefix: str,
|
|
encoding: str = "utf-8",
|
|
mode: "int | None" = None,
|
|
preserve_owner: bool = True,
|
|
) -> None:
|
|
"""Temp file + fsync + :func:`atomic_replace`, then re-apply owner/mode.
|
|
|
|
*write(f)* emits the payload into the open text handle. *mode* (when not ``None``) is fchmod'd
|
|
onto the temp fd BEFORE the replace so the target never transits through mkstemp's 0600 (fchmod
|
|
is Unix-only; the post-replace chmod is the sole path on Windows and harmless elsewhere). The
|
|
temp file is removed on any failure — ``BaseException`` on purpose, so KeyboardInterrupt /
|
|
SystemExit still clean up before re-raising.
|
|
"""
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
original_owner = _preserve_file_owner(path) if preserve_owner else None
|
|
fd, tmp_path = tempfile.mkstemp(dir=str(path.parent), prefix=prefix, suffix=".tmp")
|
|
try:
|
|
with os.fdopen(fd, "w", encoding=encoding) as f:
|
|
if mode is not None and hasattr(os, "fchmod"):
|
|
os.fchmod(f.fileno(), mode)
|
|
write(f)
|
|
f.flush()
|
|
os.fsync(f.fileno())
|
|
# Preserve symlinks — swap in-place on the real file (GitHub #16743).
|
|
_restore_file_metadata(Path(atomic_replace(tmp_path, path)), original_owner, mode)
|
|
except BaseException:
|
|
try:
|
|
os.unlink(tmp_path)
|
|
except OSError:
|
|
pass
|
|
raise
|
|
|
|
|
|
def _mode_for_write(
|
|
path: Path, create_mode: "int | None", preserve: bool = True
|
|
) -> "int | None":
|
|
"""Existing permission bits of *path* (when *preserve*), else *create_mode* for a new file."""
|
|
mode = _preserve_file_mode(path) if preserve else None
|
|
if mode is None and create_mode is not None and not path.exists():
|
|
mode = create_mode
|
|
return mode
|
|
|
|
|
|
def atomic_write_text(
|
|
path: Union[str, Path],
|
|
content: str,
|
|
*,
|
|
encoding: str = "utf-8",
|
|
tmp_prefix: str = ".tmp_",
|
|
preserve_mode: bool = False,
|
|
create_mode: "int | None" = None,
|
|
) -> None:
|
|
"""Write *content* to *path* via temp file + fsync + atomic rename.
|
|
|
|
Ensures the target file is never left in a partially-written state if the process crashes or is
|
|
interrupted. ``atomic_replace`` preserves symlinks and handles cross-device / busy-file
|
|
fallbacks.
|
|
|
|
Used by the memory store, skill manager, and agent importer so that every destructive file
|
|
rewrite in the codebase shares one implementation.
|
|
"""
|
|
path = Path(path)
|
|
_atomic_write(
|
|
path,
|
|
lambda f: f.write(content),
|
|
prefix=tmp_prefix,
|
|
encoding=encoding,
|
|
mode=_mode_for_write(path, create_mode, preserve=preserve_mode),
|
|
preserve_owner=preserve_mode,
|
|
)
|
|
|
|
|
|
def atomic_json_write(
|
|
path: Union[str, Path],
|
|
data: Any,
|
|
*,
|
|
indent: int = 2,
|
|
mode: int | None = None,
|
|
**dump_kwargs: Any,
|
|
) -> None:
|
|
"""Write JSON data to a file atomically.
|
|
|
|
Uses temp file + fsync + os.replace to ensure the target file is never left in a partially-
|
|
written state. If the process crashes mid-write, the previous version of the file remains
|
|
intact.
|
|
"""
|
|
path = Path(path)
|
|
_atomic_write(
|
|
path,
|
|
lambda f: json.dump(data, f, indent=indent, ensure_ascii=False, **dump_kwargs),
|
|
prefix=f".{path.stem}_",
|
|
mode=mode if mode is not None else _preserve_file_mode(path),
|
|
)
|
|
|
|
|
|
def warn_if_credential_file_broadly_readable(
|
|
path: Union[str, Path],
|
|
*,
|
|
label: str = "",
|
|
log: logging.Logger | None = None,
|
|
) -> bool:
|
|
"""Warn (once per call) when a credential file is group/world-readable.
|
|
|
|
Secret-bearing files that users create by hand (or that older Hermes versions wrote without an
|
|
explicit mode) commonly end up 0o644 under the default umask. This helper is the shared read-
|
|
time check for that class: call it before loading any token/credential file so the owner gets a
|
|
remediation hint in the logs.
|
|
|
|
Returns True when a warning was emitted. No-ops (returns False) on platforms without POSIX
|
|
permission bits semantics (best effort), when the file is missing, or when permissions are
|
|
already tight.
|
|
"""
|
|
p = Path(path)
|
|
try:
|
|
file_mode = p.stat().st_mode
|
|
except OSError:
|
|
return False
|
|
# Windows ACLs don't map onto POSIX group/other bits; st_mode there is synthesized.
|
|
if os.name != "posix" or not (file_mode & (stat.S_IRGRP | stat.S_IROTH)):
|
|
return False
|
|
(log or logger).warning(
|
|
"%s%s is group/world-readable (mode 0%o) and contains secrets. "
|
|
"Run: chmod 600 %s",
|
|
f"{label} " if label else "",
|
|
p.name,
|
|
stat.S_IMODE(file_mode),
|
|
p,
|
|
)
|
|
return True
|
|
|
|
|
|
class IndentDumper(yaml.SafeDumper):
|
|
"""PyYAML dumper that indents list items under mapping keys (2-space).
|
|
|
|
Default PyYAML emits "indentless" sequences while ``ruamel.yaml`` (used by
|
|
:func:`atomic_roundtrip_yaml_update`) indents them; mixing both in one ``config.yaml`` makes
|
|
stricter parsers like ``js-yaml`` reject it. Forcing ``indentless=False`` keeps every write
|
|
path byte-identical.
|
|
"""
|
|
|
|
def increase_indent(self, flow=False, indentless=False): # noqa: ARG002
|
|
return super().increase_indent(flow, False)
|
|
|
|
|
|
def atomic_yaml_write(
|
|
path: Union[str, Path],
|
|
data: Any,
|
|
*,
|
|
default_flow_style: bool = False,
|
|
sort_keys: bool = False,
|
|
extra_content: str | None = None,
|
|
create_mode: "int | None" = None,
|
|
) -> None:
|
|
"""Write YAML data to a file atomically.
|
|
|
|
Uses temp file + fsync + os.replace to ensure the target file is never left in a partially-
|
|
written state. If the process crashes mid-write, the previous version of the file remains
|
|
intact.
|
|
"""
|
|
path = Path(path)
|
|
|
|
def _write(f) -> None:
|
|
# allow_unicode=True writes emoji/kaomoji (e.g. personalities, skin
|
|
# cursors) as real UTF-8 instead of fragile escape sequences. Without
|
|
# it, PyYAML emits astral-plane chars as `\UXXXXXXXX` (8-digit) escapes
|
|
# inside multi-line double-quoted strings wrapped with `\`
|
|
# continuations — a structure that stricter/non-PyYAML parsers and
|
|
# hand-edits routinely break into unclosed quotes, corrupting the whole
|
|
# config (GitHub #51356).
|
|
yaml.dump(
|
|
data,
|
|
f,
|
|
Dumper=IndentDumper,
|
|
default_flow_style=default_flow_style,
|
|
sort_keys=sort_keys,
|
|
allow_unicode=True,
|
|
)
|
|
if extra_content:
|
|
f.write(extra_content)
|
|
|
|
_atomic_write(
|
|
path, _write, prefix=f".{path.stem}_", mode=_mode_for_write(path, create_mode)
|
|
)
|
|
|
|
|
|
def _roundtrip_yaml():
|
|
"""ruamel round-trip ``YAML`` configured to keep quotes/Unicode with 2-space indents."""
|
|
from ruamel.yaml import YAML
|
|
|
|
yaml_rt = YAML(typ="rt")
|
|
yaml_rt.preserve_quotes = True
|
|
yaml_rt.allow_unicode = True
|
|
yaml_rt.default_flow_style = False
|
|
yaml_rt.indent(mapping=2, sequence=4, offset=2)
|
|
return yaml_rt
|
|
|
|
|
|
def _load_commented_map(yaml_rt, path: Path):
|
|
"""Load *path* with *yaml_rt* as a ``CommentedMap`` (empty when missing/blank)."""
|
|
from ruamel.yaml.comments import CommentedMap
|
|
|
|
data = None
|
|
if path.exists():
|
|
with path.open("r", encoding="utf-8") as f:
|
|
data = yaml_rt.load(f)
|
|
return data if isinstance(data, CommentedMap) else CommentedMap(data or {})
|
|
|
|
|
|
def atomic_roundtrip_yaml_update(
|
|
path: Union[str, Path],
|
|
key_path: str,
|
|
value: Any,
|
|
) -> None:
|
|
"""Update one dotted YAML key while preserving comments and readable text.
|
|
|
|
Narrower than :func:`atomic_yaml_write` on purpose: for user-edited config files where
|
|
comments, ordering, quoting and Unicode must survive a single setting mutation. Still writes
|
|
via temp file + fsync + atomic replace.
|
|
"""
|
|
from ruamel.yaml.comments import CommentedMap
|
|
|
|
path = Path(path)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
yaml_rt = _roundtrip_yaml()
|
|
config = _load_commented_map(yaml_rt, path)
|
|
|
|
current = config
|
|
# Honor escaped dots and prefer existing literal dotted keys (e.g. model
|
|
# IDs like ``glm-5.3``) over blind splitting — same navigation as
|
|
# ``hermes config set``'s ``_set_nested`` (#91607: /model + TUI
|
|
# persistence route through here and used to write ``glm-5: {'3': ...}``
|
|
# phantom siblings while the runtime kept reading the literal key).
|
|
from hermes_cli.config import _greedy_literal_match, _split_key_path
|
|
|
|
keys = _split_key_path(key_path)
|
|
i = 0
|
|
while True:
|
|
remaining = keys[i:]
|
|
seg, consumed = remaining[0], 1
|
|
match = _greedy_literal_match(dict(current), remaining)
|
|
if match is not None:
|
|
seg, consumed = match
|
|
if i + consumed == len(keys):
|
|
current[seg] = value
|
|
break
|
|
next_value = current.get(seg)
|
|
if not isinstance(next_value, CommentedMap):
|
|
next_value = CommentedMap()
|
|
current[seg] = next_value
|
|
current = next_value
|
|
i += consumed
|
|
|
|
_atomic_write(
|
|
path,
|
|
lambda f: yaml_rt.dump(config, f),
|
|
prefix=f".{path.stem}_",
|
|
mode=_preserve_file_mode(path),
|
|
)
|
|
|
|
|
|
def atomic_roundtrip_yaml_save(
|
|
path: Union[str, Path],
|
|
new_state: dict,
|
|
) -> None:
|
|
"""Persist a full config-state dict while preserving comments and ordering.
|
|
|
|
Behaves like ``atomic_yaml_write`` (writes the whole file in one shot from ``new_state``), but
|
|
routes through ruamel.yaml round-trip mode so existing comments, key order, quotes, and readable
|
|
Unicode survive.
|
|
|
|
This is the comment-safe replacement for ``yaml.safe_dump(cfg, f)`` in callers that mutate a
|
|
deep-loaded config dict and want to persist the whole thing.
|
|
"""
|
|
from ruamel.yaml.comments import CommentedMap
|
|
from ruamel.yaml.scalarstring import DoubleQuotedScalarString
|
|
|
|
from hermes_cli.config import require_readable_config_before_write
|
|
|
|
path = Path(path)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
require_readable_config_before_write(path)
|
|
yaml_rt = _roundtrip_yaml()
|
|
existing = _load_commented_map(yaml_rt, path)
|
|
|
|
# ruamel's round-trip dumper resolves plain scalars against the YAML 1.2
|
|
# core schema, where only true/false/null are reserved words — so a plain
|
|
# python str like "off" or "yes" is emitted unquoted. Every other config
|
|
# reader in this codebase (atomic_config_write's PyYAML path, yaml.safe_load
|
|
# call sites, etc.) parses under YAML 1.1 rules, where on/off/yes/no are
|
|
# boolean synonyms. Without forcing quotes here, a freshly written
|
|
# `approvals.mode: off` silently round-trips back as `False` under
|
|
# yaml.safe_load. Force-quote any new string value that YAML 1.1 would
|
|
# otherwise misparse as bool/null.
|
|
_YAML11_AMBIGUOUS_WORDS = {"y", "n", "yes", "no", "true", "false", "on", "off", "null", "~"}
|
|
|
|
def _quote_if_yaml11_ambiguous(value):
|
|
if isinstance(value, str) and value.lower() in _YAML11_AMBIGUOUS_WORDS:
|
|
return DoubleQuotedScalarString(value)
|
|
return value
|
|
|
|
def _merge(dst: CommentedMap, src: dict) -> None:
|
|
# Update / recurse into keys present in src.
|
|
for key, value in src.items():
|
|
if isinstance(value, dict):
|
|
current = dst.get(key)
|
|
if not isinstance(current, CommentedMap):
|
|
current = CommentedMap()
|
|
dst[key] = current
|
|
_merge(current, value)
|
|
else:
|
|
dst[key] = _quote_if_yaml11_ambiguous(value)
|
|
# Delete keys missing from src — preserves "explicit absence" semantics
|
|
# of the old _save_cfg(cfg) pattern (e.g. cfg.pop("custom_prompt", None)
|
|
# then _save_cfg must actually remove the key from disk).
|
|
for key in [k for k in dst if k not in src]:
|
|
del dst[key]
|
|
|
|
_merge(existing, new_state)
|
|
|
|
_atomic_write(
|
|
path,
|
|
lambda f: yaml_rt.dump(existing, f),
|
|
prefix=f".{path.stem}_",
|
|
mode=_preserve_file_mode(path),
|
|
)
|
|
|
|
|
|
# ─── JSON Helpers ─────────────────────────────────────────────────────────────
|
|
|
|
|
|
def safe_json_loads(text: str, default: Any = None) -> Any:
|
|
"""Parse JSON, returning *default* on any parse error."""
|
|
try:
|
|
return json.loads(text)
|
|
except (json.JSONDecodeError, TypeError, ValueError):
|
|
return default
|
|
|
|
|
|
# ── Fast YAML loading ────────────────────────────────────────────────────
|
|
#
|
|
# PyYAML's pure-Python SafeLoader is ~8x slower than the libyaml-backed
|
|
# ``CSafeLoader`` C extension. Startup parses config.yaml and every plugin
|
|
# manifest with the slow path, costing ~0.9s of cold-start time. The C loader
|
|
# is a true drop-in for ``safe_load`` (same restricted tag set), so prefer it
|
|
# and fall back to the pure-Python loader only when libyaml isn't compiled in.
|
|
_fast_yaml_loader = getattr(yaml, "CSafeLoader", None) or yaml.SafeLoader
|
|
|
|
|
|
def fast_safe_load(stream: Any) -> Any:
|
|
"""``yaml.safe_load`` using the libyaml C loader when available.
|
|
|
|
Accepts the same inputs as ``yaml.safe_load`` (a ``str``/``bytes`` document or a readable file
|
|
object) and returns the same parsed structure. Falls back to PyYAML's pure-Python ``SafeLoader``
|
|
when ``CSafeLoader`` isn't available, so behavior is identical everywhere — only the speed
|
|
differs.
|
|
"""
|
|
return yaml.load(stream, Loader=_fast_yaml_loader)
|
|
|
|
|
|
# ─── Environment Variable Helpers ─────────────────────────────────────────────
|
|
|
|
|
|
def _env_number(key: str, default, cast):
|
|
raw = os.getenv(key, "").strip()
|
|
if not raw:
|
|
return default
|
|
try:
|
|
return cast(raw)
|
|
except (ValueError, TypeError):
|
|
return default
|
|
|
|
|
|
def env_int(key: str, default: int = 0) -> int:
|
|
"""Read an environment variable as an integer, with fallback."""
|
|
return _env_number(key, default, int)
|
|
|
|
|
|
def env_float(key: str, default: float = 0.0) -> float:
|
|
"""Read an environment variable as a float, with fallback."""
|
|
return _env_number(key, default, float)
|
|
|
|
|
|
def env_bool(key: str, default: bool = False) -> bool:
|
|
"""Read an environment variable as a boolean."""
|
|
return is_truthy_value(os.getenv(key, ""), default=default)
|
|
|
|
|
|
# ─── Proxy Helpers ────────────────────────────────────────────────────────────
|
|
|
|
|
|
_PROXY_ENV_KEYS = (
|
|
"HTTPS_PROXY", "HTTP_PROXY", "ALL_PROXY",
|
|
"https_proxy", "http_proxy", "all_proxy",
|
|
)
|
|
|
|
|
|
def normalize_proxy_url(proxy_url: str | None) -> str | None:
|
|
"""Normalize proxy URLs for httpx/aiohttp compatibility.
|
|
|
|
WSL/Clash-style environments export SOCKS proxies as ``socks://host:port``; httpx rejects
|
|
that alias and needs the explicit ``socks5://`` scheme.
|
|
"""
|
|
candidate = str(proxy_url or "").strip()
|
|
if candidate.lower().startswith("socks://"):
|
|
return f"socks5://{candidate[len('socks://'):]}"
|
|
return candidate or None
|
|
|
|
|
|
def normalize_proxy_env_vars() -> None:
|
|
"""Rewrite supported proxy env vars to canonical URL forms in-place."""
|
|
for key in _PROXY_ENV_KEYS:
|
|
value = os.getenv(key, "")
|
|
normalized = normalize_proxy_url(value)
|
|
if normalized and normalized != value:
|
|
os.environ[key] = normalized
|
|
|
|
|
|
# ─── URL Parsing Helpers ──────────────────────────────────────────────────────
|
|
|
|
|
|
def _parse_base_url(base_url: str):
|
|
"""``urlparse`` that tolerates a bare ``host[:port][/path]`` (no scheme)."""
|
|
raw = (base_url or "").strip()
|
|
if not raw:
|
|
return None
|
|
return urlparse(raw if "://" in raw else f"//{raw}")
|
|
|
|
|
|
def base_url_hostname(base_url: str) -> str:
|
|
"""Return the lowercased hostname for a base URL, or ``""`` if absent.
|
|
|
|
Compare exact hostnames against known provider hosts instead of substring-matching the raw
|
|
URL: substring checks treat ``https://api.openai.com.example/v1`` or
|
|
``https://proxy.test/api.openai.com/v1`` as native endpoints, mis-routing api_mode and auth.
|
|
"""
|
|
parsed = _parse_base_url(base_url)
|
|
return (parsed.hostname or "").lower().rstrip(".") if parsed else ""
|
|
|
|
|
|
# ─── Model Capability Detection ──────────────────────────────────────────────
|
|
|
|
|
|
def model_forces_max_completion_tokens(model: str) -> bool:
|
|
"""Return True for model families that require ``max_completion_tokens``.
|
|
|
|
OpenAI's newer families reject ``max_tokens`` on /v1/chat/completions with HTTP 400
|
|
``unsupported_parameter`` — the caller must send ``max_completion_tokens`` instead. This covers:
|
|
"""
|
|
m = (model or "").strip().lower().rsplit("/", 1)[-1]
|
|
return m.startswith(("gpt-4o", "gpt-4.1", "gpt-5", "o1", "o3", "o4"))
|
|
|
|
|
|
def base_url_origin(base_url: str) -> tuple[str, str, int]:
|
|
"""Return ``(scheme, hostname, effective_port)`` for a base URL.
|
|
|
|
Origin, not just host: ``https://h`` vs ``http://h`` and two ports on one host are different
|
|
trust boundaries, so any decision to hand a bearer secret to a new URL must compare all
|
|
three — hostname alone would authorise an HTTPS→HTTP downgrade. The port defaults to 443/80
|
|
when absent so ``https://h`` equals ``https://h:443``. Returns ``("", "", 0)`` on no
|
|
hostname or a bad port.
|
|
"""
|
|
parsed = _parse_base_url(base_url)
|
|
hostname = (parsed.hostname or "").lower().rstrip(".") if parsed else ""
|
|
if not hostname:
|
|
return ("", "", 0)
|
|
scheme = (parsed.scheme or "").lower()
|
|
try:
|
|
port = parsed.port
|
|
except ValueError:
|
|
# Out-of-range or non-numeric port — not a usable origin.
|
|
return ("", "", 0)
|
|
if port is None:
|
|
port = {"https": 443, "http": 80}.get(scheme, 0)
|
|
return (scheme, hostname, port)
|
|
|
|
|
|
def base_url_host_matches(base_url: str, domain: str) -> bool:
|
|
"""Return True when the base URL's hostname is ``domain`` or a subdomain.
|
|
|
|
Safer counterpart to ``domain in base_url``, which has the substring false-positive class
|
|
noted on ``base_url_hostname`` (``evil.com/moonshot.ai`` or ``moonshot.ai.evil`` must not
|
|
match). Accepts bare hosts, full URLs, and URLs with paths.
|
|
"""
|
|
hostname = base_url_hostname(base_url)
|
|
domain = (domain or "").strip().lower().rstrip(".")
|
|
return bool(hostname and domain) and (hostname == domain or hostname.endswith("." + domain))
|