Under one multiplexing ticker, every non-launch profile's in-flight claim leaked on every run. The cron home scope is a ContextVar: the claim is taken on the ticker thread inside `_profile_cron_scope`, but the pool worker's `finally` sits outside `ctx.run`, where the worker thread resolves the LAUNCH home — so the discard missed the real key. The job then skipped a fire window until the force-release backstop swept it, and the shutdown drain plus `hermes.cron.jobs.running` saw phantom work. `_submit_with_guard` now captures the registering home and passes it to `release_running_job(job_id, home=...)` on every release path. Also in this pass: - Stand down for a profile that runs its OWN gateway (`_cron_profile_gate`, the gate the serve/Desktop ticker already passes). On a host pinned to per-profile gateways both processes raced that profile's tick lock, and when the launch gateway won, delivery went through SharedRouteAdapters/fail-closed instead of the profile's live adapters. The gate compares the liveness PID against `os.getpid()`: this process holds the launch `gateway.pid` and publishes every served profile in `served_profiles`, so a bare liveness answer would have stood cron down host-wide. - Job-liveness consumers ask `is_job_running(job_id, home=...)` instead of the host-wide bare-id union, which let profile A's running `daily-brief` report profile B's idle one as running and keep B's stale one-shot alive. - `register_ticked_homes` reaps the parallel pools of homes that leave the ticked set; pools lived until `atexit`, so every home ever ticked kept a ThreadPoolExecutor and its worker threads. - `mark_running_jobs_interrupted` reads the real home Path from `_inflight_home_path` instead of rebuilding it from the normcased key half. - The ownership test imports `cron.scheduler_ownership` per test, so the file fails behaviourally on base instead of as a collection error.
138 lines
5.4 KiB
Python
138 lines
5.4 KiB
Python
"""Which profile homes THIS process ticks, and who owns the cron tick for a given home.
|
|
|
|
One gateway process per host multiplexes every profile, so "does a gateway own cron for profile
|
|
X" stopped being answerable from the launch home alone: ``gateway.status.owns_gateway_runtime_lock``
|
|
is a process-global boolean set once at boot, and every path resolver behind
|
|
``is_gateway_runtime_lock_active`` goes through ``get_process_hermes_home()``, which ignores the
|
|
per-tick ``_profile_cron_scope`` override. Both therefore answer about the LAUNCH home while the
|
|
ticker is scoped to some other profile.
|
|
|
|
The two predicates here keep those questions apart:
|
|
|
|
* :func:`owns_cron_tick_for` — this process is the host gateway AND ticks that home.
|
|
* :func:`live_gateway_ticking` — a DIFFERENT, live host gateway holds the runtime lock and its
|
|
published served set covers that home.
|
|
|
|
Every probe failure answers "no": ownership claims are certainties, never guesses.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import threading
|
|
from pathlib import Path
|
|
from typing import Any, Optional, Union
|
|
|
|
from hermes_constants import get_hermes_home, hermes_home_key
|
|
|
|
# Home key -> home path for every profile this process ticks, republished each ticker cycle so a
|
|
# profile created or tombstoned mid-run is reflected without a restart.
|
|
_ticked_homes: dict[str, Path] = {}
|
|
_ticked_lock = threading.Lock()
|
|
|
|
|
|
def register_ticked_homes(homes) -> None:
|
|
"""Publish the set of profile homes this process's cron ticker owns this cycle.
|
|
|
|
A home that leaves the set also loses its cron worker pool: pools are per home and otherwise
|
|
live until process exit, so serving (or churning) many profiles leaks one ThreadPoolExecutor
|
|
and its worker threads per home ever ticked.
|
|
"""
|
|
resolved = {}
|
|
for home in homes:
|
|
path = Path(home)
|
|
resolved[hermes_home_key(path)] = path
|
|
with _ticked_lock:
|
|
departed = set(_ticked_homes) - set(resolved)
|
|
_ticked_homes.clear()
|
|
_ticked_homes.update(resolved)
|
|
if departed:
|
|
# Late import: cron.scheduler imports this module.
|
|
from cron.scheduler import discard_parallel_pools
|
|
|
|
discard_parallel_pools(departed)
|
|
|
|
|
|
def ticked_homes() -> dict:
|
|
"""Snapshot of ``home key -> home`` for the profiles this process ticks."""
|
|
with _ticked_lock:
|
|
return dict(_ticked_homes)
|
|
|
|
|
|
def serves_profile(home: Optional[Union[Path, str]] = None) -> bool:
|
|
"""True when THIS process's cron ticker owns ``home`` (default: the active cron scope)."""
|
|
key = hermes_home_key(home if home is not None else get_hermes_home())
|
|
with _ticked_lock:
|
|
return key in _ticked_homes
|
|
|
|
|
|
def owns_cron_tick_for(home: Optional[Union[Path, str]] = None) -> bool:
|
|
"""True when this process is the host gateway multiplexer AND ticks ``home``.
|
|
|
|
The runtime lock alone is not enough: it proves only that this process is the host gateway,
|
|
not that its ticker visits the profile whose store is being ticked right now.
|
|
"""
|
|
try:
|
|
from gateway import status as gateway_status
|
|
|
|
if not gateway_status.owns_gateway_runtime_lock():
|
|
return False
|
|
except Exception:
|
|
return False
|
|
return serves_profile(home)
|
|
|
|
|
|
def record_serves_profile(record: Any, home: Optional[Union[Path, str]] = None) -> bool:
|
|
"""True when a gateway runtime-status ``record``'s served set covers ``home``.
|
|
|
|
The record is read from the holder's own home, so it always proves that home — that is the
|
|
pre-multiplex meaning, and a single-profile gateway publishes an EMPTY ``served_profiles``.
|
|
Other profiles need an explicit entry in the published served set.
|
|
"""
|
|
if not isinstance(record, dict):
|
|
return False
|
|
from gateway.status import (
|
|
_get_process_hermes_home, _profile_label_for_home, _same_hermes_home)
|
|
|
|
try:
|
|
target = Path(home) if home is not None else get_hermes_home()
|
|
record_home = record.get("hermes_home")
|
|
if not isinstance(record_home, str) or not record_home.strip():
|
|
record_home = _get_process_hermes_home()
|
|
if _same_hermes_home(record_home, target):
|
|
return True
|
|
except Exception:
|
|
return False
|
|
served = record.get("served_profiles")
|
|
if not isinstance(served, list):
|
|
return False
|
|
label = _profile_label_for_home(target)
|
|
return label is not None and label in served
|
|
|
|
|
|
def live_gateway_ticking(home: Optional[Union[Path, str]] = None) -> Optional[dict]:
|
|
"""Runtime-status record of ANOTHER live host gateway that ticks ``home``, else None.
|
|
|
|
``None`` covers every "we cannot prove it" case: this process holds the lock itself, no lock
|
|
is held, the record does not belong to the lock holder, its heartbeat is stale, or its served
|
|
set does not name ``home``.
|
|
"""
|
|
try:
|
|
from gateway import status as gateway_status
|
|
|
|
if gateway_status.owns_gateway_runtime_lock():
|
|
return None
|
|
if not gateway_status.is_gateway_runtime_lock_active():
|
|
return None
|
|
holder_pid = gateway_status.get_running_pid(cleanup_stale=False)
|
|
record = gateway_status.read_runtime_status()
|
|
if (
|
|
holder_pid is None
|
|
or not isinstance(record, dict)
|
|
or record.get("pid") != holder_pid
|
|
or gateway_status.runtime_status_is_stale(record)
|
|
):
|
|
return None
|
|
except Exception:
|
|
return None
|
|
return record if record_serves_profile(record, home) else None
|