0dcadf6f41
Reverts the in-tree org skill-marketplace: hermes_wisdom package, three model tools, CLI/gateway/desktop/dashboard/Telegram/Slack surfaces. Later non-Wisdom work on shared files (guest onboarding i18n, dashboard startup schema, Slack adapter, tui_gateway) is kept; Wisdom-only call sites and config were stripped from those files.
344 lines
17 KiB
Python
344 lines
17 KiB
Python
"""Skill Sync wire model: content-addressed objects, the HTTP client, tree walks.
|
|
|
|
Independent of local skill state (no ``~/.hermes`` reads); ``tools/skills_sync_client.py``
|
|
orchestrates push/pull on top of it and re-exports these names. Wire contract version 1:
|
|
``hsp_version`` / ``X-HSP-Object-Type`` are deployed protocol identifiers and are NOT
|
|
renamed with the product name "Skill Sync".
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import stat as _stat
|
|
from contextlib import suppress
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path, PurePosixPath
|
|
from typing import Any, Dict, List, Optional, Tuple
|
|
|
|
logger = logging.getLogger("tools.skills_sync_client")
|
|
WIRE_VERSION = "1"
|
|
DEFAULT_MAX_OBJECT_BYTES = 26214400 # 25 MiB, mirrors capabilities default
|
|
KIND_BLOB, KIND_TREE, KIND_COMMIT = "blob", "tree", "commit"
|
|
MODE_FILE, MODE_EXEC, MODE_DIR = "file", "exec", "dir"
|
|
ARTIFACT_TYPE_SKILL = "skill"
|
|
_EXEC_BITS = _stat.S_IXUSR | _stat.S_IXGRP | _stat.S_IXOTH
|
|
|
|
# `sync-manifest`: per-skill opt-in is CONTENT in the object model, not a device-local flag --
|
|
# a root-level blob in the tree at refs/user/<owner>/HEAD recording {name, enabled}. The plane
|
|
# manifest is authoritative; the local `.usage.json` `sync` flag is only the editable intent
|
|
# (reconciled FROM it on pull, TO it on push). Shape MUST match gateway-gateway src/sync/manifest.ts.
|
|
SYNC_MANIFEST_ENTRY_NAME = SYNC_MANIFEST_TYPE = "sync-manifest"
|
|
SYNC_MANIFEST_VERSION = 1
|
|
|
|
|
|
# Content addressing: the wire uses the FULL 64-hex sha256 -- a different namespace from the
|
|
# truncated 16-hex local `content_hash` (skills_guard.py).
|
|
def wire_address(data: bytes) -> str:
|
|
return "sha256:" + hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def canonical_json_bytes(obj: Dict[str, Any]) -> bytes:
|
|
"""Canonical JSON for hashing: UTF-8, sorted keys, no whitespace/trailing newline; arrays already in
|
|
contract order. Client and server MUST agree byte-for-byte or a push fails ``422 hash_mismatch``."""
|
|
return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
|
|
|
|
|
def build_sync_manifest_bytes(skills: Dict[str, bool]) -> bytes:
|
|
"""Canonical ``sync-manifest`` bytes for ``{name: enabled}`` (sorted by name for a stable address)."""
|
|
return canonical_json_bytes({
|
|
"type": SYNC_MANIFEST_TYPE, "version": SYNC_MANIFEST_VERSION,
|
|
"skills": [{"name": name, "enabled": bool(enabled)} for name, enabled in sorted(skills.items())],
|
|
})
|
|
|
|
|
|
def parse_sync_manifest(data: bytes) -> Optional[Dict[str, bool]]:
|
|
"""``{name: enabled}`` from ``sync-manifest`` bytes, or None if malformed. Strict (mirrors gateway
|
|
``parseSyncManifest``): a bad manifest must not read as "nothing opted in"."""
|
|
try:
|
|
value = json.loads(data.decode("utf-8"))
|
|
except Exception:
|
|
return None
|
|
if (not isinstance(value, dict) or value.get("type") != SYNC_MANIFEST_TYPE
|
|
or value.get("version") != SYNC_MANIFEST_VERSION or not isinstance(value.get("skills"), list)):
|
|
return None
|
|
out: Dict[str, bool] = {}
|
|
for raw in value["skills"]:
|
|
name, enabled = (raw.get("name"), raw.get("enabled")) if isinstance(raw, dict) else (None, None)
|
|
if not isinstance(name, str) or not name or not isinstance(enabled, bool):
|
|
return None
|
|
out[name] = enabled
|
|
return out
|
|
|
|
|
|
# Object building
|
|
class ObjectSet:
|
|
"""Objects to push, ``hash -> (kind, bytes)``, deduped by content address."""
|
|
|
|
def __init__(self) -> None:
|
|
self.objects: Dict[str, Tuple[str, bytes]] = {}
|
|
|
|
def add(self, kind: str, data: bytes) -> str:
|
|
addr = wire_address(data)
|
|
self.objects.setdefault(addr, (kind, data))
|
|
return addr
|
|
|
|
def __len__(self) -> int:
|
|
return len(self.objects)
|
|
|
|
|
|
def _entry(name: str, kind: str, hash_: str, mode: str) -> Dict[str, str]:
|
|
return {"name": name, "kind": kind, "hash": hash_, "mode": mode}
|
|
|
|
|
|
def _add_tree(entries: List[Dict[str, str]], objects: ObjectSet) -> str:
|
|
"""Canonicalize *entries* (sorted by name, byte order) into a tree object."""
|
|
entries.sort(key=lambda e: e["name"])
|
|
return objects.add(KIND_TREE, canonical_json_bytes({"type": KIND_TREE, "entries": entries}))
|
|
|
|
|
|
def _file_mode(path: Path) -> str:
|
|
"""``exec`` if +x else ``file``. No symlink / other modes are emitted."""
|
|
with suppress(OSError):
|
|
if path.stat().st_mode & _EXEC_BITS:
|
|
return MODE_EXEC
|
|
return MODE_FILE
|
|
|
|
|
|
def build_tree(dir_path: Path, objects: ObjectSet, *, max_object_bytes: int) -> str:
|
|
"""Build objects for *dir_path* recursively; return the tree address. Symlinks/special files are
|
|
skipped (contract). A blob over *max_object_bytes* raises ValueError (server would 413)."""
|
|
entries: List[Dict[str, str]] = []
|
|
for child in sorted(dir_path.iterdir(), key=lambda p: p.name):
|
|
if child.is_symlink():
|
|
logger.debug("skills_sync_client: skipping symlink %s", child)
|
|
elif child.is_dir():
|
|
entries.append(_entry(child.name, KIND_TREE, build_tree(child, objects, max_object_bytes=max_object_bytes),
|
|
MODE_DIR))
|
|
elif child.is_file():
|
|
data = child.read_bytes()
|
|
if len(data) > max_object_bytes:
|
|
raise ValueError(f"file {child} is {len(data)} bytes > max_object_bytes {max_object_bytes}")
|
|
entries.append(_entry(child.name, KIND_BLOB, objects.add(KIND_BLOB, data), _file_mode(child)))
|
|
return _add_tree(entries, objects)
|
|
|
|
|
|
def build_commit(tree_hash: str, parents: List[str], *, owner: str, device: str, message: str,
|
|
objects: ObjectSet, ts: Optional[str] = None) -> str:
|
|
"""Commit object address. ``parents``: 0 first commit, 1 edit, 2 merge (base first, other head second)."""
|
|
return objects.add(KIND_COMMIT, canonical_json_bytes({
|
|
"type": KIND_COMMIT, "tree": tree_hash, "parents": list(parents),
|
|
"author": {"owner": owner, "device": device},
|
|
"ts": ts or datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
"message": message, "artifact_type": ARTIFACT_TYPE_SKILL}))
|
|
|
|
|
|
def build_root_tree(node: Dict[str, Any], objects: ObjectSet, *, manifest_hash: Optional[str] = None) -> str:
|
|
"""Canonicalize a nested ``{name: {"__tree__": hash} | subdict}`` root into trees. ``manifest_hash``
|
|
adds the root ``sync-manifest`` BLOB entry (cannot collide with a skill dir: those are trees)."""
|
|
entries: List[Dict[str, str]] = []
|
|
for name, child in node.items():
|
|
leaf = isinstance(child, dict) and "__tree__" in child and len(child) == 1
|
|
entries.append(_entry(name, KIND_TREE, child["__tree__"] if leaf else build_root_tree(child, objects),
|
|
MODE_DIR))
|
|
if manifest_hash is not None:
|
|
entries.append(_entry(SYNC_MANIFEST_ENTRY_NAME, KIND_BLOB, manifest_hash, MODE_FILE))
|
|
return _add_tree(entries, objects)
|
|
|
|
|
|
def nest_skill_tree(root: Dict[str, Any], rel_parts: Tuple[str, ...], tree_hash: str) -> None:
|
|
"""Insert a skill tree leaf into the nested root structure by path parts."""
|
|
node = root
|
|
for part in rel_parts[:-1]:
|
|
node = node.setdefault(part, {})
|
|
node[rel_parts[-1]] = {"__tree__": tree_hash}
|
|
|
|
|
|
def assemble_root_from_skill_trees(skill_trees: Dict[str, str], objects: ObjectSet) -> str:
|
|
"""Profile-root tree from ``{posix_rel_path: tree_hash}``; skill trees are assumed durable, only
|
|
the new intermediate/root trees are added."""
|
|
root: Dict[str, Any] = {}
|
|
for path, tree_hash in skill_trees.items():
|
|
nest_skill_tree(root, PurePosixPath(path).parts, tree_hash)
|
|
return build_root_tree(root, objects)
|
|
|
|
|
|
# HTTP client (routes under /v1/sync/)
|
|
class SyncError(RuntimeError):
|
|
"""A non-recoverable wire error (4xx the client can't retry)."""
|
|
|
|
def __init__(self, message: str, *, status: Optional[int] = None):
|
|
super().__init__(message)
|
|
self.status = status
|
|
|
|
|
|
class SyncConflict(RuntimeError):
|
|
"""CAS lost (409); NOT a rejection, pushed objects are durable. ``actual`` = head to merge against,
|
|
or None when the ref does not exist (server sends ""): retry as a create, never fetch it."""
|
|
|
|
def __init__(self, actual: Optional[str]):
|
|
self.actual: Optional[str] = actual or None
|
|
super().__init__(f"CAS conflict; actual head {self.actual}" if self.actual
|
|
else "CAS conflict; the ref does not exist yet")
|
|
|
|
|
|
def _check_version(caps: Dict[str, Any]) -> None:
|
|
"""Reject an incompatible server major version."""
|
|
ver = str(caps.get("hsp_version") or "") # wire field name
|
|
if ver.split(".", 1)[0] != WIRE_VERSION:
|
|
raise SyncError(f"this server speaks sync version {ver!r}, but this Hermes speaks "
|
|
f"{WIRE_VERSION} — update Hermes to sync with it")
|
|
|
|
|
|
def _body(r) -> Dict[str, Any]:
|
|
return r.json() if r.content else {}
|
|
|
|
|
|
def checked_capabilities(client: "SyncClient") -> Tuple[Dict[str, Any], int]:
|
|
"""Version-checked ``(caps, max_object_bytes)`` for a sync session."""
|
|
caps = client.capabilities()
|
|
_check_version(caps)
|
|
return caps, int(caps.get("max_object_bytes") or DEFAULT_MAX_OBJECT_BYTES)
|
|
|
|
|
|
class SyncClient:
|
|
"""Sync client bound to a base URL + Nous bearer. Org refs/objects live behind SEPARATE ``org/``
|
|
routes: the personal routes are hard-scoped to the token's owner and would silently answer an
|
|
org query with personal data, so org readers MUST pass ``org_scope=True`` on every hop."""
|
|
|
|
def __init__(self, base_url: str, api_key: str, *, timeout: float = 30.0):
|
|
self.base = base_url.rstrip("/")
|
|
self.api_key = api_key
|
|
self.timeout = timeout
|
|
import requests # core dependency
|
|
self._session = requests.Session()
|
|
self._session.headers["Authorization"] = f"Bearer {api_key}"
|
|
|
|
def _request(self, method: str, path: str, op: str, *, ok=(200,), errors: Optional[Dict[int, Any]] = None, **kw):
|
|
"""One wire call to ``/v1/sync/<path>``; SyncError unless the status is in *ok*. *errors* maps a
|
|
status to a message (str or ``fn(response)``); anything else gets ``"<op> failed: <code>"``."""
|
|
r = self._session.request(method, f"{self.base}/v1/sync/{path.lstrip('/')}", timeout=self.timeout, **kw)
|
|
if r.status_code in ok:
|
|
return r
|
|
msg = (errors or {}).get(r.status_code)
|
|
raise SyncError((msg(r) if callable(msg) else msg) or f"{op} failed: {r.status_code}", status=r.status_code)
|
|
|
|
def capabilities(self) -> Dict[str, Any]:
|
|
"""GET capabilities (no auth required)."""
|
|
return self._request("GET", "capabilities", "capabilities").json()
|
|
|
|
def get_refs(self, prefix: str, *, org_scope: bool = False) -> List[Dict[str, str]]:
|
|
"""GET refs?prefix=... (or org/refs, filtered client-side by *prefix*)."""
|
|
path, params = ("org/refs", None) if org_scope else ("refs", {"prefix": prefix})
|
|
refs = (self._request("GET", path, "get_refs", params=params).json() or {}).get("refs", [])
|
|
return [r_ for r_ in refs if str(r_.get("name", "")).startswith(prefix)] if org_scope else refs
|
|
|
|
def get_object(self, obj_hash: str, *, org_scope: bool = False) -> Tuple[str, bytes]:
|
|
"""GET objects/:hash -> ``(kind, bytes)``; kind from the object-type header, blob default."""
|
|
r = self._request("GET", f"{'org/' if org_scope else ''}objects/{obj_hash}", "get_object",
|
|
errors={404: f"object {obj_hash} not found", 403: f"object {obj_hash} not readable"})
|
|
return r.headers.get("X-HSP-Object-Type") or KIND_BLOB, r.content
|
|
|
|
def _get_json_of_kind(self, obj_hash: str, expected: str, org_scope: bool) -> Dict[str, Any]:
|
|
kind, data = self.get_object(obj_hash, org_scope=org_scope)
|
|
if kind != expected:
|
|
raise SyncError(f"{obj_hash} is {kind}, expected {expected}")
|
|
return json.loads(data.decode("utf-8"))
|
|
|
|
def get_commit_json(self, commit_hash: str, *, org_scope: bool = False) -> Dict[str, Any]:
|
|
return self._get_json_of_kind(commit_hash, KIND_COMMIT, org_scope)
|
|
|
|
def get_tree_json(self, tree_hash: str, *, org_scope: bool = False) -> Dict[str, Any]:
|
|
return self._get_json_of_kind(tree_hash, KIND_TREE, org_scope)
|
|
|
|
def put_objects(self, objects: Dict[str, Tuple[str, bytes]], *, org_scope: bool = False) -> Dict[str, Any]:
|
|
"""POST objects as multipart (field = claimed ``sha256:<hex>``, filename = kind, body = raw
|
|
bytes; no base64-in-JSON). The server rehashes and 422s the whole batch on mismatch; known
|
|
hashes are no-ops. ``org_scope`` adds ``?scope=org`` (required before an org CAS/propose)."""
|
|
files = [(h, (kind, data, "application/octet-stream")) for h, (kind, data) in objects.items()]
|
|
return _body(self._request(
|
|
"POST", "objects", "put_objects", ok=(200, 201), files=files,
|
|
params={"scope": "org"} if org_scope else None,
|
|
errors={413: "object too large (413)", 422: lambda r: f"hash_mismatch (422): {r.text}"}))
|
|
|
|
def cas_ref(self, name: str, from_hash: Optional[str], to_hash: str) -> Dict[str, Any]:
|
|
"""POST refs/:name -- atomic CAS; SyncConflict on 409. A member's CAS on an org HEAD becomes a
|
|
proposal (202) -> ``{"proposal_pending": True, ...}``: success-shaped, never present as live."""
|
|
r = self._request("POST", f"refs/{name}", "cas_ref", ok=(200, 202, 409),
|
|
json={"from": from_hash, "to": to_hash}, errors={403: "forbidden (403) -- owner/permission"})
|
|
if r.status_code == 202:
|
|
return {"proposal_pending": True, **_body(r)}
|
|
if r.status_code == 409: # "" actual = the ref does not exist server-side (-> None)
|
|
raise SyncConflict((r.json() or {}).get("actual", ""))
|
|
return _body(r)
|
|
|
|
|
|
# Reading remote trees
|
|
def read_ref_hash(client: SyncClient, ref: str, *, org_scope: bool = False) -> Optional[str]:
|
|
"""Hash of *ref* (queried with itself as prefix), or None if absent."""
|
|
refs = client.get_refs(ref, org_scope=org_scope)
|
|
return next((r.get("hash") for r in refs if r.get("name") == ref), None)
|
|
|
|
|
|
def root_tree_of_commit(client: SyncClient, commit_hash: str, *, org_scope: bool = False) -> str:
|
|
return client.get_commit_json(commit_hash, org_scope=org_scope)["tree"]
|
|
|
|
|
|
def skill_trees_of_root(client: SyncClient, root_tree_hash: str, *, org_scope: bool = False) -> Dict[str, str]:
|
|
"""``{posix_rel_path: skill_tree_hash}`` for every subtree containing a ``SKILL.md`` blob."""
|
|
result: Dict[str, str] = {}
|
|
def _walk(tree_hash: str, prefix: str) -> None:
|
|
entries = client.get_tree_json(tree_hash, org_scope=org_scope).get("entries", [])
|
|
if prefix and any(e.get("name") == "SKILL.md" and e.get("kind") == KIND_BLOB for e in entries):
|
|
result[prefix] = tree_hash
|
|
return
|
|
for e in entries:
|
|
if e.get("kind") == KIND_TREE:
|
|
_walk(e["hash"], f"{prefix}/{e['name']}" if prefix else e["name"])
|
|
_walk(root_tree_hash, "")
|
|
return result
|
|
|
|
|
|
def read_manifest_of_root(client: SyncClient, root_tree_hash: str) -> Optional[Dict[str, bool]]:
|
|
"""``{name: enabled}`` from the root ``sync-manifest`` blob (how a device learns another's opt-ins)."""
|
|
try:
|
|
for e in client.get_tree_json(root_tree_hash).get("entries", []):
|
|
if e.get("name") == SYNC_MANIFEST_ENTRY_NAME and e.get("kind") == KIND_BLOB:
|
|
return parse_sync_manifest(client.get_object(e["hash"])[1])
|
|
except Exception as e:
|
|
logger.debug("skills_sync_client: manifest read failed: %s", e)
|
|
return None
|
|
|
|
|
|
def materialize_tree(client: SyncClient, tree_hash: str, dest: Path, *, org_scope: bool = False) -> None:
|
|
"""Write the tree into *dest*: blobs -> files (+x for ``exec``), trees -> subdirs. Does NOT delete
|
|
files absent from the tree (caller decides). Refuses path traversal."""
|
|
dest.mkdir(parents=True, exist_ok=True)
|
|
for entry in client.get_tree_json(tree_hash, org_scope=org_scope).get("entries", []):
|
|
name = entry.get("name", "")
|
|
if not name or "/" in name or name in (".", ".."):
|
|
logger.warning("skills_sync_client: skipping unsafe tree entry %r", name)
|
|
continue
|
|
target, kind = dest / name, entry.get("kind")
|
|
if kind == KIND_TREE:
|
|
materialize_tree(client, entry["hash"], target, org_scope=org_scope)
|
|
elif kind == KIND_BLOB:
|
|
_, data = client.get_object(entry["hash"], org_scope=org_scope)
|
|
target.write_bytes(data)
|
|
if entry.get("mode") == MODE_EXEC:
|
|
with suppress(OSError):
|
|
target.chmod(target.stat().st_mode | _EXEC_BITS)
|
|
|
|
|
|
def merge_skill(base: Optional[str], ours: Optional[str], theirs: Optional[str]) -> str:
|
|
"""Three-way decision: ``ours``/``theirs``/``either``/``overlap``/``none``. A side "modified" the
|
|
skill when its hash differs from the base (same semantics as skills_sync.py)."""
|
|
if ours == theirs:
|
|
return "either" if ours is not None else "none"
|
|
if theirs == base: # only we moved
|
|
return "ours"
|
|
if ours == base: # only they moved
|
|
return "theirs"
|
|
return "overlap"
|