a49d196b5c
# Conflicts: # hermes_cli/env_loader.py # hermes_cli/urllib_security.py # tools/terminal_scope.py
571 lines
29 KiB
Python
571 lines
29 KiB
Python
"""Skills Hub GitHub adapter: API auth, tap providers, and the Contents/Trees source."""
|
|
|
|
import json
|
|
import logging
|
|
import subprocess
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional, Tuple, Union
|
|
from urllib.parse import quote
|
|
|
|
import httpx
|
|
|
|
from hermes_cli._subprocess_compat import windows_hide_flags
|
|
from agent.retry_utils import parse_retry_after_seconds
|
|
from tools.skills_guard import TRUSTED_REPOS
|
|
from tools.skills_hub_models import (
|
|
SkillBundle, SkillMeta, SkillSource, _cache_metas, _cached_metas, _dedupe_by_trust,
|
|
_hermes_tags, _matches_query, _parse_frontmatter, _referenced_support_paths, hub,
|
|
_validate_bundle_rel_path,
|
|
)
|
|
|
|
logger = logging.getLogger("tools.skills_hub")
|
|
|
|
# GitHub tap repo (owner/repo) -> provider label used by the docs-site catalog
|
|
# (website/scripts/extract-skills.py::GITHUB_TAP_LABELS). The runtime index collapses every tap into
|
|
# source="github"; ``extra.provider`` keeps per-tap identity searchable/filterable without disturbing
|
|
# dedup / floor / index-skip logic keyed on the bare source id.
|
|
GITHUB_TAP_PROVIDERS = {
|
|
"openai/skills": "OpenAI", "anthropics/skills": "Anthropic", "huggingface/skills": "HuggingFace",
|
|
"nvidia/skills": "NVIDIA", "voltagent/awesome-agent-skills": "VoltAgent", "garrytan/gstack": "gstack",
|
|
"minimax-ai/cli": "MiniMax",
|
|
}
|
|
|
|
# Accepted ``--source`` provider filters (lowercased). Not real source ids —
|
|
# they narrow merged results to GitHub-tap skills carrying that ``extra.provider``.
|
|
_PROVIDER_FILTER_VALUES = frozenset(v.lower() for v in GITHUB_TAP_PROVIDERS.values())
|
|
|
|
_API = "https://api.github.com/repos"
|
|
_ACCEPT_JSON = "application/vnd.github.v3+json"
|
|
|
|
|
|
def github_provider_for(repo: str) -> Optional[str]:
|
|
"""Provider label for an ``owner/repo`` tap (case-insensitive), or None."""
|
|
return GITHUB_TAP_PROVIDERS.get(repo.strip().lower()) if repo else None
|
|
|
|
|
|
def _filter_results_by_provider(results: List[SkillMeta], provider: str) -> List[SkillMeta]:
|
|
"""Keep only results whose ``extra.provider`` matches ``provider``. An explicit provider filter
|
|
(``--source nvidia``) narrows to exactly that provider — the official catalog is NOT injected the
|
|
way unfiltered browse does."""
|
|
want = provider.strip().lower()
|
|
return [r for r in results if str((r.extra or {}).get("provider", "")).lower() == want]
|
|
|
|
|
|
def _provider_filter_of(source_filter: str) -> str:
|
|
"""Normalized provider filter when ``--source`` names one (nvidia/openai/...), else ``""``."""
|
|
value = source_filter.strip().lower()
|
|
return value if value in _PROVIDER_FILTER_VALUES else ""
|
|
|
|
|
|
def _tap_cache_key(repo: str, path: str, bucket: Optional[str] = None) -> str:
|
|
"""Disk-cache key for one tap's skill listing (tests seed that cache through it too)."""
|
|
return f"{repo}_{path}_{bucket or ''}".replace("/", "_").replace(" ", "_")
|
|
|
|
|
|
def _is_rate_limit_response(resp: httpx.Response) -> bool:
|
|
"""403 with exhausted quota, or any 429."""
|
|
return resp.status_code == 429 or (
|
|
resp.status_code == 403 and resp.headers.get("X-RateLimit-Remaining", "") == "0"
|
|
)
|
|
|
|
|
|
class GitHubAuth:
|
|
"""GitHub API authentication, tried in priority order: GITHUB_TOKEN / GH_TOKEN (PAT), `gh auth token`
|
|
(gh CLI), GitHub App JWT + installation token, then unauthenticated (60 req/hr, public repos only)."""
|
|
|
|
def __init__(self):
|
|
self._cached_token: Optional[str] = None
|
|
self._cached_method: Optional[str] = None
|
|
self._app_token_expiry: float = 0
|
|
|
|
def get_headers(self) -> Dict[str, str]:
|
|
token = self._resolve_token()
|
|
return {"Accept": _ACCEPT_JSON, **({"Authorization": f"token {token}"} if token else {})}
|
|
|
|
def is_authenticated(self) -> bool:
|
|
return self._resolve_token() is not None
|
|
|
|
def auth_method(self) -> str:
|
|
"""'pat', 'gh-cli', 'github-app', or 'anonymous'."""
|
|
self._resolve_token()
|
|
return self._cached_method or "anonymous"
|
|
|
|
def _resolve_token(self) -> Optional[str]:
|
|
if self._cached_token and (self._cached_method != "github-app" or time.time() < self._app_token_expiry):
|
|
return self._cached_token
|
|
for method, resolve in (
|
|
("pat", self._try_pat), ("gh-cli", self._try_gh_cli), ("github-app", self._try_github_app),
|
|
):
|
|
token = resolve()
|
|
if token:
|
|
self._cached_token, self._cached_method = token, method
|
|
if method == "github-app":
|
|
self._app_token_expiry = time.time() + 3500 # ~58 min (tokens last 1 hour)
|
|
return token
|
|
self._cached_method = "anonymous"
|
|
return None
|
|
|
|
@staticmethod
|
|
def _try_pat() -> Optional[str]:
|
|
# Profile-scoped secret lookup (multiplexed gateway safe).
|
|
from agent.secret_scope import get_secret
|
|
return get_secret("GITHUB_TOKEN") or get_secret("GH_TOKEN")
|
|
|
|
def _try_gh_cli(self) -> Optional[str]:
|
|
try:
|
|
result = subprocess.run(
|
|
["gh", "auth", "token"], capture_output=True, text=True, encoding='utf-8', errors='replace',
|
|
timeout=5, stdin=subprocess.DEVNULL, creationflags=windows_hide_flags(),
|
|
)
|
|
if result.returncode == 0 and result.stdout.strip():
|
|
return result.stdout.strip()
|
|
except (FileNotFoundError, subprocess.TimeoutExpired) as e:
|
|
logger.debug("gh CLI token lookup failed: %s", e)
|
|
return None
|
|
|
|
def _try_github_app(self) -> Optional[str]:
|
|
from agent.secret_scope import get_secret
|
|
app_id, key_path = get_secret("GITHUB_APP_ID"), get_secret("GITHUB_APP_PRIVATE_KEY_PATH")
|
|
installation_id = get_secret("GITHUB_APP_INSTALLATION_ID")
|
|
if not all([app_id, key_path, installation_id]):
|
|
return None
|
|
try:
|
|
import jwt # PyJWT
|
|
except ImportError:
|
|
logger.debug("PyJWT not installed, skipping GitHub App auth")
|
|
return None
|
|
try:
|
|
key_file = Path(key_path)
|
|
if not key_file.exists():
|
|
return None
|
|
now = int(time.time())
|
|
encoded_jwt = jwt.encode(
|
|
{"iat": now - 60, "exp": now + (10 * 60), "iss": app_id},
|
|
key_file.read_text(encoding="utf-8-sig"), algorithm="RS256",
|
|
)
|
|
resp = httpx.post(
|
|
f"https://api.github.com/app/installations/{installation_id}/access_tokens",
|
|
headers={"Authorization": f"Bearer {encoded_jwt}", "Accept": _ACCEPT_JSON}, timeout=10,
|
|
)
|
|
if resp.status_code == 201:
|
|
return resp.json().get("token")
|
|
except Exception as e:
|
|
logger.debug("GitHub App auth failed: %s", e)
|
|
return None
|
|
|
|
|
|
def _split_repo_id(identifier: str) -> Optional[Tuple[str, str]]:
|
|
"""``owner/repo/path/to/skill`` -> ``(owner/repo, path/to/skill)``; None when too short."""
|
|
parts = identifier.split("/", 2)
|
|
return (f"{parts[0]}/{parts[1]}", parts[2]) if len(parts) >= 3 else None
|
|
|
|
|
|
def _skill_file_path(skill_path: str, filename: str = "SKILL.md") -> str:
|
|
"""Path of ``filename`` inside a skill directory. An empty ``skill_path`` means the skill
|
|
directory IS the repo root — the single-skill layout some skills.sh repos use (``SKILL.md``
|
|
and its ``references/``/``scripts/`` next to ``README.md``)."""
|
|
return f"{skill_path}/{filename}" if skill_path else filename
|
|
|
|
|
|
def _skip_bundle_file(rel_path: str) -> bool:
|
|
"""Dotfiles, bytecode and __pycache__ never ship in a bundle."""
|
|
base = rel_path.rsplit("/", 1)[-1]
|
|
return base.startswith(".") or base.endswith(".pyc") or "__pycache__" in rel_path.split("/")
|
|
|
|
|
|
def _tree_members(entries: List[dict], prefix: str):
|
|
"""``(rel_path, item_path, is_regular_blob)`` for every git-tree entry under ``prefix``. Symlinks
|
|
(mode 120000) and non-blobs report ``is_regular_blob=False`` so callers can reject a SKILL.md-linked
|
|
symlink instead of silently following it."""
|
|
for item in entries:
|
|
item_path = item.get("path", "")
|
|
if item_path.startswith(prefix):
|
|
yield item_path[len(prefix):], item_path, item.get("type") == "blob" and item.get("mode") != "120000"
|
|
|
|
|
|
class GitHubSource(SkillSource):
|
|
"""Fetch skills from GitHub repos via the Contents API."""
|
|
|
|
DEFAULT_TAPS = [
|
|
# openai/skills keeps content under skills/.curated/ + skills/.system/; _list_skills_in_repo
|
|
# skips "."/"_" directories, so both entries point at the inner paths.
|
|
{"repo": "openai/skills", "path": "skills/.curated/"},
|
|
{"repo": "openai/skills", "path": "skills/.system/"},
|
|
{"repo": "anthropics/skills", "path": "skills/"},
|
|
{"repo": "huggingface/skills", "path": "skills/"},
|
|
# NVIDIA-verified skills (CUDA-X, NeMo, cuOpt, ...), each with a signed skill.oms.sig
|
|
# + governance card; `trusted` via tools/skills_guard.py::TRUSTED_REPOS.
|
|
{"repo": "NVIDIA/skills", "path": "skills/"},
|
|
{"repo": "garrytan/gstack", "path": ""},
|
|
# --- Science bucket ---
|
|
# Two scientific-skill repos share one hub category via the tap-level "bucket" key so
|
|
# their skills surface together. Both stay `community` trust on purpose (NOT in
|
|
# tools/skills_guard.py::TRUSTED_REPOS): the guard scans every skill and INSTALL_POLICY
|
|
# auto-installs only "safe" ones. Skills wrap third-party tools with their OWN licenses
|
|
# (some GPL; KEGG is commercial for non-academic use) — surfaced per skill, not vetted here.
|
|
# K-Dense-AI/scientific-agent-skills: flat skills/<name>/, MIT.
|
|
{"repo": "K-Dense-AI/scientific-agent-skills", "path": "skills/", "bucket": "science"},
|
|
# synthetic-sciences/openscience: Apache-2.0, nested backend/cli/skills/<category>/<name>/.
|
|
# _list_skills_in_repo walks ONE level under a tap path, so each category is its own tap
|
|
# (same one-entry-per-inner-path pattern as openai/skills above).
|
|
*(
|
|
{"repo": "synthetic-sciences/openscience", "path": f"backend/cli/skills/{_cat}/", "bucket": "science"}
|
|
for _cat in (
|
|
"biology", "chemistry", "cloud-compute", "coding", "data-engineering", "databases",
|
|
"document-parsing", "llm-tools", "ml-inference", "ml-training", "other", "physics",
|
|
"quantum", "research", "scholar-evaluation", "visualization", "writing",
|
|
)
|
|
),
|
|
]
|
|
|
|
SOURCE_ID = "github"
|
|
_parse_frontmatter_quick = staticmethod(_parse_frontmatter)
|
|
|
|
def __init__(self, auth: GitHubAuth, extra_taps: Optional[List[Dict]] = None):
|
|
self.auth = auth
|
|
self.taps = list(self.DEFAULT_TAPS) + list(extra_taps or [])
|
|
# Per-instance repo -> (default_branch, tree_entries); lives for one
|
|
# search/install flow so repeated tree lookups cost no API calls.
|
|
self._tree_cache: Dict[str, Optional[Tuple[str, List[dict]]]] = {}
|
|
self._tree_revisions: Dict[str, str] = {}
|
|
# repo -> skills.sh.json grouping map; None = fetched, no sidecar.
|
|
self._skillsh_groupings: Dict[str, Optional[Dict[str, str]]] = {}
|
|
self._rate_limited: bool = False
|
|
|
|
@property
|
|
def is_rate_limited(self) -> bool: # whether the GitHub API rate limit was hit during operations
|
|
return self._rate_limited
|
|
|
|
def trust_level_for(self, identifier: str) -> str:
|
|
# identifier format: "owner/repo/path/to/skill"
|
|
parts = identifier.split("/", 2)
|
|
return "trusted" if len(parts) >= 2 and f"{parts[0]}/{parts[1]}" in TRUSTED_REPOS else "community"
|
|
|
|
def search(self, query: str, limit: int = 10, *, provider_filter: str = "") -> List[SkillMeta]:
|
|
"""Substring-match taps, skip taps outside a provider filter, then dedupe by identifier
|
|
preferring higher trust and limit."""
|
|
results: List[SkillMeta] = []
|
|
query_lower = query.lower()
|
|
want = provider_filter.strip().lower()
|
|
for tap in self.taps:
|
|
# The tap repo fixes every result's provider, so wrong-provider taps can be
|
|
# skipped before their enumeration cost (cache reads or GitHub API calls).
|
|
if want and (github_provider_for(tap["repo"]) or "").lower() != want:
|
|
continue
|
|
try:
|
|
for skill in self._list_skills_in_repo(tap["repo"], tap.get("path", ""), tap.get("bucket")):
|
|
if _matches_query(query_lower, skill.name, skill.description, skill.tags):
|
|
results.append(skill)
|
|
except Exception as e:
|
|
logger.debug("Failed to search %s: %s", tap['repo'], e)
|
|
return _dedupe_by_trust(results)[:limit]
|
|
|
|
def fetch(self, identifier: str) -> Optional[SkillBundle]:
|
|
"""Download a skill; identifier format: "owner/repo/path/to/skill-dir"."""
|
|
if (split := _split_repo_id(identifier)) is None:
|
|
return None
|
|
repo, skill_path = split
|
|
skill_dir = skill_path.rstrip("/")
|
|
# Resolve the tree FIRST so every byte fetch — SKILL.md included — is pinned to the
|
|
# same revision; an unpinned /contents fetch floats to HEAD and can serve bytes newer
|
|
# than the tree the paths were validated against (TOCTOU). Idempotent + cached.
|
|
tree = self._get_repo_tree(repo)
|
|
pinned_ref = self._tree_revisions.get(repo)
|
|
skill_md = self._fetch_file_content(repo, _skill_file_path(skill_dir), ref=pinned_ref)
|
|
if skill_md is None:
|
|
return None
|
|
referenced = _referenced_support_paths(skill_md)
|
|
if referenced is None:
|
|
return None
|
|
files: Dict[str, Union[str, bytes]] = {"SKILL.md": skill_md}
|
|
if tree is not None:
|
|
complete = self._collect_tree_files(repo, skill_dir, tree[1], pinned_ref, referenced, files)
|
|
if complete is None:
|
|
return None
|
|
# A bundle with a transiently failed blob fetch must not record the tree sha: the
|
|
# update check would otherwise see "same revision" and never re-fetch the gap (#101454).
|
|
revision = (pinned_ref or tree[0]) if complete else ""
|
|
else:
|
|
for rel_path in referenced:
|
|
self._add_support_file(repo, _skill_file_path(skill_dir, rel_path), rel_path, files, rel_path)
|
|
revision = ""
|
|
url = (f"https://github.com/{repo}/tree/{revision}" + (f"/{skill_path}" if skill_path else "")
|
|
if revision else f"https://github.com/{repo}/{skill_path}")
|
|
return SkillBundle(
|
|
name=skill_dir.split("/")[-1] or repo.split("/")[-1], files=files, source="github", identifier=identifier,
|
|
trust_level=self.trust_level_for(identifier), metadata={"source_url": url, "source_revision": revision},
|
|
)
|
|
|
|
def current_revision(self, identifier: str) -> str:
|
|
"""Tree sha the default branch currently resolves to — one cached tree lookup per repo,
|
|
no blob downloads — so an update check can skip refetching unchanged skills."""
|
|
if (split := _split_repo_id(identifier)) is None:
|
|
return ""
|
|
repo, _ = split
|
|
self._get_repo_tree(repo) # populates _tree_revisions
|
|
return self._tree_revisions.get(repo, "")
|
|
|
|
def _add_support_file(self, repo: str, item_path: str, rel_path: str, files: dict, shown: str, **kw) -> bool:
|
|
"""Fetch one support file into ``files``; a failed fetch warns (naming ``shown``), is skipped, and
|
|
returns False."""
|
|
content = self._fetch_file_bytes(repo, item_path, **kw)
|
|
if content is None:
|
|
logger.warning("Failed to fetch referenced skill support file; continuing without it: %s", shown)
|
|
return False
|
|
files[rel_path] = content
|
|
return True
|
|
|
|
def _collect_tree_files(
|
|
self, repo: str, skill_path: str, entries: List[dict], ref: Optional[str], referenced: set,
|
|
files: Dict[str, Union[str, bytes]],
|
|
) -> Optional[bool]:
|
|
"""Download the FULL skill directory from the pinned tree into ``files``. Link-driven fetching
|
|
silently dropped support files under non-canonical dirs (``reference/``, ``agents/``, root
|
|
LICENSE); everything still goes through quarantine + scan, and the scanner sees MORE this way.
|
|
Returns None (bundle rejected) on an unsafe path or a SKILL.md-linked path that exists in the
|
|
tree as a symlink/non-blob — that shape is an escape attempt. A linked path that is simply absent
|
|
is a dangling link (repo-only dev tool, prose over-match): warn and install without it. Returns
|
|
False when a blob fetch failed (installed with a gap the next update check must be able to fill).
|
|
An empty ``skill_path`` is the repo-root skill layout, so the whole repo root is its directory."""
|
|
prefix = f"{skill_path}/" if skill_path else ""
|
|
symlinked: set = set()
|
|
complete = True
|
|
for rel_path, item_path, regular in _tree_members(entries, prefix):
|
|
if not regular:
|
|
symlinked.add(rel_path)
|
|
continue
|
|
if rel_path == "SKILL.md" or _skip_bundle_file(rel_path):
|
|
continue
|
|
try:
|
|
rel_path = _validate_bundle_rel_path(rel_path)
|
|
except ValueError:
|
|
logger.warning("Rejected unsafe file path in skill bundle: %s", item_path)
|
|
return None
|
|
complete &= self._add_support_file(repo, item_path, rel_path, files, item_path, ref=ref)
|
|
for rel_path in sorted(referenced):
|
|
# A SKILL.md-linked support path that isn't in the tree is a dangling link — a repo-only dev
|
|
# tool, prose over-match, or a file the author forgot to push. Warn and install without it
|
|
# rather than aborting the whole install (#66760/#90081): the skill body still works, and the
|
|
# gap is visible in the log. A referenced path that IS in the tree but as a symlink (or any
|
|
# non-regular entry) stays a hard rejection — that shape is an escape attempt, not a forgotten
|
|
# file.
|
|
if rel_path in symlinked:
|
|
logger.warning("Rejected non-regular referenced file in skill bundle: %s%s", prefix, rel_path)
|
|
return None
|
|
if rel_path not in files:
|
|
logger.warning(
|
|
"Referenced skill support file is missing; continuing without it: %s%s", prefix, rel_path)
|
|
return complete
|
|
|
|
def inspect(self, identifier: str) -> Optional[SkillMeta]:
|
|
"""Fetch just the SKILL.md metadata for preview."""
|
|
if (split := _split_repo_id(identifier)) is None:
|
|
return None
|
|
repo, skill_path = split[0], split[1].rstrip("/")
|
|
content = self._fetch_file_content(repo, _skill_file_path(skill_path))
|
|
if not content:
|
|
return None
|
|
fm = _parse_frontmatter(content)
|
|
tags = _hermes_tags(fm) or (fm["tags"] if isinstance(fm.get("tags"), list) else [])
|
|
provider = github_provider_for(repo)
|
|
return SkillMeta(
|
|
name=fm.get("name", skill_path.split("/")[-1] or repo.split("/")[-1]),
|
|
description=str(fm.get("description", "")),
|
|
source="github", identifier=identifier, trust_level=self.trust_level_for(identifier),
|
|
repo=repo, path=skill_path, tags=[str(t) for t in tags],
|
|
extra={"provider": provider} if provider else {},
|
|
)
|
|
|
|
# -- Internal helpers --
|
|
|
|
def _list_skills_in_repo(self, repo: str, path: str, bucket: Optional[str] = None) -> List[SkillMeta]:
|
|
"""List skill directories in a GitHub repo path, using cached index. ``bucket`` labels every
|
|
skill from a tap whose repo ships no ``skills.sh.json`` grouping, so several repos can share one
|
|
hub category (e.g. "science"); a sidecar grouping still wins when present."""
|
|
cache_key = _tap_cache_key(repo, path, bucket)
|
|
cached = _cached_metas(cache_key)
|
|
if cached is not None:
|
|
return cached
|
|
resp = self._github_get(f"{_API}/{repo}/contents/{path.rstrip('/')}")
|
|
if resp is None or resp.status_code != 200:
|
|
return []
|
|
entries = resp.json()
|
|
if not isinstance(entries, list):
|
|
return []
|
|
skills: List[SkillMeta] = []
|
|
groupings = self._get_skillsh_groupings(repo)
|
|
prefix = path.rstrip("/")
|
|
for entry in entries:
|
|
if entry.get("type") != "dir" or entry["name"].startswith((".", "_")):
|
|
continue
|
|
dir_name = entry["name"]
|
|
meta = self.inspect(f"{repo}/{prefix}/{dir_name}" if prefix else f"{repo}/{dir_name}")
|
|
if meta:
|
|
category = (groupings and (groupings.get(meta.name) or groupings.get(dir_name))) or bucket
|
|
if category:
|
|
meta.extra["category"] = category
|
|
skills.append(meta)
|
|
_cache_metas(cache_key, skills)
|
|
return skills
|
|
|
|
def _get_repo_tree(self, repo: str) -> Optional[Tuple[str, List[dict]]]:
|
|
"""Cached ``(default_branch, tree_entries)`` for a repo, or None. One install may need the tree
|
|
several times; caching saves the ``GET /repos/{repo}`` + ``GET .../git/trees/{branch}`` pair each
|
|
time (~12 of the 60/hr unauthenticated budget before)."""
|
|
if repo in self._tree_cache:
|
|
return self._tree_cache[repo]
|
|
# Misses are cached too: within one command a truncated/unreachable tree stays that way,
|
|
# and the update check now probes the tree before every fetch (#101454).
|
|
self._tree_cache[repo] = None
|
|
repo_data = self._github_json(f"{_API}/{repo}")
|
|
if repo_data is None:
|
|
return None
|
|
default_branch = repo_data.get("default_branch", "main")
|
|
tree_data = self._github_json(
|
|
f"{_API}/{repo}/git/trees/{default_branch}", params={"recursive": "1"}, timeout=30.0,
|
|
)
|
|
if tree_data is None:
|
|
return None
|
|
if tree_data.get("truncated"):
|
|
logger.debug("Git tree truncated for %s", repo)
|
|
return None
|
|
if isinstance(tree_data.get("sha"), str) and tree_data["sha"]:
|
|
self._tree_revisions[repo] = tree_data["sha"]
|
|
self._tree_cache[repo] = tree = (default_branch, tree_data.get("tree", []))
|
|
return tree
|
|
|
|
def _github_json(self, url: str, **kwargs) -> Optional[dict]:
|
|
"""Decoded JSON body of a 200 ``_github_get`` (which flags rate-limit exhaustion), else None."""
|
|
resp = self._github_get(url, **kwargs)
|
|
try:
|
|
return resp.json() if resp is not None and resp.status_code == 200 else None
|
|
except ValueError:
|
|
return None
|
|
|
|
def _github_get(
|
|
self, url: str, *, params: Optional[Dict] = None, headers: Optional[Dict] = None,
|
|
timeout: float = 15.0, max_retries: int = 3,
|
|
) -> Optional[httpx.Response]:
|
|
"""GET against the GitHub API with retry/backoff on transient failures. Returns the final
|
|
response (caller inspects status) or None when every attempt raised a transport error.
|
|
Retries rate-limit 403/429 (waiting until ``Retry-After`` / ``X-RateLimit-Reset`` when present,
|
|
capped 60s — one shared limit zeroes every GitHub tap at once during an index build), 5xx, and
|
|
transport errors with exponential backoff. Terminal rate-limit exhaustion flags the instance so
|
|
an index build fails loud instead of silently shipping zero GitHub skills."""
|
|
hdrs = headers if headers is not None else self.auth.get_headers()
|
|
backoff = 1.0
|
|
last_resp: Optional[httpx.Response] = None
|
|
for attempt in range(max_retries):
|
|
last_attempt = attempt >= max_retries - 1
|
|
wait = backoff
|
|
try:
|
|
resp = hub()._skills_hub_http_get(
|
|
url, params=params, headers=hdrs, timeout=timeout, follow_redirects=True
|
|
)
|
|
except httpx.HTTPError as e:
|
|
logger.debug("GitHub GET %s failed (attempt %d/%d): %s", url, attempt + 1, max_retries, e)
|
|
if last_attempt:
|
|
return None
|
|
else:
|
|
last_resp = resp
|
|
if resp.status_code == 200:
|
|
return resp
|
|
if resp.status_code in (403, 429):
|
|
limited = _is_rate_limit_response(resp)
|
|
if not limited or last_attempt:
|
|
if limited: # terminal exhaustion: flag the instance so callers fail loud
|
|
self._rate_limited = True
|
|
logger.warning("GitHub API rate limit exhausted (unauthenticated: 60 req/hr). "
|
|
"Set GITHUB_TOKEN or install the gh CLI to raise the limit to 5,000/hr.")
|
|
return resp
|
|
reset = resp.headers.get("X-RateLimit-Reset", "")
|
|
retry_after = parse_retry_after_seconds(resp.headers)
|
|
if retry_after is not None:
|
|
wait = min(retry_after, 60.0)
|
|
elif reset.isdigit():
|
|
delta = float(reset) - time.time()
|
|
if 0 < delta <= 60.0:
|
|
wait = delta
|
|
logger.debug("GitHub rate limited on %s, waiting %.1fs (attempt %d/%d)",
|
|
url, wait, attempt + 1, max_retries)
|
|
elif not (500 <= resp.status_code < 600) or last_attempt:
|
|
return resp
|
|
time.sleep(wait)
|
|
backoff = min(backoff * 2, 30.0)
|
|
return last_resp
|
|
|
|
def _find_skill_in_repo_tree(self, repo: str, skill_name: str) -> Optional[str]:
|
|
"""Locate ``<skill_name>/SKILL.md`` anywhere in the repo tree (one API call); full identifier or None."""
|
|
if (cached := self._get_repo_tree(repo)) is None:
|
|
return None
|
|
skill_md_suffix = f"/{skill_name}/SKILL.md"
|
|
for entry in cached[1]:
|
|
path = entry.get("path", "")
|
|
if entry.get("type") == "blob" and (path.endswith(skill_md_suffix) or path == skill_md_suffix[1:]):
|
|
return f"{repo}/{path[: -len('/SKILL.md')]}"
|
|
return None
|
|
|
|
def _find_repo_root_skill(self, repo: str) -> Optional[str]:
|
|
"""Identifier for a single-skill repo whose ``SKILL.md`` sits at the repo ROOT (no skill
|
|
directory) — e.g. ``orzcls/win-disk-cleaner``. The empty path segment (``owner/repo/``)
|
|
denotes the skill directory being the repo root. Only repos with EXACTLY ONE SKILL.md in
|
|
the whole tree qualify, so a categorized multi-skill repo never resolves here."""
|
|
tree = self._get_repo_tree(repo)
|
|
if tree is None:
|
|
return None
|
|
skill_mds = [
|
|
entry.get("path", "") for entry in tree[1]
|
|
if entry.get("type") == "blob" and entry.get("mode") != "120000"
|
|
and (entry.get("path", "") == "SKILL.md" or entry.get("path", "").endswith("/SKILL.md"))
|
|
]
|
|
return f"{repo}/" if skill_mds == ["SKILL.md"] else None
|
|
|
|
def _fetch_file_content(self, repo: str, path: str, ref: Optional[str] = None) -> Optional[str]:
|
|
"""Fetch a single text file from GitHub (None on miss or non-UTF-8)."""
|
|
content = self._fetch_file_bytes(repo, path, ref=ref)
|
|
try:
|
|
return None if content is None else content.decode("utf-8")
|
|
except UnicodeDecodeError:
|
|
return None
|
|
|
|
def _fetch_file_bytes(self, repo: str, path: str, ref: Optional[str] = None) -> Optional[bytes]:
|
|
"""Fetch exact file bytes. ``ref`` pins to a tree SHA (see ``fetch`` on
|
|
the TOCTOU); None keeps the legacy unpinned behavior."""
|
|
resp = self._github_get(
|
|
f"{_API}/{repo}/contents/{quote(path, safe='/')}", params={"ref": ref} if ref else None,
|
|
headers={**self.auth.get_headers(), "Accept": "application/vnd.github.v3.raw"},
|
|
)
|
|
return resp.content if resp is not None and resp.status_code == 200 else None
|
|
|
|
def _get_skillsh_groupings(self, repo: str) -> Optional[Dict[str, str]]:
|
|
"""Repo-root ``skills.sh.json`` groupings flattened to ``{skill_name: title}``. ``skills.sh.json``
|
|
is a cross-ecosystem standard (``$schema: https://skills.sh/schemas/skills.sh.schema.json``); any
|
|
tap shipping it gets category pills for free. None when absent/unparsable; cached per repo."""
|
|
if repo not in self._skillsh_groupings:
|
|
content = self._fetch_file_content(repo, "skills.sh.json")
|
|
self._skillsh_groupings[repo] = self._parse_skillsh_groupings(content) if content else None
|
|
return self._skillsh_groupings[repo]
|
|
|
|
@staticmethod
|
|
def _parse_skillsh_groupings(content: str) -> Optional[Dict[str, str]]:
|
|
"""Flatten ``{"groupings": [{"title", "skills": [...]}]}``; None if not usable."""
|
|
try:
|
|
data = json.loads(content)
|
|
except (json.JSONDecodeError, TypeError):
|
|
return None
|
|
groupings = data.get("groupings") if isinstance(data, dict) else None
|
|
if not isinstance(groupings, list):
|
|
return None
|
|
mapping: Dict[str, str] = {}
|
|
for group in groupings:
|
|
if not isinstance(group, dict):
|
|
continue
|
|
title, members = group.get("title"), group.get("skills")
|
|
if not isinstance(title, str) or not isinstance(members, list):
|
|
continue
|
|
for member in members:
|
|
if isinstance(member, str) and member:
|
|
mapping.setdefault(member, title) # first grouping wins
|
|
return mapping
|