From 4dc9988f7870b1a02e3987d6cd6ec9f93364273c Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 5 Sep 2026 11:11:23 -0700 Subject: [PATCH] feat(skills): reddit-reading and rss-feeds bundled skills, multi-platform sweep guidance Two zero-install research skills plus routing guidance, ported as ideas (not code) from the Agent Reach skill's per-platform backend routing. reddit-reading (skills/social-media): subreddit listings, site/subreddit search, threads with comments, user pages. Live-verified from a datacentre IP: www .json, api.reddit.com, old.reddit, r.jina.ai and the browser tool all return 403 / an empty shell / a humanity check; the Atom .rss endpoints are the only anonymous path and are throttled to ~1 request/min/IP. The script waits out the x-ratelimit-reset window once and retries, and switches to the OAuth API (scores, nested comments, ~100 req/min) when REDDIT_CLIENT_ID/SECRET are present. `doctor` reports the active backend. rss-feeds (skills/research): RSS 2.0 / RSS 1.0 / Atom / JSON Feed parsing with UTC-normalised dates, --since/--limit, and feed discovery behind a page URL (, then well-known paths). Stdlib only; the optional blogwatcher skill remains the stateful many-feed reader and now points at this one for one-off reads. grounded-citations gains a "Multi-Platform Sweeps" section routing "what are people saying about X" tasks across web, Reddit, feeds, video, code and X with per-platform attribution and coverage-gap reporting; competitor-news-monitor references the new sources. Tests: two invariant tests per skill (format normalisation + discovery; anonymous 429 handling + OAuth routing/flattening), no network. The reddit test caught a real bug: the feed footer's "/u/author" leaked into post bodies. --- optional-skills/research/blogwatcher/SKILL.md | 1 + .../research/competitor-news-monitor/SKILL.md | 4 +- skills/research/grounded-citations/SKILL.md | 25 +- skills/research/rss-feeds/SKILL.md | 96 ++++++ skills/research/rss-feeds/scripts/feed.py | 246 ++++++++++++++ skills/social-media/reddit-reading/SKILL.md | 107 ++++++ .../reddit-reading/scripts/reddit.py | 307 ++++++++++++++++++ tests/skills/test_reddit_reading_skill.py | 113 +++++++ tests/skills/test_rss_feeds_skill.py | 69 ++++ website/docs/reference/skills-catalog.md | 2 + .../research-competitor-news-monitor.md | 6 +- .../research/research-grounded-citations.md | 27 +- .../bundled/research/research-rss-feeds.md | 114 +++++++ .../social-media-reddit-reading.md | 125 +++++++ .../optional/research/research-blogwatcher.md | 3 +- website/sidebars.ts | 2 + 16 files changed, 1236 insertions(+), 11 deletions(-) create mode 100644 skills/research/rss-feeds/SKILL.md create mode 100644 skills/research/rss-feeds/scripts/feed.py create mode 100644 skills/social-media/reddit-reading/SKILL.md create mode 100644 skills/social-media/reddit-reading/scripts/reddit.py create mode 100644 tests/skills/test_reddit_reading_skill.py create mode 100644 tests/skills/test_rss_feeds_skill.py create mode 100644 website/docs/user-guide/skills/bundled/research/research-rss-feeds.md create mode 100644 website/docs/user-guide/skills/bundled/social-media/social-media-reddit-reading.md diff --git a/optional-skills/research/blogwatcher/SKILL.md b/optional-skills/research/blogwatcher/SKILL.md index cf4b58472a..edd2449b01 100644 --- a/optional-skills/research/blogwatcher/SKILL.md +++ b/optional-skills/research/blogwatcher/SKILL.md @@ -24,6 +24,7 @@ Track blog and RSS/Atom feed updates with the `blogwatcher-cli` tool. Supports a - **Recurring watch — use the cronjob tool's `monitor` field, not a bare schedule.** `monitor` runs a script each tick and only wakes the agent when output changes: set it to a script that runs `blogwatcher-cli scan >/dev/null 2>&1 && blogwatcher-cli articles` (deterministic output; new articles = changed output = agent wakes with the diff injected). Unchanged ticks cost zero LLM calls. Set `deliver` to route digests to a chat/channel; add `continuity: true` so consecutive digests can dedupe. - **Reading an article the user asks about**: `web_extract([url])` on the article URL from `blogwatcher-cli articles` — do not re-scrape by hand. - **One-off "watch this page for changes" without feed semantics**: skip this skill; the cronjob tool's `monitor` field accepts an http(s) URL directly. +- **One-off read of a feed or a site's latest posts, nothing to install**: the bundled `rss-feeds` skill (`scripts/feed.py read URL`); blogwatcher earns its install when you track many feeds with read/unread state. - **Company/competitor tracking with analysis and citations**: prefer the `competitor-news-monitor` skill; blogwatcher is the lighter raw-feed layer it can sit on. ## Installation diff --git a/skills/research/competitor-news-monitor/SKILL.md b/skills/research/competitor-news-monitor/SKILL.md index c480f2b18b..e1fb7ef2d8 100644 --- a/skills/research/competitor-news-monitor/SKILL.md +++ b/skills/research/competitor-news-monitor/SKILL.md @@ -8,7 +8,7 @@ platforms: [linux, macos, windows] metadata: hermes: tags: [Competitors, News, Market-Research, Monitoring] - related_skills: [blogwatcher] + related_skills: [blogwatcher, rss-feeds, reddit-reading] --- # Competitor News Monitor @@ -42,7 +42,7 @@ For each company include, where available: 5. reputable trade and financial press 6. job postings as weak supporting evidence -Use `blogwatcher` for feeds and `web_search`/`web_extract` for pages. Write the watch contract (watchlist, categories, materiality threshold, last cutoff) to a state file under `~/.hermes/competitor-watches/.json`, then create the job: +Use `rss-feeds` (bundled) or `blogwatcher` (optional, stateful) for feeds, `reddit-reading` for community discussion, and `web_search`/`web_extract` for pages. Write the watch contract (watchlist, categories, materiality threshold, last cutoff) to a state file under `~/.hermes/competitor-watches/.json`, then create the job: ``` cronjob(action="create", diff --git a/skills/research/grounded-citations/SKILL.md b/skills/research/grounded-citations/SKILL.md index ae69fb98b0..599fc6b1d7 100644 --- a/skills/research/grounded-citations/SKILL.md +++ b/skills/research/grounded-citations/SKILL.md @@ -1,7 +1,7 @@ --- name: grounded-citations description: "Ground answers and documents in cited, verifiable sources." -version: 1.1.0 +version: 1.2.0 author: Hermes Agent + Teknium license: MIT platforms: [linux, macos, windows] @@ -9,7 +9,7 @@ metadata: hermes: tags: [Research, Citations, Grounding, Sources, Web, Reports] category: research - related_skills: [arxiv, pdf] + related_skills: [arxiv, pdf, reddit-reading, rss-feeds, youtube-content] --- # Grounded Citations @@ -126,6 +126,27 @@ sources, cite inline, end with the rendered `Sources:` list. For a short answer you may render the block from `sources.py render --only ` instead of writing to a file. +## Multi-Platform Sweeps + +"What are people saying about X" / "research X across the web" is not one +`web_search`. Fan out across source types, collect in parallel, then synthesise +with every claim attributed to the platform it came from: + +| Source type | Route | What it adds | +|---|---|---| +| Open web | `web_search` → `web_extract` | official docs, articles, announcements | +| Community discussion | `reddit-reading` (`search`, `thread`) | real user experience, complaints, workarounds | +| Blogs / releases / changelogs | `rss-feeds` (`read`, `discover`) | dated primary posts, version history | +| Video | `youtube-content` | walkthroughs, demos, talks | +| Code | `terminal` with `gh search repos` / `gh search issues` | implementations, open bugs | +| X/Twitter | `xurl` (needs API access) | announcements, developer chatter | + +Register every URL from every route in the ledger as it arrives (step ②). Keep +opinion and measurement apart: a Reddit thread is evidence that users *report* +something, not that it is true; pair it with a primary source or label it as +sentiment. Report per-platform coverage gaps ("Reddit search returned nothing +newer than March") rather than silently narrowing to what worked. + ## Fact-Checking Mode For work where the reader must be able to check the chain — medical, legal, diff --git a/skills/research/rss-feeds/SKILL.md b/skills/research/rss-feeds/SKILL.md new file mode 100644 index 0000000000..48033d4028 --- /dev/null +++ b/skills/research/rss-feeds/SKILL.md @@ -0,0 +1,96 @@ +--- +name: rss-feeds +description: "Read RSS, Atom, JSON feeds; discover feeds behind a page." +version: 1.0.0 +author: Teknium (teknium1), Hermes Agent +license: MIT +platforms: [linux, macos, windows] +metadata: + hermes: + tags: [RSS, Atom, Feeds, Monitoring, Research, Blogs, Releases] + related_skills: [reddit-reading, competitor-news-monitor, grounded-citations, youtube-content, blogwatcher] +--- + +# RSS Feeds Skill + +Reads any RSS 2.0, RSS 1.0/RDF, Atom, or JSON Feed URL into a clean, date-sorted list of +entries, and discovers the feed behind an ordinary page URL (`` or +the usual `/feed`, `/rss.xml`, `/atom.xml` paths). Standard library only, nothing to +install. It does not fetch full article bodies — pass an entry's link to `web_extract` for +that. + +## When to Use + +- "What's new on ", "latest releases of ", "recent posts in + ", "read this feed", "does this site have an RSS feed". +- Building a recurring digest with `cronjob_manage` (feeds are cheaper and more stable than + scraping the HTML front page every run). For a persistent read/unread database across + many feeds install the optional `blogwatcher` skill; this skill is the zero-install read. +- Anything where a structured list of `title / link / date / author / summary` beats a + rendered page: podcasts, changelogs, YouTube channels, newsrooms, forum categories. + +## Prerequisites + +None. Python 3.10+, network access to the feed host. + +## How to Run + +Run through `terminal` with the skill-relative script path: + +```bash +python3 scripts/feed.py read https://hnrss.org/frontpage --limit 10 +python3 scripts/feed.py read https://simonwillison.net/ # page URL → discovers the feed +python3 scripts/feed.py read URL --since 2026-09-01 --json # only newer entries, machine-readable +python3 scripts/feed.py discover https://example.com/ # list candidate feed URLs +``` + +## Quick Reference + +| Source | Feed URL pattern | +|---|---| +| GitHub releases / commits / tags | `https://github.com/OWNER/REPO/releases.atom`, `…/commits/BRANCH.atom`, `…/tags.atom` | +| Subreddit / Reddit search | `https://www.reddit.com/r/NAME/.rss`, `https://www.reddit.com/search.rss?q=…` (1 req/min anon; see `reddit-reading`) | +| YouTube channel | `https://www.youtube.com/feeds/videos.xml?channel_id=UC…` | +| Hacker News | `https://hnrss.org/frontpage`, `https://hnrss.org/newest?q=TERM` | +| arXiv category | `https://rss.arxiv.org/rss/cs.CL` | +| Substack / Medium / WordPress / Ghost | `SITE/feed`, `medium.com/feed/@user`, `SITE/rss/` | +| Podcasts | the show's RSS URL from its hosting page (`discover` finds it) | + +Output fields per entry: `title`, `link`, `published` (UTC ISO 8601), `author`, `summary` +(HTML stripped, ≤ 2000 chars). Entries are sorted newest-first. + +## Procedure + +① If you only have a site URL, run `read` on it directly; the script discovers the feed +and reports which URL it used (`discovered_from`). Use `discover` when you want to choose +between several advertised feeds (comments feed vs posts feed, per-category feeds). + +② Bound the request: `--limit` for "latest N", `--since YYYY-MM-DD` for "since last +check". For a cron digest persist the last-seen `published` value and pass it as +`--since` next run. + +③ For full text, hand the entry `link` to `web_extract`; feed summaries are frequently +truncated or the first paragraph only. + +④ Cite the entry `link`, not the feed URL, when the result feeds a report +(`grounded-citations`). + +## Pitfalls + +- A 200 response with HTML means the URL is a page, not a feed; the script falls through + to discovery automatically, but a site with no `` and none of the + common paths reports `no feed found` — check the site's footer or `/sitemap.xml` before + concluding there is none. +- Reddit feeds share Reddit's anonymous throttle (about one request per minute per IP). + Chain them through `reddit-reading`, which waits out the window, when you need more + than one Reddit call. +- Dates: RSS `pubDate` is RFC 822 and Atom uses ISO 8601; the script normalises both to + UTC. Feeds that omit dates sort to the bottom and are dropped by `--since`. +- Some feeds are Cloudflare-fronted and 403 non-browser clients; `blocked-page-recovery` + handles that class. + +## Verification + +`python3 scripts/feed.py read https://github.com/NousResearch/hermes-agent/releases.atom +--limit 1` prints one entry with a `releases/tag/` link and a `[atom]` format tag; +`discover https://simonwillison.net/` prints an `/atom/` URL. diff --git a/skills/research/rss-feeds/scripts/feed.py b/skills/research/rss-feeds/scripts/feed.py new file mode 100644 index 0000000000..24edef11f1 --- /dev/null +++ b/skills/research/rss-feeds/scripts/feed.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +"""Read RSS / Atom / JSON Feed sources and discover feeds behind a page URL. + +Standard library only, so it runs in any Hermes environment without an install +step. Output is JSON (``--json``) or a compact text listing. + + python3 feed.py read https://example.com/feed.xml [--limit N] [--since 2026-09-01] + python3 feed.py discover https://example.com/ + python3 feed.py read https://example.com/ -> discovers, then reads the first feed +""" + +from __future__ import annotations + +import argparse +import html +import json +import re +import sys +import urllib.error +import urllib.parse +import urllib.request +import xml.etree.ElementTree as ET +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime + +USER_AGENT = "hermes-agent/1.0 (rss-feeds skill; +https://github.com/NousResearch/hermes-agent)" +TIMEOUT = 20 +NS = { + "atom": "http://www.w3.org/2005/Atom", + "dc": "http://purl.org/dc/elements/1.1/", + "content": "http://purl.org/rss/1.0/modules/content/", + "media": "http://search.yahoo.com/mrss/", +} +FEED_TYPES = ("application/rss+xml", "application/atom+xml", "application/feed+json", "application/json") +COMMON_FEED_PATHS = ("/feed", "/feed.xml", "/rss", "/rss.xml", "/atom.xml", "/index.xml", "/feed.json", "/blog/feed", "/blog/rss.xml") +_TAG_RE = re.compile(r"<[^>]+>") +_WS_RE = re.compile(r"\s+") + + +def fetch(url: str) -> tuple[bytes, str]: + req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, "Accept": "*/*"}) + with urllib.request.urlopen(req, timeout=TIMEOUT) as resp: + return resp.read(), resp.headers.get("Content-Type", "") + + +def strip_html(text: str | None) -> str: + if not text: + return "" + return _WS_RE.sub(" ", html.unescape(_TAG_RE.sub(" ", text))).strip() + + +def parse_date(value: str | None) -> str | None: + """Normalise RFC 822 (RSS) and ISO 8601 (Atom/JSON Feed) dates to UTC ISO.""" + if not value: + return None + value = value.strip() + try: + dt = parsedate_to_datetime(value) + except (TypeError, ValueError): + try: + dt = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return value + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc).isoformat() + + +def _text(el, *paths) -> str | None: + for p in paths: + found = el.find(p, NS) + if found is not None and (found.text or "").strip(): + return found.text + return None + + +def _atom_link(entry) -> str | None: + alternate = None + for link in entry.findall("atom:link", NS) + entry.findall("link"): + href = link.get("href") + if not href: + continue + rel = link.get("rel", "alternate") + if rel == "alternate": + return href + alternate = alternate or href + return alternate + + +def parse_xml(data: bytes) -> dict: + root = ET.fromstring(data) + tag = root.tag.rsplit("}", 1)[-1].lower() + if tag == "feed": # Atom + title = strip_html(_text(root, "atom:title")) + entries = [] + for e in root.findall("atom:entry", NS): + entries.append({ + "title": strip_html(_text(e, "atom:title")), + "link": _atom_link(e), + "published": parse_date(_text(e, "atom:published", "atom:updated")), + "author": strip_html(_text(e, "atom:author/atom:name", "dc:creator")), + "summary": strip_html(_text(e, "atom:summary", "atom:content"))[:2000], + }) + return {"format": "atom", "title": title, "entries": entries} + channel = root.find("channel") if tag == "rss" else root # RSS 2.0 vs RDF/RSS 1.0 + if channel is None: + raise ValueError(f"unrecognised XML root <{tag}>") + entries = [] + for item in channel.iter("item") if tag == "rss" else root.iter("{http://purl.org/rss/1.0/}item"): + entries.append({ + "title": strip_html(_text(item, "title", "{http://purl.org/rss/1.0/}title")), + "link": (_text(item, "link", "{http://purl.org/rss/1.0/}link") or "").strip() or None, + "published": parse_date(_text(item, "pubDate", "dc:date")), + "author": strip_html(_text(item, "dc:creator", "author")), + "summary": strip_html(_text(item, "content:encoded", "description", "{http://purl.org/rss/1.0/}description"))[:2000], + }) + return {"format": "rss", "title": strip_html(_text(channel, "title", "{http://purl.org/rss/1.0/}title")), "entries": entries} + + +def parse_json_feed(data: bytes) -> dict: + doc = json.loads(data) + entries = [] + for item in doc.get("items", []): + authors = item.get("authors") or ([item["author"]] if item.get("author") else []) + entries.append({ + "title": strip_html(item.get("title")), + "link": item.get("url") or item.get("external_url"), + "published": parse_date(item.get("date_published") or item.get("date_modified")), + "author": ", ".join(a.get("name", "") for a in authors if isinstance(a, dict)) or None, + "summary": strip_html(item.get("summary") or item.get("content_text") or item.get("content_html"))[:2000], + }) + return {"format": "jsonfeed", "title": strip_html(doc.get("title")), "entries": entries} + + +def parse_feed(data: bytes, content_type: str = "") -> dict: + head = data.lstrip()[:1] + if head == b"{" or "json" in content_type: + return parse_json_feed(data) + return parse_xml(data) + + +def discover(page_url: str, page_html: bytes | None = None) -> list[str]: + """Return candidate feed URLs for a page: first, then well-known paths.""" + if page_html is None: + page_html, _ = fetch(page_url) + text = page_html.decode("utf-8", "replace") + found: list[str] = [] + for m in re.finditer(r"]*>", text, re.I): + tag = m.group(0) + type_m = re.search(r"""type\s*=\s*["']([^"']+)""", tag, re.I) + href_m = re.search(r"""href\s*=\s*["']([^"']+)""", tag, re.I) + rel_m = re.search(r"""rel\s*=\s*["']([^"']+)""", tag, re.I) + if not href_m or not type_m or type_m.group(1).lower() not in FEED_TYPES: + continue + if rel_m and "alternate" not in rel_m.group(1).lower(): + continue + url = urllib.parse.urljoin(page_url, html.unescape(href_m.group(1))) + if url not in found: + found.append(url) + if found: + return found + parsed = urllib.parse.urlsplit(page_url) + base = f"{parsed.scheme}://{parsed.netloc}" + return [base + p for p in COMMON_FEED_PATHS] + + +def looks_like_feed(data: bytes, content_type: str) -> bool: + head = data.lstrip()[:300].lower() + return head.startswith(b"{") and b"items" in data[:2000] or b" dict: + data, ctype = fetch(url) + if looks_like_feed(data, ctype): + feed = parse_feed(data, ctype) + feed["url"] = url + return feed + candidates = discover(url, data) + errors = [] + for cand in candidates: + try: + cdata, cctype = fetch(cand) + except (urllib.error.URLError, OSError) as exc: + errors.append(f"{cand}: {exc}") + continue + if looks_like_feed(cdata, cctype): + feed = parse_feed(cdata, cctype) + feed["url"] = cand + feed["discovered_from"] = url + return feed + raise SystemExit(f"no feed found at {url}; tried {len(candidates)} candidates\n" + "\n".join(errors)) + + +def filter_entries(entries: list[dict], limit: int, since: str | None) -> list[dict]: + if since: + cutoff = datetime.fromisoformat(since).replace(tzinfo=timezone.utc) if "T" not in since else datetime.fromisoformat(since.replace("Z", "+00:00")) + if cutoff.tzinfo is None: + cutoff = cutoff.replace(tzinfo=timezone.utc) + entries = [e for e in entries if e["published"] and datetime.fromisoformat(e["published"]) >= cutoff] + entries.sort(key=lambda e: e["published"] or "", reverse=True) + return entries[:limit] + + +def render_text(feed: dict) -> str: + lines = [f"{feed.get('title') or '(untitled feed)'} [{feed['format']}] {feed['url']}"] + for e in feed["entries"]: + when = (e["published"] or "")[:10] + by = f" — {e['author']}" if e.get("author") else "" + lines.append(f"- {when} {e['title'] or '(no title)'}{by}\n {e['link'] or ''}") + if e.get("summary"): + lines.append(f" {e['summary'][:300]}") + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = ap.add_subparsers(dest="cmd", required=True) + r = sub.add_parser("read", help="read a feed (or discover one behind a page URL)") + r.add_argument("url") + r.add_argument("--limit", type=int, default=20) + r.add_argument("--since", help="ISO date/datetime; drop older entries") + r.add_argument("--json", action="store_true") + d = sub.add_parser("discover", help="list feed URLs advertised by a page") + d.add_argument("url") + d.add_argument("--json", action="store_true") + args = ap.parse_args(argv) + + try: + if args.cmd == "discover": + urls = discover(args.url) + print(json.dumps(urls, indent=2) if args.json else "\n".join(urls)) + return 0 + feed = read(args.url) + feed["entries"] = filter_entries(feed["entries"], args.limit, args.since) + print(json.dumps(feed, indent=2, ensure_ascii=False) if args.json else render_text(feed)) + return 0 + except urllib.error.HTTPError as exc: + print(f"HTTP {exc.code} for {exc.url}", file=sys.stderr) + return 2 + except (urllib.error.URLError, ET.ParseError, ValueError, json.JSONDecodeError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/social-media/reddit-reading/SKILL.md b/skills/social-media/reddit-reading/SKILL.md new file mode 100644 index 0000000000..8d97c6ece2 --- /dev/null +++ b/skills/social-media/reddit-reading/SKILL.md @@ -0,0 +1,107 @@ +--- +name: reddit-reading +description: "Read Reddit: subreddits, search, threads, users. No browser." +version: 1.0.0 +author: Teknium (teknium1), Hermes Agent +license: MIT +platforms: [linux, macos, windows] +metadata: + hermes: + tags: [Reddit, Social Media, Research, Discussions, Community] + related_skills: [rss-feeds, grounded-citations, blocked-page-recovery, xurl] +--- + +# Reddit Reading Skill + +Reads Reddit content — subreddit listings, site or subreddit search, full threads with +comments, and user activity — from a server or headless machine where the normal routes +are dead. It does not post, vote, or log in as a user. Idea credit: the per-platform +backend routing in [Agent Reach](https://github.com/Panniantong/Agent-Reach). + +## When to Use + +- "What is r/LocalLLaMA saying about X", "find Reddit threads on Y", "summarise this + Reddit thread", "what has u/someone posted lately". +- Any `reddit.com` URL the user shares. `web_extract`, `browser_navigate` and the + `.json` endpoints all fail from server IPs (403 or a "Prove your humanity" wall); + this skill is the working path. +- Not for posting, voting, messaging, or anything needing a user login. + +## Prerequisites + +None for the anonymous path. For anything beyond a handful of calls per task, create +a free Reddit "script" app at https://www.reddit.com/prefs/apps and put the two values +in `~/.hermes/.env`: + +``` +REDDIT_CLIENT_ID=... +REDDIT_CLIENT_SECRET=... +``` + +The script picks the OAuth backend automatically when both are set (~100 requests per +minute, scores, comment nesting, `num_comments`). Without them it uses Reddit's Atom +feeds, which are the only unauthenticated endpoints still served to non-residential IPs. + +## How to Run + +Run every command through `terminal` with the skill-relative script path: + +```bash +python3 scripts/reddit.py doctor # which backend, current rate-limit window +python3 scripts/reddit.py sub LocalLLaMA --sort hot --limit 15 +python3 scripts/reddit.py search "hermes agent" --sub LocalLLaMA --sort new +python3 scripts/reddit.py thread https://www.reddit.com/r/x/comments/abc123/slug/ --limit 40 +python3 scripts/reddit.py user spez --limit 10 +python3 scripts/reddit.py --json search "topic" # machine-readable +``` + +## Quick Reference + +| Need | Command | Anonymous | OAuth | +|---|---|---|---| +| Subreddit front page | `sub NAME --sort hot\|new\|top\|rising [--time week]` | ✔ | ✔ | +| Search all of Reddit | `search "q" --sort relevance\|new\|top\|comments` | ✔ | ✔ | +| Search one subreddit | `search "q" --sub NAME` | ✔ | ✔ | +| Thread + comments | `thread URL --limit N` | ✔ top-level only, no scores | ✔ nested, scores | +| User posts/comments | `user NAME` | ✔ | ✔ | +| Backend + rate limit | `doctor` | ✔ | ✔ | + +## Procedure + +① `doctor` once per task if you have not called it this session — it tells you which +backend is live and how many seconds remain in the anonymous window. + +② Plan your calls before making them. Anonymous Reddit allows roughly **one request per +minute per IP**; the script sleeps until the window resets on a 429 and retries once, so +a five-call plan costs about five minutes. Prefer one `search --sub` over several `sub` +listings, and read one thread rather than the whole listing. + +③ For "what is the community saying" questions, read the thread bodies (`thread`) rather +than stopping at titles; the listing only carries the first ~300 characters of each post. + +④ Cite the permalink (`url` field), not the listing page, when the result feeds a report. +`grounded-citations` registers these URLs like any other source. + +⑤ If the user needs sustained Reddit access (monitoring, more than ~10 calls), stop and +ask them to add the OAuth credentials rather than grinding through the throttle. + +## Pitfalls + +- `www.reddit.com/…/.json`, `api.reddit.com` and `old.reddit.com` return 403 or an + empty "Welcome to Reddit" shell for datacentre IPs. Do not fall back to them; do not + spoof a browser User-Agent (also 403). +- `r.jina.ai` and the `browser_navigate` tool hit the same block ("blocked by network + security" / humanity check). `blocked-page-recovery`'s Wayback route can still recover + an **old** thread that was archived; it cannot fetch fresh ones. +- Anonymous thread feeds only contain the post plus top-level comments (Reddit caps the + feed at a handful of entries); scores and reply nesting are OAuth-only. +- Reddit's `limit` on feeds is advisory — expect 5–25 entries regardless of what you ask. +- Never paste `REDDIT_CLIENT_SECRET` into a chat or log; the script reads it from the + environment only. + +## Verification + +`python3 scripts/reddit.py doctor` prints `anonymous_feed: ok` and an +`x-ratelimit-reset` value; `sub announcements --limit 1` returns one entry with a +`reddit.com/r/announcements/comments/` URL. With credentials set, `doctor` prints +`active_backend: oauth` and `thread …` output shows numeric scores. diff --git a/skills/social-media/reddit-reading/scripts/reddit.py b/skills/social-media/reddit-reading/scripts/reddit.py new file mode 100644 index 0000000000..3f00646437 --- /dev/null +++ b/skills/social-media/reddit-reading/scripts/reddit.py @@ -0,0 +1,307 @@ +#!/usr/bin/env python3 +"""Read Reddit without a browser: listings, search, threads with comments, user pages. + +Two backends, chosen automatically: + +* **OAuth API** (preferred when ``REDDIT_CLIENT_ID`` + ``REDDIT_CLIENT_SECRET`` are set): + a free "script" app from https://www.reddit.com/prefs/apps; ~100 requests/minute, + full JSON including scores and nested comments. +* **Anonymous Atom feeds** (``.rss`` endpoints): the only unauthenticated path Reddit + still serves to server IPs (``.json`` and old.reddit return 403 / an empty shell). + Roughly ONE request per minute per IP; the script sleeps until the window resets + when it hits a 429 and retries once. + + python3 reddit.py sub LocalLLaMA [--sort hot|new|top] [--limit N] + python3 reddit.py search "hermes agent" [--sub LocalLLaMA] [--sort new] [--limit N] + python3 reddit.py thread https://www.reddit.com/r/x/comments/abc123/... [--limit N] + python3 reddit.py user spez [--limit N] + python3 reddit.py doctor # which backend is active, and why + +Add ``--json`` to any read command for machine-readable output. Standard library only. +""" + +from __future__ import annotations + +import argparse +import base64 +import html +import json +import os +import re +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +import xml.etree.ElementTree as ET + +USER_AGENT = "hermes-agent/1.0 (reddit-reading skill; +https://github.com/NousResearch/hermes-agent)" +TIMEOUT = 25 +ATOM = {"a": "http://www.w3.org/2005/Atom"} +WWW = "https://www.reddit.com" +OAUTH = "https://oauth.reddit.com" +_TAG_RE = re.compile(r"<[^>]+>") +_WS_RE = re.compile(r"\s+") +_THREAD_RE = re.compile(r"reddit\.com/r/([^/]+)/comments/([a-z0-9]+)", re.I) + + +def strip_html(text: str | None) -> str: + if not text: + return "" + # Reddit wraps entry bodies in a with a "submitted by /u/x [link] [comments]" footer. + text = _TAG_RE.sub(" ", html.unescape(text)) + text = re.sub(r"submitted by\s+/u/\S+|\[link\]|\[comments\]", " ", text) + return _WS_RE.sub(" ", html.unescape(text)).strip() + + +# ── HTTP ───────────────────────────────────────────────────────────────────── + +def _get(url: str, headers: dict | None = None, retry_on_429: bool = True) -> tuple[bytes, dict]: + hdrs = {"User-Agent": USER_AGENT, "Accept": "*/*"} + hdrs.update(headers or {}) + req = urllib.request.Request(url, headers=hdrs) + try: + with urllib.request.urlopen(req, timeout=TIMEOUT) as resp: + return resp.read(), dict(resp.headers) + except urllib.error.HTTPError as exc: + if exc.code == 429 and retry_on_429: + wait = _reset_seconds(exc.headers) + print(f"reddit: 429 rate-limited, sleeping {wait}s until the window resets", file=sys.stderr) + time.sleep(wait) + return _get(url, headers, retry_on_429=False) + raise + + +def _reset_seconds(headers) -> int: + for key in ("x-ratelimit-reset", "retry-after"): + val = headers.get(key) if headers else None + if val: + try: + return max(1, min(int(float(val)) + 1, 120)) + except ValueError: + pass + return 61 + + +# ── OAuth backend ──────────────────────────────────────────────────────────── + +def oauth_credentials() -> tuple[str, str] | None: + cid, secret = os.environ.get("REDDIT_CLIENT_ID"), os.environ.get("REDDIT_CLIENT_SECRET") + return (cid, secret) if cid and secret else None + + +def oauth_token(cid: str, secret: str) -> str: + body = urllib.parse.urlencode({"grant_type": "client_credentials"}).encode() + auth = base64.b64encode(f"{cid}:{secret}".encode()).decode() + req = urllib.request.Request( + f"{WWW}/api/v1/access_token", data=body, + headers={"Authorization": f"Basic {auth}", "User-Agent": USER_AGENT}, + ) + with urllib.request.urlopen(req, timeout=TIMEOUT) as resp: + return json.loads(resp.read())["access_token"] + + +def _api(path: str, token: str, **params): + params.setdefault("raw_json", 1) + url = f"{OAUTH}{path}?{urllib.parse.urlencode({k: v for k, v in params.items() if v is not None})}" + data, _ = _get(url, {"Authorization": f"Bearer {token}"}) + return json.loads(data) + + +def _post_from_api(child: dict) -> dict: + d = child["data"] + return { + "title": d.get("title"), + "author": d.get("author"), + "subreddit": d.get("subreddit"), + "score": d.get("score"), + "num_comments": d.get("num_comments"), + "created_utc": d.get("created_utc"), + "url": f"{WWW}{d['permalink']}" if d.get("permalink") else d.get("url"), + "external_url": None if d.get("is_self") else d.get("url"), + "body": (d.get("selftext") or "")[:4000], + } + + +def _flatten_comments(children: list, depth: int = 0, out: list | None = None) -> list: + out = out if out is not None else [] + for c in children: + if c.get("kind") != "t1": + continue + d = c["data"] + out.append({ + "author": d.get("author"), "score": d.get("score"), "depth": depth, + "created_utc": d.get("created_utc"), "body": (d.get("body") or "")[:4000], + "url": f"{WWW}{d['permalink']}" if d.get("permalink") else None, + }) + replies = d.get("replies") + if isinstance(replies, dict): + _flatten_comments(replies["data"]["children"], depth + 1, out) + return out + + +def api_listing(token: str, path: str, limit: int, **params) -> list[dict]: + data = _api(path, token, limit=limit, **params) + return [_post_from_api(c) for c in data["data"]["children"] if c.get("kind") == "t3"] + + +def api_thread(token: str, sub: str, post_id: str, limit: int) -> dict: + data = _api(f"/r/{sub}/comments/{post_id}", token, limit=limit, depth=10, sort="top") + post = _post_from_api(data[0]["data"]["children"][0]) + post["comments"] = _flatten_comments(data[1]["data"]["children"])[:limit] + return post + + +# ── Anonymous Atom backend ─────────────────────────────────────────────────── + +def _entries(url: str) -> list[dict]: + data, _ = _get(url) + root = ET.fromstring(data) + out = [] + for e in root.findall("a:entry", ATOM): + link = e.find("a:link", ATOM) + out.append({ + "title": strip_html(e.findtext("a:title", default="", namespaces=ATOM)), + "author": (e.findtext("a:author/a:name", default="", namespaces=ATOM) or "").replace("/u/", "") or None, + "created": e.findtext("a:updated", default="", namespaces=ATOM) or None, + "url": link.get("href") if link is not None else None, + "body": strip_html(e.findtext("a:content", default="", namespaces=ATOM))[:4000], + }) + return out + + +def atom_listing(path: str, limit: int, **params) -> list[dict]: + params["limit"] = limit + return _entries(f"{WWW}{path}.rss?{urllib.parse.urlencode({k: v for k, v in params.items() if v is not None})}") + + +def atom_thread(sub: str, post_id: str, limit: int) -> dict: + entries = _entries(f"{WWW}/r/{sub}/comments/{post_id}/.rss?limit={limit}") + if not entries: + raise SystemExit("thread feed returned no entries") + post, comments = entries[0], entries[1:] + post["comments"] = [{"author": c["author"], "created": c["created"], "body": c["body"], "url": c["url"]} for c in comments] + post["note"] = "anonymous feed: scores and nesting unavailable; set REDDIT_CLIENT_ID/SECRET for full data" + return post + + +# ── Commands ───────────────────────────────────────────────────────────────── + +def parse_thread_url(url: str) -> tuple[str, str]: + m = _THREAD_RE.search(url) + if not m: + raise SystemExit(f"not a Reddit thread URL: {url}") + return m.group(1), m.group(2) + + +def cmd_sub(a, token): + path = f"/r/{a.name}/{a.sort}" + if token: + return api_listing(token, path, a.limit, t=a.time if a.sort == "top" else None) + return atom_listing(path, a.limit, t=a.time if a.sort == "top" else None) + + +def cmd_search(a, token): + path = f"/r/{a.sub}/search" if a.sub else "/search" + params = {"q": a.query, "sort": a.sort, "restrict_sr": 1 if a.sub else None, "t": a.time} + return api_listing(token, path, a.limit, **params) if token else atom_listing(path, a.limit, **params) + + +def cmd_thread(a, token): + sub, post_id = parse_thread_url(a.url) + return api_thread(token, sub, post_id, a.limit) if token else atom_thread(sub, post_id, a.limit) + + +def cmd_user(a, token): + path = f"/user/{a.name}" + if token: + data = _api(f"{path}/overview", token, limit=a.limit) + out = [] + for c in data["data"]["children"]: + out.append(_post_from_api(c) if c["kind"] == "t3" else _flatten_comments([c])[0]) + return out + return atom_listing(path, a.limit) + + +def cmd_doctor(a, token): + report = {"oauth_credentials": bool(oauth_credentials()), "user_agent": USER_AGENT} + if token: + try: + _api("/r/announcements/hot", token, limit=1) + report["active_backend"] = "oauth" + except (urllib.error.URLError, OSError, KeyError) as exc: + report["active_backend"] = "oauth (broken)" + report["oauth_error"] = str(exc) + else: + report["active_backend"] = "anonymous-atom" + try: + data, headers = _get(f"{WWW}/r/announcements/.rss?limit=1", retry_on_429=False) + report["anonymous_feed"] = "ok" if b" str: + if cmd == "doctor": + return "\n".join(f"{k}: {v}" for k, v in result.items()) + if cmd == "thread": + p = result + lines = [f"# {p.get('title')} — u/{p.get('author')} score={p.get('score', '?')} {p.get('url')}", p.get("body", "")[:1500], ""] + for c in p["comments"]: + indent = " " * c.get("depth", 0) + lines.append(f"{indent}- u/{c.get('author')} (score {c.get('score', '?')}): {c.get('body', '')[:600]}") + if p.get("note"): + lines.append(f"\n[{p['note']}]") + return "\n".join(lines) + lines = [] + for p in result: + score = f" ↑{p['score']}" if p.get("score") is not None else "" + nc = f" 💬{p['num_comments']}" if p.get("num_comments") is not None else "" + lines.append(f"- {p.get('title') or p.get('body', '')[:80]}{score}{nc} — u/{p.get('author')}\n {p.get('url')}") + if p.get("body") and p.get("title"): + lines.append(f" {p['body'][:300]}") + return "\n".join(lines) or "(no results)" + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--json", action="store_true") + sub = ap.add_subparsers(dest="cmd", required=True) + s = sub.add_parser("sub"); s.add_argument("name"); s.add_argument("--sort", default="hot", choices=["hot", "new", "top", "rising"]); s.add_argument("--time", default="week", choices=["hour", "day", "week", "month", "year", "all"]); s.add_argument("--limit", type=int, default=15) + q = sub.add_parser("search"); q.add_argument("query"); q.add_argument("--sub"); q.add_argument("--sort", default="relevance", choices=["relevance", "new", "top", "comments"]); q.add_argument("--time", default="all", choices=["hour", "day", "week", "month", "year", "all"]); q.add_argument("--limit", type=int, default=15) + t = sub.add_parser("thread"); t.add_argument("url"); t.add_argument("--limit", type=int, default=40) + u = sub.add_parser("user"); u.add_argument("name"); u.add_argument("--limit", type=int, default=15) + sub.add_parser("doctor") + args = ap.parse_args(argv) + + creds = oauth_credentials() + token = None + if creds: + try: + token = oauth_token(*creds) + except (urllib.error.URLError, OSError, KeyError) as exc: + print(f"reddit: OAuth token failed ({exc}); falling back to anonymous feeds", file=sys.stderr) + try: + result = COMMANDS[args.cmd](args, token) + except urllib.error.HTTPError as exc: + print(f"HTTP {exc.code} for {exc.url}", file=sys.stderr) + return 2 + except (urllib.error.URLError, ET.ParseError, json.JSONDecodeError, KeyError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + print(json.dumps(result, indent=2, ensure_ascii=False) if args.json else render(args.cmd, result)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/skills/test_reddit_reading_skill.py b/tests/skills/test_reddit_reading_skill.py new file mode 100644 index 0000000000..45310f098d --- /dev/null +++ b/tests/skills/test_reddit_reading_skill.py @@ -0,0 +1,113 @@ +"""Tests for skills/social-media/reddit-reading/scripts/reddit.py — backend selection and throttle handling.""" + +import io +import sys +import urllib.error +from pathlib import Path +from unittest import mock + +import pytest + +SCRIPTS_DIR = Path(__file__).resolve().parents[2] / "skills" / "social-media" / "reddit-reading" / "scripts" +sys.path.insert(0, str(SCRIPTS_DIR)) + +import reddit # noqa: E402 + +THREAD_ATOM = b""" +/u/opPost title + 2026-09-01T00:00:00+00:00 + <div>body text</div> submitted by /u/op [link] [comments] +/u/c1/u/c1 on Post title + 2026-09-01T01:00:00+00:00 + <p>first comment</p> +""" + + +def _http_error(code, headers): + return urllib.error.HTTPError("https://www.reddit.com/x", code, "msg", headers, io.BytesIO(b"")) + + +def test_anonymous_thread_uses_atom_feed_and_waits_out_a_429_exactly_once(monkeypatch): + """Without OAuth credentials the .rss endpoint is used; a 429 sleeps for x-ratelimit-reset and retries once, + and the parsed thread separates the post from its comments with feed noise stripped.""" + monkeypatch.delenv("REDDIT_CLIENT_ID", raising=False) + monkeypatch.delenv("REDDIT_CLIENT_SECRET", raising=False) + assert reddit.oauth_credentials() is None + + calls = [] + sleeps = [] + + class Resp(io.BytesIO): + headers = {"x-ratelimit-remaining": "0.0"} + + def __enter__(self): + return self + + def __exit__(self, *a): + self.close() + + def fake_urlopen(req, timeout): + calls.append(req.full_url) + if len(calls) == 1: + raise _http_error(429, {"x-ratelimit-reset": "7"}) + return Resp(THREAD_ATOM) + + with mock.patch.object(reddit.urllib.request, "urlopen", fake_urlopen), \ + mock.patch.object(reddit.time, "sleep", sleeps.append): + post = reddit.atom_thread("test", "abc123", limit=10) + + assert calls[0].startswith("https://www.reddit.com/r/test/comments/abc123/.rss") and len(calls) == 2 + assert sleeps == [8] # reset + 1s margin, one retry only + assert post["title"] == "Post title" and post["author"] == "op" + assert post["body"] == "body text" # "submitted by … [link] [comments]" footer stripped + assert [c["author"] for c in post["comments"]] == ["c1"] + + # a second 429 after the retry propagates instead of looping + with mock.patch.object(reddit.urllib.request, "urlopen", side_effect=_http_error(429, {})), \ + mock.patch.object(reddit.time, "sleep", lambda s: None), pytest.raises(urllib.error.HTTPError): + reddit._get("https://www.reddit.com/r/test/.rss") + + +def test_oauth_credentials_route_to_oauth_host_and_flatten_nested_comments(monkeypatch): + """With REDDIT_CLIENT_ID/SECRET the script talks to oauth.reddit.com with a bearer token and + returns nested comments flattened with depth and scores — data the anonymous path cannot provide.""" + monkeypatch.setenv("REDDIT_CLIENT_ID", "cid") + monkeypatch.setenv("REDDIT_CLIENT_SECRET", "sec") + assert reddit.oauth_credentials() == ("cid", "sec") + + listing = [ + {"data": {"children": [{"kind": "t3", "data": {"title": "T", "author": "op", "subreddit": "test", "score": 42, + "num_comments": 2, "created_utc": 1.0, "permalink": "/r/test/comments/abc123/t/", + "is_self": True, "url": "https://www.reddit.com/r/test/comments/abc123/t/", "selftext": "s"}}]}}, + {"data": {"children": [{"kind": "t1", "data": {"author": "a", "score": 5, "body": "top", "permalink": "/p/1", + "replies": {"data": {"children": [{"kind": "t1", "data": {"author": "b", "score": 1, "body": "reply", "replies": ""}}]}}}}, + {"kind": "more", "data": {}}]}}, + ] + seen = {} + + def fake_api(path, token, **params): + seen["path"], seen["token"] = path, token + return listing + + with mock.patch.object(reddit, "_api", fake_api): + post = reddit.api_thread("tok", "test", "abc123", limit=10) + + assert seen == {"path": "/r/test/comments/abc123", "token": "tok"} + assert post["score"] == 42 and post["url"] == "https://www.reddit.com/r/test/comments/abc123/t/" + assert [(c["author"], c["depth"], c["score"]) for c in post["comments"]] == [("a", 0, 5), ("b", 1, 1)] + + # the bearer header actually reaches the OAuth host + captured = {} + + def fake_get(url, headers=None, retry_on_429=True): + captured["url"], captured["headers"] = url, headers + return b'{"data": {"children": []}}', {} + + with mock.patch.object(reddit, "_get", fake_get): + reddit._api("/r/test/hot", "tok", limit=1) + assert captured["url"].startswith("https://oauth.reddit.com/r/test/hot?") + assert captured["headers"]["Authorization"] == "Bearer tok" + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__])) diff --git a/tests/skills/test_rss_feeds_skill.py b/tests/skills/test_rss_feeds_skill.py new file mode 100644 index 0000000000..9ce7af8748 --- /dev/null +++ b/tests/skills/test_rss_feeds_skill.py @@ -0,0 +1,69 @@ +"""Tests for skills/research/rss-feeds/scripts/feed.py — parsing and discovery contracts.""" + +import sys +from pathlib import Path +from unittest import mock + +import pytest + +SCRIPTS_DIR = Path(__file__).resolve().parents[2] / "skills" / "research" / "rss-feeds" / "scripts" +sys.path.insert(0, str(SCRIPTS_DIR)) + +import feed # noqa: E402 + +RSS = b""" +Blog & Notes +Olderhttps://ex.com/aMon, 01 Sep 2026 10:00:00 GMT + Ann<p>Hello <b>world</b></p> +Newerhttps://ex.com/bThu, 04 Sep 2026 08:30:00 +0200 +""" + +ATOM = b"""Atom Site +Entry +2026-09-03T12:00:00ZBob<p>Body</p> +""" + +JSONFEED = b'{"version":"https://jsonfeed.org/version/1.1","title":"JF","items":[{"id":"1","url":"https://ex.com/j","title":"J1","date_published":"2026-09-02T00:00:00Z","authors":[{"name":"Cy"}],"content_text":"txt"}]}' + +PAGE = b"""x + +""" + + +def test_all_three_formats_normalise_to_the_same_entry_shape_and_utc_dates(): + """RSS (RFC 822), Atom (ISO), JSON Feed (ISO) parse to identical keys with UTC-normalised dates, + and Atom picks the rel=alternate link over rel=self.""" + rss, atom, jf = feed.parse_feed(RSS), feed.parse_feed(ATOM), feed.parse_feed(JSONFEED, "application/feed+json") + keys = {"title", "link", "published", "author", "summary"} + for f in (rss, atom, jf): + assert f["entries"] and all(set(e) == keys for e in f["entries"]) + assert rss["title"] == "Blog & Notes" + assert rss["entries"][0]["summary"] == "Hello world" # HTML stripped, entities decoded + assert rss["entries"][1]["published"] == "2026-09-04T06:30:00+00:00" # +0200 → UTC + assert atom["entries"][0]["link"] == "https://ex.com/post" + assert jf["entries"][0]["author"] == "Cy" + newest = feed.filter_entries(rss["entries"], limit=1, since="2026-09-02") + assert [e["title"] for e in newest] == ["Newer"] + + +def test_page_url_discovers_advertised_feed_then_reads_it(): + """A non-feed page falls through to discovery (resolved against the page URL) + and `read` returns the parsed feed tagged with where it was discovered from.""" + responses = { + "https://ex.com/blog/": (PAGE, "text/html"), + "https://ex.com/atom/everything/": (ATOM, "application/atom+xml"), + } + with mock.patch.object(feed, "fetch", side_effect=lambda u: responses[u]): + assert feed.discover("https://ex.com/blog/") == ["https://ex.com/atom/everything/"] + result = feed.read("https://ex.com/blog/") + assert result["url"] == "https://ex.com/atom/everything/" + assert result["discovered_from"] == "https://ex.com/blog/" + assert result["entries"][0]["title"] == "Entry" + # no advertised feed → well-known paths are proposed, never an empty list + with mock.patch.object(feed, "fetch", return_value=(b"plain", "text/html")): + candidates = feed.discover("https://plain.example/") + assert candidates and all(c.startswith("https://plain.example/") for c in candidates) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__])) diff --git a/website/docs/reference/skills-catalog.md b/website/docs/reference/skills-catalog.md index b6ab7f5c21..d56a104fa9 100644 --- a/website/docs/reference/skills-catalog.md +++ b/website/docs/reference/skills-catalog.md @@ -101,11 +101,13 @@ If a skill is missing from this list but present in the repo, the catalog is reg | [`competitor-news-monitor`](/docs/user-guide/skills/bundled/research/research-competitor-news-monitor) | Watch named companies for material news; cited digests. | `research\competitor-news-monitor` | | [`grounded-citations`](/docs/user-guide/skills/bundled/research/research-grounded-citations) | Ground answers and documents in cited, verifiable sources. | `research\grounded-citations` | | [`llm-wiki`](/docs/user-guide/skills/bundled/research/research-llm-wiki) | Karpathy's LLM Wiki: build/query interlinked markdown KB. | `research\llm-wiki` | +| [`rss-feeds`](/docs/user-guide/skills/bundled/research/research-rss-feeds) | Read RSS, Atom, JSON feeds; discover feeds behind a page. | `research/rss-feeds` | ## social-media | Skill | Description | Path | |-------|-------------|------| +| [`reddit-reading`](/docs/user-guide/skills/bundled/social-media/social-media-reddit-reading) | Read Reddit: subreddits, search, threads, users. No browser. | `social-media/reddit-reading` | | [`xurl`](/docs/user-guide/skills/bundled/social-media/social-media-xurl) | X/Twitter via xurl CLI: raw post search, posting, DM, media. | `social-media\xurl` | ## software-development diff --git a/website/docs/user-guide/skills/bundled/research/research-competitor-news-monitor.md b/website/docs/user-guide/skills/bundled/research/research-competitor-news-monitor.md index dd756be8a9..5ac27e64d4 100644 --- a/website/docs/user-guide/skills/bundled/research/research-competitor-news-monitor.md +++ b/website/docs/user-guide/skills/bundled/research/research-competitor-news-monitor.md @@ -15,13 +15,13 @@ Watch named companies for material news; cited digests. | | | |---|---| | Source | Bundled (installed by default) | -| Path | `skills/research\competitor-news-monitor` | +| Path | `skills/research/competitor-news-monitor` | | Version | `0.1.0` | | Author | Ben Barclay (benbarclay), Hermes Agent | | License | MIT | | Platforms | linux, macos, windows | | Tags | `Competitors`, `News`, `Market-Research`, `Monitoring` | -| Related skills | [`blogwatcher`](/docs/user-guide/skills/optional/research/research-blogwatcher) | +| Related skills | [`blogwatcher`](/docs/user-guide/skills/optional/research/research-blogwatcher), [`rss-feeds`](/docs/user-guide/skills/bundled/research/research-rss-feeds), [`reddit-reading`](/docs/user-guide/skills/bundled/social-media/social-media-reddit-reading) | ## Reference: full SKILL.md @@ -60,7 +60,7 @@ For each company include, where available: 5. reputable trade and financial press 6. job postings as weak supporting evidence -Use `blogwatcher` for feeds and `web_search`/`web_extract` for pages. Write the watch contract (watchlist, categories, materiality threshold, last cutoff) to a state file under `~/.hermes/competitor-watches/.json`, then create the job: +Use `rss-feeds` (bundled) or `blogwatcher` (optional, stateful) for feeds, `reddit-reading` for community discussion, and `web_search`/`web_extract` for pages. Write the watch contract (watchlist, categories, materiality threshold, last cutoff) to a state file under `~/.hermes/competitor-watches/.json`, then create the job: ``` cronjob(action="create", diff --git a/website/docs/user-guide/skills/bundled/research/research-grounded-citations.md b/website/docs/user-guide/skills/bundled/research/research-grounded-citations.md index 938e223a68..8166801512 100644 --- a/website/docs/user-guide/skills/bundled/research/research-grounded-citations.md +++ b/website/docs/user-guide/skills/bundled/research/research-grounded-citations.md @@ -15,13 +15,13 @@ Ground answers and documents in cited, verifiable sources. | | | |---|---| | Source | Bundled (installed by default) | -| Path | `skills/research\grounded-citations` | -| Version | `1.1.0` | +| Path | `skills/research/grounded-citations` | +| Version | `1.2.0` | | Author | Hermes Agent + Teknium | | License | MIT | | Platforms | linux, macos, windows | | Tags | `Research`, `Citations`, `Grounding`, `Sources`, `Web`, `Reports` | -| Related skills | [`arxiv`](/docs/user-guide/skills/bundled/research/research-arxiv), [`arxiv`](/docs/user-guide/skills/bundled/research/research-arxiv), `ocr-and-documents` | +| Related skills | [`arxiv`](/docs/user-guide/skills/bundled/research/research-arxiv), [`pdf`](/docs/user-guide/skills/bundled/productivity/productivity-pdf), [`reddit-reading`](/docs/user-guide/skills/bundled/social-media/social-media-reddit-reading), [`rss-feeds`](/docs/user-guide/skills/bundled/research/research-rss-feeds), [`youtube-content`](/docs/user-guide/skills/bundled/media/media-youtube-content) | ## Reference: full SKILL.md @@ -143,6 +143,27 @@ sources, cite inline, end with the rendered `Sources:` list. For a short answer you may render the block from `sources.py render --only ` instead of writing to a file. +## Multi-Platform Sweeps + +"What are people saying about X" / "research X across the web" is not one +`web_search`. Fan out across source types, collect in parallel, then synthesise +with every claim attributed to the platform it came from: + +| Source type | Route | What it adds | +|---|---|---| +| Open web | `web_search` → `web_extract` | official docs, articles, announcements | +| Community discussion | `reddit-reading` (`search`, `thread`) | real user experience, complaints, workarounds | +| Blogs / releases / changelogs | `rss-feeds` (`read`, `discover`) | dated primary posts, version history | +| Video | `youtube-content` | walkthroughs, demos, talks | +| Code | `terminal` with `gh search repos` / `gh search issues` | implementations, open bugs | +| X/Twitter | `xurl` (needs API access) | announcements, developer chatter | + +Register every URL from every route in the ledger as it arrives (step ②). Keep +opinion and measurement apart: a Reddit thread is evidence that users *report* +something, not that it is true; pair it with a primary source or label it as +sentiment. Report per-platform coverage gaps ("Reddit search returned nothing +newer than March") rather than silently narrowing to what worked. + ## Fact-Checking Mode For work where the reader must be able to check the chain — medical, legal, diff --git a/website/docs/user-guide/skills/bundled/research/research-rss-feeds.md b/website/docs/user-guide/skills/bundled/research/research-rss-feeds.md new file mode 100644 index 0000000000..4946eaa94b --- /dev/null +++ b/website/docs/user-guide/skills/bundled/research/research-rss-feeds.md @@ -0,0 +1,114 @@ +--- +title: "Rss Feeds — Read RSS, Atom, JSON feeds; discover feeds behind a page" +sidebar_label: "Rss Feeds" +description: "Read RSS, Atom, JSON feeds; discover feeds behind a page" +--- + +{/* This page is auto-generated from the skill's SKILL.md by website/scripts/generate-skill-docs.py. Edit the source SKILL.md, not this page. */} + +# Rss Feeds + +Read RSS, Atom, JSON feeds; discover feeds behind a page. + +## Skill metadata + +| | | +|---|---| +| Source | Bundled (installed by default) | +| Path | `skills/research/rss-feeds` | +| Version | `1.0.0` | +| Author | Teknium (teknium1), Hermes Agent | +| License | MIT | +| Platforms | linux, macos, windows | +| Tags | `RSS`, `Atom`, `Feeds`, `Monitoring`, `Research`, `Blogs`, `Releases` | +| Related skills | [`reddit-reading`](/docs/user-guide/skills/bundled/social-media/social-media-reddit-reading), [`competitor-news-monitor`](/docs/user-guide/skills/bundled/research/research-competitor-news-monitor), [`grounded-citations`](/docs/user-guide/skills/bundled/research/research-grounded-citations), [`youtube-content`](/docs/user-guide/skills/bundled/media/media-youtube-content), [`blogwatcher`](/docs/user-guide/skills/optional/research/research-blogwatcher) | + +## Reference: full SKILL.md + +:::info +The following is the complete skill definition that Hermes loads when this skill is triggered. This is what the agent sees as instructions when the skill is active. +::: + +# RSS Feeds Skill + +Reads any RSS 2.0, RSS 1.0/RDF, Atom, or JSON Feed URL into a clean, date-sorted list of +entries, and discovers the feed behind an ordinary page URL (`` or +the usual `/feed`, `/rss.xml`, `/atom.xml` paths). Standard library only, nothing to +install. It does not fetch full article bodies — pass an entry's link to `web_extract` for +that. + +## When to Use + +- "What's new on <blog/site>", "latest releases of <GitHub repo>", "recent posts in + <subreddit>", "read this feed", "does this site have an RSS feed". +- Building a recurring digest with `cronjob_manage` (feeds are cheaper and more stable than + scraping the HTML front page every run). For a persistent read/unread database across + many feeds install the optional `blogwatcher` skill; this skill is the zero-install read. +- Anything where a structured list of `title / link / date / author / summary` beats a + rendered page: podcasts, changelogs, YouTube channels, newsrooms, forum categories. + +## Prerequisites + +None. Python 3.10+, network access to the feed host. + +## How to Run + +Run through `terminal` with the skill-relative script path: + +```bash +python3 scripts/feed.py read https://hnrss.org/frontpage --limit 10 +python3 scripts/feed.py read https://simonwillison.net/ # page URL → discovers the feed +python3 scripts/feed.py read URL --since 2026-09-01 --json # only newer entries, machine-readable +python3 scripts/feed.py discover https://example.com/ # list candidate feed URLs +``` + +## Quick Reference + +| Source | Feed URL pattern | +|---|---| +| GitHub releases / commits / tags | `https://github.com/OWNER/REPO/releases.atom`, `…/commits/BRANCH.atom`, `…/tags.atom` | +| Subreddit / Reddit search | `https://www.reddit.com/r/NAME/.rss`, `https://www.reddit.com/search.rss?q=…` (1 req/min anon; see `reddit-reading`) | +| YouTube channel | `https://www.youtube.com/feeds/videos.xml?channel_id=UC…` | +| Hacker News | `https://hnrss.org/frontpage`, `https://hnrss.org/newest?q=TERM` | +| arXiv category | `https://rss.arxiv.org/rss/cs.CL` | +| Substack / Medium / WordPress / Ghost | `SITE/feed`, `medium.com/feed/@user`, `SITE/rss/` | +| Podcasts | the show's RSS URL from its hosting page (`discover` finds it) | + +Output fields per entry: `title`, `link`, `published` (UTC ISO 8601), `author`, `summary` +(HTML stripped, ≤ 2000 chars). Entries are sorted newest-first. + +## Procedure + +① If you only have a site URL, run `read` on it directly; the script discovers the feed +and reports which URL it used (`discovered_from`). Use `discover` when you want to choose +between several advertised feeds (comments feed vs posts feed, per-category feeds). + +② Bound the request: `--limit` for "latest N", `--since YYYY-MM-DD` for "since last +check". For a cron digest persist the last-seen `published` value and pass it as +`--since` next run. + +③ For full text, hand the entry `link` to `web_extract`; feed summaries are frequently +truncated or the first paragraph only. + +④ Cite the entry `link`, not the feed URL, when the result feeds a report +(`grounded-citations`). + +## Pitfalls + +- A 200 response with HTML means the URL is a page, not a feed; the script falls through + to discovery automatically, but a site with no `` and none of the + common paths reports `no feed found` — check the site's footer or `/sitemap.xml` before + concluding there is none. +- Reddit feeds share Reddit's anonymous throttle (about one request per minute per IP). + Chain them through `reddit-reading`, which waits out the window, when you need more + than one Reddit call. +- Dates: RSS `pubDate` is RFC 822 and Atom uses ISO 8601; the script normalises both to + UTC. Feeds that omit dates sort to the bottom and are dropped by `--since`. +- Some feeds are Cloudflare-fronted and 403 non-browser clients; `blocked-page-recovery` + handles that class. + +## Verification + +`python3 scripts/feed.py read https://github.com/NousResearch/hermes-agent/releases.atom +--limit 1` prints one entry with a `releases/tag/` link and a `[atom]` format tag; +`discover https://simonwillison.net/` prints an `/atom/` URL. diff --git a/website/docs/user-guide/skills/bundled/social-media/social-media-reddit-reading.md b/website/docs/user-guide/skills/bundled/social-media/social-media-reddit-reading.md new file mode 100644 index 0000000000..219195c947 --- /dev/null +++ b/website/docs/user-guide/skills/bundled/social-media/social-media-reddit-reading.md @@ -0,0 +1,125 @@ +--- +title: "Reddit Reading — Read Reddit: subreddits, search, threads, users" +sidebar_label: "Reddit Reading" +description: "Read Reddit: subreddits, search, threads, users" +--- + +{/* This page is auto-generated from the skill's SKILL.md by website/scripts/generate-skill-docs.py. Edit the source SKILL.md, not this page. */} + +# Reddit Reading + +Read Reddit: subreddits, search, threads, users. No browser. + +## Skill metadata + +| | | +|---|---| +| Source | Bundled (installed by default) | +| Path | `skills/social-media/reddit-reading` | +| Version | `1.0.0` | +| Author | Teknium (teknium1), Hermes Agent | +| License | MIT | +| Platforms | linux, macos, windows | +| Tags | `Reddit`, `Social Media`, `Research`, `Discussions`, `Community` | +| Related skills | [`rss-feeds`](/docs/user-guide/skills/bundled/research/research-rss-feeds), [`grounded-citations`](/docs/user-guide/skills/bundled/research/research-grounded-citations), [`blocked-page-recovery`](/docs/user-guide/skills/bundled/web/web-blocked-page-recovery), [`xurl`](/docs/user-guide/skills/bundled/social-media/social-media-xurl) | + +## Reference: full SKILL.md + +:::info +The following is the complete skill definition that Hermes loads when this skill is triggered. This is what the agent sees as instructions when the skill is active. +::: + +# Reddit Reading Skill + +Reads Reddit content — subreddit listings, site or subreddit search, full threads with +comments, and user activity — from a server or headless machine where the normal routes +are dead. It does not post, vote, or log in as a user. Idea credit: the per-platform +backend routing in [Agent Reach](https://github.com/Panniantong/Agent-Reach). + +## When to Use + +- "What is r/LocalLLaMA saying about X", "find Reddit threads on Y", "summarise this + Reddit thread", "what has u/someone posted lately". +- Any `reddit.com` URL the user shares. `web_extract`, `browser_navigate` and the + `.json` endpoints all fail from server IPs (403 or a "Prove your humanity" wall); + this skill is the working path. +- Not for posting, voting, messaging, or anything needing a user login. + +## Prerequisites + +None for the anonymous path. For anything beyond a handful of calls per task, create +a free Reddit "script" app at https://www.reddit.com/prefs/apps and put the two values +in `~/.hermes/.env`: + +``` +REDDIT_CLIENT_ID=... +REDDIT_CLIENT_SECRET=... +``` + +The script picks the OAuth backend automatically when both are set (~100 requests per +minute, scores, comment nesting, `num_comments`). Without them it uses Reddit's Atom +feeds, which are the only unauthenticated endpoints still served to non-residential IPs. + +## How to Run + +Run every command through `terminal` with the skill-relative script path: + +```bash +python3 scripts/reddit.py doctor # which backend, current rate-limit window +python3 scripts/reddit.py sub LocalLLaMA --sort hot --limit 15 +python3 scripts/reddit.py search "hermes agent" --sub LocalLLaMA --sort new +python3 scripts/reddit.py thread https://www.reddit.com/r/x/comments/abc123/slug/ --limit 40 +python3 scripts/reddit.py user spez --limit 10 +python3 scripts/reddit.py --json search "topic" # machine-readable +``` + +## Quick Reference + +| Need | Command | Anonymous | OAuth | +|---|---|---|---| +| Subreddit front page | `sub NAME --sort hot\|new\|top\|rising [--time week]` | ✔ | ✔ | +| Search all of Reddit | `search "q" --sort relevance\|new\|top\|comments` | ✔ | ✔ | +| Search one subreddit | `search "q" --sub NAME` | ✔ | ✔ | +| Thread + comments | `thread URL --limit N` | ✔ top-level only, no scores | ✔ nested, scores | +| User posts/comments | `user NAME` | ✔ | ✔ | +| Backend + rate limit | `doctor` | ✔ | ✔ | + +## Procedure + +① `doctor` once per task if you have not called it this session — it tells you which +backend is live and how many seconds remain in the anonymous window. + +② Plan your calls before making them. Anonymous Reddit allows roughly **one request per +minute per IP**; the script sleeps until the window resets on a 429 and retries once, so +a five-call plan costs about five minutes. Prefer one `search --sub` over several `sub` +listings, and read one thread rather than the whole listing. + +③ For "what is the community saying" questions, read the thread bodies (`thread`) rather +than stopping at titles; the listing only carries the first ~300 characters of each post. + +④ Cite the permalink (`url` field), not the listing page, when the result feeds a report. +`grounded-citations` registers these URLs like any other source. + +⑤ If the user needs sustained Reddit access (monitoring, more than ~10 calls), stop and +ask them to add the OAuth credentials rather than grinding through the throttle. + +## Pitfalls + +- `www.reddit.com/…/.json`, `api.reddit.com` and `old.reddit.com` return 403 or an + empty "Welcome to Reddit" shell for datacentre IPs. Do not fall back to them; do not + spoof a browser User-Agent (also 403). +- `r.jina.ai` and the `browser_navigate` tool hit the same block ("blocked by network + security" / humanity check). `blocked-page-recovery`'s Wayback route can still recover + an **old** thread that was archived; it cannot fetch fresh ones. +- Anonymous thread feeds only contain the post plus top-level comments (Reddit caps the + feed at a handful of entries); scores and reply nesting are OAuth-only. +- Reddit's `limit` on feeds is advisory — expect 5–25 entries regardless of what you ask. +- Never paste `REDDIT_CLIENT_SECRET` into a chat or log; the script reads it from the + environment only. + +## Verification + +`python3 scripts/reddit.py doctor` prints `anonymous_feed: ok` and an +`x-ratelimit-reset` value; `sub announcements --limit 1` returns one entry with a +`reddit.com/r/announcements/comments/` URL. With credentials set, `doctor` prints +`active_backend: oauth` and `thread …` output shows numeric scores. diff --git a/website/docs/user-guide/skills/optional/research/research-blogwatcher.md b/website/docs/user-guide/skills/optional/research/research-blogwatcher.md index 84d9ac28aa..e929ea0863 100644 --- a/website/docs/user-guide/skills/optional/research/research-blogwatcher.md +++ b/website/docs/user-guide/skills/optional/research/research-blogwatcher.md @@ -15,7 +15,7 @@ Monitor blogs and RSS/Atom feeds via blogwatcher-cli tool. | | | |---|---| | Source | Optional — install with `hermes skills install official/research/blogwatcher` | -| Path | `optional-skills/research\blogwatcher` | +| Path | `optional-skills/research/blogwatcher` | | Version | `2.0.0` | | Author | JulienTant (fork of Hyaxia/blogwatcher) | | License | MIT | @@ -39,6 +39,7 @@ Track blog and RSS/Atom feed updates with the `blogwatcher-cli` tool. Supports a - **Recurring watch — use the cronjob tool's `monitor` field, not a bare schedule.** `monitor` runs a script each tick and only wakes the agent when output changes: set it to a script that runs `blogwatcher-cli scan >/dev/null 2>&1 && blogwatcher-cli articles` (deterministic output; new articles = changed output = agent wakes with the diff injected). Unchanged ticks cost zero LLM calls. Set `deliver` to route digests to a chat/channel; add `continuity: true` so consecutive digests can dedupe. - **Reading an article the user asks about**: `web_extract([url])` on the article URL from `blogwatcher-cli articles` — do not re-scrape by hand. - **One-off "watch this page for changes" without feed semantics**: skip this skill; the cronjob tool's `monitor` field accepts an http(s) URL directly. +- **One-off read of a feed or a site's latest posts, nothing to install**: the bundled `rss-feeds` skill (`scripts/feed.py read URL`); blogwatcher earns its install when you track many feeds with read/unread state. - **Company/competitor tracking with analysis and citations**: prefer the `competitor-news-monitor` skill; blogwatcher is the lighter raw-feed layer it can sit on. ## Installation diff --git a/website/sidebars.ts b/website/sidebars.ts index a50277bae6..fbe83d78c6 100644 --- a/website/sidebars.ts +++ b/website/sidebars.ts @@ -264,6 +264,7 @@ const sidebars: SidebarsConfig = { 'user-guide/skills/bundled/research/research-competitor-news-monitor', 'user-guide/skills/bundled/research/research-grounded-citations', 'user-guide/skills/bundled/research/research-llm-wiki', + 'user-guide/skills/bundled/research/research-rss-feeds', ], }, { @@ -272,6 +273,7 @@ const sidebars: SidebarsConfig = { key: 'skills-bundled-social-media', collapsed: true, items: [ + 'user-guide/skills/bundled/social-media/social-media-reddit-reading', 'user-guide/skills/bundled/social-media/social-media-xurl', ], },