diff --git a/README.md b/README.md index 3ace8f9..8d919b3 100644 --- a/README.md +++ b/README.md @@ -18,12 +18,12 @@

> **Monitor the current thing. Your dashboard for the internet.** -> Build a deck, pack it with columns, refresh on demand. Each column is a plugin: X, Bluesky, Reddit, Hacker News, Lobsters, Stack Overflow, DEV.to, npm + PyPI + crates.io packages, Hugging Face (models / datasets / spaces), arXiv (CS / stat / math.OC papers), GitHub (trending / issues / PRs / stars / forks / backlinks / search / releases / commits / Actions / Discussions), Farcaster, Mastodon, YouTube, RSS, Google News, Bing, Substack, LinkedIn, Facebook, Instagram, Apple + Google Play reviews, on-chain wallet activity, Polymarket prediction markets, CoinGecko crypto trending + prices, DeFiLlama TVL leaderboard, Dexscreener DEX pair search + watchlist, and the six biggest Chinese platforms (Weibo / Zhihu / Douyin / Bilibili / Toutiao / Baidu). +> Build a deck, pack it with columns, refresh on demand. Each column is a plugin: X, Reddit, Hacker News, Lobsters, Stack Overflow, DEV.to, npm + PyPI + crates.io packages, Hugging Face (models / datasets / spaces), arXiv (CS / stat / math.OC papers), GitHub (trending / issues / PRs / stars / forks / backlinks / search / releases / commits / Actions / Discussions), Farcaster, Mastodon, YouTube, RSS, Google News, Substack, LinkedIn, Facebook, Instagram, Apple + Google Play reviews, on-chain wallet activity, Polymarket prediction markets, CoinGecko crypto trending + prices, DeFiLlama TVL leaderboard, Dexscreener DEX pair search + watchlist, and the six biggest Chinese platforms (Weibo / Zhihu / Douyin / Bilibili / Toutiao / Baidu). ### What it does - You name a deck. Minitor packs it with whatever you're watching. -- 49 column types out of the box — social feeds, news, GitHub (including commits, CI runs, and Discussions), Hugging Face, arXiv, DEV.to, Product Hunt, npm + PyPI + crates.io packages, app reviews, on-chain transactions, prediction markets, CoinGecko prices, DeFiLlama TVL, Dexscreener DEX pairs, Chinese hot boards. +- 47 column types out of the box — social feeds, news, GitHub (including commits, CI runs, and Discussions), Hugging Face, arXiv, DEV.to, Product Hunt, npm + PyPI + crates.io packages, app reviews, on-chain transactions, prediction markets, CoinGecko prices, DeFiLlama TVL, Dexscreener DEX pairs, Chinese hot boards. - Refresh per column or auto-fetch on creation. Load more pages 10 at a time. - Shape the signal per column — highlight items with alert keywords, or filter the feed to "show only" / "hide" by keyword so a firehose column shows just what matters. - ⌘K command palette over every deck, column, and action. Drag to reorder. @@ -42,7 +42,7 @@ git clone https://github.com/aaronjmars/minitor.git && cd minitor The launcher checks Node, picks the right package manager (npm / pnpm / yarn / bun based on lockfile), installs deps, copies `.env.example` → `.env.local` if missing, runs DB migrations against PGlite, and starts the dev server at `http://localhost:3000`. Re-running it just starts the server. -For Grok / X / News / Web / Farcaster columns, paste your **[xAI API key](https://console.x.ai/)** into `XAI_API_KEY` in `.env.local`. Keyless columns (Reddit, HN, Lobsters, Stack Overflow, DEV.to, Product Hunt, npm, PyPI, crates.io, Hugging Face, arXiv, Bluesky, Mastodon, RSS, Google News, Bing, GitHub, China Hot, YouTube channel/playlist, app reviews, wallet transactions, Polymarket, CoinGecko, DeFiLlama, Dexscreener) work out of the box with no keys. +For Grok / X / News / Web / Farcaster columns, paste your **[xAI API key](https://console.x.ai/)** into `XAI_API_KEY` in `.env.local`. Keyless columns (Reddit, HN, Lobsters, Stack Overflow, DEV.to, Product Hunt, npm, PyPI, crates.io, Hugging Face, arXiv, Mastodon, RSS, Google News, GitHub, China Hot, YouTube channel/playlist, app reviews, wallet transactions, Polymarket, CoinGecko, DeFiLlama, Dexscreener) work out of the box with no keys. **Other launcher subcommands:** @@ -59,9 +59,9 @@ For Grok / X / News / Web / Farcaster columns, paste your **[xAI API key](https: | Category | Columns | |----------|---------| -| **Social — X / Bluesky / Reddit / HN / Farcaster / Mastodon** (7) | `x-search`, `x-trending`, `bluesky`, `reddit`, `hacker-news`, `farcaster`, `mastodon` | +| **Social — X / Reddit / HN / Farcaster / Mastodon** (6) | `x-search`, `x-trending`, `reddit`, `hacker-news`, `farcaster`, `mastodon` | | **GitHub** (11) | `github-trending`, `github-releases`, `github-issues`, `github-prs`, `github-commits`, `github-stars`, `github-forks`, `github-search`, `github-backlinks`, `github-actions`, `github-discussions` | -| **News & web** (11) | `bing` (Web search), `google-news`, `news-search`, `rss`, `lobsters`, `stack-overflow`, `devto`, `npm`, `pypi`, `crates`, `producthunt` | +| **News & web** (10) | `google-news`, `news-search`, `rss`, `lobsters`, `stack-overflow`, `devto`, `npm`, `pypi`, `crates`, `producthunt` | | **AI / ML** (2) | `huggingface` (trending models, datasets, spaces), `arxiv` (CS / stat / math.OC papers) | | **Long-form & video** (3) | `substack`, `youtube`, `linkedin` | | **Mention monitors** (2) | `facebook`, `instagram` | @@ -89,7 +89,7 @@ Full plugin manifest: [`lib/columns/plugins/manifest.ts`](lib/columns/plugins/ma - **Founder dashboard** — your X mentions, Hacker News, Product Hunt RSS, GitHub stars on your repo, App Store reviews, Substack analytics, all on one screen - **Crypto desk** — wallet activity across 9 chains, X trending in crypto, Reddit r/cryptocurrency, news search for protocol names - **Open-source maintainer** — GitHub trending in your language, new issues / PRs across watched repos, new stargazers + forks, backlinks from HN / Reddit / news for your repo URL -- **Journalist** — Google News + Bing + xAI news search on a topic, paired with X trending and Reddit threads, plus Substack publications you trust +- **Journalist** — Google News + xAI news search on a topic, paired with X trending and Reddit threads, plus Substack publications you trust - **PM / marketing** — Apple + Google Play reviews on your app, Twitter mentions, LinkedIn / Facebook / Instagram mentions, Substack write-ups - **China-watcher** — Weibo, Zhihu, Douyin, Bilibili, Toutiao, and Baidu hot boards in one column stack - **Personal radar** — RSS for the blogs that matter, Farcaster for the casts that matter, no algorithmic feed in sight diff --git a/lib/columns/README.md b/lib/columns/README.md index ca4578a..779502c 100644 --- a/lib/columns/README.md +++ b/lib/columns/README.md @@ -70,7 +70,7 @@ client bundle and vice versa. if (!q) throw new Error("Search query is required."); ``` - Most input-driven columns follow this (`linkedin`, `youtube`, `bluesky`, + Most input-driven columns follow this (`linkedin`, `youtube`, `reddit`, `mastodon`, and the `repo`-based `github-*` columns). Throwing here is caught by the shared API route and surfaced to the user as a fetch-error toast. - Keep upstream HTTP clients in `lib/integrations/.ts` and import diff --git a/lib/columns/plugins/bing/client.tsx b/lib/columns/plugins/bing/client.tsx deleted file mode 100644 index 73a71aa..0000000 --- a/lib/columns/plugins/bing/client.tsx +++ /dev/null @@ -1,73 +0,0 @@ -"use client"; - -import { ExternalLink } from "lucide-react"; -import { RelativeTime } from "@/components/relative-time"; -import { Input } from "@/components/ui/input"; -import { Label } from "@/components/ui/label"; -import { - defineColumnUI, - type ConfigFormProps, - type ItemRendererProps, -} from "@/lib/columns/types"; -import { meta, type BingConfig, type BingMeta } from "./plugin"; - -function ConfigForm({ value, onChange }: ConfigFormProps) { - return ( -
- - onChange({ query: e.target.value })} - /> -

- Backed by Bing News RSS — keyless. Quotes pin exact phrases. -

-
- ); -} - -function ItemRenderer({ item }: ItemRendererProps) { - const source = item.meta?.source ?? ""; - return ( - -
- - Web - - {source && {source}} - · - - - - -
-

- {item.content.split("\n\n")[0]} -

- {item.content.includes("\n\n") && ( -

- {item.content.split("\n\n").slice(1).join("\n\n")} -

- )} - - ); -} - -export const column = defineColumnUI({ - ...meta, - ConfigForm, - ItemRenderer, -}); diff --git a/lib/columns/plugins/bing/plugin.ts b/lib/columns/plugins/bing/plugin.ts deleted file mode 100644 index 1d7c1b2..0000000 --- a/lib/columns/plugins/bing/plugin.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { z } from "zod"; -import { Globe } from "lucide-react"; -import type { PluginMeta } from "@/lib/columns/types"; - -export const schema = z.object({ - query: z.string().default(""), -}); - -export type BingConfig = z.infer; - -export interface BingMeta { - source: string; - feedTitle?: string; -} - -export const meta: PluginMeta = { - id: "bing", - label: "Web search", - description: - "Search the web via Bing News (RSS, no key required).", - icon: Globe, - accent: "#5b8def", - category: "news", - schema, - defaultConfig: schema.parse({}), - defaultTitle: (c) => - c.query.trim() ? `Web · ${c.query.trim()}` : "Web search", - capabilities: { paginated: true }, -}; diff --git a/lib/columns/plugins/bing/server.ts b/lib/columns/plugins/bing/server.ts deleted file mode 100644 index 1e110e2..0000000 --- a/lib/columns/plugins/bing/server.ts +++ /dev/null @@ -1,23 +0,0 @@ -import "server-only"; - -import { - defineColumnServer, - type FeedItem, - type ServerFetcher, -} from "@/lib/columns/types"; -import { fetchFeed } from "@/lib/integrations/rss"; -import { sliceForPage } from "@/lib/columns/paginate"; -import { meta, type BingConfig, type BingMeta } from "./plugin"; - -const fetch: ServerFetcher = async (config, cursor) => { - const q = config.query.trim(); - if (!q) throw new Error("Search query is required."); - const url = `https://www.bing.com/news/search?q=${encodeURIComponent(q)}&format=rss`; - const items = (await fetchFeed(url, 50)) as FeedItem[]; - return sliceForPage(items, cursor); -}; - -export const server = defineColumnServer({ - meta, - fetch, -}); diff --git a/lib/columns/plugins/bluesky/client.tsx b/lib/columns/plugins/bluesky/client.tsx deleted file mode 100644 index 04d17fb..0000000 --- a/lib/columns/plugins/bluesky/client.tsx +++ /dev/null @@ -1,139 +0,0 @@ -"use client"; - -import { Heart, MessageCircle, Repeat2 } from "lucide-react"; -import { RelativeTime } from "@/components/relative-time"; -import { Avatar, AvatarFallback, AvatarImage } from "@/components/ui/avatar"; -import { Input } from "@/components/ui/input"; -import { Label } from "@/components/ui/label"; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, -} from "@/components/ui/select"; -import { - defineColumnUI, - type ConfigFormProps, - type ItemRendererProps, -} from "@/lib/columns/types"; -import { formatCompactCount } from "@/lib/utils"; -import { meta, type BlueskyConfig, type BlueskyMeta } from "./plugin"; - -function ConfigForm({ value, onChange }: ConfigFormProps) { - return ( -
-
- - -
- {value.mode === "search" ? ( -
- - onChange({ ...value, query: e.target.value })} - /> -

- Latest posts on Bluesky matching the keyword. Sorted newest first. -

-
- ) : ( -
- - onChange({ ...value, handle: e.target.value })} - /> -

- Bare usernames default to .bsky.social — e.g.{" "} - jay resolves to jay.bsky.social. Use the - full handle for custom domains. -

-
- )} -
- ); -} - -function ItemRenderer({ item }: ItemRendererProps) { - const m = item.meta; - const likes = m?.likes ?? 0; - const reposts = m?.reposts ?? 0; - const replies = m?.replies ?? 0; - const handle = item.author.handle ?? item.author.name; - const display = item.author.name; - - return ( - -
- - - {display.slice(0, 1).toUpperCase()} - -
-
- - {display} - - @{handle} - · - - - -
- {item.content && ( -

- {item.content} -

- )} -
- - - {formatCompactCount(replies)} - - - - {formatCompactCount(reposts)} - - - - {formatCompactCount(likes)} - -
-
-
- - ); -} - -export const column = defineColumnUI({ - ...meta, - ConfigForm, - ItemRenderer, -}); diff --git a/lib/columns/plugins/bluesky/plugin.ts b/lib/columns/plugins/bluesky/plugin.ts deleted file mode 100644 index 6c8b80e..0000000 --- a/lib/columns/plugins/bluesky/plugin.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { z } from "zod"; -import { Cloud } from "lucide-react"; -import type { PluginMeta } from "@/lib/columns/types"; - -export const schema = z.object({ - mode: z.enum(["search", "author"]).default("search"), - query: z.string().default(""), - handle: z.string().default(""), -}); - -export type BlueskyConfig = z.infer; - -export interface BlueskyMeta { - likes: number; - reposts: number; - replies: number; - postUrl: string; -} - -export const meta: PluginMeta = { - id: "bluesky", - label: "Bluesky", - description: - "Latest posts from Bluesky — keyword search or by author handle. Keyless via the public AppView.", - icon: Cloud, - accent: "#0085ff", - category: "social", - schema, - defaultConfig: schema.parse({}), - defaultTitle: (c) => { - if (c.mode === "author") { - const h = c.handle.trim().replace(/^@/, ""); - return h ? `Bluesky · @${h}` : "Bluesky · Author"; - } - const q = c.query.trim(); - return q ? `Bluesky · ${q}` : "Bluesky · Search"; - }, - capabilities: { paginated: true }, -}; diff --git a/lib/columns/plugins/bluesky/server.ts b/lib/columns/plugins/bluesky/server.ts deleted file mode 100644 index 2e5e7a2..0000000 --- a/lib/columns/plugins/bluesky/server.ts +++ /dev/null @@ -1,38 +0,0 @@ -import "server-only"; - -import { - defineColumnServer, - type FeedItem, - type ServerFetcher, -} from "@/lib/columns/types"; -import { fetchBlueskyPage } from "@/lib/integrations/bluesky"; -import { PAGE_SIZE } from "@/lib/columns/constants"; -import { meta, type BlueskyConfig, type BlueskyMeta } from "./plugin"; - -const fetch: ServerFetcher = async ( - config, - cursor, -) => { - if (config.mode === "author") { - if (!config.handle.trim()) throw new Error("Author handle is required."); - } else if (!config.query.trim()) { - throw new Error("Search query is required."); - } - - const r = await fetchBlueskyPage( - config.mode, - config.query, - config.handle, - PAGE_SIZE, - cursor, - ); - return { - items: r.items as FeedItem[], - nextCursor: r.nextCursor, - }; -}; - -export const server = defineColumnServer({ - meta, - fetch, -}); diff --git a/lib/columns/plugins/github-discussions/plugin.ts b/lib/columns/plugins/github-discussions/plugin.ts index 43ebf98..c712118 100644 --- a/lib/columns/plugins/github-discussions/plugin.ts +++ b/lib/columns/plugins/github-discussions/plugin.ts @@ -58,6 +58,7 @@ export const meta: PluginMeta = { }, capabilities: { paginated: true, - rateLimitHint: "60 req/hr without GITHUB_TOKEN, 5000 with.", + rateLimitHint: + "Requires GITHUB_TOKEN — the GraphQL API gives unauthenticated requests ~0 quota (403). 5000 req/hr with one.", }, }; diff --git a/lib/columns/plugins/github-stars/plugin.ts b/lib/columns/plugins/github-stars/plugin.ts index d5c1d73..14cacb8 100644 --- a/lib/columns/plugins/github-stars/plugin.ts +++ b/lib/columns/plugins/github-stars/plugin.ts @@ -27,6 +27,6 @@ export const meta: PluginMeta = { capabilities: { paginated: true, rateLimitHint: - "60 req/hr without GITHUB_TOKEN, 5000 with. Token unlocks GraphQL for newest-first ordering.", + "Requires GITHUB_TOKEN — GitHub auth-gates the stargazers list (401 without a token). 5000 req/hr with one.", }, }; diff --git a/lib/columns/plugins/manifest.ts b/lib/columns/plugins/manifest.ts index 00decc5..9125bd8 100644 --- a/lib/columns/plugins/manifest.ts +++ b/lib/columns/plugins/manifest.ts @@ -13,7 +13,6 @@ import { meta as githubTrending } from "./github-trending/plugin"; import { meta as githubIssues } from "./github-issues/plugin"; import { meta as rss } from "./rss/plugin"; import { meta as googleNews } from "./google-news/plugin"; -import { meta as bing } from "./bing/plugin"; import { meta as farcaster } from "./farcaster/plugin"; import { meta as mastodon } from "./mastodon/plugin"; import { meta as youtube } from "./youtube/plugin"; @@ -36,7 +35,6 @@ import { meta as playReviews } from "./play-reviews/plugin"; import { meta as githubStars } from "./github-stars/plugin"; import { meta as githubForks } from "./github-forks/plugin"; import { meta as githubReleases } from "./github-releases/plugin"; -import { meta as bluesky } from "./bluesky/plugin"; import { meta as lobsters } from "./lobsters/plugin"; import { meta as polymarket } from "./polymarket/plugin"; import { meta as stackOverflow } from "./stack-overflow/plugin"; @@ -64,7 +62,6 @@ export const PLUGIN_METAS = [ githubIssues, rss, googleNews, - bing, farcaster, mastodon, youtube, @@ -87,7 +84,6 @@ export const PLUGIN_METAS = [ githubStars, githubForks, githubReleases, - bluesky, lobsters, polymarket, stackOverflow, diff --git a/lib/columns/registry.ts b/lib/columns/registry.ts index e7af0bf..14165f9 100644 --- a/lib/columns/registry.ts +++ b/lib/columns/registry.ts @@ -21,7 +21,6 @@ import { column as githubTrending } from "@/lib/columns/plugins/github-trending/ import { column as githubIssues } from "@/lib/columns/plugins/github-issues/client"; import { column as rss } from "@/lib/columns/plugins/rss/client"; import { column as googleNews } from "@/lib/columns/plugins/google-news/client"; -import { column as bing } from "@/lib/columns/plugins/bing/client"; import { column as farcaster } from "@/lib/columns/plugins/farcaster/client"; import { column as mastodon } from "@/lib/columns/plugins/mastodon/client"; import { column as youtube } from "@/lib/columns/plugins/youtube/client"; @@ -44,7 +43,6 @@ import { column as playReviews } from "@/lib/columns/plugins/play-reviews/client import { column as githubStars } from "@/lib/columns/plugins/github-stars/client"; import { column as githubForks } from "@/lib/columns/plugins/github-forks/client"; import { column as githubReleases } from "@/lib/columns/plugins/github-releases/client"; -import { column as bluesky } from "@/lib/columns/plugins/bluesky/client"; import { column as lobsters } from "@/lib/columns/plugins/lobsters/client"; import { column as polymarket } from "@/lib/columns/plugins/polymarket/client"; import { column as stackOverflow } from "@/lib/columns/plugins/stack-overflow/client"; @@ -75,7 +73,6 @@ const COLUMNS_BY_ID: Record = { "github-issues": githubIssues, rss, "google-news": googleNews, - bing, farcaster, mastodon, youtube, @@ -98,7 +95,6 @@ const COLUMNS_BY_ID: Record = { "github-stars": githubStars, "github-forks": githubForks, "github-releases": githubReleases, - bluesky, lobsters, polymarket, "stack-overflow": stackOverflow, diff --git a/lib/columns/server-registry.ts b/lib/columns/server-registry.ts index 99b1141..1086983 100644 --- a/lib/columns/server-registry.ts +++ b/lib/columns/server-registry.ts @@ -17,7 +17,6 @@ import { server as githubTrending } from "@/lib/columns/plugins/github-trending/ import { server as githubIssues } from "@/lib/columns/plugins/github-issues/server"; import { server as rss } from "@/lib/columns/plugins/rss/server"; import { server as googleNews } from "@/lib/columns/plugins/google-news/server"; -import { server as bing } from "@/lib/columns/plugins/bing/server"; import { server as farcaster } from "@/lib/columns/plugins/farcaster/server"; import { server as mastodon } from "@/lib/columns/plugins/mastodon/server"; import { server as youtube } from "@/lib/columns/plugins/youtube/server"; @@ -40,7 +39,6 @@ import { server as playReviews } from "@/lib/columns/plugins/play-reviews/server import { server as githubStars } from "@/lib/columns/plugins/github-stars/server"; import { server as githubForks } from "@/lib/columns/plugins/github-forks/server"; import { server as githubReleases } from "@/lib/columns/plugins/github-releases/server"; -import { server as bluesky } from "@/lib/columns/plugins/bluesky/server"; import { server as lobsters } from "@/lib/columns/plugins/lobsters/server"; import { server as polymarket } from "@/lib/columns/plugins/polymarket/server"; import { server as stackOverflow } from "@/lib/columns/plugins/stack-overflow/server"; @@ -68,7 +66,6 @@ const SERVERS_BY_ID: Record = { "github-issues": githubIssues, rss, "google-news": googleNews, - bing, farcaster, mastodon, youtube, @@ -91,7 +88,6 @@ const SERVERS_BY_ID: Record = { "github-stars": githubStars, "github-forks": githubForks, "github-releases": githubReleases, - bluesky, lobsters, polymarket, "stack-overflow": stackOverflow, diff --git a/lib/env-keys.ts b/lib/env-keys.ts index 7545208..88e042b 100644 --- a/lib/env-keys.ts +++ b/lib/env-keys.ts @@ -40,7 +40,7 @@ export const ENV_KEYS: EnvKeySpec[] = [ key: "GITHUB_TOKEN", label: "GitHub", description: - "Optional — raises every GitHub plugin's rate limit from 60 → 5000 req/hr. GitHub code search additionally requires a token.", + "Optional — raises every GitHub plugin's rate limit from 60 → 5000 req/hr. GitHub code search, stars, and discussions additionally require a token (GitHub auth-gates those endpoints).", signupUrl: "https://github.com/settings/tokens", required: false, }, diff --git a/lib/integrations/bluesky.ts b/lib/integrations/bluesky.ts deleted file mode 100644 index d1a8a28..0000000 --- a/lib/integrations/bluesky.ts +++ /dev/null @@ -1,182 +0,0 @@ -import { fetchUpstream } from "@/lib/integrations/fetch"; -import type { FeedItem } from "@/lib/columns/types"; - -// Bluesky public AppView — keyless, no auth, generous quota for read endpoints. -// https://docs.bsky.app/docs/api -// Both endpoints used here are documented as public. -const BLUESKY = "https://public.api.bsky.app/xrpc"; - -export type BlueskyMode = "search" | "author"; - -interface BlueskyAuthor { - did?: string; - handle?: string; - displayName?: string; - avatar?: string; -} - -interface BlueskyRecord { - text?: string; - createdAt?: string; -} - -interface BlueskyPost { - uri?: string; - cid?: string; - author?: BlueskyAuthor; - record?: BlueskyRecord; - indexedAt?: string; - likeCount?: number; - replyCount?: number; - repostCount?: number; - quoteCount?: number; -} - -interface SearchResponse { - posts?: BlueskyPost[]; - cursor?: string; -} - -interface AuthorFeedItem { - post?: BlueskyPost; - reason?: { $type?: string }; -} - -interface AuthorFeedResponse { - feed?: AuthorFeedItem[]; - cursor?: string; -} - -// at://did:plc:abc/app.bsky.feed.post/3kxyz → 3kxyz (the rkey). -// Falls back to the full uri if the format doesn't match — better a -// non-clickable id than throwing on a future record-type change. -function rkeyFromUri(uri: string): string { - const m = /\/([^/]+)$/.exec(uri); - return m ? m[1] : uri; -} - -function postUrl(post: BlueskyPost): string { - const handle = post.author?.handle; - const uri = post.uri ?? ""; - if (!handle || !uri) return "https://bsky.app"; - return `https://bsky.app/profile/${handle}/post/${rkeyFromUri(uri)}`; -} - -function mapPost(post: BlueskyPost): FeedItem | null { - const handle = post.author?.handle; - const uri = post.uri; - // Skip posts that lack either an author handle or a uri — the renderer - // can't link to them and there's no useful id, so they'd just render as - // dead content. Bluesky's API normally returns both, this guards against - // schema drift from future record types. - if (!handle || !uri) return null; - - const text = post.record?.text ?? ""; - const createdAt = - post.record?.createdAt ?? post.indexedAt ?? new Date().toISOString(); - - return { - id: post.cid ?? uri, - author: { - name: post.author?.displayName?.trim() || handle, - handle, - avatarUrl: post.author?.avatar, - }, - content: text, - url: postUrl(post), - createdAt, - meta: { - likes: post.likeCount ?? 0, - reposts: (post.repostCount ?? 0) + (post.quoteCount ?? 0), - replies: post.replyCount ?? 0, - postUrl: postUrl(post), - }, - }; -} - -async function fetchSearch( - query: string, - limit: number, - cursor?: string, -): Promise<{ items: FeedItem[]; nextCursor?: string }> { - const params = new URLSearchParams({ - q: query, - limit: String(limit), - sort: "latest", - }); - if (cursor) params.set("cursor", cursor); - const res = await fetchUpstream(`${BLUESKY}/app.bsky.feed.searchPosts?${params}`, { - headers: { accept: "application/json" }, - cache: "no-store", - }); - if (!res.ok) { - throw new Error( - `Bluesky search ${res.status}: ${(await res.text()).slice(0, 200)}`, - ); - } - const json = (await res.json()) as SearchResponse; - const items = (json.posts ?? []) - .map(mapPost) - .filter((x): x is FeedItem => x !== null); - return { items, nextCursor: json.cursor }; -} - -async function fetchAuthor( - actor: string, - limit: number, - cursor?: string, -): Promise<{ items: FeedItem[]; nextCursor?: string }> { - const params = new URLSearchParams({ - actor, - limit: String(limit), - filter: "posts_no_replies", - }); - if (cursor) params.set("cursor", cursor); - const res = await fetchUpstream(`${BLUESKY}/app.bsky.feed.getAuthorFeed?${params}`, { - headers: { accept: "application/json" }, - cache: "no-store", - }); - if (!res.ok) { - throw new Error( - `Bluesky author ${res.status}: ${(await res.text()).slice(0, 200)}`, - ); - } - const json = (await res.json()) as AuthorFeedResponse; - // Skip reposts — Bluesky author feeds include them but the resulting card - // would attribute the original poster, which is confusing in a column titled - // "by @actor". `posts_no_replies` removes replies but not reposts. - const items = (json.feed ?? []) - .filter((entry) => !entry.reason) - .map((entry) => entry.post) - .filter((p): p is BlueskyPost => Boolean(p)) - .map(mapPost) - .filter((x): x is FeedItem => x !== null); - return { items, nextCursor: json.cursor }; -} - -function normalizeHandle(input: string): string { - // Accept "@example.bsky.social", "example.bsky.social", or a bare username - // ("example"). The bare username gets ".bsky.social" appended — the canonical - // suffix for users who haven't set a custom handle. - const trimmed = input.trim().replace(/^@/, ""); - if (!trimmed) return trimmed; - if (trimmed.includes(".")) return trimmed; - return `${trimmed}.bsky.social`; -} - -export async function fetchBlueskyPage( - mode: BlueskyMode, - query: string, - handle: string, - limit: number, - cursor?: string, -): Promise<{ items: FeedItem[]; nextCursor?: string }> { - if (mode === "author") { - const actor = normalizeHandle(handle); - if (!actor) return { items: [] }; - return fetchAuthor(actor, limit, cursor); - } - const q = query.trim(); - if (!q) return { items: [] }; - return fetchSearch(q, limit, cursor); -} diff --git a/lib/integrations/github-discussions.ts b/lib/integrations/github-discussions.ts index 8213f18..38d16ca 100644 --- a/lib/integrations/github-discussions.ts +++ b/lib/integrations/github-discussions.ts @@ -8,11 +8,11 @@ import { identiconUrl } from "@/lib/utils"; // `github.ts` rather than inside it. The REST module is intentionally left // untouched so its API surface (legacy callers) stays stable. // -// Auth: optional `GITHUB_TOKEN`. With a token we get the standard 5000 req/hr -// authenticated GraphQL quota; without one we degrade to the 60 req/hr -// unauthenticated public quota (same as the rest of the github-* plugins). -// The column never fails because of a missing token — that's a deliberate -// product decision: keyless dashboards are a first-class use case. +// Auth: `GITHUB_TOKEN` is REQUIRED. Unlike GitHub's REST API (which serves the +// other github-* plugins keyless at 60 req/hr), the GraphQL API gives +// unauthenticated requests ~0 quota — it returns HTTP 403 "API rate limit +// exceeded" immediately. Discussions have no REST surface, so there is no +// keyless path; we fail fast with a clear message instead of a raw 403. const GRAPHQL_ENDPOINT = "https://api.github.com/graphql"; @@ -133,6 +133,11 @@ export async function fetchDiscussions( mode: GHDiscussionMode, first: number, ): Promise[]> { + if (!process.env.GITHUB_TOKEN) { + throw new Error( + "GitHub Discussions requires a token. Set GITHUB_TOKEN in your env (read-only public scope is enough).", + ); + } const { owner, name } = parseRepo(repo); const fullRepo = `${owner}/${name}`; // We always pull a generous batch (caller controls `first`, usually 50) so diff --git a/lib/integrations/github.ts b/lib/integrations/github.ts index a76f01e..eec7087 100644 --- a/lib/integrations/github.ts +++ b/lib/integrations/github.ts @@ -686,78 +686,6 @@ export function normalizeGitHubRepo(input: string): string { return clean; } -function parseLastPage(linkHeader: string | null): number | undefined { - if (!linkHeader) return undefined; - // Link: <...&page=42>; rel="last", <...&page=2>; rel="next" - for (const part of linkHeader.split(",")) { - const m = /<([^>]+)>;\s*rel="last"/.exec(part.trim()); - if (m) { - try { - const u = new URL(m[1]); - const p = u.searchParams.get("page"); - if (p) return Number(p); - } catch { - // The Link-header URL is matched by regex; a malformed match shouldn't - // be fatal — fall through and let pagination return undefined. - } - } - } - return undefined; -} - -interface GHStargazerEdgeREST { - starred_at: string; - user: { login: string; avatar_url?: string; html_url?: string }; -} - -async function ghFetchStargazersPageREST( - fullRepo: string, - page: number, - perPage: number, -): Promise<{ items: GHWatcherItem[]; lastPage?: number }> { - const params = new URLSearchParams({ - per_page: String(perPage), - page: String(page), - }); - const url = `${API}/repos/${fullRepo}/stargazers?${params}`; - const res = await fetchUpstream(url, { - headers: { - ...headers(), - // star+json is required to receive `starred_at` timestamps. - accept: "application/vnd.github.star+json", - }, - cache: "no-store", - }); - if (!res.ok) { - const body = await res.text().catch(() => ""); - throw new Error(`GitHub ${res.status}: ${body.slice(0, 200)}`); - } - const lastPage = parseLastPage(res.headers.get("link")); - const json = (await res.json()) as GHStargazerEdgeREST[]; - const items = json.map((edge) => { - const u = edge.user; - return { - id: `gh-star-${fullRepo}-${u.login}`, - author: { - name: u.login, - handle: u.login, - avatarUrl: - u.avatar_url ?? - identiconUrl(u.login), - }, - content: `${u.login} starred ${fullRepo}`, - url: u.html_url ?? `https://github.com/${u.login}`, - createdAt: edge.starred_at, - meta: { - kind: "star", - repo: fullRepo, - starredAt: edge.starred_at, - }, - } satisfies GHWatcherItem; - }); - return { items, lastPage }; -} - interface GHGraphQLStargazersResponse { data?: { repository: { @@ -841,49 +769,27 @@ async function fetchStargazersGraphQL( }; } -async function fetchStargazersREST( - fullRepo: string, - limit: number, - cursor?: string, -): Promise { - // REST sorts oldest-first. To surface newest-first we must walk the Link - // header to find the last page, then page backwards from there. - let pageToFetch: number; - if (cursor) { - pageToFetch = Number(cursor); - if (!Number.isFinite(pageToFetch) || pageToFetch < 1) { - return { items: [] }; - } - } else { - const probe = await ghFetchStargazersPageREST(fullRepo, 1, limit); - pageToFetch = probe.lastPage ?? 1; - } - const { items } = await ghFetchStargazersPageREST( - fullRepo, - pageToFetch, - limit, - ); - items.reverse(); - return { - items, - nextCursor: pageToFetch > 1 ? String(pageToFetch - 1) : undefined, - }; -} - export async function fetchStargazers( repo: string, limit = 12, cursor?: string, ): Promise { - const fullRepo = normalizeGitHubRepo(repo); - if (process.env.GITHUB_TOKEN) { - return fetchStargazersGraphQL( - fullRepo, - limit, - cursor?.startsWith("gql:") ? cursor.slice(4) : undefined, + // GitHub now returns 401 on the REST `/stargazers` endpoint for + // unauthenticated requests (the plain repo endpoint still works keyless, but + // the stargazer list is auth-gated), so this column requires a token — there + // is no keyless way to list recent stargazers. With a token we use GraphQL, + // which orders by STARRED_AT and gives real newest-first pagination. + if (!process.env.GITHUB_TOKEN) { + throw new Error( + "GitHub stargazers requires a token. Set GITHUB_TOKEN in your env (read-only public scope is enough).", ); } - return fetchStargazersREST(fullRepo, limit, cursor); + const fullRepo = normalizeGitHubRepo(repo); + return fetchStargazersGraphQL( + fullRepo, + limit, + cursor?.startsWith("gql:") ? cursor.slice(4) : undefined, + ); } interface GHForkREST { diff --git a/lib/integrations/reddit.ts b/lib/integrations/reddit.ts index f748807..af1a3d8 100644 --- a/lib/integrations/reddit.ts +++ b/lib/integrations/reddit.ts @@ -1,47 +1,87 @@ import { fetchUpstream } from "@/lib/integrations/fetch"; import type { FeedItem } from "@/lib/columns/types"; import { identiconUrl } from "@/lib/utils"; +import { decodeEntities, stripHtml } from "@/lib/integrations/text"; +// Reddit's JSON API (`/r//.json`, `/search.json`) now returns HTTP +// 403 with an HTML anti-bot page to unauthenticated clients from most IPs — +// datacenter and residential alike. Reddit's Atom feeds (`/.rss`) are still +// served keyless to a descriptive User-Agent, so we read those instead. +// +// The trade-off vs. the old JSON path: the feed carries no score / comment +// count and no `after` cursor. Score and comments degrade to 0 (the UI renders +// them with a `?? 0` fallback), and each column shows a single feed page +// (~25 items) instead of paginating. Everything else — title, author, +// permalink, outbound link, timestamp — comes through. const UA = "minitor/0.1 (https://github.com/anthropics/claude-code dashboard)"; +const ACCEPT = "application/atom+xml, application/xml;q=0.9, */*;q=0.8"; -interface RedditChild { - data: { - id: string; - name?: string; - author?: string; - title?: string; - selftext?: string; - permalink?: string; - url?: string; - created_utc?: number; - score?: number; - num_comments?: number; - subreddit?: string; - thumbnail?: string; - is_self?: boolean; - over_18?: boolean; - stickied?: boolean; - }; -} +const SORTS = new Set(["hot", "new", "top", "rising"]); -interface RedditListing { - data?: { children?: RedditChild[] }; - message?: string; - error?: number; +// --- tiny Atom helpers (regex, matching the lib/integrations/rss.ts style) --- + +function getTag(xml: string, tag: string): string { + const re = new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)`, "i"); + return xml.match(re)?.[1] ?? ""; } -const SORTS = new Set(["hot", "new", "top", "rising"]); +// The first `` in an entry with no `rel` (or rel="alternate") is the +// submission's permalink (the Reddit comments page). +function permalinkOf(entry: string): string { + const re = /]*?)\/?>/gi; + let m: RegExpExecArray | null; + while ((m = re.exec(entry))) { + const href = m[1].match(/\bhref=["']([^"']+)["']/)?.[1]; + const rel = m[1].match(/\brel=["']([^"']+)["']/)?.[1]; + if (href && (!rel || rel === "alternate")) return href; + } + return ""; +} -interface RedditListingDataExt { - children?: RedditChild[]; - after?: string | null; - before?: string | null; +// Each entry body embeds `[link]` (the submission's outbound +// URL) alongside `[comments]`. Pull the `[link]` +// href so we can tell a link post from a self post. +function outboundLink(contentHtml: string): string | undefined { + const html = decodeEntities(contentHtml); + return html.match(/href=["']([^"']+)["'][^>]*>\s*\[link\]/i)?.[1]; } -interface RedditListingExt { - data?: RedditListingDataExt; - message?: string; - error?: number; +function parseRedditFeed(xml: string, fallbackSub: string): FeedItem[] { + const items: FeedItem[] = []; + const entryRe = /]*>([\s\S]*?)<\/entry>/gi; + let m: RegExpExecArray | null; + while ((m = entryRe.exec(xml))) { + const entry = m[1]; + const id = getTag(entry, "id").trim().replace(/^t3_/, "") || permalinkOf(entry); + const title = decodeEntities(stripHtml(getTag(entry, "title"))).trim(); + const permalink = permalinkOf(entry); + const author = + stripHtml(getTag(getTag(entry, "author"), "name")).replace(/^\/u\//, "").trim() || + "unknown"; + const subreddit = + entry.match(/]*\bterm=["']([^"']+)["']/i)?.[1] || fallbackSub; + const published = getTag(entry, "published") || getTag(entry, "updated"); + const outbound = outboundLink(getTag(entry, "content")); + // A self post's `[link]` points back at its own comments page. + const isSelf = !outbound || /reddit\.com\/r\/[^/]+\/comments\//i.test(outbound); + + items.push({ + id, + author: { name: author, handle: author, avatarUrl: identiconUrl(author) }, + content: title, + url: permalink || outbound, + createdAt: new Date(published || Date.now()).toISOString(), + meta: { + score: 0, + comments: 0, + subreddit, + isSelf, + externalUrl: isSelf ? undefined : outbound, + nsfw: false, + }, + }); + } + return items; } export async function fetchSubredditPage( @@ -50,62 +90,24 @@ export async function fetchSubredditPage( limit = 12, after?: string, ): Promise<{ items: FeedItem[]; nextAfter?: string }> { + // The Atom feed is a single fixed page with no cursor, so a "load more" + // (non-empty `after`) has nothing further to return. + if (after) return { items: [], nextAfter: undefined }; + const sub = subreddit.trim().replace(/^r\//, "") || "popular"; const sort = SORTS.has(sortBy) ? sortBy : "hot"; - const params = new URLSearchParams({ - limit: String(limit), - raw_json: "1", - }); - if (after) params.set("after", after); - const url = `https://www.reddit.com/r/${encodeURIComponent(sub)}/${sort}.json?${params}`; + const url = `https://www.reddit.com/r/${encodeURIComponent(sub)}/${sort}/.rss?limit=${limit}`; const res = await fetchUpstream(url, { - headers: { "user-agent": UA, accept: "application/json" }, + headers: { "user-agent": UA, accept: ACCEPT }, cache: "no-store", }); - if (!res.ok) { throw new Error(`Reddit ${res.status}: ${(await res.text()).slice(0, 200)}`); } - const json = (await res.json()) as RedditListingExt; - if (json.error || !json.data?.children) { - throw new Error(`Reddit error: ${json.message ?? "no listing data"}`); - } - - const items = json.data.children - .filter((c) => !c.data.stickied) - .slice(0, limit) - .map((c) => toFeedItem(c, sub)); - const nextAfter = json.data.after ?? undefined; - return { items, nextAfter: nextAfter || undefined }; -} - -function toFeedItem(c: RedditChild, fallbackSub: string): FeedItem { - const d = c.data; - const author = d.author ?? "unknown"; - const permalink = d.permalink - ? `https://reddit.com${d.permalink}` - : d.url; - return { - id: d.id, - author: { - name: author, - handle: author, - avatarUrl: identiconUrl(author), - }, - content: d.title ?? "", - url: permalink, - createdAt: new Date(((d.created_utc ?? 0) * 1000) || Date.now()).toISOString(), - meta: { - score: d.score ?? 0, - comments: d.num_comments ?? 0, - subreddit: d.subreddit ?? fallbackSub, - isSelf: !!d.is_self, - externalUrl: !d.is_self ? d.url : undefined, - nsfw: !!d.over_18, - }, - }; + const items = parseRedditFeed(await res.text(), sub).slice(0, limit); + return { items, nextAfter: undefined }; } export async function searchReddit( @@ -114,17 +116,13 @@ export async function searchReddit( ): Promise { const q = query.trim(); if (!q) return []; - const url = `https://www.reddit.com/search.json?q=${encodeURIComponent(q)}&sort=new&limit=${limit}&raw_json=1`; + const url = `https://www.reddit.com/search.rss?q=${encodeURIComponent(q)}&sort=new&limit=${limit}`; const res = await fetchUpstream(url, { - headers: { "user-agent": UA, accept: "application/json" }, + headers: { "user-agent": UA, accept: ACCEPT }, cache: "no-store", }); if (!res.ok) { throw new Error(`Reddit ${res.status}: ${(await res.text()).slice(0, 200)}`); } - const json = (await res.json()) as RedditListing; - if (!json.data?.children) return []; - return json.data.children - .slice(0, limit) - .map((c) => toFeedItem(c, c.data.subreddit ?? "all")); + return parseRedditFeed(await res.text(), "all").slice(0, limit); }