import type { Page } from 'playwright'; import { ResolveError, oneSegment, type Comment, type Media, type Post } from '../types.ts'; import { readOpenGraph } from '../browser/capture.ts'; import { stripTracking } from './url.ts'; import type { PlatformSpec, ResolveContext } from './types.ts'; const HOST = 'https://www.reddit.com'; /** Path roots that are really Reddit paths, so anything else that arrives as * a single opaque segment is a `redd.it` share code. */ const KNOWN_ROOTS = new Set(['r', 'u', 'user', 'comments', 'gallery', 'media', 'wiki']); type Listing = { kind?: string; data?: { children?: Array> } }; type Thing = { kind?: string; data?: T }; type RedditVideo = { fallback_url?: string; hls_url?: string; has_audio?: boolean; width?: number; height?: number; duration?: number; }; type MediaMeta = { status?: string; e?: string; m?: string; s?: { u?: string; gif?: string; mp4?: string; x?: number; y?: number }; }; type Link = { title?: string; author?: string; subreddit_name_prefixed?: string; created_utc?: number; selftext?: string; permalink?: string; url?: string; url_overridden_by_dest?: string; domain?: string; post_hint?: string; is_self?: boolean; is_video?: boolean; is_gallery?: boolean; over_18?: boolean; spoiler?: boolean; link_flair_text?: string | null; num_comments?: number; score?: number; gallery_data?: { items?: Array<{ media_id?: string }> }; media_metadata?: Record; secure_media?: { reddit_video?: RedditVideo } | null; media?: { reddit_video?: RedditVideo } | null; preview?: { images?: Array<{ source?: { url?: string; width?: number; height?: number } }>; reddit_video_preview?: RedditVideo; }; crosspost_parent_list?: Link[]; }; type CommentData = { author?: string; body?: string; media_metadata?: Record; created_utc?: number; score?: number; score_hidden?: boolean; is_submitter?: boolean; distinguished?: string | null; replies?: Listing | ''; }; type MoreData = { count?: number }; /** `[deleted]` and `[removed]` are the platform's own words, kept as they are. */ function authorName(author: string | undefined): string { if (!author || author.startsWith('[')) return author ?? '[unknown]'; return `u/${author}`; } function isoFrom(seconds: number | undefined): string | undefined { return seconds ? new Date(seconds * 1000).toISOString() : undefined; } function sized(width: number | undefined, height: number | undefined) { return { ...(width ? { width } : {}), ...(height ? { height } : {}), }; } /** * Reddit's own video, which arrives in three forms at once. * * `fallback_url` is a plain MP4 and the nicer thing to serve — it proxies, * so the viewer never talks to Reddit, and the native scrubber can seek it. * But when a post has sound that MP4 is the video track alone: the audio is * a separate file the DASH manifest stitches back on. So a silent post gets * the MP4 and one with audio gets the HLS playlist, linked directly because * proxying it would mean rewriting the manifest and every segment. */ function fromRedditVideo(video: RedditVideo, poster: string | undefined): Media[] { const common = { ...sized(video.width, video.height), ...(video.duration ? { durationSec: video.duration } : {}), }; if (video.has_audio !== false && video.hls_url) { return [{ kind: 'video', url: video.hls_url, hls: true, direct: true, ...common }]; } const url = video.fallback_url ?? video.hls_url; if (!url) return []; if (url === video.hls_url) { return [{ kind: 'video', url, hls: true, direct: true, ...common }]; } return [{ kind: 'video', url, ...(poster ? { poster: { url: poster } } : {}), ...common, }]; } /** One entry of a gallery post. Reddit keeps the pictures somewhere other * than the order it wants them shown in, so the two have to be joined. */ function fromGalleryItem(meta: MediaMeta | undefined): Media[] { if (!meta || meta.status !== 'valid') return []; const size = sized(meta.s?.x, meta.s?.y); // An animated entry ships as a video as well; that is smaller and seekable. if (meta.s?.mp4) return [{ kind: 'video', url: meta.s.mp4, ...size }]; const url = meta.s?.gif ?? meta.s?.u; return url ? [{ kind: 'image', url, ...size }] : []; } export function mediaFromLink(link: Link): Media[] { // A crosspost is a wrapper with nothing of its own in it. const crossposted = link.crosspost_parent_list?.[0]; if (crossposted && !link.is_self) { const inner = mediaFromLink(crossposted); if (inner.length) return inner; } // `scrubber_media_url` looks like a poster and is not one: it is a second, // tiny MP4 for the timeline thumbnails. The still is in the preview. const still = link.preview?.images?.[0]?.source?.url; const video = link.secure_media?.reddit_video ?? link.media?.reddit_video; if (video) return fromRedditVideo(video, still); if (link.is_gallery) { const items = link.gallery_data?.items ?? []; const gallery = items.flatMap((item) => fromGalleryItem(item.media_id ? link.media_metadata?.[item.media_id] : undefined)); if (gallery.length) return gallery; } // A GIF link is served as a video by the preview, which is the version // worth having: the original is often tens of megabytes. const asVideo = link.preview?.reddit_video_preview; if (asVideo) return fromRedditVideo(asVideo, still); const direct = link.url_overridden_by_dest ?? link.url; if (direct && /\.(jpe?g|png|webp|gif)(\?|$)/i.test(direct)) { const source = link.preview?.images?.[0]?.source; return [{ kind: 'image', url: direct, ...sized(source?.width, source?.height) }]; } // A link to somewhere else: the preview is all Reddit has, and it is still // worth showing above the link itself. const preview = link.preview?.images?.[0]?.source; if (preview?.url) { return [{ kind: 'image', url: preview.url, ...sized(preview.width, preview.height) }]; } return []; } /** * The words of the post: the self text, or — for a link post — the * destination, which is the whole content of the post and needs to be * tappable. */ function bodyOf(link: Link): string | undefined { if (link.selftext?.trim()) return link.selftext; const dest = link.url_overridden_by_dest; if (dest && !dest.includes('redd.it') && !dest.startsWith(`${HOST}/gallery/`)) return dest; return undefined; } /** The whole of `![...](...)`, with the target captured. */ const INLINE_IMAGE = /!\[([^\]\n]*)\]\(([^)\s]+)\)/g; /** * A Giphy token, which unlike the others says what it points at. * * Reddit writes a variant name after the id on some of them * (`giphy|abc123|downsized`); only the id is kept, because Giphy does not * serve every variant for every gif but always serves the full one. */ const GIPHY_TOKEN = /^giphy\|([A-Za-z0-9]+)(?:\|[a-z_]+)?$/; /** * Point a comment's inline images at something fetchable. * * Reddit writes them as `![gif](giphy|Ve7wX45)`, `![img](emote|t5_2th52|4358)` * or `![img](jo8gf0ca92zd1)` — a token rather than an address. Usually the * token is a key in that same comment's `media_metadata`, which is where the * real URL is, so one lookup covers all three and none of them needs naming * here. * * Usually, not always: Reddit ships plenty of comments carrying a Giphy token * and no `media_metadata` at all, and with nothing to look the token up in * those showed the token itself where the gif should have been. Giphy is the * one kind that can be resolved without the lookup, the id in it being Giphy's * own, so it falls back to Giphy's address for that id. The other two cannot: * their ids mean nothing off Reddit. * * A target that is already an address is not a key, so it falls through * untouched. */ export function resolveInlineImages( body: string, meta: Record | undefined, ): string { return body.replace(INLINE_IMAGE, (whole, alt: string, token: string) => { const entry = meta?.[token]; if (entry?.status === 'valid') { // An animated one has both; the GIF plays in an `` on its own, // which an MP4 does not. const url = entry.s?.gif ?? entry.s?.u; if (url) return `![${alt}](${url})`; } const giphy = GIPHY_TOKEN.exec(token)?.[1]; return giphy ? `![${alt}](https://i.giphy.com/media/${giphy}/giphy.gif)` : whole; }); } export function commentsFrom(listing: Listing | undefined): { comments: Comment[]; more: number; } { const comments: Comment[] = []; let more = 0; for (const child of listing?.data?.children ?? []) { if (child.kind === 'more') { more += (child.data as MoreData | undefined)?.count ?? 0; continue; } const data = child.data; if (!data || data.body === undefined) continue; const replies = data.replies && typeof data.replies === 'object' ? commentsFrom(data.replies) : { comments: [], more: 0 }; comments.push({ author: authorName(data.author), ...(data.body ? { text: resolveInlineImages(data.body, data.media_metadata) } : {}), ...(isoFrom(data.created_utc) ? { postedAt: isoFrom(data.created_utc) } : {}), // Reddit hides the score on a new comment so an early downvote cannot // steer the rest. Showing a placeholder 1 would be a lie. ...(data.score_hidden !== true && typeof data.score === 'number' ? { score: data.score } : {}), ...(data.is_submitter ? { isAuthor: true } : {}), ...(data.distinguished ? { distinguished: data.distinguished } : {}), replies: replies.comments, ...(replies.more ? { moreReplies: replies.more } : {}), }); } return { comments, more }; } export function toPost(body: [Listing, Listing], originalUrl: string): Post { const link = body[0]?.data?.children?.[0]?.data; if (!link) throw new ResolveError('Reddit returned no post', 'reddit', originalUrl); const { comments, more } = commentsFrom(body[1]); const flags = [ link.link_flair_text?.trim(), link.over_18 ? 'NSFW' : undefined, link.spoiler ? 'Spoiler' : undefined, ].filter(Boolean); return { platform: 'reddit', platformLabel: 'Reddit', originalUrl, author: { // The subreddit is the identity a Reddit link carries — the poster is // a detail underneath it, so it goes in the display name. handle: link.subreddit_name_prefixed ?? 'reddit', displayName: [authorName(link.author), ...flags].join(' · '), }, textPosition: 'above', segments: oneSegment({ ...(link.title ? { title: link.title } : {}), ...(bodyOf(link) ? { text: bodyOf(link) } : {}), media: mediaFromLink(link), ...(isoFrom(link.created_utc) ? { postedAt: isoFrom(link.created_utc) } : {}), }), comments, ...(more ? { moreComments: more } : {}), ...(typeof link.num_comments === 'number' ? { commentCount: link.num_comments } : {}), }; } export type FlatComment = { depth: number; author: string; score: number; created: string; text: string; }; /** * Rebuild the nesting from a flat list of comments that each know only their * own depth, which is how the rendered page presents them. */ export function treeFromDepths(flat: FlatComment[]): Comment[] { const roots: Comment[] = []; // The comment last seen at each depth, so a reply can be attached to // whatever it appeared underneath. const openAt: Comment[] = []; for (const row of flat) { if (!row.text) continue; const comment: Comment = { author: authorName(row.author), text: row.text, ...(row.created && !Number.isNaN(Date.parse(row.created)) ? { postedAt: new Date(row.created).toISOString() } : {}), ...(Number.isFinite(row.score) ? { score: row.score } : {}), replies: [], }; const parent = openAt[row.depth - 1]; if (row.depth > 0 && parent) parent.replies.push(comment); else roots.push(comment); // Anything deeper belonged to the comment this one just replaced. openAt.length = row.depth + 1; openAt[row.depth] = comment; } return roots; } /** * Reads the comment tree back out of the rendered page. * * The floor, for when the JSON is refused. Reddit renders every comment as a * `` carrying its own depth, flat in document order. */ async function commentsFromDom(page: Page): Promise { const flat = await page .evaluate(() => [...document.querySelectorAll('shreddit-comment')].map((el) => ({ depth: Number(el.getAttribute('depth') ?? 0), author: el.getAttribute('author') ?? '', score: Number(el.getAttribute('score') ?? Number.NaN), created: el.getAttribute('created') ?? '', text: el.querySelector('[slot="comment"]')?.textContent?.trim() ?? '', }))) .catch((): FlatComment[] => []); return treeFromDepths(flat); } /** Whatever the page will admit to, when the JSON will not come. */ async function fromPage(page: Page, originalUrl: string): Promise { const attrs = await page .evaluate(() => { const post = document.querySelector('shreddit-post'); if (!post) return null; const get = (name: string): string | undefined => post.getAttribute(name) ?? undefined; return { title: get('post-title'), author: get('author'), subreddit: get('subreddit-prefixed-name'), created: get('created-timestamp'), score: get('score'), comments: get('comment-count'), image: get('content-href'), }; }) .catch(() => null); const og = await readOpenGraph(page); const title = attrs?.title ?? og['og:title']; if (!title) throw new ResolveError('Reddit served neither the post nor its data', 'reddit', originalUrl); const image = attrs?.image ?? og['og:image']; const media: Media[] = image && /\.(jpe?g|png|webp|gif)(\?|$)/i.test(image) ? [{ kind: 'image', url: image }] : []; const count = Number(attrs?.comments); return { platform: 'reddit', platformLabel: 'Reddit', originalUrl, author: { handle: attrs?.subreddit ?? 'reddit', displayName: authorName(attrs?.author), }, textPosition: 'above', segments: oneSegment({ title, media, ...(attrs?.created && !Number.isNaN(Date.parse(attrs.created)) ? { postedAt: new Date(attrs.created).toISOString() } : {}), }), comments: await commentsFromDom(page), ...(Number.isFinite(count) ? { commentCount: count } : {}), }; } /** The `.json` twin of a post URL, asked for at the size of one page. */ function jsonUrlFor(postUrl: string): string { const url = new URL(postUrl); url.pathname = `${url.pathname.replace(/\/+$/, '')}/.json`; url.search = ''; url.searchParams.set('raw_json', '1'); url.searchParams.set('limit', '100'); url.searchParams.set('sort', 'confidence'); return url.href; } function isShareLink(url: URL): boolean { return url.hostname === 'redd.it' || /^\/r\/[^/]+\/s\/[^/]+/.test(url.pathname); } /** * Share links from the app are opaque, and the post id is only on the other * side of a redirect. One request that is told not to follow it is enough — * far cheaper than loading the page to find out where it went. */ async function followShareLink(page: Page, originalUrl: string): Promise { const response = await page.request .get(originalUrl, { maxRedirects: 0 }) .catch(() => undefined); const location = response?.headers()['location']; if (!location) return originalUrl; try { return stripTracking(new URL(location, originalUrl)).href; } catch { return originalUrl; } } async function fetchJson(page: Page, url: string): Promise<[Listing, Listing] | undefined> { const response = await page.request .get(url, { headers: { Accept: 'application/json' } }) .catch(() => undefined); if (!response?.ok()) return undefined; if (!response.headers()['content-type']?.includes('json')) return undefined; const body = (await response.json().catch(() => undefined)) as unknown; return Array.isArray(body) && body.length >= 2 ? (body as [Listing, Listing]) : undefined; } async function resolve({ page, originalUrl }: ResolveContext): Promise { let postUrl = originalUrl; try { if (isShareLink(new URL(originalUrl))) postUrl = await followShareLink(page, originalUrl); } catch { // Not a URL we can take apart; the request below will say so. } // Reddit's JSON is the whole post and the whole first page of comments in // one response, which no amount of reading the page can match. let body = await fetchJson(page, jsonUrlFor(postUrl)); if (!body) { // A cold context gets a JavaScript challenge instead, which the page // solves by itself on any ordinary navigation. Doing that once leaves the // cookie behind and the JSON answers from then on, including for every // later post. await page.goto(postUrl, { waitUntil: 'domcontentloaded' }).catch(() => undefined); await page.waitForTimeout(1500); body = await fetchJson(page, jsonUrlFor(page.url() || postUrl)); if (!body) { // Still refused. The page in front of us is the post, so read that. await page.waitForSelector('shreddit-comment', { timeout: 5_000 }).catch(() => undefined); return fromPage(page, originalUrl); } } // The resolved permalink, not the share code we came in with: an opaque // `/s/` link says nothing about what it points at, and handing it back is // the same as handing back a tracking parameter. return toPost(body, postUrl); } export const reddit: PlatformSpec = { id: 'reddit', label: 'Reddit', prefix: 'reddit', canonicalHost: 'www.reddit.com', textPosition: 'above', matchesHost: (host) => host === 'reddit.com' || host.endsWith('.reddit.com') || host === 'redd.it', /** A single opaque segment is a `redd.it` share code — the rewrite rule * drops the host, so `redd.it/abc123` arrives as just `abc123`. */ buildOriginalUrl: (pathRest, search) => { const first = pathRest.split('/')[0] ?? ''; if (first && !pathRest.includes('/') && !KNOWN_ROOTS.has(first)) { return `https://redd.it/${first}${search}`; } return `${HOST}/${pathRest}${search}`; }, resolve, };