import { config } from '../config.ts'; import { ResolveError, oneSegment, type Author, type Media, type Post, type Quoted } from '../types.ts'; import { captureJson } from '../browser/capture.ts'; import { hostMatcher } from './url.ts'; import type { PlatformSpec, ResolveContext } from './types.ts'; type Size = { w?: number; h?: number }; type VideoVariant = { bitrate?: number; content_type?: string; url?: string }; type MediaDetail = { type?: 'photo' | 'video' | 'animated_gif'; media_url_https?: string; ext_alt_text?: string; original_info?: Size; video_info?: { duration_millis?: number; variants?: VideoVariant[] }; }; type TweetUser = { screen_name?: string; name?: string; profile_image_url_https?: string }; type TweetResult = { id_str?: string; text?: string; /** Which slice of `text` X itself shows. UTF-16 indices. */ display_text_range?: [number, number]; entities?: { urls?: Array<{ url?: string; expanded_url?: string }> }; created_at?: string; user?: TweetUser; mediaDetails?: MediaDetail[]; quoted_tweet?: TweetResult; tombstone?: unknown; }; /** * X hands back post text with the markup-significant characters already * escaped — an ampersand someone typed arrives as `&`. Everything * downstream escapes on the way out, so leaving them would put the entity * itself on screen. */ const ENTITIES: Record = { '&': '&', '<': '<', '>': '>', '"': '"', ''': "'", }; function decodeEntities(text: string): string { // `&` last would double-decode `&lt;`; one pass over the lot cannot. return text.replace(/&(?:amp|lt|gt|quot|#39);/g, (entity) => ENTITIES[entity] ?? entity); } const SHORTLINK = /https?:\/\/t\.co\/[A-Za-z0-9]+/g; /** * Put back the addresses `t.co` stands in for. * * Every link in a post is rewritten to a shortlink before it is stored, so * left alone the page shows `t.co/QdJhOVu4En` — which says nothing about * where it goes, and routes the reader through X's click tracker to find * out. The payload carries the real address alongside, so there is no reason * to show the stand-in. * * Matched on the shortlink itself rather than by `indices`, which are offsets * into a string this function is in the middle of changing the length of. */ function expandShortlinks(text: string, result: TweetResult): string { const expansions = new Map(); for (const entity of result.entities?.urls ?? []) { if (!entity.url || !entity.expanded_url) continue; if (!/^https?:\/\//.test(entity.expanded_url)) continue; expansions.set(entity.url, entity.expanded_url); } if (expansions.size === 0) return text; return text.replace(SHORTLINK, (link) => expansions.get(link) ?? link); } /** * The words of the post, as X itself would show them. * * A quote post gets a `t.co` link to the post it quotes stapled to the end of * its text, and X hides it — the quoted post is on the page, so a shortlink * to it is noise. `display_text_range` is where that boundary is, and it also * keeps a link the author put there on purpose, which is why the range is * used rather than stripping trailing URLs. * * Sliced before decoding: the indices are into the escaped text, where an * ampersand is five characters. They are UTF-16 units, so a plain `slice` is * right and splitting to codepoints first would overshoot past an emoji. */ function visibleText(result: TweetResult): string | undefined { const raw = result.text; if (!raw) return undefined; const range = result.display_text_range; const shown = Array.isArray(range) ? raw.slice(range[0], range[1]) : raw; // Expanded after the slice for the same reason: the range is measured // against the text as it arrived. const text = expandShortlinks(decodeEntities(shown), result).trimEnd(); return text || undefined; } /** Every X post URL ends in `/status/`, whatever precedes it. */ function tweetId(pathRest: string): string | undefined { return /\/status(?:es)?\/(\d+)/.exec(`/${pathRest}`)?.[1]; } /** * The syndication endpoint wants a token derived from the id. This is the * same arithmetic the embed script does before calling it. */ function syndicationToken(id: string): string { return ((Number(id) / 1e15) * Math.PI).toString(36).replace(/(0+|\.)/g, ''); } function bestVariant(variants: VideoVariant[] | undefined): VideoVariant | undefined { return (variants ?? []) .filter((v) => v.content_type === 'video/mp4' && v.url) .sort((a, b) => (b.bitrate ?? 0) - (a.bitrate ?? 0))[0]; } export function mediaFromDetails(details: MediaDetail[] | undefined): Media[] { return (details ?? []).flatMap((detail) => { const size = { ...(detail.original_info?.w ? { width: detail.original_info.w } : {}), ...(detail.original_info?.h ? { height: detail.original_info.h } : {}), }; if (detail.type === 'photo') { if (!detail.media_url_https) return []; return [{ kind: 'image', // `?name=orig` asks twimg for the untouched upload. url: `${detail.media_url_https}?name=orig`, ...(detail.ext_alt_text ? { alt: detail.ext_alt_text } : {}), ...size, }]; } const variant = bestVariant(detail.video_info?.variants); if (!variant?.url) return []; const duration = detail.video_info?.duration_millis; return [{ kind: 'video', url: variant.url, ...(detail.media_url_https ? { poster: { url: detail.media_url_https } } : {}), ...(duration ? { durationSec: Math.round(duration / 1000) } : {}), ...size, }]; }); } function authorOf(user: TweetUser | undefined): Author { const avatar = user?.profile_image_url_https; return { handle: `@${user?.screen_name ?? 'unknown'}`, ...(user?.name ? { displayName: user.name } : {}), // `_normal` is a 48px thumbnail; `_x96` is the same file at 96. ...(avatar ? { avatar: { url: avatar.replace('_normal', '_x96') } } : {}), }; } /** * The post a quote post is talking about. * * Shown in full — whose it is, what it said and what it showed — rather than * having its picture lifted out and hung under the quoter's name. Half the * quote posts people share are someone answering a stranger, and the other * half are someone continuing their own thought from an earlier post; both * are unreadable with only one side of them on the page. * * The syndication payload carries no permalink for the quoted post, so its * URL is rebuilt from the handle and the id, the same shape every X post has. */ export function quotedFrom(quoted: TweetResult | undefined): Quoted | undefined { if (!quoted?.user) return undefined; const handle = quoted.user.screen_name; return { author: authorOf(quoted.user), ...(visibleText(quoted) ? { text: visibleText(quoted) } : {}), media: mediaFromDetails(quoted.mediaDetails), ...(quoted.created_at ? { postedAt: quoted.created_at } : {}), ...(handle && quoted.id_str ? { url: `https://x.com/${handle}/status/${quoted.id_str}` } : {}), }; } export function toPost(result: TweetResult, originalUrl: string): Post { const quoted = quotedFrom(result.quoted_tweet); return { platform: 'x', platformLabel: 'X', originalUrl, author: authorOf(result.user), textPosition: 'above', segments: oneSegment({ ...(visibleText(result) ? { text: visibleText(result) } : {}), // Only its own. The quoted post's pictures are shown with the quoted // post, where it is clear whose they are. media: mediaFromDetails(result.mediaDetails), ...(result.created_at ? { postedAt: result.created_at } : {}), ...(quoted ? { quoted } : {}), }), }; } async function resolve({ page, originalUrl, pathRest }: ResolveContext): Promise { const id = tweetId(pathRest); if (!id) throw new ResolveError('No post id in that X URL', 'x', originalUrl); // The official embed renders logged out, and calls the syndication // endpoint on our behalf. Catching that response gives structured media // with alt text, rather than whatever the embed chose to draw. const captured = captureJson(page, (url) => url.includes('/tweet-result')); const embedUrl = new URL('https://platform.twitter.com/embed/Tweet.html'); embedUrl.searchParams.set('id', id); embedUrl.searchParams.set('lang', 'en'); await page.goto(embedUrl.href, { waitUntil: 'domcontentloaded' }).catch(() => undefined); let result = await captured.wait(config.navigationTimeoutMs); if (!result?.user) { // Ask for it ourselves if the embed did not, or was slow. const api = new URL('https://cdn.syndication.twimg.com/tweet-result'); api.searchParams.set('id', id); api.searchParams.set('lang', 'en'); api.searchParams.set('token', syndicationToken(id)); result = await page.request .get(api.href, { headers: { Referer: 'https://platform.twitter.com/' } }) .then((r) => (r.ok() ? (r.json() as Promise) : undefined)) .catch(() => undefined); } if (!result?.user || result.tombstone) { throw new ResolveError( 'X would not return that post — it may be deleted, protected, or age restricted', 'x', originalUrl, ); } return toPost(result, originalUrl); } export const x: PlatformSpec = { id: 'x', label: 'X', prefix: 'x', canonicalHost: 'x.com', textPosition: 'above', matchesHost: hostMatcher('x.com', 'twitter.com'), resolve, };