Initial commit: read social posts back without the app
antisocial is the other half of a StopTheMadness redirect rule. Links to X, Threads, Instagram, TikTok and Bluesky get rewritten to /<prefix>/<original path>, and this resolves the post and shows the media and the words, with a badge saying where it came from and a button to copy the original URL. Every request drives a real headless Chromium, logged out, from a residential IP. One code path, and it survives markup changes better than parsing from the outside would. Extraction is layered, most structured first: the platform's own API response caught in flight, then an inline payload, then the rendered DOM, then Open Graph tags. Media is never linked straight at a CDN. Instagram and TikTok reject requests without a matching Referer and cookies, and proxying keeps the viewer's browser from talking to the platform at all. Range is forwarded so the native video scrubber can seek. HLS is the exception, since proxying it would mean rewriting playlists. TikTok sometimes answers with a slider puzzle. Rather than reporting that as a failure, the page is parked and the viewer is handed the puzzle: screenshots stream out, pointer events are replayed back. Solving it leaves the cookie in the shared browser context, so the retry is an ordinary request. A failed resolve is never a blank error page. The card carries the platform, the original URL and the copy button, so a broken adapter still leaves the link one tap away. Verified end to end against real shared links on all five platforms, in the container, including multi-image carousels, reels, TikTok short links and photo posts. 49 tests run the adapters against captured payloads with no network. Co-Authored-By: Claude Opus 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01BGkRmLfiWuJHx6tQ12EELY
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,87 @@
|
||||
import type { Page, Response } from 'playwright';
|
||||
|
||||
/**
|
||||
* Watch for the page's own API responses going past and hand back the first
|
||||
* one that matches. Reading the JSON a platform serves its own front end is
|
||||
* far steadier than scraping the DOM it renders from that JSON.
|
||||
*
|
||||
* Attach this *before* navigating.
|
||||
*/
|
||||
export function captureJson<T = unknown>(
|
||||
page: Page,
|
||||
matches: (url: string, response: Response) => boolean,
|
||||
): { wait: (timeoutMs: number) => Promise<T | undefined> } {
|
||||
let settled: T | undefined;
|
||||
const listeners: Array<(value: T) => void> = [];
|
||||
|
||||
const onResponse = (response: Response): void => {
|
||||
if (settled !== undefined) return;
|
||||
if (!matches(response.url(), response)) return;
|
||||
void response
|
||||
.json()
|
||||
.then((body: T) => {
|
||||
if (settled !== undefined) return;
|
||||
settled = body;
|
||||
for (const notify of listeners.splice(0)) notify(body);
|
||||
})
|
||||
.catch(() => {
|
||||
// A matching response that isn't JSON, or was aborted. Keep waiting
|
||||
// for a better one rather than failing the whole resolve.
|
||||
});
|
||||
};
|
||||
|
||||
page.on('response', onResponse);
|
||||
|
||||
return {
|
||||
async wait(timeoutMs: number): Promise<T | undefined> {
|
||||
if (settled !== undefined) return settled;
|
||||
return new Promise<T | undefined>((resolve) => {
|
||||
const timer = setTimeout(() => resolve(undefined), timeoutMs);
|
||||
listeners.push((value) => {
|
||||
clearTimeout(timer);
|
||||
resolve(value);
|
||||
});
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Read an inline `<script>` payload out of the DOM.
|
||||
*
|
||||
* Waits for the element rather than reading straight after navigation: a
|
||||
* platform that bounces through an interstitial first will have an empty
|
||||
* document at `domcontentloaded`.
|
||||
*/
|
||||
export async function readInlineJson<T = unknown>(
|
||||
page: Page,
|
||||
selector: string,
|
||||
timeoutMs = 15_000,
|
||||
): Promise<T | undefined> {
|
||||
const raw = await page
|
||||
.locator(selector)
|
||||
.first()
|
||||
.textContent({ timeout: timeoutMs })
|
||||
.catch(() => null);
|
||||
if (!raw) return undefined;
|
||||
try {
|
||||
return JSON.parse(raw) as T;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** Every `<meta property="og:*">` on the page, as a plain object. */
|
||||
export async function readOpenGraph(page: Page): Promise<Record<string, string>> {
|
||||
return page
|
||||
.evaluate(() => {
|
||||
const out: Record<string, string> = {};
|
||||
for (const el of document.querySelectorAll('meta[property], meta[name]')) {
|
||||
const key = el.getAttribute('property') ?? el.getAttribute('name');
|
||||
const value = el.getAttribute('content');
|
||||
if (key && value && !(key in out)) out[key] = value;
|
||||
}
|
||||
return out;
|
||||
})
|
||||
.catch(() => ({}));
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { chromium, type Browser, type BrowserContext, type Page } from 'playwright';
|
||||
import { config } from '../config.ts';
|
||||
import { Semaphore } from './queue.ts';
|
||||
|
||||
export type PageSession = {
|
||||
page: Page;
|
||||
/**
|
||||
* Keep this page open after the callback returns or throws, and hand
|
||||
* responsibility for closing it to the caller. Used to park a page that is
|
||||
* showing a challenge so the viewer can finish it by hand.
|
||||
*/
|
||||
detach: () => void;
|
||||
};
|
||||
|
||||
/** Pure telemetry and advertising. Dropping these is faster and quieter. */
|
||||
const BLOCKED_HOSTS = [
|
||||
'google-analytics.com',
|
||||
'googletagmanager.com',
|
||||
'googlesyndication.com',
|
||||
'doubleclick.net',
|
||||
'scorecardresearch.com',
|
||||
'quantserve.com',
|
||||
'analytics.tiktok.com',
|
||||
'mcs.tiktokv.com',
|
||||
'log-va.tiktokv.com',
|
||||
'log.tiktokv.com',
|
||||
];
|
||||
|
||||
const HEAVY_RESOURCE_TYPES = new Set(['font', 'media', 'image']);
|
||||
|
||||
let browser: Browser | undefined;
|
||||
let context: BrowserContext | undefined;
|
||||
let starting: Promise<BrowserContext> | undefined;
|
||||
let consecutiveFailures = 0;
|
||||
let lastStateSave = 0;
|
||||
|
||||
const gate = new Semaphore(config.maxConcurrent);
|
||||
|
||||
function statePath(): string {
|
||||
return join(config.profileDir, 'state.json');
|
||||
}
|
||||
|
||||
/**
|
||||
* Playwright's Chromium announces itself as `HeadlessChrome`, which is the
|
||||
* one thing about this setup that does not look like someone opening a link.
|
||||
* The rest of the string is left exactly as the bundled build reports it, so
|
||||
* it stays honest and tracks Playwright upgrades on its own.
|
||||
*/
|
||||
function userAgentFor(b: Browser): string {
|
||||
const version = b.version();
|
||||
return `Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${version} Safari/537.36`;
|
||||
}
|
||||
|
||||
async function loadStorageState(): Promise<string | undefined> {
|
||||
try {
|
||||
const raw = await readFile(statePath(), 'utf8');
|
||||
JSON.parse(raw); // reject a truncated file rather than failing the launch
|
||||
return statePath();
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
async function start(): Promise<BrowserContext> {
|
||||
await mkdir(config.profileDir, { recursive: true });
|
||||
|
||||
const launched = await chromium.launch({
|
||||
args: [
|
||||
// The container is unprivileged and single-user; the kernel sandbox
|
||||
// cannot start without extra capabilities we would rather not grant.
|
||||
'--no-sandbox',
|
||||
'--disable-gpu',
|
||||
'--disable-blink-features=AutomationControlled',
|
||||
'--mute-audio',
|
||||
],
|
||||
});
|
||||
|
||||
launched.on('disconnected', () => {
|
||||
if (browser === launched) {
|
||||
browser = undefined;
|
||||
context = undefined;
|
||||
}
|
||||
});
|
||||
|
||||
const storageState = await loadStorageState();
|
||||
const created = await launched.newContext({
|
||||
userAgent: userAgentFor(launched),
|
||||
locale: 'en-CA',
|
||||
timezoneId: 'America/Halifax',
|
||||
viewport: { width: 1280, height: 900 },
|
||||
// Retina-sized assets are what these platforms serve a real laptop.
|
||||
deviceScaleFactor: 2,
|
||||
serviceWorkers: 'block',
|
||||
extraHTTPHeaders: { 'Accept-Language': 'en-CA,en;q=0.9' },
|
||||
...(storageState ? { storageState } : {}),
|
||||
});
|
||||
created.setDefaultNavigationTimeout(config.navigationTimeoutMs);
|
||||
created.setDefaultTimeout(config.navigationTimeoutMs);
|
||||
|
||||
browser = launched;
|
||||
context = created;
|
||||
return created;
|
||||
}
|
||||
|
||||
async function getContext(): Promise<BrowserContext> {
|
||||
if (context) return context;
|
||||
starting ??= start().finally(() => {
|
||||
starting = undefined;
|
||||
});
|
||||
return starting;
|
||||
}
|
||||
|
||||
/** Cookies and localStorage are persisted so the browser accumulates the
|
||||
* same everyday state a real one does. Throttled: this is a nicety, not a
|
||||
* thing worth writing on every request. */
|
||||
async function saveState(): Promise<void> {
|
||||
if (!context) return;
|
||||
const now = Date.now();
|
||||
if (now - lastStateSave < 30_000) return;
|
||||
lastStateSave = now;
|
||||
try {
|
||||
const state = await context.storageState();
|
||||
await writeFile(statePath(), JSON.stringify(state), 'utf8');
|
||||
} catch {
|
||||
// Losing the profile costs nothing but a fresh set of cookies.
|
||||
}
|
||||
}
|
||||
|
||||
export async function warmUp(): Promise<void> {
|
||||
await getContext();
|
||||
}
|
||||
|
||||
export function isReady(): boolean {
|
||||
return context !== undefined && browser?.isConnected() === true;
|
||||
}
|
||||
|
||||
export function queueDepth(): number {
|
||||
return gate.queueDepth;
|
||||
}
|
||||
|
||||
export async function shutdown(): Promise<void> {
|
||||
const b = browser;
|
||||
browser = undefined;
|
||||
context = undefined;
|
||||
await b?.close().catch(() => undefined);
|
||||
}
|
||||
|
||||
async function restart(): Promise<void> {
|
||||
consecutiveFailures = 0;
|
||||
await shutdown();
|
||||
}
|
||||
|
||||
/**
|
||||
* Run `fn` against a fresh page, queued behind the concurrency gate. The
|
||||
* page is always closed, and two consecutive failures recycle Chromium
|
||||
* rather than leaving a wedged browser to fail every later request.
|
||||
*/
|
||||
export async function withPage<T>(fn: (session: PageSession) => Promise<T>): Promise<T> {
|
||||
return gate.run(async () => {
|
||||
const ctx = await getContext();
|
||||
const page = await ctx.newPage();
|
||||
let detached = false;
|
||||
|
||||
await page.route('**/*', (route) => {
|
||||
const request = route.request();
|
||||
const url = request.url();
|
||||
|
||||
if (BLOCKED_HOSTS.some((host) => url.includes(host))) {
|
||||
return route.abort();
|
||||
}
|
||||
|
||||
// Only the URLs matter here: the bytes are fetched later, by the
|
||||
// viewer's browser, through our own media proxy. Aborting still leaves
|
||||
// the `src` attribute in the DOM for the adapters to read.
|
||||
if (HEAVY_RESOURCE_TYPES.has(request.resourceType())) {
|
||||
return route.abort();
|
||||
}
|
||||
|
||||
return route.continue();
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await fn({ page, detach: () => { detached = true; } });
|
||||
consecutiveFailures = 0;
|
||||
void saveState();
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (detached) throw error;
|
||||
consecutiveFailures += 1;
|
||||
if (consecutiveFailures >= 2) await restart();
|
||||
throw error;
|
||||
} finally {
|
||||
if (!detached) await page.close().catch(() => undefined);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** The `Cookie` header the browser context would send to `url`, if any.
|
||||
* Handed to the media proxy so its fetches look like the browser's. */
|
||||
export async function cookieHeaderFor(url: string): Promise<string | undefined> {
|
||||
if (!context) return undefined;
|
||||
const cookies = await context.cookies(url).catch(() => []);
|
||||
if (cookies.length === 0) return undefined;
|
||||
return cookies.map((c) => `${c.name}=${c.value}`).join('; ');
|
||||
}
|
||||
|
||||
/** The user agent the context is running with, so proxied fetches match. */
|
||||
export function currentUserAgent(): string | undefined {
|
||||
return browser ? userAgentFor(browser) : undefined;
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
/**
|
||||
* A counting semaphore. Chromium is the memory ceiling on this box, so page
|
||||
* loads queue behind a fixed number of permits rather than opening an
|
||||
* unbounded number of tabs when several links arrive at once.
|
||||
*/
|
||||
export class Semaphore {
|
||||
#permits: number;
|
||||
readonly #waiters: Array<() => void> = [];
|
||||
|
||||
constructor(permits: number) {
|
||||
if (permits < 1) throw new Error('Semaphore needs at least one permit');
|
||||
this.#permits = permits;
|
||||
}
|
||||
|
||||
get queueDepth(): number {
|
||||
return this.#waiters.length;
|
||||
}
|
||||
|
||||
async run<T>(fn: () => Promise<T>): Promise<T> {
|
||||
await this.#acquire();
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
this.#release();
|
||||
}
|
||||
}
|
||||
|
||||
#acquire(): Promise<void> {
|
||||
if (this.#permits > 0) {
|
||||
this.#permits -= 1;
|
||||
return Promise.resolve();
|
||||
}
|
||||
return new Promise<void>((resolve) => this.#waiters.push(resolve));
|
||||
}
|
||||
|
||||
#release(): void {
|
||||
const next = this.#waiters.shift();
|
||||
if (next) {
|
||||
next();
|
||||
return;
|
||||
}
|
||||
this.#permits += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/** Reject with a clear message rather than hanging if a resolve overruns. */
|
||||
export async function withTimeout<T>(
|
||||
promise: Promise<T>,
|
||||
ms: number,
|
||||
label: string,
|
||||
): Promise<T> {
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
const timeout = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms);
|
||||
});
|
||||
try {
|
||||
return await Promise.race([promise, timeout]);
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import type { Page } from 'playwright';
|
||||
import type { PlatformId } from '../types.ts';
|
||||
|
||||
/**
|
||||
* Pages parked mid-challenge, waiting for a person to finish them.
|
||||
*
|
||||
* A parked page is outside the pool's normal lifecycle: it holds no
|
||||
* concurrency permit, so parking one never blocks anything else, but it does
|
||||
* hold a browser tab, hence the cap and the expiry.
|
||||
*/
|
||||
export type Challenge = {
|
||||
id: string;
|
||||
page: Page;
|
||||
platform: PlatformId;
|
||||
platformLabel: string;
|
||||
originalUrl: string;
|
||||
/** Where to send the viewer once the puzzle is done. */
|
||||
returnTo: string;
|
||||
expiresAt: number;
|
||||
};
|
||||
|
||||
/** The selectors a challenge tends to live in, biggest hint first. */
|
||||
export const CHALLENGE_SELECTORS = '[id*="captcha"], [class*="captcha_verify"], [class*="captcha-verify"]';
|
||||
|
||||
const MAX_PARKED = 3;
|
||||
const TTL_MS = 5 * 60 * 1000;
|
||||
|
||||
const parked = new Map<string, Challenge>();
|
||||
|
||||
async function close(challenge: Challenge): Promise<void> {
|
||||
parked.delete(challenge.id);
|
||||
await challenge.page.close().catch(() => undefined);
|
||||
}
|
||||
|
||||
function sweep(): void {
|
||||
const now = Date.now();
|
||||
for (const challenge of [...parked.values()]) {
|
||||
if (challenge.expiresAt <= now) void close(challenge);
|
||||
}
|
||||
}
|
||||
|
||||
export async function park(options: {
|
||||
page: Page;
|
||||
platform: PlatformId;
|
||||
platformLabel: string;
|
||||
originalUrl: string;
|
||||
}): Promise<string> {
|
||||
sweep();
|
||||
|
||||
// Never sit on more than a few tabs. The oldest is the least likely to
|
||||
// still have someone looking at it.
|
||||
while (parked.size >= MAX_PARKED) {
|
||||
const oldest = [...parked.values()].sort((a, b) => a.expiresAt - b.expiresAt)[0];
|
||||
if (!oldest) break;
|
||||
await close(oldest);
|
||||
}
|
||||
|
||||
const id = randomBytes(9).toString('base64url');
|
||||
parked.set(id, { ...options, id, returnTo: '/', expiresAt: Date.now() + TTL_MS });
|
||||
return id;
|
||||
}
|
||||
|
||||
export function get(id: string): Challenge | undefined {
|
||||
sweep();
|
||||
const challenge = parked.get(id);
|
||||
if (!challenge) return undefined;
|
||||
// Someone is still working on it.
|
||||
challenge.expiresAt = Date.now() + TTL_MS;
|
||||
return challenge;
|
||||
}
|
||||
|
||||
export function setReturnTo(id: string, returnTo: string): void {
|
||||
const challenge = parked.get(id);
|
||||
if (challenge) challenge.returnTo = returnTo;
|
||||
}
|
||||
|
||||
export async function release(id: string): Promise<string | undefined> {
|
||||
const challenge = parked.get(id);
|
||||
if (!challenge) return undefined;
|
||||
const { returnTo } = challenge;
|
||||
await close(challenge);
|
||||
return returnTo;
|
||||
}
|
||||
|
||||
export function parkedCount(): number {
|
||||
sweep();
|
||||
return parked.size;
|
||||
}
|
||||
|
||||
/** Has the puzzle gone away? */
|
||||
export async function isSolved(challenge: Challenge): Promise<boolean> {
|
||||
return challenge.page
|
||||
.evaluate((selectors: string) => {
|
||||
const el = document.querySelector(selectors);
|
||||
if (!el) return true;
|
||||
const style = getComputedStyle(el);
|
||||
return style.display === 'none' || style.visibility === 'hidden';
|
||||
}, CHALLENGE_SELECTORS)
|
||||
.catch(() => false);
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
/**
|
||||
* Every knob is an environment variable with a working default, so the
|
||||
* container runs correctly with no configuration at all.
|
||||
*/
|
||||
|
||||
function int(name: string, fallback: number): number {
|
||||
const raw = process.env[name];
|
||||
if (raw === undefined || raw === '') return fallback;
|
||||
const parsed = Number.parseInt(raw, 10);
|
||||
if (!Number.isFinite(parsed)) {
|
||||
throw new Error(`${name} must be an integer, got ${JSON.stringify(raw)}`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function str(name: string, fallback: string): string {
|
||||
const raw = process.env[name];
|
||||
return raw === undefined || raw === '' ? fallback : raw;
|
||||
}
|
||||
|
||||
export const config = {
|
||||
host: str('HOST', '0.0.0.0'),
|
||||
port: int('PORT', 8080),
|
||||
logLevel: str('LOG_LEVEL', 'info'),
|
||||
|
||||
/** Where the persistent Chromium profile lives. Survives restarts so that
|
||||
* cookies and dismissed consent banners accumulate the way a real
|
||||
* browser's do. */
|
||||
profileDir: str('PROFILE_DIR', './profile'),
|
||||
|
||||
/** Concurrent page loads. Chromium is the memory ceiling on this box, so
|
||||
* requests queue rather than opening an unbounded number of tabs. */
|
||||
maxConcurrent: int('MAX_CONCURRENT', 2),
|
||||
|
||||
/** Budget for a single navigation, and for a whole resolve including
|
||||
* extraction. */
|
||||
navigationTimeoutMs: int('NAVIGATION_TIMEOUT_MS', 20_000),
|
||||
resolveTimeoutMs: int('RESOLVE_TIMEOUT_MS', 30_000),
|
||||
|
||||
/** Resolved posts are cached in memory only. Reloading or hitting back
|
||||
* must not drive Chromium again. */
|
||||
cacheTtlMs: int('CACHE_TTL_MS', 60 * 60 * 1000),
|
||||
cacheMax: int('CACHE_MAX', 200),
|
||||
|
||||
/** How long a signed /m/ media token stays valid. Long enough to finish
|
||||
* watching a video, short enough that a leaked URL goes stale. */
|
||||
mediaTokenTtlMs: int('MEDIA_TOKEN_TTL_MS', 6 * 60 * 60 * 1000),
|
||||
|
||||
/** Public origin, used to build absolute URLs. Only needs setting if
|
||||
* something ever cares about absolute links. */
|
||||
publicOrigin: str('PUBLIC_ORIGIN', 'https://antisocial.unsupervised.studio'),
|
||||
} as const;
|
||||
|
||||
export type Config = typeof config;
|
||||
@@ -0,0 +1,38 @@
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import { LRUCache } from 'lru-cache';
|
||||
import { config } from '../config.ts';
|
||||
import type { Asset } from '../types.ts';
|
||||
|
||||
/**
|
||||
* Maps an opaque id to the upstream asset it stands for.
|
||||
*
|
||||
* A server-side registry rather than a signed self-describing token, for two
|
||||
* reasons: the upstream URLs carry short-lived CDN signatures and session
|
||||
* hints that have no business being in page source, and TikTok and Instagram
|
||||
* need a `Cookie` header alongside the `Referer`, which would either bloat
|
||||
* the token or need a persisted encryption key.
|
||||
*
|
||||
* The cost is that ids do not survive a restart. For a single-user app whose
|
||||
* post cache is also in memory, a reload is the whole recovery story.
|
||||
*/
|
||||
const entries = new LRUCache<string, Asset>({
|
||||
max: 4_000,
|
||||
ttl: config.mediaTokenTtlMs,
|
||||
updateAgeOnGet: true,
|
||||
});
|
||||
|
||||
/** Register an asset and get back the path the page should point at. */
|
||||
export function proxyUrlFor(asset: Asset): string {
|
||||
if (asset.direct) return asset.url;
|
||||
const id = randomBytes(16).toString('base64url');
|
||||
entries.set(id, asset);
|
||||
return `/m/${id}`;
|
||||
}
|
||||
|
||||
export function lookupAsset(id: string): Asset | undefined {
|
||||
return entries.get(id);
|
||||
}
|
||||
|
||||
export function registrySize(): number {
|
||||
return entries.size;
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
import { config } from '../config.ts';
|
||||
import { ResolveError, type Media, type Post } from '../types.ts';
|
||||
import { captureJson } from '../browser/capture.ts';
|
||||
import { hostMatcher } from './url.ts';
|
||||
import type { PlatformSpec, ResolveContext } from './types.ts';
|
||||
|
||||
const API = 'https://public.api.bsky.app/xrpc';
|
||||
|
||||
type Aspect = { width: number; height: number };
|
||||
|
||||
type Embed = {
|
||||
$type?: string;
|
||||
images?: Array<{ thumb?: string; fullsize?: string; alt?: string; aspectRatio?: Aspect }>;
|
||||
playlist?: string;
|
||||
thumbnail?: string;
|
||||
aspectRatio?: Aspect;
|
||||
media?: Embed;
|
||||
record?: { embeds?: Embed[] };
|
||||
};
|
||||
|
||||
type BskyPost = {
|
||||
uri?: string;
|
||||
author?: { handle?: string; displayName?: string; avatar?: string };
|
||||
record?: { text?: string; createdAt?: string };
|
||||
embed?: Embed;
|
||||
indexedAt?: string;
|
||||
};
|
||||
|
||||
type ThreadResponse = { thread?: { post?: BskyPost } };
|
||||
|
||||
/** `profile/<actor>/post/<rkey>` is the only shape that carries a post. */
|
||||
function parsePath(pathRest: string): { actor: string; rkey: string } | undefined {
|
||||
const match = /^profile\/([^/]+)\/post\/([^/?#]+)/.exec(pathRest);
|
||||
if (!match?.[1] || !match[2]) return undefined;
|
||||
return { actor: decodeURIComponent(match[1]), rkey: match[2] };
|
||||
}
|
||||
|
||||
export function mediaFromEmbed(embed: Embed | undefined): Media[] {
|
||||
if (!embed) return [];
|
||||
|
||||
// A quote post with media attached nests the real embed one level down.
|
||||
if (embed.media) return mediaFromEmbed(embed.media);
|
||||
|
||||
if (embed.images?.length) {
|
||||
return embed.images.flatMap<Media>((image) => {
|
||||
const url = image.fullsize ?? image.thumb;
|
||||
if (!url) return [];
|
||||
return [{
|
||||
kind: 'image',
|
||||
url,
|
||||
...(image.alt ? { alt: image.alt } : {}),
|
||||
...(image.aspectRatio ?? {}),
|
||||
}];
|
||||
});
|
||||
}
|
||||
|
||||
if (embed.playlist) {
|
||||
return [{
|
||||
kind: 'video',
|
||||
url: embed.playlist,
|
||||
// An HLS playlist: proxying it would mean rewriting the manifest and
|
||||
// every segment URL, and Bluesky's CDN is happy to be linked directly.
|
||||
hls: true,
|
||||
direct: true,
|
||||
...(embed.thumbnail ? { poster: { url: embed.thumbnail } } : {}),
|
||||
...(embed.aspectRatio ?? {}),
|
||||
}];
|
||||
}
|
||||
|
||||
// A quote post carries the quoted post's media one level in. That is
|
||||
// usually the thing being pointed at, so it is what to show.
|
||||
const quoted = embed.record?.embeds?.[0];
|
||||
if (quoted) return mediaFromEmbed(quoted);
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
export function toPost(post: BskyPost, originalUrl: string): Post {
|
||||
const handle = post.author?.handle ?? 'unknown';
|
||||
return {
|
||||
platform: 'bluesky',
|
||||
platformLabel: 'Bluesky',
|
||||
originalUrl,
|
||||
author: {
|
||||
handle: `@${handle}`,
|
||||
...(post.author?.displayName ? { displayName: post.author.displayName } : {}),
|
||||
...(post.author?.avatar ? { avatar: { url: post.author.avatar } } : {}),
|
||||
},
|
||||
...(post.record?.text ? { text: post.record.text } : {}),
|
||||
textPosition: 'above',
|
||||
media: mediaFromEmbed(post.embed),
|
||||
...(post.record?.createdAt ? { postedAt: post.record.createdAt } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
async function resolve({ page, originalUrl, pathRest }: ResolveContext): Promise<Post> {
|
||||
const parsed = parsePath(pathRest);
|
||||
if (!parsed) {
|
||||
throw new ResolveError('Not a Bluesky post URL', 'bluesky', originalUrl);
|
||||
}
|
||||
|
||||
// Bluesky publishes a genuinely public API, so asking it directly beats
|
||||
// booting the whole single-page app. Still the browser's network stack,
|
||||
// so it carries the same identity as every other request we make.
|
||||
let did = parsed.actor;
|
||||
if (!did.startsWith('did:')) {
|
||||
const resolved = await page.request
|
||||
.get(`${API}/com.atproto.identity.resolveHandle?handle=${encodeURIComponent(did)}`)
|
||||
.then((r) => (r.ok() ? (r.json() as Promise<{ did?: string }>) : undefined))
|
||||
.catch(() => undefined);
|
||||
if (resolved?.did) did = resolved.did;
|
||||
}
|
||||
|
||||
const uri = `at://${did}/app.bsky.feed.post/${parsed.rkey}`;
|
||||
const direct = await page.request
|
||||
.get(`${API}/app.bsky.feed.getPostThread?depth=0&parentHeight=0&uri=${encodeURIComponent(uri)}`)
|
||||
.then((r) => (r.ok() ? (r.json() as Promise<ThreadResponse>) : undefined))
|
||||
.catch(() => undefined);
|
||||
|
||||
if (direct?.thread?.post) return toPost(direct.thread.post, originalUrl);
|
||||
|
||||
// Fall back to loading the post the way a person would and catching the
|
||||
// same call as it goes past.
|
||||
// `getPostThread?` and not `getPostThreadV2?`: the app calls V2 now, whose
|
||||
// shape is different, and settling on it would mean waiting for a payload
|
||||
// we cannot read.
|
||||
const captured = captureJson<ThreadResponse>(page, (url) => url.includes('getPostThread?'));
|
||||
await page.goto(originalUrl, { waitUntil: 'domcontentloaded' });
|
||||
const body = await captured.wait(config.navigationTimeoutMs);
|
||||
if (body?.thread?.post) return toPost(body.thread.post, originalUrl);
|
||||
|
||||
throw new ResolveError('Bluesky returned no post for that URL', 'bluesky', originalUrl);
|
||||
}
|
||||
|
||||
export const bluesky: PlatformSpec = {
|
||||
id: 'bluesky',
|
||||
label: 'Bluesky',
|
||||
prefix: 'bsky',
|
||||
canonicalHost: 'bsky.app',
|
||||
textPosition: 'above',
|
||||
matchesHost: hostMatcher('bsky.app'),
|
||||
resolve,
|
||||
};
|
||||
@@ -0,0 +1,42 @@
|
||||
import type { PlatformSpec } from './types.ts';
|
||||
import { stripTracking } from './url.ts';
|
||||
import { x } from './x.ts';
|
||||
import { threads } from './threads.ts';
|
||||
import { instagram } from './instagram.ts';
|
||||
import { tiktok } from './tiktok.ts';
|
||||
import { bluesky } from './bluesky.ts';
|
||||
|
||||
export const platforms: readonly PlatformSpec[] = [x, threads, instagram, tiktok, bluesky];
|
||||
|
||||
const byPrefix = new Map(platforms.map((p) => [p.prefix, p]));
|
||||
|
||||
export function platformByPrefix(prefix: string): PlatformSpec | undefined {
|
||||
return byPrefix.get(prefix.toLowerCase());
|
||||
}
|
||||
|
||||
/** Route a pasted original URL to its platform. Used by bin/resolve.ts. */
|
||||
export function platformForUrl(url: string): PlatformSpec | undefined {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
const host = parsed.hostname.toLowerCase();
|
||||
return platforms.find((p) => p.matchesHost(host));
|
||||
}
|
||||
|
||||
/**
|
||||
* Turn an inbound request into the canonical original URL.
|
||||
*
|
||||
* `/x/user/status/123?s=20` -> `https://x.com/user/status/123`
|
||||
*/
|
||||
export function originalUrlFor(spec: PlatformSpec, pathRest: string, search: string): string {
|
||||
const rest = pathRest.replace(/^\/+/, '');
|
||||
const raw = spec.buildOriginalUrl
|
||||
? spec.buildOriginalUrl(rest, search)
|
||||
: `https://${spec.canonicalHost}/${rest}${search}`;
|
||||
return stripTracking(new URL(raw)).href;
|
||||
}
|
||||
|
||||
export { stripTracking, hostMatcher } from './url.ts';
|
||||
@@ -0,0 +1,312 @@
|
||||
import { ResolveError, type Media, type Post } from '../types.ts';
|
||||
import { readOpenGraph } from '../browser/capture.ts';
|
||||
import { findCaption, findMetaMedia, mediaFromMetaNode, metaHeaders } from './meta-media.ts';
|
||||
import { extractJsonObjectAfterKey } from './scan.ts';
|
||||
import { hostMatcher } from './url.ts';
|
||||
import type { PlatformSpec, ResolveContext } from './types.ts';
|
||||
|
||||
const REFERER = 'https://www.instagram.com/';
|
||||
|
||||
type GraphNode = {
|
||||
display_url?: string;
|
||||
video_url?: string;
|
||||
is_video?: boolean;
|
||||
dimensions?: { width?: number; height?: number };
|
||||
accessibility_caption?: string;
|
||||
video_duration?: number;
|
||||
edge_sidecar_to_children?: { edges?: Array<{ node?: GraphNode }> };
|
||||
edge_media_to_caption?: { edges?: Array<{ node?: { text?: string } }> };
|
||||
owner?: { username?: string; full_name?: string; profile_pic_url?: string };
|
||||
taken_at_timestamp?: number;
|
||||
};
|
||||
|
||||
type Scraped = {
|
||||
media: Media[];
|
||||
text?: string;
|
||||
handle?: string;
|
||||
displayName?: string;
|
||||
avatar?: string;
|
||||
postedAt?: string;
|
||||
};
|
||||
|
||||
/** Shared Instagram links usually lead with the username. */
|
||||
function handleFromPath(pathRest: string): string | undefined {
|
||||
const first = pathRest.split('/')[0];
|
||||
if (!first || ['p', 'reel', 'reels', 'tv', 'share'].includes(first)) return undefined;
|
||||
return first;
|
||||
}
|
||||
|
||||
function shortcode(pathRest: string): string | undefined {
|
||||
return /(?:^|\/)(?:p|reel|reels|tv)\/([A-Za-z0-9_-]+)/.exec(pathRest)?.[1];
|
||||
}
|
||||
|
||||
export function mediaFromGraph(node: GraphNode): Media[] {
|
||||
const children = node.edge_sidecar_to_children?.edges;
|
||||
if (children?.length) {
|
||||
return children.flatMap((edge) => (edge.node ? mediaFromGraph(edge.node) : []));
|
||||
}
|
||||
|
||||
const fetchHeaders = metaHeaders(REFERER);
|
||||
const size = {
|
||||
...(node.dimensions?.width ? { width: node.dimensions.width } : {}),
|
||||
...(node.dimensions?.height ? { height: node.dimensions.height } : {}),
|
||||
};
|
||||
|
||||
if (node.is_video && node.video_url) {
|
||||
return [{
|
||||
kind: 'video',
|
||||
url: node.video_url,
|
||||
fetchHeaders,
|
||||
...(node.display_url ? { poster: { url: node.display_url, fetchHeaders } } : {}),
|
||||
...(node.video_duration ? { durationSec: Math.round(node.video_duration) } : {}),
|
||||
...size,
|
||||
}];
|
||||
}
|
||||
|
||||
if (!node.display_url) return [];
|
||||
return [{
|
||||
kind: 'image',
|
||||
url: node.display_url,
|
||||
fetchHeaders,
|
||||
...(node.accessibility_caption ? { alt: node.accessibility_caption } : {}),
|
||||
...size,
|
||||
}];
|
||||
}
|
||||
|
||||
export function fromGraph(node: GraphNode): Scraped {
|
||||
return {
|
||||
media: mediaFromGraph(node),
|
||||
...(node.edge_media_to_caption?.edges?.[0]?.node?.text
|
||||
? { text: node.edge_media_to_caption.edges[0].node.text }
|
||||
: {}),
|
||||
...(node.owner?.username ? { handle: node.owner.username } : {}),
|
||||
...(node.owner?.full_name ? { displayName: node.owner.full_name } : {}),
|
||||
...(node.owner?.profile_pic_url ? { avatar: node.owner.profile_pic_url } : {}),
|
||||
...(node.taken_at_timestamp
|
||||
? { postedAt: new Date(node.taken_at_timestamp * 1000).toISOString() }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** The text of every script on the page, big ones only. */
|
||||
async function scriptTexts(ctx: ResolveContext): Promise<string[]> {
|
||||
return ctx.page
|
||||
.evaluate(() =>
|
||||
[...document.querySelectorAll('script')]
|
||||
.map((el) => el.textContent ?? '')
|
||||
.filter((text) => text.length > 256),
|
||||
)
|
||||
.catch(() => [] as string[]);
|
||||
}
|
||||
|
||||
/** JSON blobs that are actually addressable as elements. */
|
||||
function parseAll(texts: string[]): unknown[] {
|
||||
return texts.flatMap((text) => {
|
||||
try {
|
||||
return [JSON.parse(text) as unknown];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function findShortcodeMedia(texts: string[]): GraphNode | undefined {
|
||||
for (const text of texts) {
|
||||
const node = extractJsonObjectAfterKey<GraphNode>(text, 'shortcode_media');
|
||||
if (node?.display_url ?? node?.video_url ?? node?.edge_sidecar_to_children) return node;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read what the embed actually drew. This is the steadiest signal on the
|
||||
* page — a reel renders a real progressive `<video>` with its poster — but
|
||||
* it only ever shows the first item of a carousel.
|
||||
*/
|
||||
async function fromEmbedDom(ctx: ResolveContext, expectVideo: boolean): Promise<Scraped> {
|
||||
// The embed hydrates its player a beat after the document is ready, so
|
||||
// reading straight away hands back the cover frame instead of the reel.
|
||||
await ctx.page
|
||||
.waitForSelector(expectVideo ? 'video' : 'video, img.EmbeddedMediaImage', { timeout: 8_000 })
|
||||
.catch(() => undefined);
|
||||
|
||||
const found = await ctx.page
|
||||
.evaluate(() => {
|
||||
const videos = [...document.querySelectorAll('video')]
|
||||
.map((el) => ({ src: el.currentSrc || el.src, poster: el.poster }))
|
||||
.filter((v) => v.src && !v.src.startsWith('blob:'));
|
||||
|
||||
const images = [...document.querySelectorAll('img.EmbeddedMediaImage')]
|
||||
.map((el) => (el as HTMLImageElement).src)
|
||||
.filter(Boolean);
|
||||
|
||||
const caption = document.querySelector('.Caption');
|
||||
let text: string | undefined;
|
||||
if (caption) {
|
||||
const clone = caption.cloneNode(true) as HTMLElement;
|
||||
for (const drop of clone.querySelectorAll('.CaptionUsername, .CaptionComments')) {
|
||||
drop.remove();
|
||||
}
|
||||
document.body.append(clone);
|
||||
// innerText keeps the line breaks the markup implies; textContent
|
||||
// would run every paragraph together.
|
||||
text = clone.innerText.trim() || undefined;
|
||||
clone.remove();
|
||||
}
|
||||
|
||||
const handle =
|
||||
document.querySelector('.UsernameText')?.textContent?.trim() ||
|
||||
document.querySelector('.CaptionUsername')?.textContent?.trim() ||
|
||||
undefined;
|
||||
|
||||
return { videos, images, text, handle };
|
||||
})
|
||||
.catch(() => ({ videos: [], images: [], text: undefined, handle: undefined }));
|
||||
|
||||
const fetchHeaders = metaHeaders(REFERER);
|
||||
const media: Media[] = found.videos.length
|
||||
? found.videos.map<Media>((video) => ({
|
||||
kind: 'video',
|
||||
url: video.src,
|
||||
fetchHeaders,
|
||||
...(video.poster ? { poster: { url: video.poster, fetchHeaders } } : {}),
|
||||
}))
|
||||
: found.images.map<Media>((url) => ({ kind: 'image', url, fetchHeaders }));
|
||||
|
||||
return {
|
||||
media,
|
||||
...(found.text ? { text: found.text } : {}),
|
||||
...(found.handle ? { handle: found.handle } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Whatever structured payload the current page is carrying, if any. */
|
||||
function fromPayloads(texts: string[]): Scraped | undefined {
|
||||
const graph = findShortcodeMedia(texts);
|
||||
if (graph) {
|
||||
const scraped = fromGraph(graph);
|
||||
if (scraped.media.length) return scraped;
|
||||
}
|
||||
|
||||
const payloads = parseAll(texts);
|
||||
const meta = findMetaMedia(payloads);
|
||||
if (!meta) return undefined;
|
||||
|
||||
const caption = findCaption(payloads);
|
||||
return {
|
||||
media: mediaFromMetaNode(meta, REFERER),
|
||||
...(caption ? { text: caption } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Absolute floor: whatever the page offers a link preview. */
|
||||
async function fromOpenGraph(ctx: ResolveContext): Promise<Scraped> {
|
||||
const og = await readOpenGraph(ctx.page);
|
||||
const fetchHeaders = metaHeaders(REFERER);
|
||||
const media: Media[] = [];
|
||||
|
||||
if (og['og:video']) {
|
||||
media.push({
|
||||
kind: 'video',
|
||||
url: og['og:video'],
|
||||
fetchHeaders,
|
||||
...(og['og:image'] ? { poster: { url: og['og:image'], fetchHeaders } } : {}),
|
||||
});
|
||||
} else if (og['og:image']) {
|
||||
media.push({ kind: 'image', url: og['og:image'], fetchHeaders });
|
||||
}
|
||||
|
||||
return {
|
||||
media,
|
||||
...(og['og:description'] ? { text: og['og:description'] } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
async function resolve(ctx: ResolveContext): Promise<Post> {
|
||||
const { page, originalUrl } = ctx;
|
||||
let code = shortcode(ctx.pathRest);
|
||||
|
||||
// `/share/...` links are redirect stubs. Follow one to find the real post.
|
||||
if (!code || ctx.pathRest.startsWith('share/')) {
|
||||
await page.goto(originalUrl, { waitUntil: 'domcontentloaded' }).catch(() => undefined);
|
||||
code = shortcode(new URL(page.url()).pathname.replace(/^\//, '')) ?? code;
|
||||
}
|
||||
|
||||
if (!code) {
|
||||
throw new ResolveError('No post code in that Instagram URL', 'instagram', originalUrl);
|
||||
}
|
||||
|
||||
// The captioned embed renders logged out and, unlike the post page,
|
||||
// carries the caption with it.
|
||||
await page.goto(`https://www.instagram.com/p/${code}/embed/captioned/`, {
|
||||
waitUntil: 'domcontentloaded',
|
||||
});
|
||||
|
||||
// The embed renders the media, but the payload behind it carries the
|
||||
// carousel, the dimensions and the alt text, so try that first.
|
||||
const texts = await scriptTexts(ctx);
|
||||
|
||||
let scraped = fromPayloads(texts);
|
||||
|
||||
// The DOM is the reliable floor: whatever is on screen is real, even when
|
||||
// the payload moved again.
|
||||
// A reel is a video whatever else goes wrong, so say so before falling
|
||||
// back to the DOM.
|
||||
const expectsVideo =
|
||||
/(?:^|\/)(?:reel|reels|tv)\//.test(ctx.pathRest) ||
|
||||
scraped?.media.some((item) => item.kind === 'video') === true;
|
||||
|
||||
const dom = await fromEmbedDom(ctx, expectsVideo);
|
||||
if (!scraped?.media.length && dom.media.length) scraped = dom;
|
||||
// The rendered caption beats a payload that only had a truncated one.
|
||||
if (scraped && !scraped.text && dom.text) scraped.text = dom.text;
|
||||
if (scraped && !scraped.handle && dom.handle) scraped.handle = dom.handle;
|
||||
|
||||
if (!scraped?.media.length) {
|
||||
// The embed refuses some posts outright ("the link may be broken").
|
||||
// Try the post itself: its payload first, then its link preview.
|
||||
await page.goto(originalUrl, { waitUntil: 'domcontentloaded' }).catch(() => undefined);
|
||||
scraped = fromPayloads(await scriptTexts(ctx)) ?? scraped;
|
||||
|
||||
if (!scraped?.media.length) {
|
||||
const og = await fromOpenGraph(ctx);
|
||||
if (og.media.length) scraped = { ...og, ...(scraped?.handle ? { handle: scraped.handle } : {}) };
|
||||
}
|
||||
}
|
||||
|
||||
if (!scraped?.media.length) {
|
||||
throw new ResolveError(
|
||||
'Instagram would not show that post logged out',
|
||||
'instagram',
|
||||
originalUrl,
|
||||
);
|
||||
}
|
||||
|
||||
const handle = scraped.handle ?? handleFromPath(ctx.pathRest) ?? 'instagram';
|
||||
return {
|
||||
platform: 'instagram',
|
||||
platformLabel: 'Instagram',
|
||||
originalUrl,
|
||||
author: {
|
||||
handle: handle.startsWith('@') ? handle : `@${handle}`,
|
||||
...(scraped.displayName ? { displayName: scraped.displayName } : {}),
|
||||
...(scraped.avatar
|
||||
? { avatar: { url: scraped.avatar, fetchHeaders: metaHeaders(REFERER) } }
|
||||
: {}),
|
||||
},
|
||||
...(scraped.text ? { text: scraped.text } : {}),
|
||||
textPosition: 'below',
|
||||
media: scraped.media,
|
||||
...(scraped.postedAt ? { postedAt: scraped.postedAt } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export const instagram: PlatformSpec = {
|
||||
id: 'instagram',
|
||||
label: 'Instagram',
|
||||
prefix: 'ig',
|
||||
canonicalHost: 'www.instagram.com',
|
||||
textPosition: 'below',
|
||||
matchesHost: hostMatcher('instagram.com'),
|
||||
resolve,
|
||||
};
|
||||
@@ -0,0 +1,129 @@
|
||||
import type { Media } from '../types.ts';
|
||||
|
||||
/**
|
||||
* Instagram and Threads are the same product underneath and serve the same
|
||||
* media schema, so both adapters normalize through here.
|
||||
*/
|
||||
|
||||
type Candidate = { url?: string; width?: number; height?: number };
|
||||
|
||||
export type MetaMediaNode = {
|
||||
image_versions2?: { candidates?: Candidate[] };
|
||||
video_versions?: Candidate[];
|
||||
carousel_media?: MetaMediaNode[];
|
||||
accessibility_caption?: string;
|
||||
original_width?: number;
|
||||
original_height?: number;
|
||||
video_duration?: number;
|
||||
};
|
||||
|
||||
/** These CDNs 403 anything that arrives without a matching referrer. */
|
||||
export function metaHeaders(referer: string): Record<string, string> {
|
||||
return { Referer: referer, Origin: new URL(referer).origin };
|
||||
}
|
||||
|
||||
function largest(candidates: Candidate[] | undefined): Candidate | undefined {
|
||||
return (candidates ?? [])
|
||||
.filter((c) => c.url)
|
||||
.sort((a, b) => (b.width ?? 0) - (a.width ?? 0))[0];
|
||||
}
|
||||
|
||||
export function mediaFromMetaNode(node: MetaMediaNode, referer: string): Media[] {
|
||||
if (node.carousel_media?.length) {
|
||||
return node.carousel_media.flatMap((child) => mediaFromMetaNode(child, referer));
|
||||
}
|
||||
|
||||
const fetchHeaders = metaHeaders(referer);
|
||||
const poster = largest(node.image_versions2?.candidates);
|
||||
const size = {
|
||||
...(node.original_width ? { width: node.original_width } : {}),
|
||||
...(node.original_height ? { height: node.original_height } : {}),
|
||||
};
|
||||
|
||||
const video = largest(node.video_versions);
|
||||
if (video?.url) {
|
||||
return [{
|
||||
kind: 'video',
|
||||
url: video.url,
|
||||
fetchHeaders,
|
||||
...(poster?.url ? { poster: { url: poster.url, fetchHeaders } } : {}),
|
||||
...(node.video_duration ? { durationSec: Math.round(node.video_duration) } : {}),
|
||||
...size,
|
||||
}];
|
||||
}
|
||||
|
||||
if (poster?.url) {
|
||||
return [{
|
||||
kind: 'image',
|
||||
url: poster.url,
|
||||
fetchHeaders,
|
||||
...(node.accessibility_caption ? { alt: node.accessibility_caption } : {}),
|
||||
...(poster.width ? { width: poster.width } : size),
|
||||
...(poster.height ? { height: poster.height } : {}),
|
||||
}];
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
function nonEmptyArray(value: unknown): boolean {
|
||||
return Array.isArray(value) && value.length > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Carrying the key is not the same as carrying media: these payloads are
|
||||
* full of stub nodes whose `image_versions2.candidates` is an empty array.
|
||||
* Only a node with something in it counts.
|
||||
*/
|
||||
function looksLikeMedia(value: Record<string, unknown>): boolean {
|
||||
if (nonEmptyArray(value['carousel_media'])) return true;
|
||||
if (nonEmptyArray(value['video_versions'])) return true;
|
||||
const images = value['image_versions2'] as { candidates?: unknown } | undefined;
|
||||
return nonEmptyArray(images?.candidates);
|
||||
}
|
||||
|
||||
/**
|
||||
* Threads buries the post inside a pile of Relay payloads whose shape moves
|
||||
* around. Rather than chase a path, walk the tree for the first node that
|
||||
* carries media and, separately, the first caption.
|
||||
*/
|
||||
export function findMetaMedia(root: unknown): MetaMediaNode | undefined {
|
||||
return walk(root, (record) => (looksLikeMedia(record) ? (record as MetaMediaNode) : undefined));
|
||||
}
|
||||
|
||||
/** Same walk, looking for `caption.text`. */
|
||||
export function findCaption(root: unknown): string | undefined {
|
||||
return walk(root, (record) => {
|
||||
const caption = record['caption'];
|
||||
if (!caption || typeof caption !== 'object') return undefined;
|
||||
const text = (caption as Record<string, unknown>)['text'];
|
||||
return typeof text === 'string' && text.length > 0 ? text : undefined;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Breadth-first so the node nearest the root — the post itself — wins over
|
||||
* anything quoted or recommended below it.
|
||||
*/
|
||||
function walk<T>(root: unknown, visit: (record: Record<string, unknown>) => T | undefined): T | undefined {
|
||||
const seen = new Set<unknown>();
|
||||
const queue: unknown[] = [root];
|
||||
|
||||
for (let i = 0; i < queue.length; i += 1) {
|
||||
const value = queue[i];
|
||||
if (value === null || typeof value !== 'object' || seen.has(value)) continue;
|
||||
seen.add(value);
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
queue.push(...value);
|
||||
continue;
|
||||
}
|
||||
|
||||
const record = value as Record<string, unknown>;
|
||||
const hit = visit(record);
|
||||
if (hit !== undefined) return hit;
|
||||
queue.push(...Object.values(record));
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
/**
|
||||
* Instagram parks its post payload inside a `requireLazy(...)` bootstrap
|
||||
* call rather than a JSON script tag, so there is no element to read and no
|
||||
* stable path to walk. What is stable is the key itself: find it, then take
|
||||
* the balanced object that follows.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Pull the JSON object that follows `"key":` out of a larger source text.
|
||||
*
|
||||
* The payload is often a JSON string inside the outer JSON, so the quotes
|
||||
* arrive escaped. When the plain form misses, the text is unescaped once and
|
||||
* scanned again.
|
||||
*/
|
||||
export function extractJsonObjectAfterKey<T = unknown>(
|
||||
source: string,
|
||||
key: string,
|
||||
): T | undefined {
|
||||
const plain = scanFor<T>(source, key);
|
||||
if (plain !== undefined) return plain;
|
||||
|
||||
if (!source.includes(`\\"${key}\\"`)) return undefined;
|
||||
|
||||
// Unescaping makes the *structure* parseable, but its string leaves are
|
||||
// still carrying the encoding of the level they came from, so they need
|
||||
// one more decode each.
|
||||
const found = scanFor<unknown>(unescapeEmbeddedJson(source), key);
|
||||
return found === undefined ? undefined : (decodeStringLeaves(found) as T);
|
||||
}
|
||||
|
||||
/** Reverse one level of JSON string encoding across the structure. */
|
||||
function unescapeEmbeddedJson(source: string): string {
|
||||
return source.replace(/\\(.)/g, (_, char: string) => (char === 'n' ? '\n' : char));
|
||||
}
|
||||
|
||||
const NEEDS_DECODE = /\\["\\/bfnrtu]/;
|
||||
|
||||
function decodeStringLeaves(value: unknown): unknown {
|
||||
if (typeof value === 'string') {
|
||||
if (!NEEDS_DECODE.test(value)) return value;
|
||||
try {
|
||||
return JSON.parse(`"${value.replace(/"/g, '\\"')}"`) as string;
|
||||
} catch {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
if (Array.isArray(value)) return value.map(decodeStringLeaves);
|
||||
if (value !== null && typeof value === 'object') {
|
||||
return Object.fromEntries(
|
||||
Object.entries(value as Record<string, unknown>).map(([k, v]) => [k, decodeStringLeaves(v)]),
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function scanFor<T>(source: string, key: string): T | undefined {
|
||||
const needle = `"${key}":`;
|
||||
let from = 0;
|
||||
|
||||
for (;;) {
|
||||
const at = source.indexOf(needle, from);
|
||||
if (at === -1) return undefined;
|
||||
from = at + needle.length;
|
||||
|
||||
const open = source.indexOf('{', from);
|
||||
if (open === -1) return undefined;
|
||||
// Only accept an object that starts right after the key, not one that
|
||||
// happens to appear later because the value was null.
|
||||
if (source.slice(from, open).trim() !== '') continue;
|
||||
|
||||
const end = matchingBrace(source, open);
|
||||
if (end === -1) continue;
|
||||
|
||||
try {
|
||||
return JSON.parse(source.slice(open, end + 1)) as T;
|
||||
} catch {
|
||||
// Keep looking: the same key may appear again in a payload that does
|
||||
// parse.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Index of the `}` closing the `{` at `open`, respecting strings and escapes. */
|
||||
function matchingBrace(source: string, open: number): number {
|
||||
let depth = 0;
|
||||
let inString = false;
|
||||
let escaped = false;
|
||||
|
||||
for (let i = open; i < source.length; i += 1) {
|
||||
const char = source[i];
|
||||
|
||||
if (escaped) {
|
||||
escaped = false;
|
||||
continue;
|
||||
}
|
||||
if (char === '\\') {
|
||||
if (inString) escaped = true;
|
||||
continue;
|
||||
}
|
||||
if (char === '"') {
|
||||
inString = !inString;
|
||||
continue;
|
||||
}
|
||||
if (inString) continue;
|
||||
|
||||
if (char === '{') depth += 1;
|
||||
else if (char === '}') {
|
||||
depth -= 1;
|
||||
if (depth === 0) return i;
|
||||
}
|
||||
}
|
||||
|
||||
return -1;
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
import { ResolveError, type Media, type Post } from '../types.ts';
|
||||
import { readOpenGraph } from '../browser/capture.ts';
|
||||
import { findCaption, findMetaMedia, mediaFromMetaNode, metaHeaders } from './meta-media.ts';
|
||||
import { hostMatcher } from './url.ts';
|
||||
import type { PlatformSpec, ResolveContext } from './types.ts';
|
||||
|
||||
const REFERER = 'https://www.threads.com/';
|
||||
|
||||
function handleFromPath(pathRest: string): string | undefined {
|
||||
const match = /^(@[^/]+)/.exec(pathRest);
|
||||
return match?.[1];
|
||||
}
|
||||
|
||||
/** Threads ships its Relay payloads in `<script type="application/json">`
|
||||
* tags, several dozen of them, in an order that is not worth relying on. */
|
||||
async function inlinePayloads(ctx: ResolveContext): Promise<unknown[]> {
|
||||
const raw = await ctx.page
|
||||
.evaluate(() =>
|
||||
[...document.querySelectorAll('script[type="application/json"]')]
|
||||
.map((el) => el.textContent ?? '')
|
||||
.filter((text) => text.length > 64),
|
||||
)
|
||||
.catch(() => [] as string[]);
|
||||
|
||||
return raw.flatMap((text) => {
|
||||
try {
|
||||
return [JSON.parse(text) as unknown];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function fromOpenGraph(ctx: ResolveContext): Promise<{ media: Media[]; text?: string }> {
|
||||
const og = await readOpenGraph(ctx.page);
|
||||
const fetchHeaders = metaHeaders(REFERER);
|
||||
|
||||
if (og['og:video']) {
|
||||
return {
|
||||
media: [{
|
||||
kind: 'video',
|
||||
url: og['og:video'],
|
||||
fetchHeaders,
|
||||
...(og['og:image'] ? { poster: { url: og['og:image'], fetchHeaders } } : {}),
|
||||
}],
|
||||
...(og['og:description'] ? { text: og['og:description'] } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
media: og['og:image'] ? [{ kind: 'image', url: og['og:image'], fetchHeaders }] : [],
|
||||
...(og['og:description'] ? { text: og['og:description'] } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
async function resolve(ctx: ResolveContext): Promise<Post> {
|
||||
const { page, originalUrl, pathRest } = ctx;
|
||||
|
||||
await page.goto(originalUrl, { waitUntil: 'domcontentloaded' });
|
||||
|
||||
const payloads = await inlinePayloads(ctx);
|
||||
const node = findMetaMedia(payloads);
|
||||
|
||||
let media = node ? mediaFromMetaNode(node, REFERER) : [];
|
||||
let text = findCaption(payloads);
|
||||
|
||||
if (media.length === 0 || !text) {
|
||||
const og = await fromOpenGraph(ctx);
|
||||
if (media.length === 0) media = og.media;
|
||||
text ??= og.text;
|
||||
}
|
||||
|
||||
// A Threads post is often only words, so unlike the others an empty media
|
||||
// list is a perfectly good result — but nothing at all is a failure.
|
||||
if (media.length === 0 && !text) {
|
||||
throw new ResolveError('Threads returned nothing for that URL', 'threads', originalUrl);
|
||||
}
|
||||
|
||||
const handle = handleFromPath(pathRest) ?? '@threads';
|
||||
return {
|
||||
platform: 'threads',
|
||||
platformLabel: 'Threads',
|
||||
originalUrl,
|
||||
author: { handle },
|
||||
...(text ? { text } : {}),
|
||||
textPosition: 'above',
|
||||
media,
|
||||
};
|
||||
}
|
||||
|
||||
export const threads: PlatformSpec = {
|
||||
id: 'threads',
|
||||
label: 'Threads',
|
||||
prefix: 'threads',
|
||||
canonicalHost: 'www.threads.com',
|
||||
textPosition: 'above',
|
||||
matchesHost: hostMatcher('threads.net', 'threads.com'),
|
||||
resolve,
|
||||
};
|
||||
@@ -0,0 +1,219 @@
|
||||
import type { Page } from 'playwright';
|
||||
import { ChallengeError, ResolveError, type Media, type Post } from '../types.ts';
|
||||
import { cookieHeaderFor } from '../browser/pool.ts';
|
||||
import { CHALLENGE_SELECTORS, park } from '../challenge/registry.ts';
|
||||
import { config } from '../config.ts';
|
||||
import { readInlineJson, readOpenGraph } from '../browser/capture.ts';
|
||||
import { hostMatcher } from './url.ts';
|
||||
import type { PlatformSpec, ResolveContext } from './types.ts';
|
||||
|
||||
const REFERER = 'https://www.tiktok.com/';
|
||||
|
||||
type UrlList = { UrlList?: string[] };
|
||||
|
||||
type ItemStruct = {
|
||||
id?: string;
|
||||
desc?: string;
|
||||
createTime?: number | string;
|
||||
author?: { uniqueId?: string; nickname?: string; avatarMedium?: string; avatarLarger?: string };
|
||||
video?: {
|
||||
playAddr?: string;
|
||||
downloadAddr?: string;
|
||||
cover?: string;
|
||||
duration?: number;
|
||||
width?: number;
|
||||
height?: number;
|
||||
bitrateInfo?: Array<{ Bitrate?: number; PlayAddr?: UrlList }>;
|
||||
};
|
||||
imagePost?: {
|
||||
images?: Array<{ imageURL?: UrlList; imageWidth?: number; imageHeight?: number }>;
|
||||
};
|
||||
};
|
||||
|
||||
type Rehydration = {
|
||||
__DEFAULT_SCOPE__?: Record<string, { itemInfo?: { itemStruct?: ItemStruct } } | undefined>;
|
||||
};
|
||||
|
||||
/**
|
||||
* The post lives under `webapp.video-detail` for a video and a different key
|
||||
* for a photo post, so take whichever scope actually carries one rather than
|
||||
* naming them.
|
||||
*/
|
||||
function findItemStruct(data: Rehydration | undefined): ItemStruct | undefined {
|
||||
for (const scope of Object.values(data?.__DEFAULT_SCOPE__ ?? {})) {
|
||||
const item = scope?.itemInfo?.itemStruct;
|
||||
if (item) return item;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* TikTok's CDN checks the referrer and, for some regions, the session
|
||||
* cookies the page was served with. The proxy replays both.
|
||||
*/
|
||||
async function tiktokHeaders(url: string): Promise<Record<string, string>> {
|
||||
const cookie = await cookieHeaderFor(url);
|
||||
return {
|
||||
Referer: REFERER,
|
||||
...(cookie ? { Cookie: cookie } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Shared TikTok links lead with the author, e.g. `@user/video/123`. */
|
||||
function handleFrom(url: string): string | undefined {
|
||||
const match = /tiktok\.com\/(@[\w.-]+)\//.exec(url);
|
||||
return match?.[1];
|
||||
}
|
||||
|
||||
function bestPlayAddr(video: NonNullable<ItemStruct['video']>): string | undefined {
|
||||
const ranked = (video.bitrateInfo ?? [])
|
||||
.filter((entry) => entry.PlayAddr?.UrlList?.length)
|
||||
.sort((a, b) => (b.Bitrate ?? 0) - (a.Bitrate ?? 0))[0];
|
||||
return ranked?.PlayAddr?.UrlList?.[0] ?? video.playAddr ?? video.downloadAddr;
|
||||
}
|
||||
|
||||
export async function mediaFromItem(item: ItemStruct): Promise<Media[]> {
|
||||
// A TikTok photo post carries no video at all.
|
||||
const photos = item.imagePost?.images;
|
||||
if (photos?.length) {
|
||||
const results: Media[] = [];
|
||||
for (const photo of photos) {
|
||||
const url = photo.imageURL?.UrlList?.[0];
|
||||
if (!url) continue;
|
||||
results.push({
|
||||
kind: 'image',
|
||||
url,
|
||||
fetchHeaders: await tiktokHeaders(url),
|
||||
...(photo.imageWidth ? { width: photo.imageWidth } : {}),
|
||||
...(photo.imageHeight ? { height: photo.imageHeight } : {}),
|
||||
});
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
const video = item.video;
|
||||
const src = video ? bestPlayAddr(video) : undefined;
|
||||
if (!video || !src) return [];
|
||||
|
||||
return [{
|
||||
kind: 'video',
|
||||
url: src,
|
||||
fetchHeaders: await tiktokHeaders(src),
|
||||
...(video.cover
|
||||
? { poster: { url: video.cover, fetchHeaders: await tiktokHeaders(video.cover) } }
|
||||
: {}),
|
||||
...(video.duration ? { durationSec: video.duration } : {}),
|
||||
...(video.width ? { width: video.width } : {}),
|
||||
...(video.height ? { height: video.height } : {}),
|
||||
}];
|
||||
}
|
||||
|
||||
/**
|
||||
* TikTok sometimes answers with a slider puzzle instead of the post.
|
||||
*
|
||||
* It renders a moment after the document is ready, so this waits rather than
|
||||
* looking once — checking too early reports a broken adapter when the real
|
||||
* answer is "there is a puzzle in the way".
|
||||
*/
|
||||
async function isChallenged(page: Page): Promise<boolean> {
|
||||
const appeared = await page
|
||||
.waitForSelector(CHALLENGE_SELECTORS, { state: 'visible', timeout: 5_000 })
|
||||
.then(() => true)
|
||||
.catch(() => false);
|
||||
if (appeared) return true;
|
||||
|
||||
return page
|
||||
.evaluate(() => /drag the slider|verify to continue/i.test(document.body?.innerText ?? ''))
|
||||
.catch(() => false);
|
||||
}
|
||||
|
||||
async function resolve(ctx: ResolveContext): Promise<Post> {
|
||||
const { page, originalUrl, detach } = ctx;
|
||||
|
||||
// Short links (`vm.`/`vt.`) are redirect stubs; loading one lands on the
|
||||
// real post, and `page.url()` is then the canonical address.
|
||||
await page.goto(originalUrl, { waitUntil: 'domcontentloaded' });
|
||||
const landed = page.url();
|
||||
|
||||
const data = await readInlineJson<Rehydration>(
|
||||
page,
|
||||
'script#__UNIVERSAL_DATA_FOR_REHYDRATION__',
|
||||
config.navigationTimeoutMs,
|
||||
);
|
||||
const item = findItemStruct(data);
|
||||
|
||||
if (item) {
|
||||
const media = await mediaFromItem(item);
|
||||
if (media.length > 0) {
|
||||
const handle = item.author?.uniqueId ?? 'tiktok';
|
||||
const avatar = item.author?.avatarLarger ?? item.author?.avatarMedium;
|
||||
const created = Number(item.createTime);
|
||||
return {
|
||||
platform: 'tiktok',
|
||||
platformLabel: 'TikTok',
|
||||
originalUrl: landed.startsWith('https://www.tiktok.com/') ? landed : originalUrl,
|
||||
author: {
|
||||
handle: `@${handle}`,
|
||||
...(item.author?.nickname ? { displayName: item.author.nickname } : {}),
|
||||
...(avatar ? { avatar: { url: avatar, fetchHeaders: await tiktokHeaders(avatar) } } : {}),
|
||||
},
|
||||
...(item.desc ? { text: item.desc } : {}),
|
||||
textPosition: 'below',
|
||||
media,
|
||||
...(Number.isFinite(created) && created > 0
|
||||
? { postedAt: new Date(created * 1000).toISOString() }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to the link preview, which at least gets a poster on screen.
|
||||
if (await isChallenged(page)) {
|
||||
// Keep the page open and hand the puzzle to whoever is reading. Solving
|
||||
// it deposits the cookie in the shared browser context, so the retry
|
||||
// afterwards is an ordinary request.
|
||||
detach();
|
||||
const id = await park({
|
||||
page,
|
||||
platform: 'tiktok',
|
||||
platformLabel: 'TikTok',
|
||||
originalUrl,
|
||||
});
|
||||
throw new ChallengeError(id, 'tiktok', originalUrl);
|
||||
}
|
||||
|
||||
const og = await readOpenGraph(page);
|
||||
if (og['og:video'] ?? og['og:image']) {
|
||||
const url = og['og:video'] ?? (og['og:image'] as string);
|
||||
return {
|
||||
platform: 'tiktok',
|
||||
platformLabel: 'TikTok',
|
||||
originalUrl,
|
||||
author: { handle: handleFrom(landed) ?? handleFrom(originalUrl) ?? '@tiktok' },
|
||||
...(og['og:description'] ? { text: og['og:description'] } : {}),
|
||||
textPosition: 'below',
|
||||
media: [{
|
||||
kind: og['og:video'] ? 'video' : 'image',
|
||||
url,
|
||||
fetchHeaders: await tiktokHeaders(url),
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
throw new ResolveError('TikTok would not return that post', 'tiktok', originalUrl);
|
||||
}
|
||||
|
||||
export const tiktok: PlatformSpec = {
|
||||
id: 'tiktok',
|
||||
label: 'TikTok',
|
||||
prefix: 'tiktok',
|
||||
canonicalHost: 'www.tiktok.com',
|
||||
textPosition: 'below',
|
||||
matchesHost: hostMatcher('tiktok.com'),
|
||||
// A single opaque segment is a `vm.`/`vt.` share code, not a post path.
|
||||
buildOriginalUrl: (rest, search) =>
|
||||
/^[A-Za-z0-9]+\/?$/.test(rest)
|
||||
? `https://vm.tiktok.com/${rest.replace(/\/$/, '')}/`
|
||||
: `https://www.tiktok.com/${rest}${search}`,
|
||||
resolve,
|
||||
};
|
||||
@@ -0,0 +1,36 @@
|
||||
import type { Page } from 'playwright';
|
||||
import type { PlatformId, Post } from '../types.ts';
|
||||
|
||||
export type PlatformSpec = {
|
||||
id: PlatformId;
|
||||
/** Human label for the badge. */
|
||||
label: string;
|
||||
/** The path segment antisocial serves this platform under. */
|
||||
prefix: string;
|
||||
/** Host used when rebuilding the original URL from our path. */
|
||||
canonicalHost: string;
|
||||
textPosition: 'above' | 'below';
|
||||
/** Matches the host of an original URL, so a pasted link can be routed. */
|
||||
matchesHost: (host: string) => boolean;
|
||||
/**
|
||||
* Rebuild the canonical original URL from the path remainder and query
|
||||
* string we were handed. Overridden where a straight host swap is wrong
|
||||
* (TikTok short links, for one).
|
||||
*/
|
||||
buildOriginalUrl?: (pathRest: string, search: string) => string;
|
||||
resolve: Adapter;
|
||||
};
|
||||
|
||||
export type ResolveContext = {
|
||||
page: Page;
|
||||
/** Keep the page open past this resolve; the caller closes it instead.
|
||||
* Used to hand a challenge page to the viewer. */
|
||||
detach: () => void;
|
||||
spec: PlatformSpec;
|
||||
/** Canonical original URL, tracking parameters already stripped. */
|
||||
originalUrl: string;
|
||||
/** Path remainder after the platform prefix, no leading slash. */
|
||||
pathRest: string;
|
||||
};
|
||||
|
||||
export type Adapter = (ctx: ResolveContext) => Promise<Post>;
|
||||
@@ -0,0 +1,28 @@
|
||||
/**
|
||||
* Query parameters that only exist to record where a link travelled. They
|
||||
* are dropped so the copy button hands back something clean and the cache
|
||||
* does not store one post under a dozen keys.
|
||||
*/
|
||||
const TRACKING_PARAMS = new Set([
|
||||
'igsh', 'igshid', 'img_index', 'fbclid', 'gclid', 'mibextid',
|
||||
'ref_src', 'ref_url', 'ref', 'source', 'checksum', 'social_sharing',
|
||||
's', 't', 'si', 'xmt', '_r', '_t', '_d',
|
||||
'is_from_webapp', 'sender_device', 'sender_web_id', 'web_id',
|
||||
'share_app_id', 'share_link_id', 'share_item_id', 'tt_from',
|
||||
]);
|
||||
|
||||
export function stripTracking(url: URL): URL {
|
||||
const cleaned = new URL(url.href);
|
||||
for (const key of [...cleaned.searchParams.keys()]) {
|
||||
if (TRACKING_PARAMS.has(key) || key.startsWith('utm_')) {
|
||||
cleaned.searchParams.delete(key);
|
||||
}
|
||||
}
|
||||
cleaned.hash = '';
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
/** Exact-host or subdomain match, e.g. `tiktok.com` and `vm.tiktok.com`. */
|
||||
export function hostMatcher(...domains: string[]): (host: string) => boolean {
|
||||
return (host) => domains.some((d) => host === d || host.endsWith(`.${d}`));
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
import { config } from '../config.ts';
|
||||
import { ResolveError, type Media, type Post } from '../types.ts';
|
||||
import { captureJson } from '../browser/capture.ts';
|
||||
import { hostMatcher } from './url.ts';
|
||||
import type { PlatformSpec, ResolveContext } from './types.ts';
|
||||
|
||||
type Size = { w?: number; h?: number };
|
||||
|
||||
type VideoVariant = { bitrate?: number; content_type?: string; url?: string };
|
||||
|
||||
type MediaDetail = {
|
||||
type?: 'photo' | 'video' | 'animated_gif';
|
||||
media_url_https?: string;
|
||||
ext_alt_text?: string;
|
||||
original_info?: Size;
|
||||
video_info?: { duration_millis?: number; variants?: VideoVariant[] };
|
||||
};
|
||||
|
||||
type TweetResult = {
|
||||
text?: string;
|
||||
created_at?: string;
|
||||
user?: { screen_name?: string; name?: string; profile_image_url_https?: string };
|
||||
mediaDetails?: MediaDetail[];
|
||||
quoted_tweet?: TweetResult;
|
||||
tombstone?: unknown;
|
||||
};
|
||||
|
||||
/** Every X post URL ends in `/status/<id>`, whatever precedes it. */
|
||||
function tweetId(pathRest: string): string | undefined {
|
||||
return /\/status(?:es)?\/(\d+)/.exec(`/${pathRest}`)?.[1];
|
||||
}
|
||||
|
||||
/**
|
||||
* The syndication endpoint wants a token derived from the id. This is the
|
||||
* same arithmetic the embed script does before calling it.
|
||||
*/
|
||||
function syndicationToken(id: string): string {
|
||||
return ((Number(id) / 1e15) * Math.PI).toString(36).replace(/(0+|\.)/g, '');
|
||||
}
|
||||
|
||||
function bestVariant(variants: VideoVariant[] | undefined): VideoVariant | undefined {
|
||||
return (variants ?? [])
|
||||
.filter((v) => v.content_type === 'video/mp4' && v.url)
|
||||
.sort((a, b) => (b.bitrate ?? 0) - (a.bitrate ?? 0))[0];
|
||||
}
|
||||
|
||||
export function mediaFromDetails(details: MediaDetail[] | undefined): Media[] {
|
||||
return (details ?? []).flatMap<Media>((detail) => {
|
||||
const size = {
|
||||
...(detail.original_info?.w ? { width: detail.original_info.w } : {}),
|
||||
...(detail.original_info?.h ? { height: detail.original_info.h } : {}),
|
||||
};
|
||||
|
||||
if (detail.type === 'photo') {
|
||||
if (!detail.media_url_https) return [];
|
||||
return [{
|
||||
kind: 'image',
|
||||
// `?name=orig` asks twimg for the untouched upload.
|
||||
url: `${detail.media_url_https}?name=orig`,
|
||||
...(detail.ext_alt_text ? { alt: detail.ext_alt_text } : {}),
|
||||
...size,
|
||||
}];
|
||||
}
|
||||
|
||||
const variant = bestVariant(detail.video_info?.variants);
|
||||
if (!variant?.url) return [];
|
||||
const duration = detail.video_info?.duration_millis;
|
||||
return [{
|
||||
kind: 'video',
|
||||
url: variant.url,
|
||||
...(detail.media_url_https ? { poster: { url: detail.media_url_https } } : {}),
|
||||
...(duration ? { durationSec: Math.round(duration / 1000) } : {}),
|
||||
...size,
|
||||
}];
|
||||
});
|
||||
}
|
||||
|
||||
export function toPost(result: TweetResult, originalUrl: string): Post {
|
||||
const handle = result.user?.screen_name ?? 'unknown';
|
||||
const avatar = result.user?.profile_image_url_https;
|
||||
return {
|
||||
platform: 'x',
|
||||
platformLabel: 'X',
|
||||
originalUrl,
|
||||
author: {
|
||||
handle: `@${handle}`,
|
||||
...(result.user?.name ? { displayName: result.user.name } : {}),
|
||||
// `_normal` is a 48px thumbnail; `_x96` is the same file at 96.
|
||||
...(avatar ? { avatar: { url: avatar.replace('_normal', '_x96') } } : {}),
|
||||
},
|
||||
...(result.text ? { text: result.text } : {}),
|
||||
textPosition: 'above',
|
||||
// A quote post usually carries no media of its own — the picture being
|
||||
// talked about belongs to the post it quotes, and that is the thing
|
||||
// worth showing.
|
||||
media: mediaFromDetails(
|
||||
result.mediaDetails?.length ? result.mediaDetails : result.quoted_tweet?.mediaDetails,
|
||||
),
|
||||
...(result.created_at ? { postedAt: result.created_at } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
async function resolve({ page, originalUrl, pathRest }: ResolveContext): Promise<Post> {
|
||||
const id = tweetId(pathRest);
|
||||
if (!id) throw new ResolveError('No post id in that X URL', 'x', originalUrl);
|
||||
|
||||
// The official embed renders logged out, and calls the syndication
|
||||
// endpoint on our behalf. Catching that response gives structured media
|
||||
// with alt text, rather than whatever the embed chose to draw.
|
||||
const captured = captureJson<TweetResult>(page, (url) => url.includes('/tweet-result'));
|
||||
|
||||
const embedUrl = new URL('https://platform.twitter.com/embed/Tweet.html');
|
||||
embedUrl.searchParams.set('id', id);
|
||||
embedUrl.searchParams.set('lang', 'en');
|
||||
await page.goto(embedUrl.href, { waitUntil: 'domcontentloaded' }).catch(() => undefined);
|
||||
|
||||
let result = await captured.wait(config.navigationTimeoutMs);
|
||||
|
||||
if (!result?.user) {
|
||||
// Ask for it ourselves if the embed did not, or was slow.
|
||||
const api = new URL('https://cdn.syndication.twimg.com/tweet-result');
|
||||
api.searchParams.set('id', id);
|
||||
api.searchParams.set('lang', 'en');
|
||||
api.searchParams.set('token', syndicationToken(id));
|
||||
result = await page.request
|
||||
.get(api.href, { headers: { Referer: 'https://platform.twitter.com/' } })
|
||||
.then((r) => (r.ok() ? (r.json() as Promise<TweetResult>) : undefined))
|
||||
.catch(() => undefined);
|
||||
}
|
||||
|
||||
if (!result?.user || result.tombstone) {
|
||||
throw new ResolveError(
|
||||
'X would not return that post — it may be deleted, protected, or age restricted',
|
||||
'x',
|
||||
originalUrl,
|
||||
);
|
||||
}
|
||||
|
||||
return toPost(result, originalUrl);
|
||||
}
|
||||
|
||||
export const x: PlatformSpec = {
|
||||
id: 'x',
|
||||
label: 'X',
|
||||
prefix: 'x',
|
||||
canonicalHost: 'x.com',
|
||||
textPosition: 'above',
|
||||
matchesHost: hostMatcher('x.com', 'twitter.com'),
|
||||
resolve,
|
||||
};
|
||||
@@ -0,0 +1,40 @@
|
||||
import type { Challenge } from '../challenge/registry.ts';
|
||||
import { html } from './html.ts';
|
||||
import { badge, layout } from './layout.ts';
|
||||
|
||||
type Box = { x: number; y: number; width: number; height: number };
|
||||
|
||||
export function renderChallenge(challenge: Challenge, box: Box | undefined): string {
|
||||
const body = html`<article class="post post--challenge" data-challenge="${challenge.id}">
|
||||
<header class="post__head">
|
||||
${badge(challenge.platform, challenge.platformLabel)}
|
||||
<div class="who"><span class="who__handle">wants you to prove you are a person</span></div>
|
||||
</header>
|
||||
|
||||
<p class="challenge__lede">
|
||||
Solve it here and the post loads. Drag on the picture exactly as you would on
|
||||
${challenge.platformLabel}.
|
||||
</p>
|
||||
|
||||
<div
|
||||
class="challenge__stage"
|
||||
id="stage"
|
||||
data-box="${box ? `${box.x},${box.y},${box.width},${box.height}` : ''}"
|
||||
>
|
||||
<img id="frame" alt="The verification puzzle" draggable="false">
|
||||
<p class="challenge__status" id="status">Loading…</p>
|
||||
</div>
|
||||
|
||||
<footer class="post__foot">
|
||||
<div class="original">
|
||||
<button type="button" class="copy" id="give-up">Give up and show me the link</button>
|
||||
<a class="original__open" href="${challenge.originalUrl}" rel="noopener noreferrer nofollow" target="_blank">
|
||||
Open on ${challenge.platformLabel}
|
||||
</a>
|
||||
<p class="original__url"><code>${challenge.originalUrl}</code></p>
|
||||
</div>
|
||||
</footer>
|
||||
</article>`;
|
||||
|
||||
return layout(`Verify — ${challenge.platformLabel}`, body);
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { html } from './html.ts';
|
||||
import { badge, layout, originalUrlBlock } from './layout.ts';
|
||||
|
||||
/**
|
||||
* A resolve can always fail — someone else's markup changed, the post was
|
||||
* deleted, a platform tightened a screw. Failing still has to leave the
|
||||
* link one tap away, so this is a card rather than a status page.
|
||||
*/
|
||||
export function renderError(options: {
|
||||
platform?: string;
|
||||
platformLabel?: string;
|
||||
originalUrl?: string;
|
||||
heading: string;
|
||||
detail?: string;
|
||||
}): string {
|
||||
const { platform, platformLabel = 'the original', originalUrl, heading, detail } = options;
|
||||
|
||||
const body = html`<article class="post post--error">
|
||||
<header class="post__head">
|
||||
${platform ? badge(platform, platformLabel) : ''}
|
||||
</header>
|
||||
<h1 class="error__heading">${heading}</h1>
|
||||
${detail ? html`<p class="error__detail">${detail}</p>` : ''}
|
||||
${originalUrl
|
||||
? html`<footer class="post__foot">${originalUrlBlock(originalUrl, platformLabel)}</footer>`
|
||||
: ''}
|
||||
</article>`;
|
||||
|
||||
return layout(heading, body);
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
/** A string that is already safe to drop into the document. */
|
||||
export class Raw {
|
||||
readonly value: string;
|
||||
constructor(value: string) {
|
||||
this.value = value;
|
||||
}
|
||||
toString(): string {
|
||||
return this.value;
|
||||
}
|
||||
}
|
||||
|
||||
export function raw(value: string): Raw {
|
||||
return new Raw(value);
|
||||
}
|
||||
|
||||
const ESCAPES: Record<string, string> = {
|
||||
'&': '&',
|
||||
'<': '<',
|
||||
'>': '>',
|
||||
'"': '"',
|
||||
"'": ''',
|
||||
};
|
||||
|
||||
export function escapeHtml(value: string): string {
|
||||
return value.replace(/[&<>"']/g, (char) => ESCAPES[char] ?? char);
|
||||
}
|
||||
|
||||
function render(value: unknown): string {
|
||||
if (value === null || value === undefined || value === false) return '';
|
||||
if (value instanceof Raw) return value.value;
|
||||
if (Array.isArray(value)) return value.map(render).join('');
|
||||
return escapeHtml(String(value));
|
||||
}
|
||||
|
||||
/**
|
||||
* Tagged template that escapes every interpolation unless it is already
|
||||
* `Raw`. Post text comes from strangers, so nothing reaches the document
|
||||
* without passing through here.
|
||||
*/
|
||||
export function html(strings: TemplateStringsArray, ...values: unknown[]): Raw {
|
||||
let out = strings[0] ?? '';
|
||||
for (let i = 0; i < values.length; i += 1) {
|
||||
out += render(values[i]) + (strings[i + 1] ?? '');
|
||||
}
|
||||
return new Raw(out);
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import { config } from '../config.ts';
|
||||
import type { PlatformSpec } from '../platforms/types.ts';
|
||||
import { html } from './html.ts';
|
||||
import { badge, layout } from './layout.ts';
|
||||
|
||||
/**
|
||||
* There is no useful landing page for a redirect target, so this is the
|
||||
* reference card instead: which prefixes exist, and the rewrite rules to
|
||||
* paste into StopTheMadness.
|
||||
*/
|
||||
export function renderIndex(platforms: readonly PlatformSpec[]): string {
|
||||
const origin = config.publicOrigin;
|
||||
|
||||
const body = html`<article class="post post--index">
|
||||
<h1 class="index__title">antisocial</h1>
|
||||
<p class="index__lede">
|
||||
Point StopTheMadness at these and shared links open here instead of in an app.
|
||||
</p>
|
||||
<table class="index__table">
|
||||
<thead>
|
||||
<tr><th>Platform</th><th>Matches</th><th>Rewrites to</th></tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
${platforms.map(
|
||||
(p) => html`<tr>
|
||||
<td>${badge(p.id, p.label)}</td>
|
||||
<td><code>${p.canonicalHost}</code></td>
|
||||
<td><code>${origin}/${p.prefix}/…</code></td>
|
||||
</tr>`,
|
||||
)}
|
||||
</tbody>
|
||||
</table>
|
||||
</article>`;
|
||||
|
||||
return layout('antisocial', body);
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { html, type Raw } from './html.ts';
|
||||
|
||||
export function layout(title: string, body: Raw): string {
|
||||
return `<!doctype html>
|
||||
${html`<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover">
|
||||
<meta name="robots" content="noindex, nofollow">
|
||||
<meta name="referrer" content="no-referrer">
|
||||
<meta name="color-scheme" content="light dark">
|
||||
<title>${title}</title>
|
||||
<link rel="stylesheet" href="/static/app.css">
|
||||
<link rel="icon" href="/static/favicon.svg" type="image/svg+xml">
|
||||
</head>
|
||||
<body>
|
||||
<main>${body}</main>
|
||||
<script src="/static/app.js" type="module"></script>
|
||||
</body>
|
||||
</html>`}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Indicative brand accents. The written label is what actually identifies
|
||||
* the platform; the colour is only there to make it recognisable at a
|
||||
* glance.
|
||||
*/
|
||||
const ACCENT: Record<string, { color: string; monogram: string }> = {
|
||||
x: { color: '#0f1419', monogram: '✕' },
|
||||
threads: { color: '#3b3b3b', monogram: '@' },
|
||||
instagram: { color: '#c13584', monogram: 'IG' },
|
||||
tiktok: { color: '#fe2c55', monogram: '♪' },
|
||||
bluesky: { color: '#0085ff', monogram: 'B' },
|
||||
};
|
||||
|
||||
export function badge(platform: string, label: string): Raw {
|
||||
const accent = ACCENT[platform] ?? { color: '#666', monogram: '?' };
|
||||
return html`<span class="badge" style="--accent: ${accent.color}">
|
||||
<span class="badge__mark" aria-hidden="true">${accent.monogram}</span>
|
||||
<span class="badge__label">${label}</span>
|
||||
</span>`;
|
||||
}
|
||||
|
||||
/** The copy control, plus the URL itself so it is always selectable even if
|
||||
* the clipboard API is unavailable. */
|
||||
export function originalUrlBlock(originalUrl: string, platformLabel: string): Raw {
|
||||
return html`<div class="original">
|
||||
<button type="button" class="copy" data-url="${originalUrl}">
|
||||
<span class="copy__idle">Copy original link</span>
|
||||
<span class="copy__done" hidden>Copied</span>
|
||||
</button>
|
||||
<a class="original__open" href="${originalUrl}" rel="noopener noreferrer nofollow" target="_blank">Open on ${platformLabel}</a>
|
||||
<p class="original__url"><code>${originalUrl}</code></p>
|
||||
</div>`;
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
import { proxyUrlFor } from '../media/registry.ts';
|
||||
import type { Media, Post } from '../types.ts';
|
||||
import { html, type Raw } from './html.ts';
|
||||
import { badge, layout, originalUrlBlock } from './layout.ts';
|
||||
import { linkify } from './text.ts';
|
||||
|
||||
function aspect(item: Media): string {
|
||||
return item.width && item.height ? `aspect-ratio: ${item.width} / ${item.height};` : '';
|
||||
}
|
||||
|
||||
function renderItem(item: Media, index: number): Raw {
|
||||
const src = proxyUrlFor(item);
|
||||
|
||||
if (item.kind === 'video') {
|
||||
const poster = item.poster ? proxyUrlFor(item.poster) : undefined;
|
||||
return html`<figure class="item item--video" data-index="${index}" style="${aspect(item)}">
|
||||
<video
|
||||
controls
|
||||
playsinline
|
||||
preload="metadata"
|
||||
${poster ? html`poster="${poster}"` : ''}
|
||||
><source src="${src}"></video>
|
||||
${item.hls
|
||||
? html`<figcaption class="item__note">Streaming format — plays natively in Safari.</figcaption>`
|
||||
: ''}
|
||||
</figure>`;
|
||||
}
|
||||
|
||||
return html`<figure class="item item--image" data-index="${index}" style="${aspect(item)}">
|
||||
<img
|
||||
src="${src}"
|
||||
alt="${item.alt ?? ''}"
|
||||
loading="${index === 0 ? 'eager' : 'lazy'}"
|
||||
decoding="async"
|
||||
>
|
||||
</figure>`;
|
||||
}
|
||||
|
||||
function renderMedia(media: Media[]): Raw {
|
||||
if (media.length === 0) return html``;
|
||||
|
||||
const many = media.length > 1;
|
||||
return html`<section class="media" data-count="${media.length}">
|
||||
${many
|
||||
? html`<div class="media__bar">
|
||||
<span class="media__count"><span class="media__at">1</span> / ${media.length}</span>
|
||||
<div class="media__views" role="group" aria-label="Layout">
|
||||
<button type="button" class="media__view" data-view="swipe" aria-pressed="true">Swipe</button>
|
||||
<button type="button" class="media__view" data-view="grid" aria-pressed="false">Grid</button>
|
||||
</div>
|
||||
</div>`
|
||||
: ''}
|
||||
<div class="media__rail">${media.map((item, index) => renderItem(item, index))}</div>
|
||||
</section>`;
|
||||
}
|
||||
|
||||
function renderText(post: Post): Raw {
|
||||
if (!post.text) return html``;
|
||||
return html`<div class="text">${linkify(post.text, post.platform)}</div>`;
|
||||
}
|
||||
|
||||
function renderWhen(post: Post): Raw {
|
||||
if (!post.postedAt) return html``;
|
||||
const date = new Date(post.postedAt);
|
||||
if (Number.isNaN(date.getTime())) return html``;
|
||||
return html`<time datetime="${date.toISOString()}">${date.toLocaleString('en-CA', {
|
||||
dateStyle: 'medium',
|
||||
timeStyle: 'short',
|
||||
})}</time>`;
|
||||
}
|
||||
|
||||
export function renderPost(post: Post): string {
|
||||
const avatar = post.author.avatar ? proxyUrlFor(post.author.avatar) : undefined;
|
||||
const above = post.textPosition === 'above';
|
||||
|
||||
const body = html`<article class="post" data-platform="${post.platform}">
|
||||
<header class="post__head">
|
||||
${badge(post.platform, post.platformLabel)}
|
||||
<div class="who">
|
||||
${avatar ? html`<img class="who__avatar" src="${avatar}" alt="" loading="lazy">` : ''}
|
||||
<div class="who__names">
|
||||
${post.author.displayName ? html`<span class="who__name">${post.author.displayName}</span>` : ''}
|
||||
<span class="who__handle">${post.author.handle}</span>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
${above ? renderText(post) : ''}
|
||||
${renderMedia(post.media)}
|
||||
${above ? '' : renderText(post)}
|
||||
|
||||
<footer class="post__foot">
|
||||
${renderWhen(post)}
|
||||
${originalUrlBlock(post.originalUrl, post.platformLabel)}
|
||||
</footer>
|
||||
</article>`;
|
||||
|
||||
const who = post.author.displayName ?? post.author.handle;
|
||||
return layout(`${who} on ${post.platformLabel}`, body);
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import { escapeHtml, raw, type Raw } from './html.ts';
|
||||
import type { PlatformId } from '../types.ts';
|
||||
|
||||
/** Where an @mention or #hashtag points on each platform. */
|
||||
const PROFILE_BASE: Record<PlatformId, string> = {
|
||||
x: 'https://x.com/',
|
||||
threads: 'https://www.threads.com/@',
|
||||
instagram: 'https://www.instagram.com/',
|
||||
tiktok: 'https://www.tiktok.com/@',
|
||||
bluesky: 'https://bsky.app/profile/',
|
||||
};
|
||||
|
||||
const TAG_BASE: Partial<Record<PlatformId, string>> = {
|
||||
x: 'https://x.com/hashtag/',
|
||||
instagram: 'https://www.instagram.com/explore/tags/',
|
||||
tiktok: 'https://www.tiktok.com/tag/',
|
||||
};
|
||||
|
||||
const TOKEN = /(https?:\/\/[^\s<]+)|(^|\s)(@[\w.-]+)|(^|\s)(#[\w]+)/g;
|
||||
|
||||
function link(href: string, label: string): string {
|
||||
return `<a href="${escapeHtml(href)}" rel="noopener noreferrer nofollow" target="_blank">${escapeHtml(label)}</a>`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Turn post text into markup: escape everything, then make links, mentions
|
||||
* and hashtags clickable back to the platform they came from.
|
||||
*/
|
||||
export function linkify(text: string, platform: PlatformId): Raw {
|
||||
let out = '';
|
||||
let cursor = 0;
|
||||
|
||||
for (const match of text.matchAll(TOKEN)) {
|
||||
const index = match.index;
|
||||
out += escapeHtml(text.slice(cursor, index));
|
||||
|
||||
const [whole, url, mentionLead, mention, tagLead, tag] = match;
|
||||
if (url) {
|
||||
out += link(url, url.replace(/^https?:\/\/(www\.)?/, ''));
|
||||
} else if (mention) {
|
||||
out += escapeHtml(mentionLead ?? '') + link(PROFILE_BASE[platform] + mention.slice(1), mention);
|
||||
} else if (tag) {
|
||||
const base = TAG_BASE[platform];
|
||||
out += escapeHtml(tagLead ?? '') + (base ? link(base + tag.slice(1), tag) : escapeHtml(tag));
|
||||
} else {
|
||||
out += escapeHtml(whole);
|
||||
}
|
||||
|
||||
cursor = index + whole.length;
|
||||
}
|
||||
|
||||
out += escapeHtml(text.slice(cursor));
|
||||
return raw(out.replace(/\n/g, '<br>'));
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import { LRUCache } from 'lru-cache';
|
||||
import { config } from './config.ts';
|
||||
import { withPage } from './browser/pool.ts';
|
||||
import { withTimeout } from './browser/queue.ts';
|
||||
import { ResolveError, type Post } from './types.ts';
|
||||
import type { PlatformSpec } from './platforms/types.ts';
|
||||
|
||||
/**
|
||||
* Resolved posts, keyed by canonical URL. Reloading, hitting back, or
|
||||
* opening the same link twice must not drive Chromium again.
|
||||
*/
|
||||
const cache = new LRUCache<string, Post>({
|
||||
max: config.cacheMax,
|
||||
ttl: config.cacheTtlMs,
|
||||
});
|
||||
|
||||
export function cacheSize(): number {
|
||||
return cache.size;
|
||||
}
|
||||
|
||||
export async function resolvePost(
|
||||
spec: PlatformSpec,
|
||||
originalUrl: string,
|
||||
pathRest: string,
|
||||
): Promise<Post> {
|
||||
const hit = cache.get(originalUrl);
|
||||
if (hit) return hit;
|
||||
|
||||
const post = await withTimeout(
|
||||
withPage(({ page, detach }) => spec.resolve({ page, detach, spec, originalUrl, pathRest })),
|
||||
config.resolveTimeoutMs,
|
||||
`${spec.label} resolve`,
|
||||
).catch((error: unknown) => {
|
||||
if (error instanceof ResolveError) throw error;
|
||||
throw new ResolveError(
|
||||
error instanceof Error ? error.message : String(error),
|
||||
spec.id,
|
||||
originalUrl,
|
||||
{ cause: error },
|
||||
);
|
||||
});
|
||||
|
||||
cache.set(originalUrl, post);
|
||||
return post;
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
import {
|
||||
CHALLENGE_SELECTORS,
|
||||
get,
|
||||
isSolved,
|
||||
release,
|
||||
type Challenge,
|
||||
} from '../challenge/registry.ts';
|
||||
import { renderChallenge } from '../render/challenge.ts';
|
||||
import { renderError } from '../render/error.ts';
|
||||
|
||||
type Box = { x: number; y: number; width: number; height: number };
|
||||
|
||||
/**
|
||||
* The puzzle is rendered in a real browser here, not in the viewer's. So
|
||||
* frames go out as screenshots and pointer events come back in and get
|
||||
* replayed onto the page. Solving it leaves the cookie in the shared browser
|
||||
* context, which is the whole point: the retry then behaves like any other
|
||||
* request.
|
||||
*/
|
||||
async function frameBox(challenge: Challenge): Promise<Box | undefined> {
|
||||
// Prefer the puzzle itself over the whole viewport — on a phone the
|
||||
// difference is between a usable slider and a postage stamp.
|
||||
const box = await challenge.page
|
||||
.locator(CHALLENGE_SELECTORS)
|
||||
.first()
|
||||
.boundingBox({ timeout: 2_000 })
|
||||
.catch(() => null);
|
||||
return box && box.width > 40 && box.height > 40 ? box : undefined;
|
||||
}
|
||||
|
||||
export function challengeRoutes(app: FastifyInstance): void {
|
||||
app.get<{ Params: { id: string } }>('/challenge/:id', async (request, reply) => {
|
||||
const challenge = get(request.params.id);
|
||||
if (!challenge) {
|
||||
return reply
|
||||
.code(410)
|
||||
.type('text/html; charset=utf-8')
|
||||
.send(
|
||||
renderError({
|
||||
heading: 'That puzzle has expired',
|
||||
detail: 'Open the link again to get a fresh one.',
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
const box = await frameBox(challenge);
|
||||
return reply
|
||||
.type('text/html; charset=utf-8')
|
||||
.header('cache-control', 'no-store')
|
||||
.send(renderChallenge(challenge, box));
|
||||
});
|
||||
|
||||
/** One frame of the parked page. */
|
||||
app.get<{ Params: { id: string } }>('/challenge/:id/frame', async (request, reply) => {
|
||||
const challenge = get(request.params.id);
|
||||
if (!challenge) return reply.code(410).send();
|
||||
|
||||
const box = await frameBox(challenge);
|
||||
const shot = await challenge.page
|
||||
.screenshot({ type: 'jpeg', quality: 70, ...(box ? { clip: box } : {}) })
|
||||
.catch(() => null);
|
||||
if (!shot) return reply.code(502).send();
|
||||
|
||||
return reply.type('image/jpeg').header('cache-control', 'no-store').send(shot);
|
||||
});
|
||||
|
||||
app.get<{ Params: { id: string } }>('/challenge/:id/state', async (request, reply) => {
|
||||
const challenge = get(request.params.id);
|
||||
if (!challenge) return reply.code(410).send({ gone: true });
|
||||
|
||||
const box = await frameBox(challenge);
|
||||
return reply.header('cache-control', 'no-store').send({
|
||||
solved: await isSolved(challenge),
|
||||
box: box ?? null,
|
||||
returnTo: challenge.returnTo,
|
||||
});
|
||||
});
|
||||
|
||||
/** Replay one pointer event onto the parked page. */
|
||||
app.post<{ Params: { id: string }; Body: { type: string; x: number; y: number } }>(
|
||||
'/challenge/:id/input',
|
||||
async (request, reply) => {
|
||||
const challenge = get(request.params.id);
|
||||
if (!challenge) return reply.code(410).send({ gone: true });
|
||||
|
||||
const { type, x, y } = request.body;
|
||||
if (!Number.isFinite(x) || !Number.isFinite(y)) {
|
||||
return reply.code(400).send({ error: 'x and y must be numbers' });
|
||||
}
|
||||
|
||||
const mouse = challenge.page.mouse;
|
||||
try {
|
||||
if (type === 'down') {
|
||||
await mouse.move(x, y);
|
||||
await mouse.down();
|
||||
} else if (type === 'move') {
|
||||
await mouse.move(x, y);
|
||||
} else if (type === 'up') {
|
||||
await mouse.move(x, y);
|
||||
await mouse.up();
|
||||
} else {
|
||||
return reply.code(400).send({ error: `unknown event ${type}` });
|
||||
}
|
||||
} catch (error) {
|
||||
request.log.warn({ err: error }, 'challenge input failed');
|
||||
return reply.code(502).send({ error: 'the page went away' });
|
||||
}
|
||||
|
||||
return reply.send({ ok: true });
|
||||
},
|
||||
);
|
||||
|
||||
/** Give up, or finish: close the tab and go back to the post. */
|
||||
app.post<{ Params: { id: string } }>('/challenge/:id/done', async (request, reply) => {
|
||||
const returnTo = await release(request.params.id);
|
||||
return reply.send({ returnTo: returnTo ?? '/' });
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
import { isReady, queueDepth } from '../browser/pool.ts';
|
||||
import { cacheSize } from '../resolve.ts';
|
||||
import { registrySize } from '../media/registry.ts';
|
||||
import { parkedCount } from '../challenge/registry.ts';
|
||||
|
||||
export function healthRoutes(app: FastifyInstance): void {
|
||||
// Liveness: the process is answering. Restarting on anything less would
|
||||
// throw away a warm browser for no reason.
|
||||
app.get('/healthz', async () => ({ ok: true }));
|
||||
|
||||
// Readiness: Chromium is up, so a request routed here can be served
|
||||
// without paying the cold start.
|
||||
app.get('/readyz', async (_request, reply) => {
|
||||
const ready = isReady();
|
||||
return reply.code(ready ? 200 : 503).send({
|
||||
ready,
|
||||
queueDepth: queueDepth(),
|
||||
cachedPosts: cacheSize(),
|
||||
mediaRefs: registrySize(),
|
||||
parkedChallenges: parkedCount(),
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
import { Readable } from 'node:stream';
|
||||
import type { ReadableStream as WebReadableStream } from 'node:stream/web';
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
import { currentUserAgent } from '../browser/pool.ts';
|
||||
import { lookupAsset } from '../media/registry.ts';
|
||||
|
||||
/** Headers worth carrying back so the browser can cache and, crucially,
|
||||
* seek within a video. */
|
||||
const PASS_THROUGH = [
|
||||
'content-type',
|
||||
'content-length',
|
||||
'content-range',
|
||||
'accept-ranges',
|
||||
'last-modified',
|
||||
'etag',
|
||||
];
|
||||
|
||||
const FALLBACK_UA =
|
||||
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36';
|
||||
|
||||
export function mediaRoutes(app: FastifyInstance): void {
|
||||
/**
|
||||
* Streams one upstream asset.
|
||||
*
|
||||
* The id only ever refers to something an adapter registered, so there is
|
||||
* no user-supplied URL here to be tricked into fetching.
|
||||
*/
|
||||
app.get<{ Params: { id: string } }>('/m/:id', async (request, reply) => {
|
||||
const asset = lookupAsset(request.params.id);
|
||||
if (!asset) {
|
||||
return reply.code(404).type('text/plain').send('Unknown media reference — reload the page.');
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
...asset.fetchHeaders,
|
||||
'User-Agent': currentUserAgent() ?? FALLBACK_UA,
|
||||
Accept: '*/*',
|
||||
'Accept-Language': 'en-CA,en;q=0.9',
|
||||
};
|
||||
|
||||
// Forwarding Range is what makes the native video scrubber work.
|
||||
const range = request.headers.range;
|
||||
if (typeof range === 'string') headers['Range'] = range;
|
||||
|
||||
let upstream: Response;
|
||||
try {
|
||||
upstream = await fetch(asset.url, { headers, redirect: 'follow' });
|
||||
} catch (error) {
|
||||
request.log.warn({ err: error, url: asset.url }, 'media fetch failed');
|
||||
return reply.code(502).type('text/plain').send('Could not reach that media.');
|
||||
}
|
||||
|
||||
if (!upstream.ok && upstream.status !== 206) {
|
||||
request.log.warn({ status: upstream.status, url: asset.url }, 'media upstream rejected');
|
||||
return reply.code(502).type('text/plain').send(`Upstream returned ${upstream.status}.`);
|
||||
}
|
||||
|
||||
for (const name of PASS_THROUGH) {
|
||||
const value = upstream.headers.get(name);
|
||||
if (value) reply.header(name, value);
|
||||
}
|
||||
reply.header('cache-control', 'private, max-age=3600');
|
||||
reply.code(upstream.status);
|
||||
|
||||
if (!upstream.body) return reply.send();
|
||||
return reply.send(Readable.fromWeb(upstream.body as WebReadableStream));
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
import { originalUrlFor, platformByPrefix, platforms } from '../platforms/index.ts';
|
||||
import { renderError } from '../render/error.ts';
|
||||
import { renderPost } from '../render/post.ts';
|
||||
import { renderIndex } from '../render/index-page.ts';
|
||||
import { resolvePost } from '../resolve.ts';
|
||||
import { ChallengeError, ResolveError } from '../types.ts';
|
||||
import { setReturnTo } from '../challenge/registry.ts';
|
||||
|
||||
export function postRoutes(app: FastifyInstance): void {
|
||||
app.get('/', async (_request, reply) =>
|
||||
reply.type('text/html; charset=utf-8').send(renderIndex(platforms)),
|
||||
);
|
||||
|
||||
app.get<{ Params: { prefix: string; '*': string } }>(
|
||||
'/:prefix/*',
|
||||
async (request, reply) => {
|
||||
const spec = platformByPrefix(request.params.prefix);
|
||||
if (!spec) {
|
||||
return reply
|
||||
.code(404)
|
||||
.type('text/html; charset=utf-8')
|
||||
.send(
|
||||
renderError({
|
||||
heading: 'Not a platform antisocial knows',
|
||||
detail: `No adapter is registered for "/${request.params.prefix}". Known prefixes: ${platforms
|
||||
.map((p) => `/${p.prefix}`)
|
||||
.join(', ')}.`,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
const queryIndex = request.url.indexOf('?');
|
||||
const search = queryIndex === -1 ? '' : request.url.slice(queryIndex);
|
||||
const pathRest = request.params['*'];
|
||||
|
||||
let originalUrl: string;
|
||||
try {
|
||||
originalUrl = originalUrlFor(spec, pathRest, search);
|
||||
} catch {
|
||||
return reply
|
||||
.code(400)
|
||||
.type('text/html; charset=utf-8')
|
||||
.send(
|
||||
renderError({
|
||||
platform: spec.id,
|
||||
platformLabel: spec.label,
|
||||
heading: 'That URL did not make sense',
|
||||
detail: `Could not rebuild a ${spec.label} link from "/${spec.prefix}/${pathRest}".`,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const post = await resolvePost(spec, originalUrl, pathRest);
|
||||
return reply
|
||||
.type('text/html; charset=utf-8')
|
||||
.header('cache-control', 'private, max-age=300')
|
||||
.send(renderPost(post));
|
||||
} catch (error) {
|
||||
if (error instanceof ChallengeError) {
|
||||
// Send them to the puzzle rather than reporting it. Solving it is
|
||||
// the one thing that actually gets the post on screen.
|
||||
setReturnTo(error.challengeId, request.url);
|
||||
return reply.redirect(`/challenge/${error.challengeId}`, 302);
|
||||
}
|
||||
|
||||
const detail = error instanceof ResolveError ? error.message : 'Something went wrong.';
|
||||
request.log.warn({ err: error, originalUrl }, 'resolve failed');
|
||||
|
||||
// Never a bare error page: the link still has to be one tap away.
|
||||
return reply
|
||||
.code(502)
|
||||
.type('text/html; charset=utf-8')
|
||||
.send(
|
||||
renderError({
|
||||
platform: spec.id,
|
||||
platformLabel: spec.label,
|
||||
originalUrl,
|
||||
heading: `Could not read that ${spec.label} post`,
|
||||
detail,
|
||||
}),
|
||||
);
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import { join } from 'node:path';
|
||||
import Fastify from 'fastify';
|
||||
import fastifyStatic from '@fastify/static';
|
||||
import { shutdown, warmUp } from './browser/pool.ts';
|
||||
import { config } from './config.ts';
|
||||
import { challengeRoutes } from './routes/challenge.ts';
|
||||
import { healthRoutes } from './routes/health.ts';
|
||||
import { mediaRoutes } from './routes/media.ts';
|
||||
import { postRoutes } from './routes/post.ts';
|
||||
import { renderError } from './render/error.ts';
|
||||
|
||||
export function build() {
|
||||
const app = Fastify({
|
||||
logger: { level: config.logLevel },
|
||||
trustProxy: true,
|
||||
// Instagram links arrive with a trailing slash, X links without one.
|
||||
routerOptions: { ignoreTrailingSlash: true },
|
||||
});
|
||||
|
||||
app.register(fastifyStatic, {
|
||||
root: join(import.meta.dirname, '..', 'public'),
|
||||
prefix: '/static/',
|
||||
maxAge: '1h',
|
||||
});
|
||||
|
||||
healthRoutes(app);
|
||||
challengeRoutes(app);
|
||||
mediaRoutes(app);
|
||||
postRoutes(app);
|
||||
|
||||
app.setNotFoundHandler(async (_request, reply) =>
|
||||
reply
|
||||
.code(404)
|
||||
.type('text/html; charset=utf-8')
|
||||
.send(renderError({ heading: 'Nothing here', detail: 'That is not a post URL.' })),
|
||||
);
|
||||
|
||||
return app;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const app = build();
|
||||
|
||||
// Start Chromium alongside the server rather than on the first request,
|
||||
// so readiness means "can serve now".
|
||||
void warmUp().catch((error: unknown) => {
|
||||
app.log.error({ err: error }, 'browser failed to start');
|
||||
});
|
||||
|
||||
for (const signal of ['SIGINT', 'SIGTERM'] as const) {
|
||||
process.once(signal, () => {
|
||||
app.log.info({ signal }, 'shutting down');
|
||||
void app
|
||||
.close()
|
||||
.then(() => shutdown())
|
||||
.then(() => process.exit(0));
|
||||
});
|
||||
}
|
||||
|
||||
await app.listen({ host: config.host, port: config.port });
|
||||
}
|
||||
|
||||
// Only run the server when this file is the entry point, so tests can
|
||||
// import `build()` without opening a socket.
|
||||
if (process.argv[1] && import.meta.filename === process.argv[1]) {
|
||||
await main();
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
/** The five platforms v1 understands. */
|
||||
export type PlatformId = 'x' | 'threads' | 'instagram' | 'tiktok' | 'bluesky';
|
||||
|
||||
/**
|
||||
* Something fetchable that lives on someone else's CDN.
|
||||
*
|
||||
* Most of these get rewritten to a `/m/` proxy URL before they reach the
|
||||
* page, because Instagram and TikTok reject requests that arrive without the
|
||||
* right `Referer` and cookies, and because proxying keeps the viewer's
|
||||
* browser from talking to the platform at all.
|
||||
*/
|
||||
export type Asset = {
|
||||
url: string;
|
||||
/** Headers the upstream CDN insists on. Held server-side, never in the page. */
|
||||
fetchHeaders?: Record<string, string>;
|
||||
/** Point the page straight at the CDN instead of proxying. Used for HLS,
|
||||
* where proxying would mean rewriting playlists and every segment. */
|
||||
direct?: boolean;
|
||||
};
|
||||
|
||||
type MediaBase = Asset & {
|
||||
width?: number;
|
||||
height?: number;
|
||||
};
|
||||
|
||||
export type Media =
|
||||
| (MediaBase & { kind: 'image'; alt?: string })
|
||||
| (MediaBase & {
|
||||
kind: 'video';
|
||||
poster?: Asset;
|
||||
durationSec?: number;
|
||||
/** An HLS playlist rather than a progressive file. Safari plays these
|
||||
* natively; other browsers get a note. */
|
||||
hls?: boolean;
|
||||
});
|
||||
|
||||
export type Author = {
|
||||
handle: string;
|
||||
displayName?: string;
|
||||
avatar?: Asset;
|
||||
};
|
||||
|
||||
/**
|
||||
* The single shape every adapter produces and the renderer consumes. Adding
|
||||
* a platform means producing one of these; nothing downstream changes.
|
||||
*/
|
||||
export type Post = {
|
||||
platform: PlatformId;
|
||||
/** Human label for the badge: "X", "Threads", ... */
|
||||
platformLabel: string;
|
||||
/** Canonical original URL, cleaned of tracking parameters. This is what
|
||||
* the copy button hands back. */
|
||||
originalUrl: string;
|
||||
author: Author;
|
||||
text?: string;
|
||||
/** Fixed per platform: the ones that lead with words put the text above
|
||||
* the media, the ones that lead with pictures put it below. */
|
||||
textPosition: 'above' | 'below';
|
||||
media: Media[];
|
||||
/** ISO 8601. */
|
||||
postedAt?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Thrown when a post cannot be resolved. Carries enough for the error card
|
||||
* to still be useful: which platform, and the link to hand back.
|
||||
*/
|
||||
export class ResolveError extends Error {
|
||||
readonly platform: PlatformId;
|
||||
readonly originalUrl: string;
|
||||
|
||||
constructor(message: string, platform: PlatformId, originalUrl: string, options?: ErrorOptions) {
|
||||
super(message, options);
|
||||
this.name = 'ResolveError';
|
||||
this.platform = platform;
|
||||
this.originalUrl = originalUrl;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A platform answered with a human-verification puzzle rather than the post.
|
||||
*
|
||||
* The page showing it is parked and still open, so the viewer can be handed
|
||||
* the puzzle to solve. Solving it leaves the resulting cookie in the shared
|
||||
* browser context, which is what makes the retry work.
|
||||
*/
|
||||
export class ChallengeError extends ResolveError {
|
||||
readonly challengeId: string;
|
||||
|
||||
constructor(challengeId: string, platform: PlatformId, originalUrl: string) {
|
||||
super('A verification puzzle is in the way.', platform, originalUrl);
|
||||
this.name = 'ChallengeError';
|
||||
this.challengeId = challengeId;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user