Initial commit: read social posts back without the app
antisocial is the other half of a StopTheMadness redirect rule. Links to X, Threads, Instagram, TikTok and Bluesky get rewritten to /<prefix>/<original path>, and this resolves the post and shows the media and the words, with a badge saying where it came from and a button to copy the original URL. Every request drives a real headless Chromium, logged out, from a residential IP. One code path, and it survives markup changes better than parsing from the outside would. Extraction is layered, most structured first: the platform's own API response caught in flight, then an inline payload, then the rendered DOM, then Open Graph tags. Media is never linked straight at a CDN. Instagram and TikTok reject requests without a matching Referer and cookies, and proxying keeps the viewer's browser from talking to the platform at all. Range is forwarded so the native video scrubber can seek. HLS is the exception, since proxying it would mean rewriting playlists. TikTok sometimes answers with a slider puzzle. Rather than reporting that as a failure, the page is parked and the viewer is handed the puzzle: screenshots stream out, pointer events are replayed back. Solving it leaves the cookie in the shared browser context, so the retry is an ordinary request. A failed resolve is never a blank error page. The card carries the platform, the original URL and the copy button, so a broken adapter still leaves the link one tap away. Verified end to end against real shared links on all five platforms, in the container, including multi-image carousels, reels, TikTok short links and photo posts. 49 tests run the adapters against captured payloads with no network. Co-Authored-By: Claude Opus 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01BGkRmLfiWuJHx6tQ12EELY
This commit is contained in:
@@ -0,0 +1,95 @@
|
||||
/**
|
||||
* Dump what a platform's page is actually serving us.
|
||||
*
|
||||
* npm run probe -- 'https://www.instagram.com/reel/ABC123/embed/captioned/'
|
||||
*
|
||||
* `npm run resolve` answers "did it work". This answers "why not": which
|
||||
* payload keys are present, what the DOM ended up with, whether we are
|
||||
* looking at a challenge or a login wall. Every adapter fix so far started
|
||||
* here.
|
||||
*
|
||||
* Pass a URL exactly as the adapter would load it — for Instagram that is
|
||||
* usually the `/embed/captioned/` form, not the post URL.
|
||||
*/
|
||||
import { shutdown, withPage } from '../src/browser/pool.ts';
|
||||
|
||||
const url = process.argv[2];
|
||||
if (!url) {
|
||||
console.error("usage: npm run probe -- '<url>' [waitMs]");
|
||||
process.exit(2);
|
||||
}
|
||||
const waitMs = Number(process.argv[3] ?? 5000);
|
||||
|
||||
/** The keys each adapter reaches for, so a missing one is obvious. */
|
||||
const KEYS = [
|
||||
'shortcode_media',
|
||||
'edge_sidecar_to_children',
|
||||
'image_versions2',
|
||||
'video_versions',
|
||||
'carousel_media',
|
||||
'__UNIVERSAL_DATA_FOR_REHYDRATION__',
|
||||
'itemStruct',
|
||||
'imagePost',
|
||||
'getPostThread',
|
||||
'tweet-result',
|
||||
];
|
||||
|
||||
await withPage(async ({ page }) => {
|
||||
// Only the calls that might carry the post. Bundle and asset traffic
|
||||
// drowns out the one response worth seeing.
|
||||
const INTERESTING = /api|graphql|xrpc|syndication|tweet-result|\.json(\?|$)/;
|
||||
const NOISE = /rsrc\.php|\/static\/|static\.|\.css|\.woff|\.svg|\.png|\.jpg/;
|
||||
|
||||
const responses: string[] = [];
|
||||
page.on('response', (r) => {
|
||||
const u = r.url();
|
||||
if (INTERESTING.test(u) && !NOISE.test(u)) {
|
||||
responses.push(`${r.status()} ${u.slice(0, 130)}`);
|
||||
}
|
||||
});
|
||||
|
||||
const response = await page.goto(url, { waitUntil: 'domcontentloaded' });
|
||||
console.log('status ', response?.status());
|
||||
console.log('landed ', page.url());
|
||||
|
||||
await page.waitForTimeout(waitMs);
|
||||
|
||||
const found = await page.evaluate(
|
||||
({ keys, selectors }: { keys: string[]; selectors: string }) => {
|
||||
const scripts = [...document.querySelectorAll('script')].map((s) => s.textContent ?? '');
|
||||
const all = scripts.join('');
|
||||
return {
|
||||
title: document.title,
|
||||
keys: keys.filter((k) => all.includes(k)),
|
||||
scripts: scripts.filter((s) => s.length > 256).map((s) => s.length),
|
||||
videos: [...document.querySelectorAll('video')].map((v) => ({
|
||||
src: (v.currentSrc || v.src).slice(0, 100),
|
||||
poster: v.poster.slice(0, 80),
|
||||
})),
|
||||
images: [...document.querySelectorAll('img')].length,
|
||||
og: Object.fromEntries(
|
||||
[...document.querySelectorAll('meta[property^="og:"]')].map((m) => [
|
||||
m.getAttribute('property'),
|
||||
(m.getAttribute('content') ?? '').slice(0, 90),
|
||||
]),
|
||||
),
|
||||
challenge: !!document.querySelector(selectors),
|
||||
body: (document.body?.innerText ?? '').slice(0, 240),
|
||||
};
|
||||
},
|
||||
{ keys: KEYS, selectors: '[id*="captcha"], [class*="captcha_verify"]' },
|
||||
);
|
||||
|
||||
console.log('title ', found.title);
|
||||
console.log('keys ', found.keys.length ? found.keys.join(', ') : '(none of the ones we read)');
|
||||
console.log('scripts ', found.scripts.join(', '));
|
||||
console.log('videos ', JSON.stringify(found.videos));
|
||||
console.log('images ', found.images);
|
||||
console.log('og ', JSON.stringify(found.og, null, 1));
|
||||
console.log('challenge', found.challenge);
|
||||
console.log('body ', JSON.stringify(found.body));
|
||||
console.log('traffic ');
|
||||
for (const line of responses.slice(0, 20)) console.log(' ', line);
|
||||
});
|
||||
|
||||
await shutdown();
|
||||
@@ -0,0 +1,37 @@
|
||||
/**
|
||||
* Resolve one URL against the real browser and print the Post as JSON.
|
||||
*
|
||||
* npm run resolve -- 'https://www.tiktok.com/@user/video/123'
|
||||
*
|
||||
* This is the fastest way to find out whether a platform still works, and
|
||||
* the first thing to reach for when one stops.
|
||||
*/
|
||||
import { shutdown } from '../src/browser/pool.ts';
|
||||
import { originalUrlFor, platformForUrl } from '../src/platforms/index.ts';
|
||||
import { resolvePost } from '../src/resolve.ts';
|
||||
|
||||
const input = process.argv[2];
|
||||
if (!input) {
|
||||
console.error('usage: npm run resolve -- <post url>');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const spec = platformForUrl(input);
|
||||
if (!spec) {
|
||||
console.error(`No adapter handles ${input}`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const parsed = new URL(input);
|
||||
const pathRest = parsed.pathname.replace(/^\/+/, '');
|
||||
const originalUrl = originalUrlFor(spec, pathRest, parsed.search);
|
||||
|
||||
try {
|
||||
const post = await resolvePost(spec, originalUrl, pathRest);
|
||||
console.log(JSON.stringify(post, null, 2));
|
||||
} catch (error) {
|
||||
console.error(error instanceof Error ? error.message : error);
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
await shutdown();
|
||||
}
|
||||
Reference in New Issue
Block a user