antisocial/bin/probe.ts

/**
 * Dump what a platform's page is actually serving us.
 *
 *   npm run probe -- 'https://www.instagram.com/reel/ABC123/embed/captioned/'
 *
 * `npm run resolve` answers "did it work". This answers "why not": which
 * payload keys are present, what the DOM ended up with, whether we are
 * looking at a challenge or a login wall. Every adapter fix so far started
 * here.
 *
 * Pass a URL exactly as the adapter would load it — for Instagram that is
 * usually the `/embed/captioned/` form, not the post URL.
 */
import { shutdown, withPage } from '../src/browser/pool.ts';

const url = process.argv[2];
if (!url) {
  console.error("usage: npm run probe -- '<url>' [waitMs]");
  process.exit(2);
}
const waitMs = Number(process.argv[3] ?? 5000);

/** The keys each adapter reaches for, so a missing one is obvious. */
const KEYS = [
  'shortcode_media',
  'edge_sidecar_to_children',
  'image_versions2',
  'video_versions',
  'carousel_media',
  '__UNIVERSAL_DATA_FOR_REHYDRATION__',
  'itemStruct',
  'imagePost',
  'getPostThread',
  'tweet-result',
];

await withPage(async ({ page }) => {
  // Only the calls that might carry the post. Bundle and asset traffic
  // drowns out the one response worth seeing.
  const INTERESTING = /api|graphql|xrpc|syndication|tweet-result|\.json(\?|$)/;
  const NOISE = /rsrc\.php|\/static\/|static\.|\.css|\.woff|\.svg|\.png|\.jpg/;

  const responses: string[] = [];
  page.on('response', (r) => {
    const u = r.url();
    if (INTERESTING.test(u) && !NOISE.test(u)) {
      responses.push(`${r.status()} ${u.slice(0, 130)}`);
    }
  });

  const response = await page.goto(url, { waitUntil: 'domcontentloaded' });
  console.log('status  ', response?.status());
  console.log('landed  ', page.url());

  await page.waitForTimeout(waitMs);

  const found = await page.evaluate(
    ({ keys, selectors }: { keys: string[]; selectors: string }) => {
      const scripts = [...document.querySelectorAll('script')].map((s) => s.textContent ?? '');
      const all = scripts.join('');
      return {
        title: document.title,
        keys: keys.filter((k) => all.includes(k)),
        scripts: scripts.filter((s) => s.length > 256).map((s) => s.length),
        videos: [...document.querySelectorAll('video')].map((v) => ({
          src: (v.currentSrc || v.src).slice(0, 100),
          poster: v.poster.slice(0, 80),
        })),
        images: [...document.querySelectorAll('img')].length,
        og: Object.fromEntries(
          [...document.querySelectorAll('meta[property^="og:"]')].map((m) => [
            m.getAttribute('property'),
            (m.getAttribute('content') ?? '').slice(0, 90),
          ]),
        ),
        challenge: !!document.querySelector(selectors),
        body: (document.body?.innerText ?? '').slice(0, 240),
      };
    },
    { keys: KEYS, selectors: '[id*="captcha"], [class*="captcha_verify"]' },
  );

  console.log('title   ', found.title);
  console.log('keys    ', found.keys.length ? found.keys.join(', ') : '(none of the ones we read)');
  console.log('scripts ', found.scripts.join(', '));
  console.log('videos  ', JSON.stringify(found.videos));
  console.log('images  ', found.images);
  console.log('og      ', JSON.stringify(found.og, null, 1));
  console.log('challenge', found.challenge);
  console.log('body    ', JSON.stringify(found.body));
  console.log('traffic ');
  for (const line of responses.slice(0, 20)) console.log('  ', line);
});

await shutdown();