diff --git a/CLAUDE.md b/CLAUDE.md index 4ecd53a..ff4ea07 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -62,6 +62,10 @@ Request → `src/routes/post.ts` → `src/platforms/index.ts` maps prefix to ada Adding a platform is one file in `src/platforms/` plus one row in the table in `index.ts`. Everything downstream already handles a `Post`. +A `Post` is a list of `Segment`s, not a single body. Most platforms produce one +(`oneSegment` in `types.ts`); Bluesky and Threads produce the author's whole chain, +with `isAnchor` marking the post that was linked — which need not be the first. + Things worth knowing before editing: - **One Chromium, one context, persistent.** Cookies and dismissed banners accumulate on @@ -88,7 +92,9 @@ Things worth knowing before editing: - **Bluesky** — asks the public API directly (still through the browser context), so it is the most reliable. The web app calls `getPostThreadV2` now; the page fallback - deliberately matches only V1. + deliberately matches only V1. Threads are built by walking `parent` up and the + author's own `replies` down; `depth`/`parentHeight` are what make that possible, at + the cost of dragging the whole reply tree along (a few hundred KB on a busy post). - **X** — the `platform.twitter.com` embed calls the syndication endpoint; we catch that response. A quote post carries no media of its own, so the quoted post's media is used. - **Instagram** — the least reliable. It ships the structured payload only some of the @@ -102,7 +108,12 @@ Things worth knowing before editing: segment and are rebuilt in `buildOriginalUrl`. - **Threads** — same media schema as Instagram (`src/platforms/meta-media.ts`). Its payloads are full of empty stub nodes, so the finder only accepts a node with actual - candidates in it. + candidates in it. The page ships the linked post, the author's follow-ups, other + people's replies and unrelated recommendations all as flat `thread_items` + containers. A follow-up is the author replying to *themselves*, which is what + separates it from a stranger's reply carrying the same `reply_to_author`. The first + post of a chain replies to nothing, so it is only reachable by walking backwards + from the one that answers it. ## Verification puzzles diff --git a/README.md b/README.md index 6171e78..77027b0 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,12 @@ Each adapter layers its extraction, most structured first: | TikTok | the post page | `__UNIVERSAL_DATA_FOR_REHYDRATION__` | | Threads | the post page | the Relay payloads in `'}

`.value, @@ -42,13 +42,19 @@ function post(overrides: Partial = {}): Post { platformLabel: 'Instagram', originalUrl: 'https://www.instagram.com/p/ABC/', author: { handle: '@nasa' }, - text: 'caption', textPosition: 'below', - media: [{ kind: 'image', url: 'https://cdn/1.jpg' }, { kind: 'image', url: 'https://cdn/2.jpg' }], + segments: oneSegment({ + text: 'caption', + media: [{ kind: 'image', url: 'https://cdn/1.jpg' }, { kind: 'image', url: 'https://cdn/2.jpg' }], + }), ...overrides, }; } +function withMedia(media: Media[], overrides: Partial = {}): Post { + return post({ segments: oneSegment({ text: 'caption', media }), ...overrides }); +} + test('media is proxied, never linked straight at the CDN', () => { const page = renderPost(post()); assert.ok(!page.includes('https://cdn/1.jpg'), 'upstream URLs must not reach the page'); @@ -56,9 +62,9 @@ test('media is proxied, never linked straight at the CDN', () => { }); test('an HLS video is linked directly, because a proxy cannot rewrite a playlist', () => { - const page = renderPost(post({ - media: [{ kind: 'video', url: 'https://video.bsky.app/x/playlist.m3u8', hls: true, direct: true }], - })); + const page = renderPost(withMedia([ + { kind: 'video', url: 'https://video.bsky.app/x/playlist.m3u8', hls: true, direct: true }, + ])); assert.ok(page.includes('https://video.bsky.app/x/playlist.m3u8')); }); @@ -72,12 +78,12 @@ test('text sits below the media for Instagram and above it for X', () => { test('the layout toggle only appears when there is more than one item', () => { assert.ok(renderPost(post()).includes('data-view="grid"')); - assert.ok(!renderPost(post({ media: [{ kind: 'image', url: 'https://cdn/1.jpg' }] })) + assert.ok(!renderPost(withMedia([{ kind: 'image', url: 'https://cdn/1.jpg' }])) .includes('data-view="grid"')); }); test('a video gets native controls and a source, not an iframe', () => { - const page = renderPost(post({ media: [{ kind: 'video', url: 'https://cdn/v.mp4' }] })); + const page = renderPost(withMedia([{ kind: 'video', url: 'https://cdn/v.mp4' }])); assert.ok(page.includes('[0][number]; + +// A real capture of a two-post NASA thread. The page ships eighteen posts: +// the two that belong together, other people's replies to them, and a pile of +// unrelated recommendations. Only the first two may survive. +const page = () => fixture('threads/thread-chain.json'); + +test('the author\'s own follow-up is kept, in order', () => { + const chain = selfThread(page(), 'DccGgKuEW-E'); + assert.equal(chain.length, 2); + assert.deepEqual(chain.map((p) => p.code), ['DccGgKuEW-E', 'DccGhYSkXYY']); + assert.match(chain[0]?.caption?.text ?? '', /Home improvement/); + assert.match(chain[1]?.caption?.text ?? '', /where you can watch/); +}); + +test('every other author on the page is dropped', () => { + const chain = selfThread(page(), 'DccGgKuEW-E'); + assert.ok(chain.every((p) => p.user?.username === 'nasa'), + 'a post by someone else survived the filter'); + assert.ok(page().length > chain.length + 10, 'the fixture should be mostly noise'); +}); + +test('a reply from someone else to the same post is not a continuation', () => { + // This one is the trap: it carries reply_to_author = nasa, exactly like a + // real follow-up does. Only the author of the *post* separates them. + const esa = page().find((p) => p.user?.username === 'europeanspaceagency'); + assert.ok(esa, 'fixture should contain a reply from another account'); + assert.equal(esa?.text_post_app_info?.reply_to_author?.username, 'nasa'); + assert.ok(!selfThread(page(), 'DccGgKuEW-E').includes(esa as Post)); +}); + +test('linking the second post still returns the whole chain', () => { + const chain = selfThread(page(), 'DccGhYSkXYY'); + assert.deepEqual(chain.map((p) => p.code), ['DccGgKuEW-E', 'DccGhYSkXYY']); +}); + +test('an unknown code yields nothing rather than guessing', () => { + assert.deepEqual(selfThread(page(), 'NoSuchCode'), []); +}); + +test('a lone post is a chain of one', () => { + const chain = selfThread( + [ + { code: 'A', taken_at: 1, user: { username: 'me' }, caption: { text: 'only' } }, + { code: 'B', taken_at: 2, user: { username: 'you' }, caption: { text: 'reply' }, + text_post_app_info: { reply_to_author: { username: 'me' } } }, + ], + 'A', + ); + assert.deepEqual(chain.map((p) => p.code), ['A']); +}); + +test('the chain is ordered by when each was posted, not by payload order', () => { + const chain = selfThread( + [ + { code: 'C', taken_at: 30, user: { username: 'me' }, + text_post_app_info: { reply_to_author: { username: 'me' } } }, + { code: 'A', taken_at: 10, user: { username: 'me' } }, + { code: 'B', taken_at: 20, user: { username: 'me' }, + text_post_app_info: { reply_to_author: { username: 'me' } } }, + ], + 'A', + ); + assert.deepEqual(chain.map((p) => p.code), ['A', 'B', 'C']); +}); diff --git a/test/x.test.ts b/test/x.test.ts index 77f4846..b26d921 100644 --- a/test/x.test.ts +++ b/test/x.test.ts @@ -9,16 +9,16 @@ test('a text-only post carries no media', () => { const post = toPost(fixture('x/text-only.json'), URL_); assert.equal(post.platform, 'x'); assert.equal(post.textPosition, 'above'); - assert.deepEqual(post.media, []); + assert.deepEqual(post.segments[0]?.media, []); assert.equal(post.author.handle, '@jack'); - assert.ok(post.postedAt); + assert.ok(post.segments[0]?.postedAt); }); test('a photo post keeps every photo and asks for the original size', () => { - const post = toPost(fixture('x/photo.json'), URL_); - assert.ok(post.media.length > 1, 'expected more than one photo'); - assert.ok(post.media.every((m) => m.kind === 'image')); - assert.ok(post.media.every((m) => m.url.endsWith('?name=orig'))); + const media = toPost(fixture('x/photo.json'), URL_).segments[0]?.media ?? []; + assert.ok(media.length > 1, 'expected more than one photo'); + assert.ok(media.every((m) => m.kind === 'image')); + assert.ok(media.every((m) => m.url.endsWith('?name=orig'))); }); test('a video picks the highest-bitrate mp4 and ignores the streaming variants', () => { @@ -64,6 +64,6 @@ test('a quote post shows the media of the post it quotes', () => { }, URL_, ); - assert.equal(post.media.length, 1); - assert.equal(post.media[0]?.url, 'https://pbs.twimg.com/q.jpg?name=orig'); + assert.equal(post.segments[0]?.media.length, 1); + assert.equal(post.segments[0]?.media[0]?.url, 'https://pbs.twimg.com/q.jpg?name=orig'); });