From 9eba4c34c67eae4db272136060e5c611deec9770 Mon Sep 17 00:00:00 2001 From: Julien Herr Date: Thu, 21 May 2026 08:26:09 +0200 Subject: [PATCH] feat: replace fixed 50-entry cap with size-based feed trimming Emails are now trimmed from the oldest end when total serialised size exceeds FEED_MAX_SIZE_BYTES (default 512 KB). Each EmailMetadata entry stores its size so future trims are computed without re-reading KV. Adds FEED_MAX_SIZE_BYTES, PROXY_TRUSTED_IPS and PROXY_AUTH_SECRET to Env. Co-Authored-By: Claude Sonnet 4.6 --- src/lib/email-processor.test.ts | 51 ++++++++++++++++++++++++++++++++- src/lib/email-processor.ts | 27 +++++++++++++++-- src/types/index.ts | 4 +++ 3 files changed, 78 insertions(+), 4 deletions(-) diff --git a/src/lib/email-processor.test.ts b/src/lib/email-processor.test.ts index e9f1cbc..21b7a49 100644 --- a/src/lib/email-processor.test.ts +++ b/src/lib/email-processor.test.ts @@ -115,7 +115,7 @@ describe("processEmail", () => { await env.EMAIL_STORAGE.put( `feed:${VALID_FEED_ID}:metadata`, JSON.stringify({ - emails: [{ key: "old-key", subject: "Old", receivedAt: 1 }], + emails: [{ key: "old-key", subject: "Old", receivedAt: 1, size: 100 }], }), ); @@ -129,4 +129,53 @@ describe("processEmail", () => { expect(metadata.emails[0].subject).toBe("New"); expect(metadata.emails[1].subject).toBe("Old"); }); + + it("trims oldest emails when total size exceeds FEED_MAX_SIZE_BYTES", async () => { + await env.EMAIL_STORAGE.put( + `feed:${VALID_FEED_ID}:config`, + JSON.stringify({}), + ); + + const oldKey1 = `feed:${VALID_FEED_ID}:111`; + const oldKey2 = `feed:${VALID_FEED_ID}:222`; + const bigContent = "x".repeat(200); + const email1 = JSON.stringify({ subject: "Old1", from: "a@b.com", content: bigContent, receivedAt: 111, headers: {} }); + const email2 = JSON.stringify({ subject: "Old2", from: "a@b.com", content: bigContent, receivedAt: 222, headers: {} }); + await env.EMAIL_STORAGE.put(oldKey1, email1); + await env.EMAIL_STORAGE.put(oldKey2, email2); + await env.EMAIL_STORAGE.put( + `feed:${VALID_FEED_ID}:metadata`, + JSON.stringify({ + emails: [ + { key: oldKey2, subject: "Old2", receivedAt: 222, size: email2.length }, + { key: oldKey1, subject: "Old1", receivedAt: 111, size: email1.length }, + ], + }), + ); + + const tinyEnv = { ...env, FEED_MAX_SIZE_BYTES: "50" }; + const res = await processEmail(makeInput({ subject: "New" }), tinyEnv as any); + expect(res.status).toBe(200); + + const metadata = await env.EMAIL_STORAGE.get(`feed:${VALID_FEED_ID}:metadata`, "json"); + expect(metadata.emails).toHaveLength(1); + expect(metadata.emails[0].subject).toBe("New"); + + const deleted1 = await env.EMAIL_STORAGE.get(oldKey1, "json"); + const deleted2 = await env.EMAIL_STORAGE.get(oldKey2, "json"); + expect(deleted1).toBeNull(); + expect(deleted2).toBeNull(); + }); + + it("keeps entries within size budget untouched", async () => { + await env.EMAIL_STORAGE.put( + `feed:${VALID_FEED_ID}:config`, + JSON.stringify({}), + ); + const bigEnv = { ...env, FEED_MAX_SIZE_BYTES: String(10 * 1024 * 1024) }; + await processEmail(makeInput({ subject: "First" }), bigEnv as any); + await processEmail(makeInput({ subject: "Second" }), bigEnv as any); + const metadata = await env.EMAIL_STORAGE.get(`feed:${VALID_FEED_ID}:metadata`, "json"); + expect(metadata.emails).toHaveLength(2); + }); }); diff --git a/src/lib/email-processor.ts b/src/lib/email-processor.ts index 6fa8925..cf53f5a 100644 --- a/src/lib/email-processor.ts +++ b/src/lib/email-processor.ts @@ -90,18 +90,39 @@ export async function processEmail( env.EMAIL_STORAGE.get(feedMetadataKey, "json"), ]); + // Note: KV has no atomic compare-and-swap. Concurrent invocations for the + // same feed can read stale metadata and produce orphaned KV entries or + // duplicate trim deletions. This is an accepted limitation given Cloudflare + // KV's eventual-consistency model. const feedMetadata = ((rawMetadata as FeedMetadata | null) || { emails: [], }) as FeedMetadata; + + const DEFAULT_MAX_BYTES = 524288; // 512 KB + const maxBytes = + parseInt(env.FEED_MAX_SIZE_BYTES ?? "", 10) || DEFAULT_MAX_BYTES; + + const serialised = JSON.stringify(emailData); + const serialisedSize = new TextEncoder().encode(serialised).byteLength; feedMetadata.emails.unshift({ key: emailKey, subject: emailData.subject, receivedAt: emailData.receivedAt, + size: serialisedSize, }); - if (feedMetadata.emails.length > 50) { - feedMetadata.emails = feedMetadata.emails.slice(0, 50); + + let totalSize = feedMetadata.emails.reduce((sum, e) => sum + (e.size ?? 0), 0); + const toDelete: string[] = []; + while (totalSize > maxBytes && feedMetadata.emails.length > 1) { + const dropped = feedMetadata.emails.pop()!; + totalSize -= dropped.size ?? 0; + toDelete.push(dropped.key); } - await env.EMAIL_STORAGE.put(feedMetadataKey, JSON.stringify(feedMetadata)); + + await Promise.all([ + env.EMAIL_STORAGE.put(feedMetadataKey, JSON.stringify(feedMetadata)), + ...toDelete.map((k) => env.EMAIL_STORAGE.delete(k)), + ]); console.log(`Successfully processed email for feed ${feedId}`); return new Response("Email processed successfully", { status: 200 }); diff --git a/src/types/index.ts b/src/types/index.ts index 8f522ab..9899900 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -3,6 +3,9 @@ export interface Env { EMAIL_STORAGE: KVNamespace; ADMIN_PASSWORD: string; DOMAIN: string; + FEED_MAX_SIZE_BYTES?: string; + PROXY_TRUSTED_IPS?: string; + PROXY_AUTH_SECRET?: string; } // Email interface for stored emails @@ -37,6 +40,7 @@ export interface EmailMetadata { key: string; subject: string; receivedAt: number; + size?: number; } // Feed list interface