mirror of
https://github.com/juherr/kill-the-news.git
synced 2026-06-20 22:03:48 +00:00
feat: replace fixed 50-entry cap with size-based feed trimming
Emails are now trimmed from the oldest end when total serialised size exceeds FEED_MAX_SIZE_BYTES (default 512 KB). Each EmailMetadata entry stores its size so future trims are computed without re-reading KV. Adds FEED_MAX_SIZE_BYTES, PROXY_TRUSTED_IPS and PROXY_AUTH_SECRET to Env. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -115,7 +115,7 @@ describe("processEmail", () => {
|
|||||||
await env.EMAIL_STORAGE.put(
|
await env.EMAIL_STORAGE.put(
|
||||||
`feed:${VALID_FEED_ID}:metadata`,
|
`feed:${VALID_FEED_ID}:metadata`,
|
||||||
JSON.stringify({
|
JSON.stringify({
|
||||||
emails: [{ key: "old-key", subject: "Old", receivedAt: 1 }],
|
emails: [{ key: "old-key", subject: "Old", receivedAt: 1, size: 100 }],
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -129,4 +129,53 @@ describe("processEmail", () => {
|
|||||||
expect(metadata.emails[0].subject).toBe("New");
|
expect(metadata.emails[0].subject).toBe("New");
|
||||||
expect(metadata.emails[1].subject).toBe("Old");
|
expect(metadata.emails[1].subject).toBe("Old");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("trims oldest emails when total size exceeds FEED_MAX_SIZE_BYTES", async () => {
|
||||||
|
await env.EMAIL_STORAGE.put(
|
||||||
|
`feed:${VALID_FEED_ID}:config`,
|
||||||
|
JSON.stringify({}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const oldKey1 = `feed:${VALID_FEED_ID}:111`;
|
||||||
|
const oldKey2 = `feed:${VALID_FEED_ID}:222`;
|
||||||
|
const bigContent = "x".repeat(200);
|
||||||
|
const email1 = JSON.stringify({ subject: "Old1", from: "a@b.com", content: bigContent, receivedAt: 111, headers: {} });
|
||||||
|
const email2 = JSON.stringify({ subject: "Old2", from: "a@b.com", content: bigContent, receivedAt: 222, headers: {} });
|
||||||
|
await env.EMAIL_STORAGE.put(oldKey1, email1);
|
||||||
|
await env.EMAIL_STORAGE.put(oldKey2, email2);
|
||||||
|
await env.EMAIL_STORAGE.put(
|
||||||
|
`feed:${VALID_FEED_ID}:metadata`,
|
||||||
|
JSON.stringify({
|
||||||
|
emails: [
|
||||||
|
{ key: oldKey2, subject: "Old2", receivedAt: 222, size: email2.length },
|
||||||
|
{ key: oldKey1, subject: "Old1", receivedAt: 111, size: email1.length },
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const tinyEnv = { ...env, FEED_MAX_SIZE_BYTES: "50" };
|
||||||
|
const res = await processEmail(makeInput({ subject: "New" }), tinyEnv as any);
|
||||||
|
expect(res.status).toBe(200);
|
||||||
|
|
||||||
|
const metadata = await env.EMAIL_STORAGE.get(`feed:${VALID_FEED_ID}:metadata`, "json");
|
||||||
|
expect(metadata.emails).toHaveLength(1);
|
||||||
|
expect(metadata.emails[0].subject).toBe("New");
|
||||||
|
|
||||||
|
const deleted1 = await env.EMAIL_STORAGE.get(oldKey1, "json");
|
||||||
|
const deleted2 = await env.EMAIL_STORAGE.get(oldKey2, "json");
|
||||||
|
expect(deleted1).toBeNull();
|
||||||
|
expect(deleted2).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps entries within size budget untouched", async () => {
|
||||||
|
await env.EMAIL_STORAGE.put(
|
||||||
|
`feed:${VALID_FEED_ID}:config`,
|
||||||
|
JSON.stringify({}),
|
||||||
|
);
|
||||||
|
const bigEnv = { ...env, FEED_MAX_SIZE_BYTES: String(10 * 1024 * 1024) };
|
||||||
|
await processEmail(makeInput({ subject: "First" }), bigEnv as any);
|
||||||
|
await processEmail(makeInput({ subject: "Second" }), bigEnv as any);
|
||||||
|
const metadata = await env.EMAIL_STORAGE.get(`feed:${VALID_FEED_ID}:metadata`, "json");
|
||||||
|
expect(metadata.emails).toHaveLength(2);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -90,18 +90,39 @@ export async function processEmail(
|
|||||||
env.EMAIL_STORAGE.get(feedMetadataKey, "json"),
|
env.EMAIL_STORAGE.get(feedMetadataKey, "json"),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
// Note: KV has no atomic compare-and-swap. Concurrent invocations for the
|
||||||
|
// same feed can read stale metadata and produce orphaned KV entries or
|
||||||
|
// duplicate trim deletions. This is an accepted limitation given Cloudflare
|
||||||
|
// KV's eventual-consistency model.
|
||||||
const feedMetadata = ((rawMetadata as FeedMetadata | null) || {
|
const feedMetadata = ((rawMetadata as FeedMetadata | null) || {
|
||||||
emails: [],
|
emails: [],
|
||||||
}) as FeedMetadata;
|
}) as FeedMetadata;
|
||||||
|
|
||||||
|
const DEFAULT_MAX_BYTES = 524288; // 512 KB
|
||||||
|
const maxBytes =
|
||||||
|
parseInt(env.FEED_MAX_SIZE_BYTES ?? "", 10) || DEFAULT_MAX_BYTES;
|
||||||
|
|
||||||
|
const serialised = JSON.stringify(emailData);
|
||||||
|
const serialisedSize = new TextEncoder().encode(serialised).byteLength;
|
||||||
feedMetadata.emails.unshift({
|
feedMetadata.emails.unshift({
|
||||||
key: emailKey,
|
key: emailKey,
|
||||||
subject: emailData.subject,
|
subject: emailData.subject,
|
||||||
receivedAt: emailData.receivedAt,
|
receivedAt: emailData.receivedAt,
|
||||||
|
size: serialisedSize,
|
||||||
});
|
});
|
||||||
if (feedMetadata.emails.length > 50) {
|
|
||||||
feedMetadata.emails = feedMetadata.emails.slice(0, 50);
|
let totalSize = feedMetadata.emails.reduce((sum, e) => sum + (e.size ?? 0), 0);
|
||||||
|
const toDelete: string[] = [];
|
||||||
|
while (totalSize > maxBytes && feedMetadata.emails.length > 1) {
|
||||||
|
const dropped = feedMetadata.emails.pop()!;
|
||||||
|
totalSize -= dropped.size ?? 0;
|
||||||
|
toDelete.push(dropped.key);
|
||||||
}
|
}
|
||||||
await env.EMAIL_STORAGE.put(feedMetadataKey, JSON.stringify(feedMetadata));
|
|
||||||
|
await Promise.all([
|
||||||
|
env.EMAIL_STORAGE.put(feedMetadataKey, JSON.stringify(feedMetadata)),
|
||||||
|
...toDelete.map((k) => env.EMAIL_STORAGE.delete(k)),
|
||||||
|
]);
|
||||||
|
|
||||||
console.log(`Successfully processed email for feed ${feedId}`);
|
console.log(`Successfully processed email for feed ${feedId}`);
|
||||||
return new Response("Email processed successfully", { status: 200 });
|
return new Response("Email processed successfully", { status: 200 });
|
||||||
|
|||||||
@@ -3,6 +3,9 @@ export interface Env {
|
|||||||
EMAIL_STORAGE: KVNamespace;
|
EMAIL_STORAGE: KVNamespace;
|
||||||
ADMIN_PASSWORD: string;
|
ADMIN_PASSWORD: string;
|
||||||
DOMAIN: string;
|
DOMAIN: string;
|
||||||
|
FEED_MAX_SIZE_BYTES?: string;
|
||||||
|
PROXY_TRUSTED_IPS?: string;
|
||||||
|
PROXY_AUTH_SECRET?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Email interface for stored emails
|
// Email interface for stored emails
|
||||||
@@ -37,6 +40,7 @@ export interface EmailMetadata {
|
|||||||
key: string;
|
key: string;
|
||||||
subject: string;
|
subject: string;
|
||||||
receivedAt: number;
|
receivedAt: number;
|
||||||
|
size?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Feed list interface
|
// Feed list interface
|
||||||
|
|||||||
Reference in New Issue
Block a user