mirror of
https://github.com/juherr/kill-the-news.git
synced 2026-06-20 22:03:48 +00:00
feat: store email attachments in R2 and expose as RSS enclosures
Attachments from incoming emails are uploaded to an optional Cloudflare R2
bucket and exposed as <enclosure> elements in RSS and <link rel="enclosure">
in Atom feeds, served at /files/{id}/{filename} with immutable caching.
R2 is opt-in: if ATTACHMENT_BUCKET is not bound the feature is a no-op.
Attachments are cleaned up from R2 on email/feed deletion and during
size-based feed trimming. Adds MockR2 to the test setup.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
+58
-6
@@ -1908,7 +1908,7 @@ async function deleteFeedFastDetailed(
|
||||
async function purgeFeedKeysStep(
|
||||
emailStorage: KVNamespace,
|
||||
feedId: string,
|
||||
options: { cursor?: string; limit?: number } = {},
|
||||
options: { cursor?: string; limit?: number; bucket?: R2Bucket } = {},
|
||||
): Promise<{
|
||||
deletedKeys: string[];
|
||||
failedKeys: string[];
|
||||
@@ -1921,6 +1921,33 @@ async function purgeFeedKeysStep(
|
||||
|
||||
const listed = await emailStorage.list({ prefix, cursor, limit });
|
||||
const keys = (listed.keys || []).map((k) => k.name);
|
||||
|
||||
// Collect R2 attachment IDs from email entries before deleting
|
||||
if (options.bucket && keys.length > 0) {
|
||||
const emailKeys = keys.filter((k) => {
|
||||
const suffix = k.slice(prefix.length);
|
||||
return suffix !== "config" && suffix !== "metadata";
|
||||
});
|
||||
if (emailKeys.length > 0) {
|
||||
const emailDataResults = await Promise.allSettled(
|
||||
emailKeys.map((k) =>
|
||||
emailStorage.get(k, { type: "json" }) as Promise<EmailData | null>,
|
||||
),
|
||||
);
|
||||
const attachmentIds = emailDataResults
|
||||
.filter(
|
||||
(r): r is PromiseFulfilledResult<EmailData | null> =>
|
||||
r.status === "fulfilled",
|
||||
)
|
||||
.flatMap((r) => r.value?.attachments?.map((a) => a.id) ?? []);
|
||||
if (attachmentIds.length > 0) {
|
||||
await Promise.allSettled(
|
||||
attachmentIds.map((id) => options.bucket!.delete(id)),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const { ok, failed } = await deleteKeysWithConcurrency(
|
||||
emailStorage,
|
||||
keys,
|
||||
@@ -1949,7 +1976,7 @@ app.post("/feeds/:feedId/delete", async (c) => {
|
||||
|
||||
// Best-effort cleanup in the background so the request stays fast.
|
||||
// Use the UI purge endpoint for full, user-visible progress.
|
||||
waitUntilSafe(c, purgeFeedKeysStep(emailStorage, feedId));
|
||||
waitUntilSafe(c, purgeFeedKeysStep(emailStorage, feedId, { bucket: env.ATTACHMENT_BUCKET }));
|
||||
if (wantsJson) {
|
||||
return c.json({ ok: true, feedId });
|
||||
}
|
||||
@@ -1987,6 +2014,7 @@ app.post("/feeds/:feedId/purge", async (c) => {
|
||||
const step = await purgeFeedKeysStep(emailStorage, feedId, {
|
||||
cursor,
|
||||
limit,
|
||||
bucket: env.ATTACHMENT_BUCKET,
|
||||
});
|
||||
|
||||
return c.json({
|
||||
@@ -3420,15 +3448,18 @@ app.post("/emails/:emailKey/delete", async (c) => {
|
||||
return c.text("Feed ID is required", 400);
|
||||
}
|
||||
|
||||
// Delete the email
|
||||
await emailStorage.delete(emailKey);
|
||||
|
||||
// Remove the email from the feed metadata
|
||||
// Load metadata first to collect attachment IDs for R2 cleanup
|
||||
const feedMetadataKey = `feed:${feedId}:metadata`;
|
||||
const feedMetadata = (await emailStorage.get(feedMetadataKey, {
|
||||
type: "json",
|
||||
})) as FeedMetadata | null;
|
||||
|
||||
const attachmentIds =
|
||||
feedMetadata?.emails.find((e) => e.key === emailKey)?.attachmentIds ?? [];
|
||||
|
||||
// Delete the email
|
||||
await emailStorage.delete(emailKey);
|
||||
|
||||
if (feedMetadata) {
|
||||
// Filter out the deleted email
|
||||
feedMetadata.emails = feedMetadata.emails.filter(
|
||||
@@ -3439,6 +3470,13 @@ app.post("/emails/:emailKey/delete", async (c) => {
|
||||
await emailStorage.put(feedMetadataKey, JSON.stringify(feedMetadata));
|
||||
}
|
||||
|
||||
// Best-effort R2 attachment cleanup
|
||||
if (env.ATTACHMENT_BUCKET && attachmentIds.length > 0) {
|
||||
await Promise.allSettled(
|
||||
attachmentIds.map((id) => env.ATTACHMENT_BUCKET!.delete(id)),
|
||||
);
|
||||
}
|
||||
|
||||
if (wantsJson) {
|
||||
return c.json({ ok: true, emailKey, feedId });
|
||||
}
|
||||
@@ -3508,6 +3546,13 @@ app.post("/feeds/:feedId/emails/bulk-delete", async (c) => {
|
||||
}
|
||||
|
||||
const candidates = emailKeys.filter((key) => allowedKeys.has(key));
|
||||
|
||||
// Collect attachment IDs from metadata before deleting (no extra KV reads needed)
|
||||
const candidateSet = new Set(candidates);
|
||||
const r2AttachmentIds = feedMetadata.emails
|
||||
.filter((e) => candidateSet.has(e.key))
|
||||
.flatMap((e) => e.attachmentIds ?? []);
|
||||
|
||||
const { ok: deletedOk, failed: failedEmailKeys } =
|
||||
await deleteKeysWithConcurrency(emailStorage, candidates, 35);
|
||||
|
||||
@@ -3517,6 +3562,13 @@ app.post("/feeds/:feedId/emails/bulk-delete", async (c) => {
|
||||
);
|
||||
await emailStorage.put(feedMetadataKey, JSON.stringify(feedMetadata));
|
||||
|
||||
// Best-effort R2 attachment cleanup
|
||||
if (env.ATTACHMENT_BUCKET && r2AttachmentIds.length > 0) {
|
||||
await Promise.allSettled(
|
||||
r2AttachmentIds.map((id) => env.ATTACHMENT_BUCKET!.delete(id)),
|
||||
);
|
||||
}
|
||||
|
||||
return c.json({
|
||||
ok: failedEmailKeys.length === 0,
|
||||
deletedEmailKeys: deletedOk,
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
import { describe, it, expect, beforeEach } from "vitest";
|
||||
import "../test/setup";
|
||||
import { createMockEnv, MockR2 } from "../test/setup";
|
||||
import { Hono } from "hono";
|
||||
import { handle as handleFiles } from "./files";
|
||||
|
||||
async function request(
|
||||
env: ReturnType<typeof createMockEnv>,
|
||||
path: string,
|
||||
): Promise<Response> {
|
||||
const app = new Hono();
|
||||
const files = new Hono();
|
||||
files.get("/:attachmentId/:filename", handleFiles);
|
||||
app.route("/files", files);
|
||||
return app.request(path, {}, env as any);
|
||||
}
|
||||
|
||||
describe("GET /files/:attachmentId/:filename", () => {
|
||||
let envNoR2: ReturnType<typeof createMockEnv>;
|
||||
let envWithR2: ReturnType<typeof createMockEnv>;
|
||||
let mockR2: MockR2;
|
||||
|
||||
beforeEach(() => {
|
||||
envNoR2 = createMockEnv();
|
||||
envWithR2 = createMockEnv({ withR2: true });
|
||||
mockR2 = (envWithR2 as any).ATTACHMENT_BUCKET as unknown as MockR2;
|
||||
});
|
||||
|
||||
it("returns 404 when ATTACHMENT_BUCKET is not configured", async () => {
|
||||
const res = await request(envNoR2, "/files/some-id/file.pdf");
|
||||
expect(res.status).toBe(404);
|
||||
expect(await res.text()).toContain("not configured");
|
||||
});
|
||||
|
||||
it("returns 404 when attachment ID is not found in R2", async () => {
|
||||
const res = await request(envWithR2, "/files/unknown-id/file.pdf");
|
||||
expect(res.status).toBe(404);
|
||||
expect(await res.text()).toBe("Not found");
|
||||
});
|
||||
|
||||
it("returns 200 with stored content when attachment exists", async () => {
|
||||
const content = new TextEncoder().encode("PDF content").buffer as ArrayBuffer;
|
||||
await mockR2.put("test-uuid", content, {
|
||||
httpMetadata: { contentType: "application/pdf" },
|
||||
});
|
||||
|
||||
const res = await request(envWithR2, "/files/test-uuid/report.pdf");
|
||||
expect(res.status).toBe(200);
|
||||
});
|
||||
|
||||
it("returns correct Content-Type from stored httpMetadata", async () => {
|
||||
const content = new TextEncoder().encode("data").buffer as ArrayBuffer;
|
||||
await mockR2.put("img-uuid", content, {
|
||||
httpMetadata: { contentType: "image/png" },
|
||||
});
|
||||
|
||||
const res = await request(envWithR2, "/files/img-uuid/photo.png");
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers.get("Content-Type")).toContain("image/png");
|
||||
});
|
||||
|
||||
it("sets Cache-Control immutable header", async () => {
|
||||
const content = new TextEncoder().encode("data").buffer as ArrayBuffer;
|
||||
await mockR2.put("cache-uuid", content, {
|
||||
httpMetadata: { contentType: "application/pdf" },
|
||||
});
|
||||
|
||||
const res = await request(envWithR2, "/files/cache-uuid/doc.pdf");
|
||||
expect(res.headers.get("Cache-Control")).toBe(
|
||||
"public, max-age=31536000, immutable",
|
||||
);
|
||||
});
|
||||
|
||||
it("sets Content-Disposition from httpMetadata when present", async () => {
|
||||
const content = new TextEncoder().encode("data").buffer as ArrayBuffer;
|
||||
await mockR2.put("disp-uuid", content, {
|
||||
httpMetadata: {
|
||||
contentType: "application/pdf",
|
||||
contentDisposition: 'attachment; filename="stored.pdf"',
|
||||
},
|
||||
});
|
||||
|
||||
const res = await request(envWithR2, "/files/disp-uuid/other.pdf");
|
||||
expect(res.headers.get("Content-Disposition")).toBe(
|
||||
'attachment; filename="stored.pdf"',
|
||||
);
|
||||
});
|
||||
|
||||
it("falls back to URL filename for Content-Disposition when not in httpMetadata", async () => {
|
||||
const content = new TextEncoder().encode("data").buffer as ArrayBuffer;
|
||||
await mockR2.put("fallback-uuid", content, {
|
||||
httpMetadata: { contentType: "text/plain" },
|
||||
});
|
||||
|
||||
const res = await request(
|
||||
envWithR2,
|
||||
"/files/fallback-uuid/hello%20world.txt",
|
||||
);
|
||||
expect(res.headers.get("Content-Disposition")).toBe(
|
||||
'attachment; filename="hello world.txt"',
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,33 @@
|
||||
import { Context } from "hono";
|
||||
import { Env } from "../types";
|
||||
|
||||
export async function handle(c: Context): Promise<Response> {
|
||||
const env = c.env as unknown as Env;
|
||||
|
||||
if (!env.ATTACHMENT_BUCKET) {
|
||||
return new Response("Attachment storage not configured", { status: 404 });
|
||||
}
|
||||
|
||||
const attachmentId = c.req.param("attachmentId");
|
||||
const filename = c.req.param("filename");
|
||||
|
||||
const object = await env.ATTACHMENT_BUCKET.get(attachmentId);
|
||||
|
||||
if (!object) {
|
||||
return new Response("Not found", { status: 404 });
|
||||
}
|
||||
|
||||
const headers = new Headers();
|
||||
object.writeHttpMetadata(headers);
|
||||
headers.set("etag", object.httpEtag);
|
||||
headers.set("Cache-Control", "public, max-age=31536000, immutable");
|
||||
|
||||
if (!headers.get("Content-Disposition")) {
|
||||
headers.set(
|
||||
"Content-Disposition",
|
||||
`attachment; filename="${decodeURIComponent(filename)}"`,
|
||||
);
|
||||
}
|
||||
|
||||
return new Response(object.body, { headers });
|
||||
}
|
||||
Reference in New Issue
Block a user