From e93bbb8d3e19b6fefc1a83aa1ff0a03845f071cb Mon Sep 17 00:00:00 2001 From: Julien Herr Date: Thu, 21 May 2026 09:09:37 +0200 Subject: [PATCH] feat: store email attachments in R2 and expose as RSS enclosures Attachments from incoming emails are uploaded to an optional Cloudflare R2 bucket and exposed as elements in RSS and in Atom feeds, served at /files/{id}/{filename} with immutable caching. R2 is opt-in: if ATTACHMENT_BUCKET is not bound the feature is a no-op. Attachments are cleaned up from R2 on email/feed deletion and during size-based feed trimming. Adds MockR2 to the test setup. Co-Authored-By: Claude Sonnet 4.6 --- README.md | 35 +++++++++ TODO.md | 2 +- src/index.ts | 6 ++ src/lib/cloudflare-email.ts | 11 ++- src/lib/email-processor.test.ts | 127 ++++++++++++++++++++++++++++++- src/lib/email-processor.ts | 60 +++++++++++++-- src/lib/forwardemail.ts | 39 +++++++++- src/routes/admin.ts | 64 ++++++++++++++-- src/routes/files.test.ts | 103 +++++++++++++++++++++++++ src/routes/files.ts | 33 ++++++++ src/test/setup.ts | 82 +++++++++++++++++++- src/types/index.ts | 11 +++ src/utils/feed-generator.test.ts | 43 +++++++++++ src/utils/feed-generator.ts | 8 ++ wrangler-example.toml | 10 +++ 15 files changed, 615 insertions(+), 19 deletions(-) create mode 100644 src/routes/files.test.ts create mode 100644 src/routes/files.ts diff --git a/README.md b/README.md index 3b6f725..df3a5d4 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,8 @@ Email-to-RSS keeps the same workflow while avoiding shared domains and shared da - ForwardEmail webhook ingestion with source-IP verification (optional alternative) - Optional per-feed sender allowlist (`email@domain.com` or `domain.com`) - RSS generation on demand (`/rss/:feedId`) +- Atom feed at `/atom/:feedId` +- Email attachments stored in Cloudflare R2 and exposed as RSS enclosures (optional) - Cloudflare KV storage for feed config + email metadata/content - Password-protected admin UI @@ -45,6 +47,8 @@ Main routes: - `src/lib/cloudflare-email.ts`: Cloudflare Email Workers ingestion - `src/routes/inbound.ts`: ForwardEmail webhook ingestion - `src/routes/rss.ts`: RSS rendering +- `src/routes/atom.ts`: Atom feed rendering +- `src/routes/files.ts`: attachment file serving from R2 - `src/routes/admin.ts`: admin UI + feed CRUD ## Requirements @@ -141,6 +145,32 @@ To override the threshold, add to `wrangler.toml` under `[vars]`: FEED_MAX_SIZE_BYTES = "524288" # 512 KB — adjust as needed ``` +### Email attachments (R2) + +When an incoming email contains attachments, the Worker can store them in a Cloudflare R2 bucket and expose them as `` elements in the RSS feed (and `` in Atom). Each attachment is served at `/files/{id}/{filename}` with an immutable cache header. + +This feature is **optional**. If no R2 bucket is bound, attachments are silently ignored and nothing else changes. + +**Setup:** + +1. Create an R2 bucket in the Cloudflare dashboard (*R2 Object Storage → Create bucket*), or with Wrangler: + ```bash + npx wrangler r2 bucket create your-bucket-name + ``` +2. In `wrangler.toml`, uncomment and fill in the R2 binding (the commented block from `wrangler-example.toml`): + ```toml + r2_buckets = [ + { binding = "ATTACHMENT_BUCKET", bucket_name = "your-bucket-name", preview_bucket_name = "your-bucket-name-preview" } + ] + ``` + Do the same under `[env.production]` (without `preview_bucket_name`). +3. Redeploy: + ```bash + npm run deploy + ``` + +Attachments are deleted from R2 automatically when the corresponding email is deleted from the admin UI, or when an email is dropped during feed size trimming. + ### External auth provider (Authelia / Authentik / reverse proxy) Instead of the built-in password login you can delegate admin authentication to a reverse proxy that sets a trusted user header (`Remote-User` or `X-Forwarded-User`). @@ -188,6 +218,11 @@ npm run build Then update `compatibility_date` and redeploy. +## Acknowledgements + +- [kill-the-newsletter](https://github.com/leafac/kill-the-newsletter) by Leandro Facchinetti — the inspiration for this project and the reference implementation for feature ideas (Atom feeds, attachment enclosures, entry HTML views, and more). +- [Email-to-RSS](https://github.com/yl8976/Email-to-RSS) by yl8976 — the initial codebase this project is based on. + ## License MIT diff --git a/TODO.md b/TODO.md index 32d209a..58fcffb 100644 --- a/TODO.md +++ b/TODO.md @@ -20,6 +20,6 @@ Feature gaps identified by comparing with [kill-the-newsletter](https://github.c ## Heavy -- [ ] **Email attachments as RSS enclosures** — store attachments in Cloudflare R2 and expose them as `` elements in the feed. kill-the-newsletter serves them at `/files/{enclosureId}/{filename}`. +- [x] **Email attachments as RSS enclosures** — store attachments in Cloudflare R2 and expose them as `` elements in the feed. kill-the-newsletter serves them at `/files/{enclosureId}/{filename}`. - [ ] **WebSub (PubSubHubbub) push notifications** — notify subscribers in real time when a new email arrives, instead of requiring them to poll the feed. Requires either integrating a public WebSub hub or implementing the hub protocol directly. diff --git a/src/index.ts b/src/index.ts index 06c0e7d..4f3d6a0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,6 +4,7 @@ import { handle as handleRSS } from "./routes/rss"; import { handle as handleAtom } from "./routes/atom"; import { handle as handleAdmin } from "./routes/admin"; import { handle as handleEntry } from "./routes/entries"; +import { handle as handleFiles } from "./routes/files"; import { handleCloudflareEmail } from "./lib/cloudflare-email"; import { Env } from "./types"; @@ -105,6 +106,7 @@ const api = new Hono(); const rss = new Hono(); const atom = new Hono(); const entries = new Hono(); +const files = new Hono(); const admin = new Hono(); // Webhook security middleware for /inbound - verify ForwardEmail.net IP @@ -141,6 +143,9 @@ atom.get("/:feedId", handleAtom); // Email entry HTML view (public) entries.get("/:feedId/:entryId", handleEntry); +// Attachment file serving (public) +files.get("/:attachmentId/:filename", handleFiles); + // Admin routes (protected) admin.route("/", handleAdmin); @@ -149,6 +154,7 @@ app.route("/api", api); app.route("/rss", rss); app.route("/atom", atom); app.route("/entries", entries); +app.route("/files", files); app.route("/admin", admin); // Root path redirects to admin dashboard diff --git a/src/lib/cloudflare-email.ts b/src/lib/cloudflare-email.ts index 3c17382..2b51acf 100644 --- a/src/lib/cloudflare-email.ts +++ b/src/lib/cloudflare-email.ts @@ -1,6 +1,6 @@ import PostalMime from "postal-mime"; import { Env } from "../types"; -import { processEmail } from "./email-processor"; +import { processEmail, RawAttachment } from "./email-processor"; export async function handleCloudflareEmail( message: ForwardableEmailMessage, @@ -21,6 +21,14 @@ export async function handleCloudflareEmail( headers[h.key] = h.value; } + const rawAttachments: RawAttachment[] = (email.attachments ?? []) + .filter((a) => a.content instanceof ArrayBuffer) + .map((a) => ({ + filename: a.filename || "attachment", + contentType: a.mimeType || "application/octet-stream", + content: a.content as ArrayBuffer, + })); + await processEmail( { toAddress: message.to, @@ -30,6 +38,7 @@ export async function handleCloudflareEmail( content: email.html ?? email.text ?? "", receivedAt: email.date ? new Date(email.date).getTime() : Date.now(), headers, + attachments: rawAttachments, }, env, ); diff --git a/src/lib/email-processor.test.ts b/src/lib/email-processor.test.ts index 21b7a49..ac7163d 100644 --- a/src/lib/email-processor.test.ts +++ b/src/lib/email-processor.test.ts @@ -1,7 +1,7 @@ import { describe, it, expect, beforeEach } from "vitest"; import "../test/setup"; -import { createMockEnv } from "../test/setup"; -import { processEmail, ProcessEmailInput } from "./email-processor"; +import { createMockEnv, MockR2 } from "../test/setup"; +import { processEmail, ProcessEmailInput, RawAttachment } from "./email-processor"; const VALID_FEED_ID = "apple.mountain.42"; const VALID_TO = `${VALID_FEED_ID}@test.getmynews.app`; @@ -179,3 +179,126 @@ describe("processEmail", () => { expect(metadata.emails).toHaveLength(2); }); }); + +describe("processEmail — attachments", () => { + const pdfContent = new TextEncoder().encode("PDF bytes").buffer as ArrayBuffer; + + const pdfAttachment: RawAttachment = { + filename: "report.pdf", + contentType: "application/pdf", + content: pdfContent, + }; + + it("skips R2 upload when ATTACHMENT_BUCKET is not configured", async () => { + const env = createMockEnv(); + await env.EMAIL_STORAGE.put( + `feed:${VALID_FEED_ID}:config`, + JSON.stringify({}), + ); + const res = await processEmail( + makeInput({ attachments: [pdfAttachment] }), + env as any, + ); + expect(res.status).toBe(200); + + const metadata = await env.EMAIL_STORAGE.get( + `feed:${VALID_FEED_ID}:metadata`, + "json", + ); + const emailData = await env.EMAIL_STORAGE.get(metadata.emails[0].key, "json"); + expect(emailData.attachments).toBeUndefined(); + }); + + it("uploads attachments to R2 and stores AttachmentData in emailData", async () => { + const env = createMockEnv({ withR2: true }); + const mockR2 = (env as any).ATTACHMENT_BUCKET as unknown as MockR2; + await env.EMAIL_STORAGE.put( + `feed:${VALID_FEED_ID}:config`, + JSON.stringify({}), + ); + const res = await processEmail( + makeInput({ attachments: [pdfAttachment] }), + env as any, + ); + expect(res.status).toBe(200); + + const metadata = await env.EMAIL_STORAGE.get( + `feed:${VALID_FEED_ID}:metadata`, + "json", + ); + const emailData = await env.EMAIL_STORAGE.get(metadata.emails[0].key, "json"); + + expect(emailData.attachments).toHaveLength(1); + expect(emailData.attachments[0].filename).toBe("report.pdf"); + expect(emailData.attachments[0].contentType).toBe("application/pdf"); + expect(emailData.attachments[0].size).toBe(pdfContent.byteLength); + + const id = emailData.attachments[0].id; + expect(mockR2._has(id)).toBe(true); + }); + + it("stores attachmentIds in EmailMetadata for trim-time cleanup", async () => { + const env = createMockEnv({ withR2: true }); + await env.EMAIL_STORAGE.put( + `feed:${VALID_FEED_ID}:config`, + JSON.stringify({}), + ); + await processEmail(makeInput({ attachments: [pdfAttachment] }), env as any); + + const metadata = await env.EMAIL_STORAGE.get( + `feed:${VALID_FEED_ID}:metadata`, + "json", + ); + expect(metadata.emails[0].attachmentIds).toHaveLength(1); + expect(typeof metadata.emails[0].attachmentIds[0]).toBe("string"); + }); + + it("deletes R2 objects when a trimmed email had attachments", async () => { + const env = createMockEnv({ withR2: true }); + const mockR2 = (env as any).ATTACHMENT_BUCKET as unknown as MockR2; + await env.EMAIL_STORAGE.put( + `feed:${VALID_FEED_ID}:config`, + JSON.stringify({}), + ); + + // Store an old email with attachment in KV and metadata + const oldKey = `feed:${VALID_FEED_ID}:111`; + const oldAttachmentId = "old-attachment-uuid"; + const bigContent = "x".repeat(200); + const oldEmail = JSON.stringify({ + subject: "Old", + from: "a@b.com", + content: bigContent, + receivedAt: 111, + headers: {}, + attachments: [{ id: oldAttachmentId, filename: "old.pdf", contentType: "application/pdf", size: 100 }], + }); + await env.EMAIL_STORAGE.put(oldKey, oldEmail); + + // Also put the attachment in mock R2 + await mockR2.put(oldAttachmentId, new ArrayBuffer(100)); + + await env.EMAIL_STORAGE.put( + `feed:${VALID_FEED_ID}:metadata`, + JSON.stringify({ + emails: [ + { + key: oldKey, + subject: "Old", + receivedAt: 111, + size: oldEmail.length, + attachmentIds: [oldAttachmentId], + }, + ], + }), + ); + + // Process with tight size budget to force trimming + const tinyEnv = { ...env, FEED_MAX_SIZE_BYTES: "50" }; + const res = await processEmail(makeInput({ subject: "New" }), tinyEnv as any); + expect(res.status).toBe(200); + + // Old attachment should be deleted from R2 + expect(mockR2._has(oldAttachmentId)).toBe(false); + }); +}); diff --git a/src/lib/email-processor.ts b/src/lib/email-processor.ts index cf53f5a..d94f281 100644 --- a/src/lib/email-processor.ts +++ b/src/lib/email-processor.ts @@ -1,5 +1,11 @@ import { EmailParser } from "../utils/email-parser"; -import { Env, FeedConfig, FeedMetadata } from "../types"; +import { AttachmentData, EmailMetadata, Env, FeedConfig, FeedMetadata } from "../types"; + +export interface RawAttachment { + filename: string; + contentType: string; + content: ArrayBuffer; +} export interface ProcessEmailInput { toAddress: string; @@ -9,6 +15,7 @@ export interface ProcessEmailInput { content: string; receivedAt: number; headers?: Record; + attachments?: RawAttachment[]; } function normalizeEmail(value: string): string { @@ -34,6 +41,29 @@ function senderMatchesAllowlist( return senderDomain === normalizedDomain; } +async function uploadAttachments( + attachments: RawAttachment[], + bucket: R2Bucket, +): Promise { + return Promise.all( + attachments.map(async (att) => { + const id = crypto.randomUUID(); + await bucket.put(id, att.content, { + httpMetadata: { + contentType: att.contentType, + contentDisposition: `attachment; filename="${att.filename}"`, + }, + }); + return { + id, + filename: att.filename, + contentType: att.contentType, + size: att.content.byteLength, + }; + }), + ); +} + export async function processEmail( input: ProcessEmailInput, env: Env, @@ -74,12 +104,18 @@ export async function processEmail( } } + const storedAttachments: AttachmentData[] = + env.ATTACHMENT_BUCKET && input.attachments?.length + ? await uploadAttachments(input.attachments, env.ATTACHMENT_BUCKET) + : []; + const emailData = { subject: input.subject, from: input.from, content: input.content, receivedAt: input.receivedAt, headers: input.headers ?? {}, + ...(storedAttachments.length > 0 ? { attachments: storedAttachments } : {}), }; const emailKey = `feed:${feedId}:${Date.now()}`; @@ -104,24 +140,36 @@ export async function processEmail( const serialised = JSON.stringify(emailData); const serialisedSize = new TextEncoder().encode(serialised).byteLength; - feedMetadata.emails.unshift({ + const newEntry: EmailMetadata = { key: emailKey, subject: emailData.subject, receivedAt: emailData.receivedAt, size: serialisedSize, - }); + ...(storedAttachments.length > 0 + ? { attachmentIds: storedAttachments.map((a) => a.id) } + : {}), + }; + feedMetadata.emails.unshift(newEntry); let totalSize = feedMetadata.emails.reduce((sum, e) => sum + (e.size ?? 0), 0); - const toDelete: string[] = []; + const toDelete: EmailMetadata[] = []; while (totalSize > maxBytes && feedMetadata.emails.length > 1) { const dropped = feedMetadata.emails.pop()!; totalSize -= dropped.size ?? 0; - toDelete.push(dropped.key); + toDelete.push(dropped); } + const r2Deletions = + env.ATTACHMENT_BUCKET && toDelete.length > 0 + ? toDelete + .flatMap((e) => e.attachmentIds ?? []) + .map((id) => env.ATTACHMENT_BUCKET!.delete(id)) + : []; + await Promise.all([ env.EMAIL_STORAGE.put(feedMetadataKey, JSON.stringify(feedMetadata)), - ...toDelete.map((k) => env.EMAIL_STORAGE.delete(k)), + ...toDelete.map((e) => env.EMAIL_STORAGE.delete(e.key)), + ...r2Deletions, ]); console.log(`Successfully processed email for feed ${feedId}`); diff --git a/src/lib/forwardemail.ts b/src/lib/forwardemail.ts index 87e40a2..5488938 100644 --- a/src/lib/forwardemail.ts +++ b/src/lib/forwardemail.ts @@ -1,6 +1,16 @@ import { EmailParser } from "../utils/email-parser"; import { Env } from "../types"; -import { processEmail } from "./email-processor"; +import { processEmail, RawAttachment } from "./email-processor"; + +export interface ForwardEmailAttachment { + filename?: string; + contentType?: string; + size?: number; + content?: + | { type: "Buffer"; data: number[] } + | ArrayBuffer + | ArrayBufferView; +} export interface ForwardEmailPayload { recipients?: string[]; @@ -17,7 +27,7 @@ export interface ForwardEmailPayload { headerLines?: Array<{ key: string; line: string }>; headers?: string; raw?: string; - attachments?: Array; + attachments?: ForwardEmailAttachment[]; } function normalizeEmail(value: string): string { @@ -41,12 +51,36 @@ function extractSenderAddresses(payload: ForwardEmailPayload): string[] { return Array.from(new Set(matches.map(normalizeEmail))); } +function toArrayBuffer( + content: ForwardEmailAttachment["content"], +): ArrayBuffer | null { + if (!content) return null; + if (content instanceof ArrayBuffer) return content; + if (ArrayBuffer.isView(content)) return (content as ArrayBufferView).buffer as ArrayBuffer; + if (typeof content === "object" && content.type === "Buffer" && Array.isArray(content.data)) { + return Uint8Array.from(content.data).buffer as ArrayBuffer; + } + return null; +} + export async function handleForwardEmail( payload: ForwardEmailPayload, env: Env, ): Promise { const emailData = EmailParser.parseForwardEmailPayload(payload); + const rawAttachments: RawAttachment[] = (payload.attachments ?? []) + .map((a) => { + const buffer = toArrayBuffer(a.content); + if (!buffer) return null; + return { + filename: a.filename || "attachment", + contentType: a.contentType || "application/octet-stream", + content: buffer, + }; + }) + .filter((a): a is RawAttachment => a !== null); + return processEmail( { toAddress: payload.recipients?.[0] || "", @@ -56,6 +90,7 @@ export async function handleForwardEmail( content: emailData.content, receivedAt: emailData.receivedAt, headers: emailData.headers, + attachments: rawAttachments, }, env, ); diff --git a/src/routes/admin.ts b/src/routes/admin.ts index 701042a..e3efa8d 100644 --- a/src/routes/admin.ts +++ b/src/routes/admin.ts @@ -1908,7 +1908,7 @@ async function deleteFeedFastDetailed( async function purgeFeedKeysStep( emailStorage: KVNamespace, feedId: string, - options: { cursor?: string; limit?: number } = {}, + options: { cursor?: string; limit?: number; bucket?: R2Bucket } = {}, ): Promise<{ deletedKeys: string[]; failedKeys: string[]; @@ -1921,6 +1921,33 @@ async function purgeFeedKeysStep( const listed = await emailStorage.list({ prefix, cursor, limit }); const keys = (listed.keys || []).map((k) => k.name); + + // Collect R2 attachment IDs from email entries before deleting + if (options.bucket && keys.length > 0) { + const emailKeys = keys.filter((k) => { + const suffix = k.slice(prefix.length); + return suffix !== "config" && suffix !== "metadata"; + }); + if (emailKeys.length > 0) { + const emailDataResults = await Promise.allSettled( + emailKeys.map((k) => + emailStorage.get(k, { type: "json" }) as Promise, + ), + ); + const attachmentIds = emailDataResults + .filter( + (r): r is PromiseFulfilledResult => + r.status === "fulfilled", + ) + .flatMap((r) => r.value?.attachments?.map((a) => a.id) ?? []); + if (attachmentIds.length > 0) { + await Promise.allSettled( + attachmentIds.map((id) => options.bucket!.delete(id)), + ); + } + } + } + const { ok, failed } = await deleteKeysWithConcurrency( emailStorage, keys, @@ -1949,7 +1976,7 @@ app.post("/feeds/:feedId/delete", async (c) => { // Best-effort cleanup in the background so the request stays fast. // Use the UI purge endpoint for full, user-visible progress. - waitUntilSafe(c, purgeFeedKeysStep(emailStorage, feedId)); + waitUntilSafe(c, purgeFeedKeysStep(emailStorage, feedId, { bucket: env.ATTACHMENT_BUCKET })); if (wantsJson) { return c.json({ ok: true, feedId }); } @@ -1987,6 +2014,7 @@ app.post("/feeds/:feedId/purge", async (c) => { const step = await purgeFeedKeysStep(emailStorage, feedId, { cursor, limit, + bucket: env.ATTACHMENT_BUCKET, }); return c.json({ @@ -3420,15 +3448,18 @@ app.post("/emails/:emailKey/delete", async (c) => { return c.text("Feed ID is required", 400); } - // Delete the email - await emailStorage.delete(emailKey); - - // Remove the email from the feed metadata + // Load metadata first to collect attachment IDs for R2 cleanup const feedMetadataKey = `feed:${feedId}:metadata`; const feedMetadata = (await emailStorage.get(feedMetadataKey, { type: "json", })) as FeedMetadata | null; + const attachmentIds = + feedMetadata?.emails.find((e) => e.key === emailKey)?.attachmentIds ?? []; + + // Delete the email + await emailStorage.delete(emailKey); + if (feedMetadata) { // Filter out the deleted email feedMetadata.emails = feedMetadata.emails.filter( @@ -3439,6 +3470,13 @@ app.post("/emails/:emailKey/delete", async (c) => { await emailStorage.put(feedMetadataKey, JSON.stringify(feedMetadata)); } + // Best-effort R2 attachment cleanup + if (env.ATTACHMENT_BUCKET && attachmentIds.length > 0) { + await Promise.allSettled( + attachmentIds.map((id) => env.ATTACHMENT_BUCKET!.delete(id)), + ); + } + if (wantsJson) { return c.json({ ok: true, emailKey, feedId }); } @@ -3508,6 +3546,13 @@ app.post("/feeds/:feedId/emails/bulk-delete", async (c) => { } const candidates = emailKeys.filter((key) => allowedKeys.has(key)); + + // Collect attachment IDs from metadata before deleting (no extra KV reads needed) + const candidateSet = new Set(candidates); + const r2AttachmentIds = feedMetadata.emails + .filter((e) => candidateSet.has(e.key)) + .flatMap((e) => e.attachmentIds ?? []); + const { ok: deletedOk, failed: failedEmailKeys } = await deleteKeysWithConcurrency(emailStorage, candidates, 35); @@ -3517,6 +3562,13 @@ app.post("/feeds/:feedId/emails/bulk-delete", async (c) => { ); await emailStorage.put(feedMetadataKey, JSON.stringify(feedMetadata)); + // Best-effort R2 attachment cleanup + if (env.ATTACHMENT_BUCKET && r2AttachmentIds.length > 0) { + await Promise.allSettled( + r2AttachmentIds.map((id) => env.ATTACHMENT_BUCKET!.delete(id)), + ); + } + return c.json({ ok: failedEmailKeys.length === 0, deletedEmailKeys: deletedOk, diff --git a/src/routes/files.test.ts b/src/routes/files.test.ts new file mode 100644 index 0000000..0480362 --- /dev/null +++ b/src/routes/files.test.ts @@ -0,0 +1,103 @@ +import { describe, it, expect, beforeEach } from "vitest"; +import "../test/setup"; +import { createMockEnv, MockR2 } from "../test/setup"; +import { Hono } from "hono"; +import { handle as handleFiles } from "./files"; + +async function request( + env: ReturnType, + path: string, +): Promise { + const app = new Hono(); + const files = new Hono(); + files.get("/:attachmentId/:filename", handleFiles); + app.route("/files", files); + return app.request(path, {}, env as any); +} + +describe("GET /files/:attachmentId/:filename", () => { + let envNoR2: ReturnType; + let envWithR2: ReturnType; + let mockR2: MockR2; + + beforeEach(() => { + envNoR2 = createMockEnv(); + envWithR2 = createMockEnv({ withR2: true }); + mockR2 = (envWithR2 as any).ATTACHMENT_BUCKET as unknown as MockR2; + }); + + it("returns 404 when ATTACHMENT_BUCKET is not configured", async () => { + const res = await request(envNoR2, "/files/some-id/file.pdf"); + expect(res.status).toBe(404); + expect(await res.text()).toContain("not configured"); + }); + + it("returns 404 when attachment ID is not found in R2", async () => { + const res = await request(envWithR2, "/files/unknown-id/file.pdf"); + expect(res.status).toBe(404); + expect(await res.text()).toBe("Not found"); + }); + + it("returns 200 with stored content when attachment exists", async () => { + const content = new TextEncoder().encode("PDF content").buffer as ArrayBuffer; + await mockR2.put("test-uuid", content, { + httpMetadata: { contentType: "application/pdf" }, + }); + + const res = await request(envWithR2, "/files/test-uuid/report.pdf"); + expect(res.status).toBe(200); + }); + + it("returns correct Content-Type from stored httpMetadata", async () => { + const content = new TextEncoder().encode("data").buffer as ArrayBuffer; + await mockR2.put("img-uuid", content, { + httpMetadata: { contentType: "image/png" }, + }); + + const res = await request(envWithR2, "/files/img-uuid/photo.png"); + expect(res.status).toBe(200); + expect(res.headers.get("Content-Type")).toContain("image/png"); + }); + + it("sets Cache-Control immutable header", async () => { + const content = new TextEncoder().encode("data").buffer as ArrayBuffer; + await mockR2.put("cache-uuid", content, { + httpMetadata: { contentType: "application/pdf" }, + }); + + const res = await request(envWithR2, "/files/cache-uuid/doc.pdf"); + expect(res.headers.get("Cache-Control")).toBe( + "public, max-age=31536000, immutable", + ); + }); + + it("sets Content-Disposition from httpMetadata when present", async () => { + const content = new TextEncoder().encode("data").buffer as ArrayBuffer; + await mockR2.put("disp-uuid", content, { + httpMetadata: { + contentType: "application/pdf", + contentDisposition: 'attachment; filename="stored.pdf"', + }, + }); + + const res = await request(envWithR2, "/files/disp-uuid/other.pdf"); + expect(res.headers.get("Content-Disposition")).toBe( + 'attachment; filename="stored.pdf"', + ); + }); + + it("falls back to URL filename for Content-Disposition when not in httpMetadata", async () => { + const content = new TextEncoder().encode("data").buffer as ArrayBuffer; + await mockR2.put("fallback-uuid", content, { + httpMetadata: { contentType: "text/plain" }, + }); + + const res = await request( + envWithR2, + "/files/fallback-uuid/hello%20world.txt", + ); + expect(res.headers.get("Content-Disposition")).toBe( + 'attachment; filename="hello world.txt"', + ); + }); +}); diff --git a/src/routes/files.ts b/src/routes/files.ts new file mode 100644 index 0000000..a825646 --- /dev/null +++ b/src/routes/files.ts @@ -0,0 +1,33 @@ +import { Context } from "hono"; +import { Env } from "../types"; + +export async function handle(c: Context): Promise { + const env = c.env as unknown as Env; + + if (!env.ATTACHMENT_BUCKET) { + return new Response("Attachment storage not configured", { status: 404 }); + } + + const attachmentId = c.req.param("attachmentId"); + const filename = c.req.param("filename"); + + const object = await env.ATTACHMENT_BUCKET.get(attachmentId); + + if (!object) { + return new Response("Not found", { status: 404 }); + } + + const headers = new Headers(); + object.writeHttpMetadata(headers); + headers.set("etag", object.httpEtag); + headers.set("Cache-Control", "public, max-age=31536000, immutable"); + + if (!headers.get("Content-Disposition")) { + headers.set( + "Content-Disposition", + `attachment; filename="${decodeURIComponent(filename)}"`, + ); + } + + return new Response(object.body, { headers }); +} diff --git a/src/test/setup.ts b/src/test/setup.ts index c48a9a0..3f281cb 100644 --- a/src/test/setup.ts +++ b/src/test/setup.ts @@ -157,12 +157,92 @@ afterEach(() => { server.resetHandlers(); }); +/** + * Mock R2 bucket implementation + * Simulates Cloudflare Workers R2 storage using an in-memory Map + */ +export class MockR2 { + private store: Map< + string, + { + body: ArrayBuffer; + httpMetadata?: { contentType?: string; contentDisposition?: string }; + httpEtag: string; + } + > = new Map(); + + async put( + key: string, + value: ReadableStream | ArrayBuffer | ArrayBufferView | string | null | Blob, + options?: { + httpMetadata?: { contentType?: string; contentDisposition?: string }; + }, + ) { + let buffer: ArrayBuffer; + if (value instanceof ArrayBuffer) { + buffer = value; + } else if (typeof value === "string") { + buffer = new TextEncoder().encode(value).buffer as ArrayBuffer; + } else if (ArrayBuffer.isView(value)) { + buffer = value.buffer as ArrayBuffer; + } else { + buffer = new ArrayBuffer(0); + } + this.store.set(key, { + body: buffer, + httpMetadata: options?.httpMetadata, + httpEtag: `"mock-etag-${key}"`, + }); + } + + async get(key: string) { + const entry = this.store.get(key); + if (!entry) return null; + return { + key, + body: new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(entry.body)); + controller.close(); + }, + }), + httpEtag: entry.httpEtag, + httpMetadata: entry.httpMetadata, + size: entry.body.byteLength, + arrayBuffer: async () => entry.body, + text: async () => new TextDecoder().decode(entry.body), + writeHttpMetadata: (headers: Headers) => { + if (entry.httpMetadata?.contentType) { + headers.set("Content-Type", entry.httpMetadata.contentType); + } + if (entry.httpMetadata?.contentDisposition) { + headers.set("Content-Disposition", entry.httpMetadata.contentDisposition); + } + }, + }; + } + + async delete(keys: string | string[]) { + const arr = Array.isArray(keys) ? keys : [keys]; + for (const k of arr) { + this.store.delete(k); + } + } + + _has(key: string) { + return this.store.has(key); + } +} + /** * Create a mock environment for testing * @returns Mock environment with KV storage and configuration */ -export const createMockEnv = () => ({ +export const createMockEnv = (options: { withR2?: boolean } = {}) => ({ EMAIL_STORAGE: new MockKV(), DOMAIN: "test.getmynews.app", ADMIN_PASSWORD: "test-password", + ...(options.withR2 + ? { ATTACHMENT_BUCKET: new MockR2() as unknown as R2Bucket } + : {}), }); diff --git a/src/types/index.ts b/src/types/index.ts index 9899900..4200e82 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -3,11 +3,20 @@ export interface Env { EMAIL_STORAGE: KVNamespace; ADMIN_PASSWORD: string; DOMAIN: string; + ATTACHMENT_BUCKET?: R2Bucket; FEED_MAX_SIZE_BYTES?: string; PROXY_TRUSTED_IPS?: string; PROXY_AUTH_SECRET?: string; } +// Stored attachment metadata (bytes live in R2, keyed by id) +export interface AttachmentData { + id: string; + filename: string; + contentType: string; + size: number; +} + // Email interface for stored emails export interface EmailData { subject: string; @@ -15,6 +24,7 @@ export interface EmailData { content: string; receivedAt: number; headers: Record; + attachments?: AttachmentData[]; } // Feed configuration interface @@ -41,6 +51,7 @@ export interface EmailMetadata { subject: string; receivedAt: number; size?: number; + attachmentIds?: string[]; } // Feed list interface diff --git a/src/utils/feed-generator.test.ts b/src/utils/feed-generator.test.ts index 50deca1..8b80e00 100644 --- a/src/utils/feed-generator.test.ts +++ b/src/utils/feed-generator.test.ts @@ -21,6 +21,18 @@ const mockEmails: EmailData[] = [ }, ]; +const mockEmailWithAttachment: EmailData = { + ...mockEmails[0], + attachments: [ + { + id: "550e8400-e29b-41d4-a716-446655440000", + filename: "report.pdf", + contentType: "application/pdf", + size: 12345, + }, + ], +}; + const BASE_URL = "https://test.getmynews.app"; const FEED_ID = "abc123"; @@ -31,6 +43,25 @@ describe("generateRssFeed", () => { expect(result).toContain("Test Newsletter"); }); + it("includes element for email with attachment", () => { + const result = generateRssFeed(mockFeedConfig, [mockEmailWithAttachment], BASE_URL, FEED_ID); + expect(result).toContain(" for email without attachments", () => { + const result = generateRssFeed(mockFeedConfig, mockEmails, BASE_URL, FEED_ID); + expect(result).not.toContain(" { + const result = generateRssFeed(mockFeedConfig, [mockEmailWithAttachment], BASE_URL, FEED_ID); + expect(result).toContain(`${BASE_URL}/files/550e8400-e29b-41d4-a716-446655440000/report.pdf`); + }); + it("includes rss self-link in RSS output", () => { const result = generateRssFeed(mockFeedConfig, mockEmails, BASE_URL, FEED_ID); expect(result).toContain(`${BASE_URL}/rss/${FEED_ID}`); @@ -104,4 +135,16 @@ describe("generateAtomFeed", () => { const result = generateAtomFeed(configWithAuthor, mockEmails, BASE_URL, FEED_ID); expect(result).toContain("Bob"); }); + + it("includes enclosure link for email with attachment in Atom feed", () => { + const result = generateAtomFeed(mockFeedConfig, [mockEmailWithAttachment], BASE_URL, FEED_ID); + expect(result).toContain('rel="enclosure"'); + expect(result).toContain("550e8400-e29b-41d4-a716-446655440000"); + expect(result).toContain("report.pdf"); + }); + + it("does not include enclosure link for email without attachments in Atom feed", () => { + const result = generateAtomFeed(mockFeedConfig, mockEmails, BASE_URL, FEED_ID); + expect(result).not.toContain('rel="enclosure"'); + }); }); diff --git a/src/utils/feed-generator.ts b/src/utils/feed-generator.ts index 363ff5f..6f6968d 100644 --- a/src/utils/feed-generator.ts +++ b/src/utils/feed-generator.ts @@ -42,6 +42,7 @@ function buildFeed( for (const email of emails) { const uniqueId = `${email.receivedAt}-${Buffer.from(email.subject).toString("base64").substring(0, 10)}`; + const firstAttachment = email.attachments?.[0]; feed.addItem({ title: email.subject, id: uniqueId, @@ -50,6 +51,13 @@ function buildFeed( content: email.content, author: [parseFromAddress(email.from)], date: new Date(email.receivedAt), + enclosure: firstAttachment + ? { + url: `${baseUrl}/files/${firstAttachment.id}/${encodeURIComponent(firstAttachment.filename)}`, + type: firstAttachment.contentType, + length: firstAttachment.size, + } + : undefined, }); } diff --git a/wrangler-example.toml b/wrangler-example.toml index 8bcb8b7..6447cd4 100644 --- a/wrangler-example.toml +++ b/wrangler-example.toml @@ -8,6 +8,11 @@ kv_namespaces = [ { binding = "EMAIL_STORAGE", id = "REPLACE_WITH_YOUR_KV_NAMESPACE_ID", preview_id = "REPLACE_WITH_YOUR_PREVIEW_KV_NAMESPACE_ID" } ] +# Optional: R2 bucket for storing email attachment enclosures +# r2_buckets = [ +# { binding = "ATTACHMENT_BUCKET", bucket_name = "REPLACE_WITH_YOUR_BUCKET_NAME", preview_bucket_name = "REPLACE_WITH_YOUR_PREVIEW_BUCKET_NAME" } +# ] + # Workers Observability (keeps config in sync with dashboard toggle) [observability.logs] enabled = true @@ -42,6 +47,11 @@ kv_namespaces = [ { binding = "EMAIL_STORAGE", id = "REPLACE_WITH_YOUR_KV_NAMESPACE_ID" } ] +# Optional: R2 bucket for storing email attachment enclosures +# r2_buckets = [ +# { binding = "ATTACHMENT_BUCKET", bucket_name = "REPLACE_WITH_YOUR_BUCKET_NAME" } +# ] + routes = [ { pattern = "REPLACE_WITH_YOUR_DOMAIN", custom_domain = true }, { pattern = "www.REPLACE_WITH_YOUR_DOMAIN", custom_domain = true }