diff --git a/README.md b/README.md index 3b6f725..df3a5d4 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,8 @@ Email-to-RSS keeps the same workflow while avoiding shared domains and shared da - ForwardEmail webhook ingestion with source-IP verification (optional alternative) - Optional per-feed sender allowlist (`email@domain.com` or `domain.com`) - RSS generation on demand (`/rss/:feedId`) +- Atom feed at `/atom/:feedId` +- Email attachments stored in Cloudflare R2 and exposed as RSS enclosures (optional) - Cloudflare KV storage for feed config + email metadata/content - Password-protected admin UI @@ -45,6 +47,8 @@ Main routes: - `src/lib/cloudflare-email.ts`: Cloudflare Email Workers ingestion - `src/routes/inbound.ts`: ForwardEmail webhook ingestion - `src/routes/rss.ts`: RSS rendering +- `src/routes/atom.ts`: Atom feed rendering +- `src/routes/files.ts`: attachment file serving from R2 - `src/routes/admin.ts`: admin UI + feed CRUD ## Requirements @@ -141,6 +145,32 @@ To override the threshold, add to `wrangler.toml` under `[vars]`: FEED_MAX_SIZE_BYTES = "524288" # 512 KB — adjust as needed ``` +### Email attachments (R2) + +When an incoming email contains attachments, the Worker can store them in a Cloudflare R2 bucket and expose them as `` elements in the RSS feed (and `` in Atom). Each attachment is served at `/files/{id}/{filename}` with an immutable cache header. + +This feature is **optional**. If no R2 bucket is bound, attachments are silently ignored and nothing else changes. + +**Setup:** + +1. Create an R2 bucket in the Cloudflare dashboard (*R2 Object Storage → Create bucket*), or with Wrangler: + ```bash + npx wrangler r2 bucket create your-bucket-name + ``` +2. In `wrangler.toml`, uncomment and fill in the R2 binding (the commented block from `wrangler-example.toml`): + ```toml + r2_buckets = [ + { binding = "ATTACHMENT_BUCKET", bucket_name = "your-bucket-name", preview_bucket_name = "your-bucket-name-preview" } + ] + ``` + Do the same under `[env.production]` (without `preview_bucket_name`). +3. Redeploy: + ```bash + npm run deploy + ``` + +Attachments are deleted from R2 automatically when the corresponding email is deleted from the admin UI, or when an email is dropped during feed size trimming. + ### External auth provider (Authelia / Authentik / reverse proxy) Instead of the built-in password login you can delegate admin authentication to a reverse proxy that sets a trusted user header (`Remote-User` or `X-Forwarded-User`). @@ -188,6 +218,11 @@ npm run build Then update `compatibility_date` and redeploy. +## Acknowledgements + +- [kill-the-newsletter](https://github.com/leafac/kill-the-newsletter) by Leandro Facchinetti — the inspiration for this project and the reference implementation for feature ideas (Atom feeds, attachment enclosures, entry HTML views, and more). +- [Email-to-RSS](https://github.com/yl8976/Email-to-RSS) by yl8976 — the initial codebase this project is based on. + ## License MIT diff --git a/TODO.md b/TODO.md index 32d209a..58fcffb 100644 --- a/TODO.md +++ b/TODO.md @@ -20,6 +20,6 @@ Feature gaps identified by comparing with [kill-the-newsletter](https://github.c ## Heavy -- [ ] **Email attachments as RSS enclosures** — store attachments in Cloudflare R2 and expose them as `` elements in the feed. kill-the-newsletter serves them at `/files/{enclosureId}/{filename}`. +- [x] **Email attachments as RSS enclosures** — store attachments in Cloudflare R2 and expose them as `` elements in the feed. kill-the-newsletter serves them at `/files/{enclosureId}/{filename}`. - [ ] **WebSub (PubSubHubbub) push notifications** — notify subscribers in real time when a new email arrives, instead of requiring them to poll the feed. Requires either integrating a public WebSub hub or implementing the hub protocol directly. diff --git a/src/index.ts b/src/index.ts index 06c0e7d..4f3d6a0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,6 +4,7 @@ import { handle as handleRSS } from "./routes/rss"; import { handle as handleAtom } from "./routes/atom"; import { handle as handleAdmin } from "./routes/admin"; import { handle as handleEntry } from "./routes/entries"; +import { handle as handleFiles } from "./routes/files"; import { handleCloudflareEmail } from "./lib/cloudflare-email"; import { Env } from "./types"; @@ -105,6 +106,7 @@ const api = new Hono(); const rss = new Hono(); const atom = new Hono(); const entries = new Hono(); +const files = new Hono(); const admin = new Hono(); // Webhook security middleware for /inbound - verify ForwardEmail.net IP @@ -141,6 +143,9 @@ atom.get("/:feedId", handleAtom); // Email entry HTML view (public) entries.get("/:feedId/:entryId", handleEntry); +// Attachment file serving (public) +files.get("/:attachmentId/:filename", handleFiles); + // Admin routes (protected) admin.route("/", handleAdmin); @@ -149,6 +154,7 @@ app.route("/api", api); app.route("/rss", rss); app.route("/atom", atom); app.route("/entries", entries); +app.route("/files", files); app.route("/admin", admin); // Root path redirects to admin dashboard diff --git a/src/lib/cloudflare-email.ts b/src/lib/cloudflare-email.ts index 3c17382..2b51acf 100644 --- a/src/lib/cloudflare-email.ts +++ b/src/lib/cloudflare-email.ts @@ -1,6 +1,6 @@ import PostalMime from "postal-mime"; import { Env } from "../types"; -import { processEmail } from "./email-processor"; +import { processEmail, RawAttachment } from "./email-processor"; export async function handleCloudflareEmail( message: ForwardableEmailMessage, @@ -21,6 +21,14 @@ export async function handleCloudflareEmail( headers[h.key] = h.value; } + const rawAttachments: RawAttachment[] = (email.attachments ?? []) + .filter((a) => a.content instanceof ArrayBuffer) + .map((a) => ({ + filename: a.filename || "attachment", + contentType: a.mimeType || "application/octet-stream", + content: a.content as ArrayBuffer, + })); + await processEmail( { toAddress: message.to, @@ -30,6 +38,7 @@ export async function handleCloudflareEmail( content: email.html ?? email.text ?? "", receivedAt: email.date ? new Date(email.date).getTime() : Date.now(), headers, + attachments: rawAttachments, }, env, ); diff --git a/src/lib/email-processor.test.ts b/src/lib/email-processor.test.ts index 21b7a49..ac7163d 100644 --- a/src/lib/email-processor.test.ts +++ b/src/lib/email-processor.test.ts @@ -1,7 +1,7 @@ import { describe, it, expect, beforeEach } from "vitest"; import "../test/setup"; -import { createMockEnv } from "../test/setup"; -import { processEmail, ProcessEmailInput } from "./email-processor"; +import { createMockEnv, MockR2 } from "../test/setup"; +import { processEmail, ProcessEmailInput, RawAttachment } from "./email-processor"; const VALID_FEED_ID = "apple.mountain.42"; const VALID_TO = `${VALID_FEED_ID}@test.getmynews.app`; @@ -179,3 +179,126 @@ describe("processEmail", () => { expect(metadata.emails).toHaveLength(2); }); }); + +describe("processEmail — attachments", () => { + const pdfContent = new TextEncoder().encode("PDF bytes").buffer as ArrayBuffer; + + const pdfAttachment: RawAttachment = { + filename: "report.pdf", + contentType: "application/pdf", + content: pdfContent, + }; + + it("skips R2 upload when ATTACHMENT_BUCKET is not configured", async () => { + const env = createMockEnv(); + await env.EMAIL_STORAGE.put( + `feed:${VALID_FEED_ID}:config`, + JSON.stringify({}), + ); + const res = await processEmail( + makeInput({ attachments: [pdfAttachment] }), + env as any, + ); + expect(res.status).toBe(200); + + const metadata = await env.EMAIL_STORAGE.get( + `feed:${VALID_FEED_ID}:metadata`, + "json", + ); + const emailData = await env.EMAIL_STORAGE.get(metadata.emails[0].key, "json"); + expect(emailData.attachments).toBeUndefined(); + }); + + it("uploads attachments to R2 and stores AttachmentData in emailData", async () => { + const env = createMockEnv({ withR2: true }); + const mockR2 = (env as any).ATTACHMENT_BUCKET as unknown as MockR2; + await env.EMAIL_STORAGE.put( + `feed:${VALID_FEED_ID}:config`, + JSON.stringify({}), + ); + const res = await processEmail( + makeInput({ attachments: [pdfAttachment] }), + env as any, + ); + expect(res.status).toBe(200); + + const metadata = await env.EMAIL_STORAGE.get( + `feed:${VALID_FEED_ID}:metadata`, + "json", + ); + const emailData = await env.EMAIL_STORAGE.get(metadata.emails[0].key, "json"); + + expect(emailData.attachments).toHaveLength(1); + expect(emailData.attachments[0].filename).toBe("report.pdf"); + expect(emailData.attachments[0].contentType).toBe("application/pdf"); + expect(emailData.attachments[0].size).toBe(pdfContent.byteLength); + + const id = emailData.attachments[0].id; + expect(mockR2._has(id)).toBe(true); + }); + + it("stores attachmentIds in EmailMetadata for trim-time cleanup", async () => { + const env = createMockEnv({ withR2: true }); + await env.EMAIL_STORAGE.put( + `feed:${VALID_FEED_ID}:config`, + JSON.stringify({}), + ); + await processEmail(makeInput({ attachments: [pdfAttachment] }), env as any); + + const metadata = await env.EMAIL_STORAGE.get( + `feed:${VALID_FEED_ID}:metadata`, + "json", + ); + expect(metadata.emails[0].attachmentIds).toHaveLength(1); + expect(typeof metadata.emails[0].attachmentIds[0]).toBe("string"); + }); + + it("deletes R2 objects when a trimmed email had attachments", async () => { + const env = createMockEnv({ withR2: true }); + const mockR2 = (env as any).ATTACHMENT_BUCKET as unknown as MockR2; + await env.EMAIL_STORAGE.put( + `feed:${VALID_FEED_ID}:config`, + JSON.stringify({}), + ); + + // Store an old email with attachment in KV and metadata + const oldKey = `feed:${VALID_FEED_ID}:111`; + const oldAttachmentId = "old-attachment-uuid"; + const bigContent = "x".repeat(200); + const oldEmail = JSON.stringify({ + subject: "Old", + from: "a@b.com", + content: bigContent, + receivedAt: 111, + headers: {}, + attachments: [{ id: oldAttachmentId, filename: "old.pdf", contentType: "application/pdf", size: 100 }], + }); + await env.EMAIL_STORAGE.put(oldKey, oldEmail); + + // Also put the attachment in mock R2 + await mockR2.put(oldAttachmentId, new ArrayBuffer(100)); + + await env.EMAIL_STORAGE.put( + `feed:${VALID_FEED_ID}:metadata`, + JSON.stringify({ + emails: [ + { + key: oldKey, + subject: "Old", + receivedAt: 111, + size: oldEmail.length, + attachmentIds: [oldAttachmentId], + }, + ], + }), + ); + + // Process with tight size budget to force trimming + const tinyEnv = { ...env, FEED_MAX_SIZE_BYTES: "50" }; + const res = await processEmail(makeInput({ subject: "New" }), tinyEnv as any); + expect(res.status).toBe(200); + + // Old attachment should be deleted from R2 + expect(mockR2._has(oldAttachmentId)).toBe(false); + }); +}); diff --git a/src/lib/email-processor.ts b/src/lib/email-processor.ts index cf53f5a..d94f281 100644 --- a/src/lib/email-processor.ts +++ b/src/lib/email-processor.ts @@ -1,5 +1,11 @@ import { EmailParser } from "../utils/email-parser"; -import { Env, FeedConfig, FeedMetadata } from "../types"; +import { AttachmentData, EmailMetadata, Env, FeedConfig, FeedMetadata } from "../types"; + +export interface RawAttachment { + filename: string; + contentType: string; + content: ArrayBuffer; +} export interface ProcessEmailInput { toAddress: string; @@ -9,6 +15,7 @@ export interface ProcessEmailInput { content: string; receivedAt: number; headers?: Record; + attachments?: RawAttachment[]; } function normalizeEmail(value: string): string { @@ -34,6 +41,29 @@ function senderMatchesAllowlist( return senderDomain === normalizedDomain; } +async function uploadAttachments( + attachments: RawAttachment[], + bucket: R2Bucket, +): Promise { + return Promise.all( + attachments.map(async (att) => { + const id = crypto.randomUUID(); + await bucket.put(id, att.content, { + httpMetadata: { + contentType: att.contentType, + contentDisposition: `attachment; filename="${att.filename}"`, + }, + }); + return { + id, + filename: att.filename, + contentType: att.contentType, + size: att.content.byteLength, + }; + }), + ); +} + export async function processEmail( input: ProcessEmailInput, env: Env, @@ -74,12 +104,18 @@ export async function processEmail( } } + const storedAttachments: AttachmentData[] = + env.ATTACHMENT_BUCKET && input.attachments?.length + ? await uploadAttachments(input.attachments, env.ATTACHMENT_BUCKET) + : []; + const emailData = { subject: input.subject, from: input.from, content: input.content, receivedAt: input.receivedAt, headers: input.headers ?? {}, + ...(storedAttachments.length > 0 ? { attachments: storedAttachments } : {}), }; const emailKey = `feed:${feedId}:${Date.now()}`; @@ -104,24 +140,36 @@ export async function processEmail( const serialised = JSON.stringify(emailData); const serialisedSize = new TextEncoder().encode(serialised).byteLength; - feedMetadata.emails.unshift({ + const newEntry: EmailMetadata = { key: emailKey, subject: emailData.subject, receivedAt: emailData.receivedAt, size: serialisedSize, - }); + ...(storedAttachments.length > 0 + ? { attachmentIds: storedAttachments.map((a) => a.id) } + : {}), + }; + feedMetadata.emails.unshift(newEntry); let totalSize = feedMetadata.emails.reduce((sum, e) => sum + (e.size ?? 0), 0); - const toDelete: string[] = []; + const toDelete: EmailMetadata[] = []; while (totalSize > maxBytes && feedMetadata.emails.length > 1) { const dropped = feedMetadata.emails.pop()!; totalSize -= dropped.size ?? 0; - toDelete.push(dropped.key); + toDelete.push(dropped); } + const r2Deletions = + env.ATTACHMENT_BUCKET && toDelete.length > 0 + ? toDelete + .flatMap((e) => e.attachmentIds ?? []) + .map((id) => env.ATTACHMENT_BUCKET!.delete(id)) + : []; + await Promise.all([ env.EMAIL_STORAGE.put(feedMetadataKey, JSON.stringify(feedMetadata)), - ...toDelete.map((k) => env.EMAIL_STORAGE.delete(k)), + ...toDelete.map((e) => env.EMAIL_STORAGE.delete(e.key)), + ...r2Deletions, ]); console.log(`Successfully processed email for feed ${feedId}`); diff --git a/src/lib/forwardemail.ts b/src/lib/forwardemail.ts index 87e40a2..5488938 100644 --- a/src/lib/forwardemail.ts +++ b/src/lib/forwardemail.ts @@ -1,6 +1,16 @@ import { EmailParser } from "../utils/email-parser"; import { Env } from "../types"; -import { processEmail } from "./email-processor"; +import { processEmail, RawAttachment } from "./email-processor"; + +export interface ForwardEmailAttachment { + filename?: string; + contentType?: string; + size?: number; + content?: + | { type: "Buffer"; data: number[] } + | ArrayBuffer + | ArrayBufferView; +} export interface ForwardEmailPayload { recipients?: string[]; @@ -17,7 +27,7 @@ export interface ForwardEmailPayload { headerLines?: Array<{ key: string; line: string }>; headers?: string; raw?: string; - attachments?: Array; + attachments?: ForwardEmailAttachment[]; } function normalizeEmail(value: string): string { @@ -41,12 +51,36 @@ function extractSenderAddresses(payload: ForwardEmailPayload): string[] { return Array.from(new Set(matches.map(normalizeEmail))); } +function toArrayBuffer( + content: ForwardEmailAttachment["content"], +): ArrayBuffer | null { + if (!content) return null; + if (content instanceof ArrayBuffer) return content; + if (ArrayBuffer.isView(content)) return (content as ArrayBufferView).buffer as ArrayBuffer; + if (typeof content === "object" && content.type === "Buffer" && Array.isArray(content.data)) { + return Uint8Array.from(content.data).buffer as ArrayBuffer; + } + return null; +} + export async function handleForwardEmail( payload: ForwardEmailPayload, env: Env, ): Promise { const emailData = EmailParser.parseForwardEmailPayload(payload); + const rawAttachments: RawAttachment[] = (payload.attachments ?? []) + .map((a) => { + const buffer = toArrayBuffer(a.content); + if (!buffer) return null; + return { + filename: a.filename || "attachment", + contentType: a.contentType || "application/octet-stream", + content: buffer, + }; + }) + .filter((a): a is RawAttachment => a !== null); + return processEmail( { toAddress: payload.recipients?.[0] || "", @@ -56,6 +90,7 @@ export async function handleForwardEmail( content: emailData.content, receivedAt: emailData.receivedAt, headers: emailData.headers, + attachments: rawAttachments, }, env, ); diff --git a/src/routes/admin.ts b/src/routes/admin.ts index 701042a..e3efa8d 100644 --- a/src/routes/admin.ts +++ b/src/routes/admin.ts @@ -1908,7 +1908,7 @@ async function deleteFeedFastDetailed( async function purgeFeedKeysStep( emailStorage: KVNamespace, feedId: string, - options: { cursor?: string; limit?: number } = {}, + options: { cursor?: string; limit?: number; bucket?: R2Bucket } = {}, ): Promise<{ deletedKeys: string[]; failedKeys: string[]; @@ -1921,6 +1921,33 @@ async function purgeFeedKeysStep( const listed = await emailStorage.list({ prefix, cursor, limit }); const keys = (listed.keys || []).map((k) => k.name); + + // Collect R2 attachment IDs from email entries before deleting + if (options.bucket && keys.length > 0) { + const emailKeys = keys.filter((k) => { + const suffix = k.slice(prefix.length); + return suffix !== "config" && suffix !== "metadata"; + }); + if (emailKeys.length > 0) { + const emailDataResults = await Promise.allSettled( + emailKeys.map((k) => + emailStorage.get(k, { type: "json" }) as Promise, + ), + ); + const attachmentIds = emailDataResults + .filter( + (r): r is PromiseFulfilledResult => + r.status === "fulfilled", + ) + .flatMap((r) => r.value?.attachments?.map((a) => a.id) ?? []); + if (attachmentIds.length > 0) { + await Promise.allSettled( + attachmentIds.map((id) => options.bucket!.delete(id)), + ); + } + } + } + const { ok, failed } = await deleteKeysWithConcurrency( emailStorage, keys, @@ -1949,7 +1976,7 @@ app.post("/feeds/:feedId/delete", async (c) => { // Best-effort cleanup in the background so the request stays fast. // Use the UI purge endpoint for full, user-visible progress. - waitUntilSafe(c, purgeFeedKeysStep(emailStorage, feedId)); + waitUntilSafe(c, purgeFeedKeysStep(emailStorage, feedId, { bucket: env.ATTACHMENT_BUCKET })); if (wantsJson) { return c.json({ ok: true, feedId }); } @@ -1987,6 +2014,7 @@ app.post("/feeds/:feedId/purge", async (c) => { const step = await purgeFeedKeysStep(emailStorage, feedId, { cursor, limit, + bucket: env.ATTACHMENT_BUCKET, }); return c.json({ @@ -3420,15 +3448,18 @@ app.post("/emails/:emailKey/delete", async (c) => { return c.text("Feed ID is required", 400); } - // Delete the email - await emailStorage.delete(emailKey); - - // Remove the email from the feed metadata + // Load metadata first to collect attachment IDs for R2 cleanup const feedMetadataKey = `feed:${feedId}:metadata`; const feedMetadata = (await emailStorage.get(feedMetadataKey, { type: "json", })) as FeedMetadata | null; + const attachmentIds = + feedMetadata?.emails.find((e) => e.key === emailKey)?.attachmentIds ?? []; + + // Delete the email + await emailStorage.delete(emailKey); + if (feedMetadata) { // Filter out the deleted email feedMetadata.emails = feedMetadata.emails.filter( @@ -3439,6 +3470,13 @@ app.post("/emails/:emailKey/delete", async (c) => { await emailStorage.put(feedMetadataKey, JSON.stringify(feedMetadata)); } + // Best-effort R2 attachment cleanup + if (env.ATTACHMENT_BUCKET && attachmentIds.length > 0) { + await Promise.allSettled( + attachmentIds.map((id) => env.ATTACHMENT_BUCKET!.delete(id)), + ); + } + if (wantsJson) { return c.json({ ok: true, emailKey, feedId }); } @@ -3508,6 +3546,13 @@ app.post("/feeds/:feedId/emails/bulk-delete", async (c) => { } const candidates = emailKeys.filter((key) => allowedKeys.has(key)); + + // Collect attachment IDs from metadata before deleting (no extra KV reads needed) + const candidateSet = new Set(candidates); + const r2AttachmentIds = feedMetadata.emails + .filter((e) => candidateSet.has(e.key)) + .flatMap((e) => e.attachmentIds ?? []); + const { ok: deletedOk, failed: failedEmailKeys } = await deleteKeysWithConcurrency(emailStorage, candidates, 35); @@ -3517,6 +3562,13 @@ app.post("/feeds/:feedId/emails/bulk-delete", async (c) => { ); await emailStorage.put(feedMetadataKey, JSON.stringify(feedMetadata)); + // Best-effort R2 attachment cleanup + if (env.ATTACHMENT_BUCKET && r2AttachmentIds.length > 0) { + await Promise.allSettled( + r2AttachmentIds.map((id) => env.ATTACHMENT_BUCKET!.delete(id)), + ); + } + return c.json({ ok: failedEmailKeys.length === 0, deletedEmailKeys: deletedOk, diff --git a/src/routes/files.test.ts b/src/routes/files.test.ts new file mode 100644 index 0000000..0480362 --- /dev/null +++ b/src/routes/files.test.ts @@ -0,0 +1,103 @@ +import { describe, it, expect, beforeEach } from "vitest"; +import "../test/setup"; +import { createMockEnv, MockR2 } from "../test/setup"; +import { Hono } from "hono"; +import { handle as handleFiles } from "./files"; + +async function request( + env: ReturnType, + path: string, +): Promise { + const app = new Hono(); + const files = new Hono(); + files.get("/:attachmentId/:filename", handleFiles); + app.route("/files", files); + return app.request(path, {}, env as any); +} + +describe("GET /files/:attachmentId/:filename", () => { + let envNoR2: ReturnType; + let envWithR2: ReturnType; + let mockR2: MockR2; + + beforeEach(() => { + envNoR2 = createMockEnv(); + envWithR2 = createMockEnv({ withR2: true }); + mockR2 = (envWithR2 as any).ATTACHMENT_BUCKET as unknown as MockR2; + }); + + it("returns 404 when ATTACHMENT_BUCKET is not configured", async () => { + const res = await request(envNoR2, "/files/some-id/file.pdf"); + expect(res.status).toBe(404); + expect(await res.text()).toContain("not configured"); + }); + + it("returns 404 when attachment ID is not found in R2", async () => { + const res = await request(envWithR2, "/files/unknown-id/file.pdf"); + expect(res.status).toBe(404); + expect(await res.text()).toBe("Not found"); + }); + + it("returns 200 with stored content when attachment exists", async () => { + const content = new TextEncoder().encode("PDF content").buffer as ArrayBuffer; + await mockR2.put("test-uuid", content, { + httpMetadata: { contentType: "application/pdf" }, + }); + + const res = await request(envWithR2, "/files/test-uuid/report.pdf"); + expect(res.status).toBe(200); + }); + + it("returns correct Content-Type from stored httpMetadata", async () => { + const content = new TextEncoder().encode("data").buffer as ArrayBuffer; + await mockR2.put("img-uuid", content, { + httpMetadata: { contentType: "image/png" }, + }); + + const res = await request(envWithR2, "/files/img-uuid/photo.png"); + expect(res.status).toBe(200); + expect(res.headers.get("Content-Type")).toContain("image/png"); + }); + + it("sets Cache-Control immutable header", async () => { + const content = new TextEncoder().encode("data").buffer as ArrayBuffer; + await mockR2.put("cache-uuid", content, { + httpMetadata: { contentType: "application/pdf" }, + }); + + const res = await request(envWithR2, "/files/cache-uuid/doc.pdf"); + expect(res.headers.get("Cache-Control")).toBe( + "public, max-age=31536000, immutable", + ); + }); + + it("sets Content-Disposition from httpMetadata when present", async () => { + const content = new TextEncoder().encode("data").buffer as ArrayBuffer; + await mockR2.put("disp-uuid", content, { + httpMetadata: { + contentType: "application/pdf", + contentDisposition: 'attachment; filename="stored.pdf"', + }, + }); + + const res = await request(envWithR2, "/files/disp-uuid/other.pdf"); + expect(res.headers.get("Content-Disposition")).toBe( + 'attachment; filename="stored.pdf"', + ); + }); + + it("falls back to URL filename for Content-Disposition when not in httpMetadata", async () => { + const content = new TextEncoder().encode("data").buffer as ArrayBuffer; + await mockR2.put("fallback-uuid", content, { + httpMetadata: { contentType: "text/plain" }, + }); + + const res = await request( + envWithR2, + "/files/fallback-uuid/hello%20world.txt", + ); + expect(res.headers.get("Content-Disposition")).toBe( + 'attachment; filename="hello world.txt"', + ); + }); +}); diff --git a/src/routes/files.ts b/src/routes/files.ts new file mode 100644 index 0000000..a825646 --- /dev/null +++ b/src/routes/files.ts @@ -0,0 +1,33 @@ +import { Context } from "hono"; +import { Env } from "../types"; + +export async function handle(c: Context): Promise { + const env = c.env as unknown as Env; + + if (!env.ATTACHMENT_BUCKET) { + return new Response("Attachment storage not configured", { status: 404 }); + } + + const attachmentId = c.req.param("attachmentId"); + const filename = c.req.param("filename"); + + const object = await env.ATTACHMENT_BUCKET.get(attachmentId); + + if (!object) { + return new Response("Not found", { status: 404 }); + } + + const headers = new Headers(); + object.writeHttpMetadata(headers); + headers.set("etag", object.httpEtag); + headers.set("Cache-Control", "public, max-age=31536000, immutable"); + + if (!headers.get("Content-Disposition")) { + headers.set( + "Content-Disposition", + `attachment; filename="${decodeURIComponent(filename)}"`, + ); + } + + return new Response(object.body, { headers }); +} diff --git a/src/test/setup.ts b/src/test/setup.ts index c48a9a0..3f281cb 100644 --- a/src/test/setup.ts +++ b/src/test/setup.ts @@ -157,12 +157,92 @@ afterEach(() => { server.resetHandlers(); }); +/** + * Mock R2 bucket implementation + * Simulates Cloudflare Workers R2 storage using an in-memory Map + */ +export class MockR2 { + private store: Map< + string, + { + body: ArrayBuffer; + httpMetadata?: { contentType?: string; contentDisposition?: string }; + httpEtag: string; + } + > = new Map(); + + async put( + key: string, + value: ReadableStream | ArrayBuffer | ArrayBufferView | string | null | Blob, + options?: { + httpMetadata?: { contentType?: string; contentDisposition?: string }; + }, + ) { + let buffer: ArrayBuffer; + if (value instanceof ArrayBuffer) { + buffer = value; + } else if (typeof value === "string") { + buffer = new TextEncoder().encode(value).buffer as ArrayBuffer; + } else if (ArrayBuffer.isView(value)) { + buffer = value.buffer as ArrayBuffer; + } else { + buffer = new ArrayBuffer(0); + } + this.store.set(key, { + body: buffer, + httpMetadata: options?.httpMetadata, + httpEtag: `"mock-etag-${key}"`, + }); + } + + async get(key: string) { + const entry = this.store.get(key); + if (!entry) return null; + return { + key, + body: new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(entry.body)); + controller.close(); + }, + }), + httpEtag: entry.httpEtag, + httpMetadata: entry.httpMetadata, + size: entry.body.byteLength, + arrayBuffer: async () => entry.body, + text: async () => new TextDecoder().decode(entry.body), + writeHttpMetadata: (headers: Headers) => { + if (entry.httpMetadata?.contentType) { + headers.set("Content-Type", entry.httpMetadata.contentType); + } + if (entry.httpMetadata?.contentDisposition) { + headers.set("Content-Disposition", entry.httpMetadata.contentDisposition); + } + }, + }; + } + + async delete(keys: string | string[]) { + const arr = Array.isArray(keys) ? keys : [keys]; + for (const k of arr) { + this.store.delete(k); + } + } + + _has(key: string) { + return this.store.has(key); + } +} + /** * Create a mock environment for testing * @returns Mock environment with KV storage and configuration */ -export const createMockEnv = () => ({ +export const createMockEnv = (options: { withR2?: boolean } = {}) => ({ EMAIL_STORAGE: new MockKV(), DOMAIN: "test.getmynews.app", ADMIN_PASSWORD: "test-password", + ...(options.withR2 + ? { ATTACHMENT_BUCKET: new MockR2() as unknown as R2Bucket } + : {}), }); diff --git a/src/types/index.ts b/src/types/index.ts index 9899900..4200e82 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -3,11 +3,20 @@ export interface Env { EMAIL_STORAGE: KVNamespace; ADMIN_PASSWORD: string; DOMAIN: string; + ATTACHMENT_BUCKET?: R2Bucket; FEED_MAX_SIZE_BYTES?: string; PROXY_TRUSTED_IPS?: string; PROXY_AUTH_SECRET?: string; } +// Stored attachment metadata (bytes live in R2, keyed by id) +export interface AttachmentData { + id: string; + filename: string; + contentType: string; + size: number; +} + // Email interface for stored emails export interface EmailData { subject: string; @@ -15,6 +24,7 @@ export interface EmailData { content: string; receivedAt: number; headers: Record; + attachments?: AttachmentData[]; } // Feed configuration interface @@ -41,6 +51,7 @@ export interface EmailMetadata { subject: string; receivedAt: number; size?: number; + attachmentIds?: string[]; } // Feed list interface diff --git a/src/utils/feed-generator.test.ts b/src/utils/feed-generator.test.ts index 50deca1..8b80e00 100644 --- a/src/utils/feed-generator.test.ts +++ b/src/utils/feed-generator.test.ts @@ -21,6 +21,18 @@ const mockEmails: EmailData[] = [ }, ]; +const mockEmailWithAttachment: EmailData = { + ...mockEmails[0], + attachments: [ + { + id: "550e8400-e29b-41d4-a716-446655440000", + filename: "report.pdf", + contentType: "application/pdf", + size: 12345, + }, + ], +}; + const BASE_URL = "https://test.getmynews.app"; const FEED_ID = "abc123"; @@ -31,6 +43,25 @@ describe("generateRssFeed", () => { expect(result).toContain("Test Newsletter"); }); + it("includes element for email with attachment", () => { + const result = generateRssFeed(mockFeedConfig, [mockEmailWithAttachment], BASE_URL, FEED_ID); + expect(result).toContain(" for email without attachments", () => { + const result = generateRssFeed(mockFeedConfig, mockEmails, BASE_URL, FEED_ID); + expect(result).not.toContain(" { + const result = generateRssFeed(mockFeedConfig, [mockEmailWithAttachment], BASE_URL, FEED_ID); + expect(result).toContain(`${BASE_URL}/files/550e8400-e29b-41d4-a716-446655440000/report.pdf`); + }); + it("includes rss self-link in RSS output", () => { const result = generateRssFeed(mockFeedConfig, mockEmails, BASE_URL, FEED_ID); expect(result).toContain(`${BASE_URL}/rss/${FEED_ID}`); @@ -104,4 +135,16 @@ describe("generateAtomFeed", () => { const result = generateAtomFeed(configWithAuthor, mockEmails, BASE_URL, FEED_ID); expect(result).toContain("Bob"); }); + + it("includes enclosure link for email with attachment in Atom feed", () => { + const result = generateAtomFeed(mockFeedConfig, [mockEmailWithAttachment], BASE_URL, FEED_ID); + expect(result).toContain('rel="enclosure"'); + expect(result).toContain("550e8400-e29b-41d4-a716-446655440000"); + expect(result).toContain("report.pdf"); + }); + + it("does not include enclosure link for email without attachments in Atom feed", () => { + const result = generateAtomFeed(mockFeedConfig, mockEmails, BASE_URL, FEED_ID); + expect(result).not.toContain('rel="enclosure"'); + }); }); diff --git a/src/utils/feed-generator.ts b/src/utils/feed-generator.ts index 363ff5f..6f6968d 100644 --- a/src/utils/feed-generator.ts +++ b/src/utils/feed-generator.ts @@ -42,6 +42,7 @@ function buildFeed( for (const email of emails) { const uniqueId = `${email.receivedAt}-${Buffer.from(email.subject).toString("base64").substring(0, 10)}`; + const firstAttachment = email.attachments?.[0]; feed.addItem({ title: email.subject, id: uniqueId, @@ -50,6 +51,13 @@ function buildFeed( content: email.content, author: [parseFromAddress(email.from)], date: new Date(email.receivedAt), + enclosure: firstAttachment + ? { + url: `${baseUrl}/files/${firstAttachment.id}/${encodeURIComponent(firstAttachment.filename)}`, + type: firstAttachment.contentType, + length: firstAttachment.size, + } + : undefined, }); } diff --git a/wrangler-example.toml b/wrangler-example.toml index 8bcb8b7..6447cd4 100644 --- a/wrangler-example.toml +++ b/wrangler-example.toml @@ -8,6 +8,11 @@ kv_namespaces = [ { binding = "EMAIL_STORAGE", id = "REPLACE_WITH_YOUR_KV_NAMESPACE_ID", preview_id = "REPLACE_WITH_YOUR_PREVIEW_KV_NAMESPACE_ID" } ] +# Optional: R2 bucket for storing email attachment enclosures +# r2_buckets = [ +# { binding = "ATTACHMENT_BUCKET", bucket_name = "REPLACE_WITH_YOUR_BUCKET_NAME", preview_bucket_name = "REPLACE_WITH_YOUR_PREVIEW_BUCKET_NAME" } +# ] + # Workers Observability (keeps config in sync with dashboard toggle) [observability.logs] enabled = true @@ -42,6 +47,11 @@ kv_namespaces = [ { binding = "EMAIL_STORAGE", id = "REPLACE_WITH_YOUR_KV_NAMESPACE_ID" } ] +# Optional: R2 bucket for storing email attachment enclosures +# r2_buckets = [ +# { binding = "ATTACHMENT_BUCKET", bucket_name = "REPLACE_WITH_YOUR_BUCKET_NAME" } +# ] + routes = [ { pattern = "REPLACE_WITH_YOUR_DOMAIN", custom_domain = true }, { pattern = "www.REPLACE_WITH_YOUR_DOMAIN", custom_domain = true }