mirror of
https://github.com/juherr/kill-the-news.git
synced 2026-06-20 22:03:48 +00:00
feat: store email attachments in R2 and expose as RSS enclosures
Attachments from incoming emails are uploaded to an optional Cloudflare R2
bucket and exposed as <enclosure> elements in RSS and <link rel="enclosure">
in Atom feeds, served at /files/{id}/{filename} with immutable caching.
R2 is opt-in: if ATTACHMENT_BUCKET is not bound the feature is a no-op.
Attachments are cleaned up from R2 on email/feed deletion and during
size-based feed trimming. Adds MockR2 to the test setup.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -4,6 +4,7 @@ import { handle as handleRSS } from "./routes/rss";
|
||||
import { handle as handleAtom } from "./routes/atom";
|
||||
import { handle as handleAdmin } from "./routes/admin";
|
||||
import { handle as handleEntry } from "./routes/entries";
|
||||
import { handle as handleFiles } from "./routes/files";
|
||||
import { handleCloudflareEmail } from "./lib/cloudflare-email";
|
||||
import { Env } from "./types";
|
||||
|
||||
@@ -105,6 +106,7 @@ const api = new Hono();
|
||||
const rss = new Hono();
|
||||
const atom = new Hono();
|
||||
const entries = new Hono();
|
||||
const files = new Hono();
|
||||
const admin = new Hono();
|
||||
|
||||
// Webhook security middleware for /inbound - verify ForwardEmail.net IP
|
||||
@@ -141,6 +143,9 @@ atom.get("/:feedId", handleAtom);
|
||||
// Email entry HTML view (public)
|
||||
entries.get("/:feedId/:entryId", handleEntry);
|
||||
|
||||
// Attachment file serving (public)
|
||||
files.get("/:attachmentId/:filename", handleFiles);
|
||||
|
||||
// Admin routes (protected)
|
||||
admin.route("/", handleAdmin);
|
||||
|
||||
@@ -149,6 +154,7 @@ app.route("/api", api);
|
||||
app.route("/rss", rss);
|
||||
app.route("/atom", atom);
|
||||
app.route("/entries", entries);
|
||||
app.route("/files", files);
|
||||
app.route("/admin", admin);
|
||||
|
||||
// Root path redirects to admin dashboard
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import PostalMime from "postal-mime";
|
||||
import { Env } from "../types";
|
||||
import { processEmail } from "./email-processor";
|
||||
import { processEmail, RawAttachment } from "./email-processor";
|
||||
|
||||
export async function handleCloudflareEmail(
|
||||
message: ForwardableEmailMessage,
|
||||
@@ -21,6 +21,14 @@ export async function handleCloudflareEmail(
|
||||
headers[h.key] = h.value;
|
||||
}
|
||||
|
||||
const rawAttachments: RawAttachment[] = (email.attachments ?? [])
|
||||
.filter((a) => a.content instanceof ArrayBuffer)
|
||||
.map((a) => ({
|
||||
filename: a.filename || "attachment",
|
||||
contentType: a.mimeType || "application/octet-stream",
|
||||
content: a.content as ArrayBuffer,
|
||||
}));
|
||||
|
||||
await processEmail(
|
||||
{
|
||||
toAddress: message.to,
|
||||
@@ -30,6 +38,7 @@ export async function handleCloudflareEmail(
|
||||
content: email.html ?? email.text ?? "",
|
||||
receivedAt: email.date ? new Date(email.date).getTime() : Date.now(),
|
||||
headers,
|
||||
attachments: rawAttachments,
|
||||
},
|
||||
env,
|
||||
);
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { describe, it, expect, beforeEach } from "vitest";
|
||||
import "../test/setup";
|
||||
import { createMockEnv } from "../test/setup";
|
||||
import { processEmail, ProcessEmailInput } from "./email-processor";
|
||||
import { createMockEnv, MockR2 } from "../test/setup";
|
||||
import { processEmail, ProcessEmailInput, RawAttachment } from "./email-processor";
|
||||
|
||||
const VALID_FEED_ID = "apple.mountain.42";
|
||||
const VALID_TO = `${VALID_FEED_ID}@test.getmynews.app`;
|
||||
@@ -179,3 +179,126 @@ describe("processEmail", () => {
|
||||
expect(metadata.emails).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("processEmail — attachments", () => {
|
||||
const pdfContent = new TextEncoder().encode("PDF bytes").buffer as ArrayBuffer;
|
||||
|
||||
const pdfAttachment: RawAttachment = {
|
||||
filename: "report.pdf",
|
||||
contentType: "application/pdf",
|
||||
content: pdfContent,
|
||||
};
|
||||
|
||||
it("skips R2 upload when ATTACHMENT_BUCKET is not configured", async () => {
|
||||
const env = createMockEnv();
|
||||
await env.EMAIL_STORAGE.put(
|
||||
`feed:${VALID_FEED_ID}:config`,
|
||||
JSON.stringify({}),
|
||||
);
|
||||
const res = await processEmail(
|
||||
makeInput({ attachments: [pdfAttachment] }),
|
||||
env as any,
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
|
||||
const metadata = await env.EMAIL_STORAGE.get(
|
||||
`feed:${VALID_FEED_ID}:metadata`,
|
||||
"json",
|
||||
);
|
||||
const emailData = await env.EMAIL_STORAGE.get(metadata.emails[0].key, "json");
|
||||
expect(emailData.attachments).toBeUndefined();
|
||||
});
|
||||
|
||||
it("uploads attachments to R2 and stores AttachmentData in emailData", async () => {
|
||||
const env = createMockEnv({ withR2: true });
|
||||
const mockR2 = (env as any).ATTACHMENT_BUCKET as unknown as MockR2;
|
||||
await env.EMAIL_STORAGE.put(
|
||||
`feed:${VALID_FEED_ID}:config`,
|
||||
JSON.stringify({}),
|
||||
);
|
||||
const res = await processEmail(
|
||||
makeInput({ attachments: [pdfAttachment] }),
|
||||
env as any,
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
|
||||
const metadata = await env.EMAIL_STORAGE.get(
|
||||
`feed:${VALID_FEED_ID}:metadata`,
|
||||
"json",
|
||||
);
|
||||
const emailData = await env.EMAIL_STORAGE.get(metadata.emails[0].key, "json");
|
||||
|
||||
expect(emailData.attachments).toHaveLength(1);
|
||||
expect(emailData.attachments[0].filename).toBe("report.pdf");
|
||||
expect(emailData.attachments[0].contentType).toBe("application/pdf");
|
||||
expect(emailData.attachments[0].size).toBe(pdfContent.byteLength);
|
||||
|
||||
const id = emailData.attachments[0].id;
|
||||
expect(mockR2._has(id)).toBe(true);
|
||||
});
|
||||
|
||||
it("stores attachmentIds in EmailMetadata for trim-time cleanup", async () => {
|
||||
const env = createMockEnv({ withR2: true });
|
||||
await env.EMAIL_STORAGE.put(
|
||||
`feed:${VALID_FEED_ID}:config`,
|
||||
JSON.stringify({}),
|
||||
);
|
||||
await processEmail(makeInput({ attachments: [pdfAttachment] }), env as any);
|
||||
|
||||
const metadata = await env.EMAIL_STORAGE.get(
|
||||
`feed:${VALID_FEED_ID}:metadata`,
|
||||
"json",
|
||||
);
|
||||
expect(metadata.emails[0].attachmentIds).toHaveLength(1);
|
||||
expect(typeof metadata.emails[0].attachmentIds[0]).toBe("string");
|
||||
});
|
||||
|
||||
it("deletes R2 objects when a trimmed email had attachments", async () => {
|
||||
const env = createMockEnv({ withR2: true });
|
||||
const mockR2 = (env as any).ATTACHMENT_BUCKET as unknown as MockR2;
|
||||
await env.EMAIL_STORAGE.put(
|
||||
`feed:${VALID_FEED_ID}:config`,
|
||||
JSON.stringify({}),
|
||||
);
|
||||
|
||||
// Store an old email with attachment in KV and metadata
|
||||
const oldKey = `feed:${VALID_FEED_ID}:111`;
|
||||
const oldAttachmentId = "old-attachment-uuid";
|
||||
const bigContent = "x".repeat(200);
|
||||
const oldEmail = JSON.stringify({
|
||||
subject: "Old",
|
||||
from: "a@b.com",
|
||||
content: bigContent,
|
||||
receivedAt: 111,
|
||||
headers: {},
|
||||
attachments: [{ id: oldAttachmentId, filename: "old.pdf", contentType: "application/pdf", size: 100 }],
|
||||
});
|
||||
await env.EMAIL_STORAGE.put(oldKey, oldEmail);
|
||||
|
||||
// Also put the attachment in mock R2
|
||||
await mockR2.put(oldAttachmentId, new ArrayBuffer(100));
|
||||
|
||||
await env.EMAIL_STORAGE.put(
|
||||
`feed:${VALID_FEED_ID}:metadata`,
|
||||
JSON.stringify({
|
||||
emails: [
|
||||
{
|
||||
key: oldKey,
|
||||
subject: "Old",
|
||||
receivedAt: 111,
|
||||
size: oldEmail.length,
|
||||
attachmentIds: [oldAttachmentId],
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
// Process with tight size budget to force trimming
|
||||
const tinyEnv = { ...env, FEED_MAX_SIZE_BYTES: "50" };
|
||||
const res = await processEmail(makeInput({ subject: "New" }), tinyEnv as any);
|
||||
expect(res.status).toBe(200);
|
||||
|
||||
// Old attachment should be deleted from R2
|
||||
expect(mockR2._has(oldAttachmentId)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
import { EmailParser } from "../utils/email-parser";
|
||||
import { Env, FeedConfig, FeedMetadata } from "../types";
|
||||
import { AttachmentData, EmailMetadata, Env, FeedConfig, FeedMetadata } from "../types";
|
||||
|
||||
export interface RawAttachment {
|
||||
filename: string;
|
||||
contentType: string;
|
||||
content: ArrayBuffer;
|
||||
}
|
||||
|
||||
export interface ProcessEmailInput {
|
||||
toAddress: string;
|
||||
@@ -9,6 +15,7 @@ export interface ProcessEmailInput {
|
||||
content: string;
|
||||
receivedAt: number;
|
||||
headers?: Record<string, string>;
|
||||
attachments?: RawAttachment[];
|
||||
}
|
||||
|
||||
function normalizeEmail(value: string): string {
|
||||
@@ -34,6 +41,29 @@ function senderMatchesAllowlist(
|
||||
return senderDomain === normalizedDomain;
|
||||
}
|
||||
|
||||
async function uploadAttachments(
|
||||
attachments: RawAttachment[],
|
||||
bucket: R2Bucket,
|
||||
): Promise<AttachmentData[]> {
|
||||
return Promise.all(
|
||||
attachments.map(async (att) => {
|
||||
const id = crypto.randomUUID();
|
||||
await bucket.put(id, att.content, {
|
||||
httpMetadata: {
|
||||
contentType: att.contentType,
|
||||
contentDisposition: `attachment; filename="${att.filename}"`,
|
||||
},
|
||||
});
|
||||
return {
|
||||
id,
|
||||
filename: att.filename,
|
||||
contentType: att.contentType,
|
||||
size: att.content.byteLength,
|
||||
};
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export async function processEmail(
|
||||
input: ProcessEmailInput,
|
||||
env: Env,
|
||||
@@ -74,12 +104,18 @@ export async function processEmail(
|
||||
}
|
||||
}
|
||||
|
||||
const storedAttachments: AttachmentData[] =
|
||||
env.ATTACHMENT_BUCKET && input.attachments?.length
|
||||
? await uploadAttachments(input.attachments, env.ATTACHMENT_BUCKET)
|
||||
: [];
|
||||
|
||||
const emailData = {
|
||||
subject: input.subject,
|
||||
from: input.from,
|
||||
content: input.content,
|
||||
receivedAt: input.receivedAt,
|
||||
headers: input.headers ?? {},
|
||||
...(storedAttachments.length > 0 ? { attachments: storedAttachments } : {}),
|
||||
};
|
||||
|
||||
const emailKey = `feed:${feedId}:${Date.now()}`;
|
||||
@@ -104,24 +140,36 @@ export async function processEmail(
|
||||
|
||||
const serialised = JSON.stringify(emailData);
|
||||
const serialisedSize = new TextEncoder().encode(serialised).byteLength;
|
||||
feedMetadata.emails.unshift({
|
||||
const newEntry: EmailMetadata = {
|
||||
key: emailKey,
|
||||
subject: emailData.subject,
|
||||
receivedAt: emailData.receivedAt,
|
||||
size: serialisedSize,
|
||||
});
|
||||
...(storedAttachments.length > 0
|
||||
? { attachmentIds: storedAttachments.map((a) => a.id) }
|
||||
: {}),
|
||||
};
|
||||
feedMetadata.emails.unshift(newEntry);
|
||||
|
||||
let totalSize = feedMetadata.emails.reduce((sum, e) => sum + (e.size ?? 0), 0);
|
||||
const toDelete: string[] = [];
|
||||
const toDelete: EmailMetadata[] = [];
|
||||
while (totalSize > maxBytes && feedMetadata.emails.length > 1) {
|
||||
const dropped = feedMetadata.emails.pop()!;
|
||||
totalSize -= dropped.size ?? 0;
|
||||
toDelete.push(dropped.key);
|
||||
toDelete.push(dropped);
|
||||
}
|
||||
|
||||
const r2Deletions =
|
||||
env.ATTACHMENT_BUCKET && toDelete.length > 0
|
||||
? toDelete
|
||||
.flatMap((e) => e.attachmentIds ?? [])
|
||||
.map((id) => env.ATTACHMENT_BUCKET!.delete(id))
|
||||
: [];
|
||||
|
||||
await Promise.all([
|
||||
env.EMAIL_STORAGE.put(feedMetadataKey, JSON.stringify(feedMetadata)),
|
||||
...toDelete.map((k) => env.EMAIL_STORAGE.delete(k)),
|
||||
...toDelete.map((e) => env.EMAIL_STORAGE.delete(e.key)),
|
||||
...r2Deletions,
|
||||
]);
|
||||
|
||||
console.log(`Successfully processed email for feed ${feedId}`);
|
||||
|
||||
+37
-2
@@ -1,6 +1,16 @@
|
||||
import { EmailParser } from "../utils/email-parser";
|
||||
import { Env } from "../types";
|
||||
import { processEmail } from "./email-processor";
|
||||
import { processEmail, RawAttachment } from "./email-processor";
|
||||
|
||||
export interface ForwardEmailAttachment {
|
||||
filename?: string;
|
||||
contentType?: string;
|
||||
size?: number;
|
||||
content?:
|
||||
| { type: "Buffer"; data: number[] }
|
||||
| ArrayBuffer
|
||||
| ArrayBufferView;
|
||||
}
|
||||
|
||||
export interface ForwardEmailPayload {
|
||||
recipients?: string[];
|
||||
@@ -17,7 +27,7 @@ export interface ForwardEmailPayload {
|
||||
headerLines?: Array<{ key: string; line: string }>;
|
||||
headers?: string;
|
||||
raw?: string;
|
||||
attachments?: Array<any>;
|
||||
attachments?: ForwardEmailAttachment[];
|
||||
}
|
||||
|
||||
function normalizeEmail(value: string): string {
|
||||
@@ -41,12 +51,36 @@ function extractSenderAddresses(payload: ForwardEmailPayload): string[] {
|
||||
return Array.from(new Set(matches.map(normalizeEmail)));
|
||||
}
|
||||
|
||||
function toArrayBuffer(
|
||||
content: ForwardEmailAttachment["content"],
|
||||
): ArrayBuffer | null {
|
||||
if (!content) return null;
|
||||
if (content instanceof ArrayBuffer) return content;
|
||||
if (ArrayBuffer.isView(content)) return (content as ArrayBufferView).buffer as ArrayBuffer;
|
||||
if (typeof content === "object" && content.type === "Buffer" && Array.isArray(content.data)) {
|
||||
return Uint8Array.from(content.data).buffer as ArrayBuffer;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function handleForwardEmail(
|
||||
payload: ForwardEmailPayload,
|
||||
env: Env,
|
||||
): Promise<Response> {
|
||||
const emailData = EmailParser.parseForwardEmailPayload(payload);
|
||||
|
||||
const rawAttachments: RawAttachment[] = (payload.attachments ?? [])
|
||||
.map((a) => {
|
||||
const buffer = toArrayBuffer(a.content);
|
||||
if (!buffer) return null;
|
||||
return {
|
||||
filename: a.filename || "attachment",
|
||||
contentType: a.contentType || "application/octet-stream",
|
||||
content: buffer,
|
||||
};
|
||||
})
|
||||
.filter((a): a is RawAttachment => a !== null);
|
||||
|
||||
return processEmail(
|
||||
{
|
||||
toAddress: payload.recipients?.[0] || "",
|
||||
@@ -56,6 +90,7 @@ export async function handleForwardEmail(
|
||||
content: emailData.content,
|
||||
receivedAt: emailData.receivedAt,
|
||||
headers: emailData.headers,
|
||||
attachments: rawAttachments,
|
||||
},
|
||||
env,
|
||||
);
|
||||
|
||||
+58
-6
@@ -1908,7 +1908,7 @@ async function deleteFeedFastDetailed(
|
||||
async function purgeFeedKeysStep(
|
||||
emailStorage: KVNamespace,
|
||||
feedId: string,
|
||||
options: { cursor?: string; limit?: number } = {},
|
||||
options: { cursor?: string; limit?: number; bucket?: R2Bucket } = {},
|
||||
): Promise<{
|
||||
deletedKeys: string[];
|
||||
failedKeys: string[];
|
||||
@@ -1921,6 +1921,33 @@ async function purgeFeedKeysStep(
|
||||
|
||||
const listed = await emailStorage.list({ prefix, cursor, limit });
|
||||
const keys = (listed.keys || []).map((k) => k.name);
|
||||
|
||||
// Collect R2 attachment IDs from email entries before deleting
|
||||
if (options.bucket && keys.length > 0) {
|
||||
const emailKeys = keys.filter((k) => {
|
||||
const suffix = k.slice(prefix.length);
|
||||
return suffix !== "config" && suffix !== "metadata";
|
||||
});
|
||||
if (emailKeys.length > 0) {
|
||||
const emailDataResults = await Promise.allSettled(
|
||||
emailKeys.map((k) =>
|
||||
emailStorage.get(k, { type: "json" }) as Promise<EmailData | null>,
|
||||
),
|
||||
);
|
||||
const attachmentIds = emailDataResults
|
||||
.filter(
|
||||
(r): r is PromiseFulfilledResult<EmailData | null> =>
|
||||
r.status === "fulfilled",
|
||||
)
|
||||
.flatMap((r) => r.value?.attachments?.map((a) => a.id) ?? []);
|
||||
if (attachmentIds.length > 0) {
|
||||
await Promise.allSettled(
|
||||
attachmentIds.map((id) => options.bucket!.delete(id)),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const { ok, failed } = await deleteKeysWithConcurrency(
|
||||
emailStorage,
|
||||
keys,
|
||||
@@ -1949,7 +1976,7 @@ app.post("/feeds/:feedId/delete", async (c) => {
|
||||
|
||||
// Best-effort cleanup in the background so the request stays fast.
|
||||
// Use the UI purge endpoint for full, user-visible progress.
|
||||
waitUntilSafe(c, purgeFeedKeysStep(emailStorage, feedId));
|
||||
waitUntilSafe(c, purgeFeedKeysStep(emailStorage, feedId, { bucket: env.ATTACHMENT_BUCKET }));
|
||||
if (wantsJson) {
|
||||
return c.json({ ok: true, feedId });
|
||||
}
|
||||
@@ -1987,6 +2014,7 @@ app.post("/feeds/:feedId/purge", async (c) => {
|
||||
const step = await purgeFeedKeysStep(emailStorage, feedId, {
|
||||
cursor,
|
||||
limit,
|
||||
bucket: env.ATTACHMENT_BUCKET,
|
||||
});
|
||||
|
||||
return c.json({
|
||||
@@ -3420,15 +3448,18 @@ app.post("/emails/:emailKey/delete", async (c) => {
|
||||
return c.text("Feed ID is required", 400);
|
||||
}
|
||||
|
||||
// Delete the email
|
||||
await emailStorage.delete(emailKey);
|
||||
|
||||
// Remove the email from the feed metadata
|
||||
// Load metadata first to collect attachment IDs for R2 cleanup
|
||||
const feedMetadataKey = `feed:${feedId}:metadata`;
|
||||
const feedMetadata = (await emailStorage.get(feedMetadataKey, {
|
||||
type: "json",
|
||||
})) as FeedMetadata | null;
|
||||
|
||||
const attachmentIds =
|
||||
feedMetadata?.emails.find((e) => e.key === emailKey)?.attachmentIds ?? [];
|
||||
|
||||
// Delete the email
|
||||
await emailStorage.delete(emailKey);
|
||||
|
||||
if (feedMetadata) {
|
||||
// Filter out the deleted email
|
||||
feedMetadata.emails = feedMetadata.emails.filter(
|
||||
@@ -3439,6 +3470,13 @@ app.post("/emails/:emailKey/delete", async (c) => {
|
||||
await emailStorage.put(feedMetadataKey, JSON.stringify(feedMetadata));
|
||||
}
|
||||
|
||||
// Best-effort R2 attachment cleanup
|
||||
if (env.ATTACHMENT_BUCKET && attachmentIds.length > 0) {
|
||||
await Promise.allSettled(
|
||||
attachmentIds.map((id) => env.ATTACHMENT_BUCKET!.delete(id)),
|
||||
);
|
||||
}
|
||||
|
||||
if (wantsJson) {
|
||||
return c.json({ ok: true, emailKey, feedId });
|
||||
}
|
||||
@@ -3508,6 +3546,13 @@ app.post("/feeds/:feedId/emails/bulk-delete", async (c) => {
|
||||
}
|
||||
|
||||
const candidates = emailKeys.filter((key) => allowedKeys.has(key));
|
||||
|
||||
// Collect attachment IDs from metadata before deleting (no extra KV reads needed)
|
||||
const candidateSet = new Set(candidates);
|
||||
const r2AttachmentIds = feedMetadata.emails
|
||||
.filter((e) => candidateSet.has(e.key))
|
||||
.flatMap((e) => e.attachmentIds ?? []);
|
||||
|
||||
const { ok: deletedOk, failed: failedEmailKeys } =
|
||||
await deleteKeysWithConcurrency(emailStorage, candidates, 35);
|
||||
|
||||
@@ -3517,6 +3562,13 @@ app.post("/feeds/:feedId/emails/bulk-delete", async (c) => {
|
||||
);
|
||||
await emailStorage.put(feedMetadataKey, JSON.stringify(feedMetadata));
|
||||
|
||||
// Best-effort R2 attachment cleanup
|
||||
if (env.ATTACHMENT_BUCKET && r2AttachmentIds.length > 0) {
|
||||
await Promise.allSettled(
|
||||
r2AttachmentIds.map((id) => env.ATTACHMENT_BUCKET!.delete(id)),
|
||||
);
|
||||
}
|
||||
|
||||
return c.json({
|
||||
ok: failedEmailKeys.length === 0,
|
||||
deletedEmailKeys: deletedOk,
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
import { describe, it, expect, beforeEach } from "vitest";
|
||||
import "../test/setup";
|
||||
import { createMockEnv, MockR2 } from "../test/setup";
|
||||
import { Hono } from "hono";
|
||||
import { handle as handleFiles } from "./files";
|
||||
|
||||
async function request(
|
||||
env: ReturnType<typeof createMockEnv>,
|
||||
path: string,
|
||||
): Promise<Response> {
|
||||
const app = new Hono();
|
||||
const files = new Hono();
|
||||
files.get("/:attachmentId/:filename", handleFiles);
|
||||
app.route("/files", files);
|
||||
return app.request(path, {}, env as any);
|
||||
}
|
||||
|
||||
describe("GET /files/:attachmentId/:filename", () => {
|
||||
let envNoR2: ReturnType<typeof createMockEnv>;
|
||||
let envWithR2: ReturnType<typeof createMockEnv>;
|
||||
let mockR2: MockR2;
|
||||
|
||||
beforeEach(() => {
|
||||
envNoR2 = createMockEnv();
|
||||
envWithR2 = createMockEnv({ withR2: true });
|
||||
mockR2 = (envWithR2 as any).ATTACHMENT_BUCKET as unknown as MockR2;
|
||||
});
|
||||
|
||||
it("returns 404 when ATTACHMENT_BUCKET is not configured", async () => {
|
||||
const res = await request(envNoR2, "/files/some-id/file.pdf");
|
||||
expect(res.status).toBe(404);
|
||||
expect(await res.text()).toContain("not configured");
|
||||
});
|
||||
|
||||
it("returns 404 when attachment ID is not found in R2", async () => {
|
||||
const res = await request(envWithR2, "/files/unknown-id/file.pdf");
|
||||
expect(res.status).toBe(404);
|
||||
expect(await res.text()).toBe("Not found");
|
||||
});
|
||||
|
||||
it("returns 200 with stored content when attachment exists", async () => {
|
||||
const content = new TextEncoder().encode("PDF content").buffer as ArrayBuffer;
|
||||
await mockR2.put("test-uuid", content, {
|
||||
httpMetadata: { contentType: "application/pdf" },
|
||||
});
|
||||
|
||||
const res = await request(envWithR2, "/files/test-uuid/report.pdf");
|
||||
expect(res.status).toBe(200);
|
||||
});
|
||||
|
||||
it("returns correct Content-Type from stored httpMetadata", async () => {
|
||||
const content = new TextEncoder().encode("data").buffer as ArrayBuffer;
|
||||
await mockR2.put("img-uuid", content, {
|
||||
httpMetadata: { contentType: "image/png" },
|
||||
});
|
||||
|
||||
const res = await request(envWithR2, "/files/img-uuid/photo.png");
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers.get("Content-Type")).toContain("image/png");
|
||||
});
|
||||
|
||||
it("sets Cache-Control immutable header", async () => {
|
||||
const content = new TextEncoder().encode("data").buffer as ArrayBuffer;
|
||||
await mockR2.put("cache-uuid", content, {
|
||||
httpMetadata: { contentType: "application/pdf" },
|
||||
});
|
||||
|
||||
const res = await request(envWithR2, "/files/cache-uuid/doc.pdf");
|
||||
expect(res.headers.get("Cache-Control")).toBe(
|
||||
"public, max-age=31536000, immutable",
|
||||
);
|
||||
});
|
||||
|
||||
it("sets Content-Disposition from httpMetadata when present", async () => {
|
||||
const content = new TextEncoder().encode("data").buffer as ArrayBuffer;
|
||||
await mockR2.put("disp-uuid", content, {
|
||||
httpMetadata: {
|
||||
contentType: "application/pdf",
|
||||
contentDisposition: 'attachment; filename="stored.pdf"',
|
||||
},
|
||||
});
|
||||
|
||||
const res = await request(envWithR2, "/files/disp-uuid/other.pdf");
|
||||
expect(res.headers.get("Content-Disposition")).toBe(
|
||||
'attachment; filename="stored.pdf"',
|
||||
);
|
||||
});
|
||||
|
||||
it("falls back to URL filename for Content-Disposition when not in httpMetadata", async () => {
|
||||
const content = new TextEncoder().encode("data").buffer as ArrayBuffer;
|
||||
await mockR2.put("fallback-uuid", content, {
|
||||
httpMetadata: { contentType: "text/plain" },
|
||||
});
|
||||
|
||||
const res = await request(
|
||||
envWithR2,
|
||||
"/files/fallback-uuid/hello%20world.txt",
|
||||
);
|
||||
expect(res.headers.get("Content-Disposition")).toBe(
|
||||
'attachment; filename="hello world.txt"',
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,33 @@
|
||||
import { Context } from "hono";
|
||||
import { Env } from "../types";
|
||||
|
||||
export async function handle(c: Context): Promise<Response> {
|
||||
const env = c.env as unknown as Env;
|
||||
|
||||
if (!env.ATTACHMENT_BUCKET) {
|
||||
return new Response("Attachment storage not configured", { status: 404 });
|
||||
}
|
||||
|
||||
const attachmentId = c.req.param("attachmentId");
|
||||
const filename = c.req.param("filename");
|
||||
|
||||
const object = await env.ATTACHMENT_BUCKET.get(attachmentId);
|
||||
|
||||
if (!object) {
|
||||
return new Response("Not found", { status: 404 });
|
||||
}
|
||||
|
||||
const headers = new Headers();
|
||||
object.writeHttpMetadata(headers);
|
||||
headers.set("etag", object.httpEtag);
|
||||
headers.set("Cache-Control", "public, max-age=31536000, immutable");
|
||||
|
||||
if (!headers.get("Content-Disposition")) {
|
||||
headers.set(
|
||||
"Content-Disposition",
|
||||
`attachment; filename="${decodeURIComponent(filename)}"`,
|
||||
);
|
||||
}
|
||||
|
||||
return new Response(object.body, { headers });
|
||||
}
|
||||
+81
-1
@@ -157,12 +157,92 @@ afterEach(() => {
|
||||
server.resetHandlers();
|
||||
});
|
||||
|
||||
/**
|
||||
* Mock R2 bucket implementation
|
||||
* Simulates Cloudflare Workers R2 storage using an in-memory Map
|
||||
*/
|
||||
export class MockR2 {
|
||||
private store: Map<
|
||||
string,
|
||||
{
|
||||
body: ArrayBuffer;
|
||||
httpMetadata?: { contentType?: string; contentDisposition?: string };
|
||||
httpEtag: string;
|
||||
}
|
||||
> = new Map();
|
||||
|
||||
async put(
|
||||
key: string,
|
||||
value: ReadableStream | ArrayBuffer | ArrayBufferView | string | null | Blob,
|
||||
options?: {
|
||||
httpMetadata?: { contentType?: string; contentDisposition?: string };
|
||||
},
|
||||
) {
|
||||
let buffer: ArrayBuffer;
|
||||
if (value instanceof ArrayBuffer) {
|
||||
buffer = value;
|
||||
} else if (typeof value === "string") {
|
||||
buffer = new TextEncoder().encode(value).buffer as ArrayBuffer;
|
||||
} else if (ArrayBuffer.isView(value)) {
|
||||
buffer = value.buffer as ArrayBuffer;
|
||||
} else {
|
||||
buffer = new ArrayBuffer(0);
|
||||
}
|
||||
this.store.set(key, {
|
||||
body: buffer,
|
||||
httpMetadata: options?.httpMetadata,
|
||||
httpEtag: `"mock-etag-${key}"`,
|
||||
});
|
||||
}
|
||||
|
||||
async get(key: string) {
|
||||
const entry = this.store.get(key);
|
||||
if (!entry) return null;
|
||||
return {
|
||||
key,
|
||||
body: new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(new Uint8Array(entry.body));
|
||||
controller.close();
|
||||
},
|
||||
}),
|
||||
httpEtag: entry.httpEtag,
|
||||
httpMetadata: entry.httpMetadata,
|
||||
size: entry.body.byteLength,
|
||||
arrayBuffer: async () => entry.body,
|
||||
text: async () => new TextDecoder().decode(entry.body),
|
||||
writeHttpMetadata: (headers: Headers) => {
|
||||
if (entry.httpMetadata?.contentType) {
|
||||
headers.set("Content-Type", entry.httpMetadata.contentType);
|
||||
}
|
||||
if (entry.httpMetadata?.contentDisposition) {
|
||||
headers.set("Content-Disposition", entry.httpMetadata.contentDisposition);
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async delete(keys: string | string[]) {
|
||||
const arr = Array.isArray(keys) ? keys : [keys];
|
||||
for (const k of arr) {
|
||||
this.store.delete(k);
|
||||
}
|
||||
}
|
||||
|
||||
_has(key: string) {
|
||||
return this.store.has(key);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a mock environment for testing
|
||||
* @returns Mock environment with KV storage and configuration
|
||||
*/
|
||||
export const createMockEnv = () => ({
|
||||
export const createMockEnv = (options: { withR2?: boolean } = {}) => ({
|
||||
EMAIL_STORAGE: new MockKV(),
|
||||
DOMAIN: "test.getmynews.app",
|
||||
ADMIN_PASSWORD: "test-password",
|
||||
...(options.withR2
|
||||
? { ATTACHMENT_BUCKET: new MockR2() as unknown as R2Bucket }
|
||||
: {}),
|
||||
});
|
||||
|
||||
@@ -3,11 +3,20 @@ export interface Env {
|
||||
EMAIL_STORAGE: KVNamespace;
|
||||
ADMIN_PASSWORD: string;
|
||||
DOMAIN: string;
|
||||
ATTACHMENT_BUCKET?: R2Bucket;
|
||||
FEED_MAX_SIZE_BYTES?: string;
|
||||
PROXY_TRUSTED_IPS?: string;
|
||||
PROXY_AUTH_SECRET?: string;
|
||||
}
|
||||
|
||||
// Stored attachment metadata (bytes live in R2, keyed by id)
|
||||
export interface AttachmentData {
|
||||
id: string;
|
||||
filename: string;
|
||||
contentType: string;
|
||||
size: number;
|
||||
}
|
||||
|
||||
// Email interface for stored emails
|
||||
export interface EmailData {
|
||||
subject: string;
|
||||
@@ -15,6 +24,7 @@ export interface EmailData {
|
||||
content: string;
|
||||
receivedAt: number;
|
||||
headers: Record<string, string>;
|
||||
attachments?: AttachmentData[];
|
||||
}
|
||||
|
||||
// Feed configuration interface
|
||||
@@ -41,6 +51,7 @@ export interface EmailMetadata {
|
||||
subject: string;
|
||||
receivedAt: number;
|
||||
size?: number;
|
||||
attachmentIds?: string[];
|
||||
}
|
||||
|
||||
// Feed list interface
|
||||
|
||||
@@ -21,6 +21,18 @@ const mockEmails: EmailData[] = [
|
||||
},
|
||||
];
|
||||
|
||||
const mockEmailWithAttachment: EmailData = {
|
||||
...mockEmails[0],
|
||||
attachments: [
|
||||
{
|
||||
id: "550e8400-e29b-41d4-a716-446655440000",
|
||||
filename: "report.pdf",
|
||||
contentType: "application/pdf",
|
||||
size: 12345,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const BASE_URL = "https://test.getmynews.app";
|
||||
const FEED_ID = "abc123";
|
||||
|
||||
@@ -31,6 +43,25 @@ describe("generateRssFeed", () => {
|
||||
expect(result).toContain("<title>Test Newsletter</title>");
|
||||
});
|
||||
|
||||
it("includes <enclosure> element for email with attachment", () => {
|
||||
const result = generateRssFeed(mockFeedConfig, [mockEmailWithAttachment], BASE_URL, FEED_ID);
|
||||
expect(result).toContain("<enclosure");
|
||||
expect(result).toContain("550e8400-e29b-41d4-a716-446655440000");
|
||||
expect(result).toContain("report.pdf");
|
||||
expect(result).toContain("application/pdf");
|
||||
expect(result).toContain("12345");
|
||||
});
|
||||
|
||||
it("does not include <enclosure> for email without attachments", () => {
|
||||
const result = generateRssFeed(mockFeedConfig, mockEmails, BASE_URL, FEED_ID);
|
||||
expect(result).not.toContain("<enclosure");
|
||||
});
|
||||
|
||||
it("enclosure URL uses /files/{id}/{filename} scheme", () => {
|
||||
const result = generateRssFeed(mockFeedConfig, [mockEmailWithAttachment], BASE_URL, FEED_ID);
|
||||
expect(result).toContain(`${BASE_URL}/files/550e8400-e29b-41d4-a716-446655440000/report.pdf`);
|
||||
});
|
||||
|
||||
it("includes rss self-link in RSS output", () => {
|
||||
const result = generateRssFeed(mockFeedConfig, mockEmails, BASE_URL, FEED_ID);
|
||||
expect(result).toContain(`${BASE_URL}/rss/${FEED_ID}`);
|
||||
@@ -104,4 +135,16 @@ describe("generateAtomFeed", () => {
|
||||
const result = generateAtomFeed(configWithAuthor, mockEmails, BASE_URL, FEED_ID);
|
||||
expect(result).toContain("Bob");
|
||||
});
|
||||
|
||||
it("includes enclosure link for email with attachment in Atom feed", () => {
|
||||
const result = generateAtomFeed(mockFeedConfig, [mockEmailWithAttachment], BASE_URL, FEED_ID);
|
||||
expect(result).toContain('rel="enclosure"');
|
||||
expect(result).toContain("550e8400-e29b-41d4-a716-446655440000");
|
||||
expect(result).toContain("report.pdf");
|
||||
});
|
||||
|
||||
it("does not include enclosure link for email without attachments in Atom feed", () => {
|
||||
const result = generateAtomFeed(mockFeedConfig, mockEmails, BASE_URL, FEED_ID);
|
||||
expect(result).not.toContain('rel="enclosure"');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -42,6 +42,7 @@ function buildFeed(
|
||||
|
||||
for (const email of emails) {
|
||||
const uniqueId = `${email.receivedAt}-${Buffer.from(email.subject).toString("base64").substring(0, 10)}`;
|
||||
const firstAttachment = email.attachments?.[0];
|
||||
feed.addItem({
|
||||
title: email.subject,
|
||||
id: uniqueId,
|
||||
@@ -50,6 +51,13 @@ function buildFeed(
|
||||
content: email.content,
|
||||
author: [parseFromAddress(email.from)],
|
||||
date: new Date(email.receivedAt),
|
||||
enclosure: firstAttachment
|
||||
? {
|
||||
url: `${baseUrl}/files/${firstAttachment.id}/${encodeURIComponent(firstAttachment.filename)}`,
|
||||
type: firstAttachment.contentType,
|
||||
length: firstAttachment.size,
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user