From f150d40c45480a64cf780264baac3e157e5a00bf Mon Sep 17 00:00:00 2001 From: Julien Herr Date: Sat, 23 May 2026 17:33:50 +0200 Subject: [PATCH] feat(attachments): R2 toggle, storage metrics, and demo R2 config Add an ATTACHMENTS_ENABLED switch (default on when R2 is bound) via a central getAttachmentBucket helper, surface R2 + estimated KV usage against the free tier on the status page and /api/stats (refreshed by the hourly cron), let setup.sh create and wire the R2 bucket, and bind the demo bucket so the deployed demo has attachments. Co-Authored-By: Claude Opus 4.7 --- README.md | 10 +++- docs/index.html | 4 ++ setup.sh | 46 ++++++++++++++++ src/config/constants.ts | 6 ++ src/index.ts | 28 +++++++++- src/lib/email-processor.test.ts | 26 +++++++++ src/lib/email-processor.ts | 10 ++-- src/routes/admin/emails.tsx | 11 ++-- src/routes/admin/feeds.tsx | 5 +- src/routes/entries.ts | 7 +-- src/routes/files.ts | 6 +- src/routes/home.tsx | 40 ++++++++++++++ src/test/setup.ts | 8 +++ src/types/index.ts | 7 +++ src/utils/attachments.ts | 9 +++ src/utils/format.ts | 8 +++ src/utils/stats.test.ts | 97 ++++++++++++++++++++++++++++++++- src/utils/stats.ts | 73 +++++++++++++++++++++++++ wrangler-example.toml | 9 +++ 19 files changed, 387 insertions(+), 23 deletions(-) create mode 100644 src/utils/attachments.ts create mode 100644 src/utils/format.ts diff --git a/README.md b/README.md index 5d2365f..3a081f9 100644 --- a/README.md +++ b/README.md @@ -179,7 +179,9 @@ When an incoming email contains attachments, the Worker can store them in a Clou This feature is **optional**. If no R2 bucket is bound, attachments are silently ignored and nothing else changes. -**Setup:** +**Setup (automated):** `setup.sh` now asks _"Enable email attachments stored in R2?"_. Answer yes and it creates the buckets (`-attachments` and `-attachments-preview`) and wires the binding into the generated `wrangler.toml` for you. + +**Setup (manual):** 1. Create an R2 bucket in the Cloudflare dashboard (_R2 Object Storage → Create bucket_), or with Wrangler: ```bash @@ -191,14 +193,18 @@ This feature is **optional**. If no R2 bucket is bound, attachments are silently { binding = "ATTACHMENT_BUCKET", bucket_name = "your-bucket-name", preview_bucket_name = "your-bucket-name-preview" } ] ``` - Do the same under `[env.production]` (without `preview_bucket_name`). + The binding is **per environment**: add it under every env you deploy (`[env.production]`, `[env.demo]`, …), each pointing at its own bucket. 3. Redeploy: ```bash npm run deploy ``` +**Turning it off:** set `ATTACHMENTS_ENABLED = "false"` in `[vars]` to disable attachments even while the R2 bucket stays bound (useful to cap usage on a demo). Any other value (or leaving it unset) keeps the feature on whenever R2 is configured. + Attachments are deleted from R2 automatically when the corresponding email is deleted from the admin UI, or when an email is dropped during feed size trimming. +**Monitoring storage / free tier:** the status page (`/`) and `/api/stats` report R2 space used (against the **10 GB** R2 free tier) and an estimate of KV space used (against the **1 GB** KV free tier). The figures are refreshed hourly by the cron trigger. KV usage is an estimate based on stored email sizes, so treat it as a lower bound. + ### External auth provider (Authelia / Authentik / reverse proxy) Instead of the built-in password login you can delegate admin authentication to a reverse proxy that sets a trusted user header (`Remote-User` or `X-Forwarded-User`). diff --git a/docs/index.html b/docs/index.html index f9686bb..8476256 100644 --- a/docs/index.html +++ b/docs/index.html @@ -1180,6 +1180,10 @@ bucket_name = "kill-the-news-attachments" How do I delete a feed?

From the password-protected admin UI — open the Feeds tab and delete it there. Its entries and attachments are removed along with it.

+
+ Does it handle attachments? +

Yes — optionally. When an R2 bucket is configured, email attachments are stored there and exposed as RSS/Atom enclosures, downloadable from each entry. It's off by default: if no R2 bucket is bound (or you set ATTACHMENTS_ENABLED = "false"), attachments are simply skipped and everything else works as usual. R2 usage is shown on the status page so you can stay within the 10 GB free tier.

+
diff --git a/setup.sh b/setup.sh index 5e73be7..a166ffb 100755 --- a/setup.sh +++ b/setup.sh @@ -134,6 +134,34 @@ if [ -z "$domain" ]; then fi echo "✅ Domain: $domain" +ENABLE_R2=false +R2_BUCKET="" +R2_PREVIEW_BUCKET="" +read -r -p "Enable email attachments stored in R2? [y/N]: " enable_r2 +if [[ "$enable_r2" =~ ^[Yy]$ ]]; then + R2_BUCKET="${WORKER_NAME}-attachments" + R2_PREVIEW_BUCKET="${R2_BUCKET}-preview" + echo "🪣 Creating R2 buckets..." + set +e + R2_OUT="$(npx wrangler r2 bucket create "$R2_BUCKET" 2>&1)" + R2_STATUS=$? + R2_PREVIEW_OUT="$(npx wrangler r2 bucket create "$R2_PREVIEW_BUCKET" 2>&1)" + R2_PREVIEW_STATUS=$? + set -e + # An existing bucket is fine; only treat real failures as blocking. + echo "$R2_OUT" | grep -qi "already exists" && R2_STATUS=0 + echo "$R2_PREVIEW_OUT" | grep -qi "already exists" && R2_PREVIEW_STATUS=0 + if [ "$R2_STATUS" -eq 0 ] && [ "$R2_PREVIEW_STATUS" -eq 0 ]; then + ENABLE_R2=true + echo " ✅ R2 bucket: $R2_BUCKET" + echo " ✅ R2 preview bucket: $R2_PREVIEW_BUCKET" + else + echo " ⚠️ Could not create R2 buckets (is R2 enabled on your account?)." + echo " Attachments will stay disabled — see README → 'Email attachments (R2)'." + echo "$R2_OUT" + fi +fi + escape_sed_replacement() { printf '%s' "$1" | sed -e 's/[\/&]/\\&/g' } @@ -158,6 +186,24 @@ else sed -i "s/REPLACE_WITH_COMPATIBILITY_DATE/$COMPATIBILITY_DATE_ESCAPED/g" wrangler.toml fi +if [ "$ENABLE_R2" = true ]; then + echo "🔗 Enabling R2 attachment binding in wrangler.toml..." + node - "wrangler.toml" "$R2_BUCKET" "$R2_PREVIEW_BUCKET" <<'NODE' +const fs = require("node:fs"); +const [file, bucket, previewBucket] = process.argv.slice(2); +let txt = fs.readFileSync(file, "utf8"); +txt = txt.split("REPLACE_WITH_YOUR_PREVIEW_BUCKET_NAME").join(previewBucket); +txt = txt.split("REPLACE_WITH_YOUR_BUCKET_NAME").join(bucket); +// Uncomment the commented r2_buckets blocks (global + [env.production]). +txt = txt.replace( + /# r2_buckets = \[\n#(\s+\{ binding = "ATTACHMENT_BUCKET".*\})\n# \]/g, + 'r2_buckets = [\n$1\n]', +); +fs.writeFileSync(file, txt); +NODE + echo " ✅ ATTACHMENT_BUCKET bound to $R2_BUCKET" +fi + echo "✅ wrangler.toml has been created and configured successfully!" echo "" echo "✅ Setup complete! Next steps:" diff --git a/src/config/constants.ts b/src/config/constants.ts index 181bdcb..5d491dc 100644 --- a/src/config/constants.ts +++ b/src/config/constants.ts @@ -1,6 +1,12 @@ /** Maximum total size of emails stored per feed (bytes). */ export const FEED_MAX_BYTES = 524288; // 512 KB +/** Cloudflare R2 free tier storage allowance (bytes). */ +export const R2_FREE_TIER_BYTES = 10 * 1024 ** 3; // 10 GB + +/** Cloudflare KV free tier storage allowance (bytes). */ +export const KV_FREE_TIER_BYTES = 1 * 1024 ** 3; // 1 GB + /** Cache TTL for ForwardEmail.net IP list (milliseconds). */ export const FORWARD_EMAIL_IPS_CACHE_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours diff --git a/src/index.ts b/src/index.ts index 6fb245c..66f70a4 100644 --- a/src/index.ts +++ b/src/index.ts @@ -18,7 +18,13 @@ import { purgeExpiredFeeds, removeFeedsFromListBulk, } from "./routes/admin/helpers"; -import { bumpCounters } from "./utils/stats"; +import { + bumpCounters, + scanR2Usage, + scanKvUsage, + setStorageSnapshot, +} from "./utils/stats"; +import { getAttachmentBucket } from "./utils/attachments"; import { FORWARD_EMAIL_IPS_CACHE_TTL_MS } from "./config/constants"; type AppEnv = { Bindings: Env }; @@ -196,6 +202,7 @@ export default { await handleCloudflareEmail(message, env, ctx); }, async scheduled(_event: ScheduledEvent, env: Env, _ctx: ExecutionContext) { + const attachmentBucket = getAttachmentBucket(env); const feeds = await listAllFeeds(env.EMAIL_STORAGE); const now = Date.now(); const expiredIds = feeds @@ -203,7 +210,7 @@ export default { .map((f) => f.id); for (const feedId of expiredIds) { - await purgeExpiredFeeds(env.EMAIL_STORAGE, feedId, env.ATTACHMENT_BUCKET); + await purgeExpiredFeeds(env.EMAIL_STORAGE, feedId, attachmentBucket); } if (expiredIds.length > 0) { await removeFeedsFromListBulk(env.EMAIL_STORAGE, expiredIds); @@ -212,5 +219,22 @@ export default { }); logger.info("Feed TTL cleanup", { deleted: expiredIds.length }); } + + // Refresh the cached storage-usage snapshot for the status page / /api/stats. + try { + const r2 = attachmentBucket + ? await scanR2Usage(attachmentBucket) + : { bytes: 0, count: 0 }; + const kv = await scanKvUsage(env.EMAIL_STORAGE); + await setStorageSnapshot(env.EMAIL_STORAGE, { + attachments_bytes: r2.bytes, + attachments_count: r2.count, + kv_bytes_estimated: kv.bytes, + }); + } catch (error) { + logger.error("Error refreshing storage snapshot", { + error: String(error), + }); + } }, }; diff --git a/src/lib/email-processor.test.ts b/src/lib/email-processor.test.ts index c14689f..f5a78ad 100644 --- a/src/lib/email-processor.test.ts +++ b/src/lib/email-processor.test.ts @@ -363,6 +363,32 @@ describe("processEmail — attachments", () => { expect(emailData.attachments).toBeUndefined(); }); + it("skips R2 upload when ATTACHMENTS_ENABLED is 'false' even with R2 bound", async () => { + const env = createMockEnv({ withR2: true }); + (env as any).ATTACHMENTS_ENABLED = "false"; + const mockR2 = (env as any).ATTACHMENT_BUCKET as unknown as MockR2; + await env.EMAIL_STORAGE.put( + `feed:${VALID_FEED_ID}:config`, + JSON.stringify({}), + ); + const res = await processEmail( + makeInput({ attachments: [pdfAttachment] }), + env as any, + ); + expect(res.status).toBe(200); + + const metadata = await env.EMAIL_STORAGE.get( + `feed:${VALID_FEED_ID}:metadata`, + "json", + ); + const emailData = await env.EMAIL_STORAGE.get( + metadata.emails[0].key, + "json", + ); + expect(emailData.attachments).toBeUndefined(); + expect((await mockR2.list()).objects).toHaveLength(0); + }); + it("uploads attachments to R2 and stores AttachmentData in emailData", async () => { const env = createMockEnv({ withR2: true }); const mockR2 = (env as any).ATTACHMENT_BUCKET as unknown as MockR2; diff --git a/src/lib/email-processor.ts b/src/lib/email-processor.ts index 2e131e7..5d2d13b 100644 --- a/src/lib/email-processor.ts +++ b/src/lib/email-processor.ts @@ -13,6 +13,7 @@ import { extractEmailDomain, } from "../utils/favicon-fetcher"; import { parseOneClickUnsubscribe } from "../utils/unsubscribe"; +import { getAttachmentBucket } from "../utils/attachments"; import { logger } from "./logger"; import { FEED_MAX_BYTES } from "../config/constants"; @@ -170,9 +171,10 @@ export async function storeEmail( env: Env, ctx?: ExecutionContext, ): Promise { + const attachmentBucket = getAttachmentBucket(env); const storedAttachments: AttachmentData[] = - env.ATTACHMENT_BUCKET && input.attachments?.length - ? await uploadAttachments(input.attachments, env.ATTACHMENT_BUCKET) + attachmentBucket && input.attachments?.length + ? await uploadAttachments(input.attachments, attachmentBucket) : []; const emailData = { @@ -249,10 +251,10 @@ export async function storeEmail( } const r2Deletions = - env.ATTACHMENT_BUCKET && toDelete.length > 0 + attachmentBucket && toDelete.length > 0 ? toDelete .flatMap((e) => e.attachmentIds ?? []) - .map((id) => env.ATTACHMENT_BUCKET!.delete(id)) + .map((id) => attachmentBucket.delete(id)) : []; await Promise.all([ diff --git a/src/routes/admin/emails.tsx b/src/routes/admin/emails.tsx index ce24dee..bf44b11 100644 --- a/src/routes/admin/emails.tsx +++ b/src/routes/admin/emails.tsx @@ -10,6 +10,7 @@ import { logger } from "../../lib/logger"; import { Layout, clampText } from "./ui"; import { deleteKeysWithConcurrency } from "./helpers"; import { feedRssUrl, feedAtomUrl, feedEmailAddress } from "../../utils/urls"; +import { getAttachmentBucket } from "../../utils/attachments"; import { emailsPageScript } from "../../scripts/generated/emails-page"; type AppEnv = { Bindings: Env }; @@ -643,9 +644,10 @@ emailsRouter.post("/emails/:emailKey/delete", async (c) => { await emailStorage.put(feedMetadataKey, JSON.stringify(feedMetadata)); } - if (env.ATTACHMENT_BUCKET && attachmentIds.length > 0) { + const attachmentBucket = getAttachmentBucket(env); + if (attachmentBucket && attachmentIds.length > 0) { await Promise.allSettled( - attachmentIds.map((id) => env.ATTACHMENT_BUCKET!.delete(id)), + attachmentIds.map((id) => attachmentBucket.delete(id)), ); } @@ -726,9 +728,10 @@ emailsRouter.post("/feeds/:feedId/emails/bulk-delete", async (c) => { ); await emailStorage.put(feedMetadataKey, JSON.stringify(feedMetadata)); - if (env.ATTACHMENT_BUCKET && r2AttachmentIds.length > 0) { + const attachmentBucket = getAttachmentBucket(env); + if (attachmentBucket && r2AttachmentIds.length > 0) { await Promise.allSettled( - r2AttachmentIds.map((id) => env.ATTACHMENT_BUCKET!.delete(id)), + r2AttachmentIds.map((id) => attachmentBucket.delete(id)), ); } diff --git a/src/routes/admin/feeds.tsx b/src/routes/admin/feeds.tsx index fae9d6a..c964937 100644 --- a/src/routes/admin/feeds.tsx +++ b/src/routes/admin/feeds.tsx @@ -7,6 +7,7 @@ import { waitUntilSafe } from "../../utils/worker"; import { feedRssUrl, feedEmailAddress } from "../../utils/urls"; import { logger } from "../../lib/logger"; import { sendUnsubscribes } from "../../utils/unsubscribe"; +import { getAttachmentBucket } from "../../utils/attachments"; import { Layout } from "./ui"; import { addFeedToList, @@ -555,7 +556,7 @@ feedsRouter.post("/:feedId/delete", async (c) => { waitUntilSafe( c, purgeFeedKeysStep(emailStorage, feedId, { - bucket: env.ATTACHMENT_BUCKET, + bucket: getAttachmentBucket(env), }), ); @@ -594,7 +595,7 @@ feedsRouter.post("/:feedId/purge", async (c) => { const step = await purgeFeedKeysStep(emailStorage, feedId, { cursor, limit, - bucket: env.ATTACHMENT_BUCKET, + bucket: getAttachmentBucket(env), }); return c.json({ diff --git a/src/routes/entries.ts b/src/routes/entries.ts index 26afc5b..df39253 100644 --- a/src/routes/entries.ts +++ b/src/routes/entries.ts @@ -2,12 +2,7 @@ import { Context } from "hono"; import { html, raw } from "hono/html"; import { Env, FeedConfig, FeedMetadata, EmailData } from "../types"; import { processEmailContent } from "../utils/html-processor"; - -function formatBytes(bytes: number): string { - if (bytes < 1024) return `${bytes} B`; - if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; - return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; -} +import { formatBytes } from "../utils/format"; export async function handle(c: Context<{ Bindings: Env }>): Promise { const feedId = c.req.param("feedId"); diff --git a/src/routes/files.ts b/src/routes/files.ts index 1af990c..16f02d9 100644 --- a/src/routes/files.ts +++ b/src/routes/files.ts @@ -1,8 +1,10 @@ import { Context } from "hono"; import { Env } from "../types"; +import { getAttachmentBucket } from "../utils/attachments"; export async function handle(c: Context<{ Bindings: Env }>): Promise { - if (!c.env.ATTACHMENT_BUCKET) { + const bucket = getAttachmentBucket(c.env); + if (!bucket) { return new Response("Attachment storage not configured", { status: 404 }); } @@ -13,7 +15,7 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise { return new Response("Not found", { status: 404 }); } - const object = await c.env.ATTACHMENT_BUCKET.get(attachmentId); + const object = await bucket.get(attachmentId); if (!object) { return new Response("Not found", { status: 404 }); diff --git a/src/routes/home.tsx b/src/routes/home.tsx index ae140a5..b683630 100644 --- a/src/routes/home.tsx +++ b/src/routes/home.tsx @@ -1,6 +1,8 @@ import { Context } from "hono"; import { Env } from "../types"; import { getStats } from "../utils/stats"; +import { formatBytes } from "../utils/format"; +import { R2_FREE_TIER_BYTES, KV_FREE_TIER_BYTES } from "../config/constants"; import { Layout } from "./admin/ui"; function formatDateTime(iso?: string): string { @@ -43,6 +45,11 @@ function formatUptime(iso?: string): string { return `${days} ${days === 1 ? "day" : "days"}`; } +function tierPercent(used: number, total: number): number { + if (total <= 0) return 0; + return Math.round((used / total) * 100); +} + type Tone = "success" | "danger"; type StatProps = { @@ -83,6 +90,11 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise { ? (stats.emails_received / stats.feeds_created).toFixed(1) : "—"; + const kvBytes = stats.kv_bytes_estimated; + const kvPercent = tierPercent(kvBytes ?? 0, KV_FREE_TIER_BYTES); + const r2Bytes = stats.attachments_bytes; + const r2Percent = tierPercent(r2Bytes ?? 0, R2_FREE_TIER_BYTES); + return c.html(
@@ -167,6 +179,34 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise {
+
+

Storage

+
+ = 80 ? "danger" : undefined} + /> + {stats.attachments_enabled ? ( + <> + + = 80 ? "danger" : undefined} + /> + + ) : ( + + )} +
+
+

Instance

diff --git a/src/test/setup.ts b/src/test/setup.ts index 1c46419..ee52ccb 100644 --- a/src/test/setup.ts +++ b/src/test/setup.ts @@ -237,6 +237,14 @@ export class MockR2 { } } + async list(_options?: { cursor?: string }) { + const objects = Array.from(this.store.entries()).map(([key, entry]) => ({ + key, + size: entry.body.byteLength, + })); + return { objects, truncated: false as const, cursor: undefined }; + } + _has(key: string) { return this.store.has(key); } diff --git a/src/types/index.ts b/src/types/index.ts index f4a5fbc..0afe603 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -5,6 +5,7 @@ export interface Env { DOMAIN: string; EMAIL_DOMAIN?: string; ATTACHMENT_BUCKET?: R2Bucket; + ATTACHMENTS_ENABLED?: string; // "false" disables attachments even when R2 is bound FEED_MAX_SIZE_BYTES?: string; PROXY_TRUSTED_IPS?: string; PROXY_AUTH_SECRET?: string; @@ -83,12 +84,18 @@ export interface Counters { last_email_at?: string; // ISO 8601 last_feed_created_at?: string; // ISO 8601 first_seen?: string; // ISO 8601 — first time counters were written (instance start) + // Storage usage snapshot, refreshed by the hourly cron (overwritten, not incremented). + attachments_bytes?: number; // Total R2 bytes used by attachments + attachments_count?: number; // Number of R2 objects + kv_bytes_estimated?: number; // Estimated KV bytes (sum of stored email sizes) + storage_scanned_at?: string; // ISO 8601 — last storage scan } // Monitoring API response: persisted counters + live-computed values export interface StatsResponse extends Counters { active_feeds: number; websub_subscriptions_active: number; + attachments_enabled: boolean; } // WebSub (PubSubHubbub) subscription configuration diff --git a/src/utils/attachments.ts b/src/utils/attachments.ts new file mode 100644 index 0000000..4a6424d --- /dev/null +++ b/src/utils/attachments.ts @@ -0,0 +1,9 @@ +import { Env } from "../types"; + +// Returns the attachment bucket only when the feature is enabled, so callers can +// narrow cleanly. Attachments are on whenever R2 is bound, unless explicitly +// turned off with ATTACHMENTS_ENABLED="false". +export function getAttachmentBucket(env: Env): R2Bucket | undefined { + if (env.ATTACHMENTS_ENABLED === "false") return undefined; + return env.ATTACHMENT_BUCKET; +} diff --git a/src/utils/format.ts b/src/utils/format.ts new file mode 100644 index 0000000..5fdd41b --- /dev/null +++ b/src/utils/format.ts @@ -0,0 +1,8 @@ +/** Human-readable byte size (B / KB / MB / GB). */ +export function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + if (bytes < 1024 * 1024 * 1024) + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; + return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`; +} diff --git a/src/utils/stats.test.ts b/src/utils/stats.test.ts index 30bb05b..6fbde1a 100644 --- a/src/utils/stats.test.ts +++ b/src/utils/stats.test.ts @@ -1,11 +1,15 @@ import { describe, it, expect } from "vitest"; -import { createMockEnv } from "../test/setup"; +import { createMockEnv, MockR2 } from "../test/setup"; import { getCounters, bumpCounters, countKeysByPrefix, getStats, + scanR2Usage, + scanKvUsage, + setStorageSnapshot, } from "./stats"; +import { getAttachmentBucket } from "./attachments"; import { STATS_KEY, FEEDS_LIST_KEY } from "../config/constants"; import { Env } from "../types"; @@ -119,4 +123,95 @@ describe("stats helper", () => { }; expect(raw.feeds_created).toBe(1); }); + + it("getStats reports attachments_enabled based on the toggle", async () => { + const off = createMockEnv() as unknown as Env; + expect((await getStats(off)).attachments_enabled).toBe(false); + + const on = createMockEnv({ withR2: true }) as unknown as Env; + expect((await getStats(on)).attachments_enabled).toBe(true); + + const disabled = createMockEnv({ withR2: true }) as unknown as Env; + (disabled as any).ATTACHMENTS_ENABLED = "false"; + expect((await getStats(disabled)).attachments_enabled).toBe(false); + }); +}); + +describe("getAttachmentBucket", () => { + it("returns the bucket when bound and not disabled", () => { + const env = createMockEnv({ withR2: true }) as unknown as Env; + expect(getAttachmentBucket(env)).toBeDefined(); + }); + + it("returns undefined when no bucket is bound", () => { + const env = createMockEnv() as unknown as Env; + expect(getAttachmentBucket(env)).toBeUndefined(); + }); + + it("returns undefined when explicitly disabled", () => { + const env = createMockEnv({ withR2: true }) as unknown as Env; + (env as any).ATTACHMENTS_ENABLED = "false"; + expect(getAttachmentBucket(env)).toBeUndefined(); + }); +}); + +describe("storage usage scans", () => { + it("scanR2Usage sums object sizes and counts", async () => { + const bucket = new MockR2(); + await bucket.put("a", new Uint8Array(100)); + await bucket.put("b", new Uint8Array(250)); + const usage = await scanR2Usage(bucket as unknown as R2Bucket); + expect(usage.count).toBe(2); + expect(usage.bytes).toBe(350); + }); + + it("scanR2Usage returns zeros for an empty bucket", async () => { + const usage = await scanR2Usage(new MockR2() as unknown as R2Bucket); + expect(usage).toEqual({ bytes: 0, count: 0 }); + }); + + it("scanKvUsage estimates KV bytes from stored email sizes", async () => { + const env = createMockEnv() as unknown as Env; + const kv = env.EMAIL_STORAGE; + await kv.put( + FEEDS_LIST_KEY, + JSON.stringify({ + feeds: [ + { id: "a", title: "A" }, + { id: "b", title: "B" }, + ], + }), + ); + await kv.put( + "feed:a:metadata", + JSON.stringify({ + emails: [ + { key: "k1", size: 100 }, + { key: "k2", size: 50 }, + ], + }), + ); + await kv.put( + "feed:b:metadata", + JSON.stringify({ emails: [{ key: "k3", size: 25 }] }), + ); + + const usage = await scanKvUsage(kv); + expect(usage.bytes).toBe(175); + }); + + it("setStorageSnapshot writes the snapshot fields", async () => { + const env = createMockEnv() as unknown as Env; + const kv = env.EMAIL_STORAGE; + await setStorageSnapshot(kv, { + attachments_bytes: 1234, + attachments_count: 5, + kv_bytes_estimated: 678, + }); + const counters = await getCounters(kv); + expect(counters.attachments_bytes).toBe(1234); + expect(counters.attachments_count).toBe(5); + expect(counters.kv_bytes_estimated).toBe(678); + expect(counters.storage_scanned_at).toBeDefined(); + }); }); diff --git a/src/utils/stats.ts b/src/utils/stats.ts index cb3fc6a..67e508f 100644 --- a/src/utils/stats.ts +++ b/src/utils/stats.ts @@ -2,6 +2,8 @@ import { Counters, Env, StatsResponse } from "../types"; import { STATS_KEY } from "../config/constants"; import { logger } from "../lib/logger"; import { listAllFeeds } from "../routes/admin/helpers"; +import { getFeedMetadata } from "./storage"; +import { getAttachmentBucket } from "./attachments"; const EMPTY_COUNTERS: Counters = { feeds_created: 0, @@ -82,5 +84,76 @@ export async function getStats(env: Env): Promise { ...counters, active_feeds: feeds.length, websub_subscriptions_active: websubCount, + attachments_enabled: !!getAttachmentBucket(env), }; } + +/** Sum the byte size and object count of every attachment stored in R2. */ +export async function scanR2Usage( + bucket: R2Bucket, +): Promise<{ bytes: number; count: number }> { + let bytes = 0; + let count = 0; + let cursor: string | undefined; + try { + do { + const listed = await bucket.list({ cursor }); + for (const obj of listed.objects) { + bytes += obj.size; + count += 1; + } + cursor = listed.truncated ? listed.cursor : undefined; + } while (cursor); + } catch (error) { + logger.error("Error scanning R2 usage", { error: String(error) }); + } + return { bytes, count }; +} + +/** + * Estimate KV storage used. KV exposes no size API, so we sum the per-email + * sizes already recorded in each feed's metadata — email bodies dominate KV + * usage. Feed config/websub/stats keys are excluded, so this is a lower-bound + * estimate. + */ +export async function scanKvUsage(kv: KVNamespace): Promise<{ bytes: number }> { + let bytes = 0; + try { + const feeds = await listAllFeeds(kv); + for (const feed of feeds) { + const metadata = await getFeedMetadata(kv, feed.id); + if (!metadata) continue; + for (const email of metadata.emails) { + bytes += email.size ?? 0; + } + } + } catch (error) { + logger.error("Error estimating KV usage", { error: String(error) }); + } + return { bytes }; +} + +/** + * Overwrite the storage-usage snapshot fields on the counters singleton. + * Unlike bumpCounters these are set (not incremented). Never throws. + */ +export async function setStorageSnapshot( + kv: KVNamespace, + snapshot: { + attachments_bytes: number; + attachments_count: number; + kv_bytes_estimated: number; + }, +): Promise { + try { + const current = await getCounters(kv); + current.attachments_bytes = snapshot.attachments_bytes; + current.attachments_count = snapshot.attachments_count; + current.kv_bytes_estimated = snapshot.kv_bytes_estimated; + current.storage_scanned_at = new Date().toISOString(); + if (!current.first_seen) current.first_seen = new Date().toISOString(); + await kv.put(STATS_KEY, JSON.stringify(current)); + } catch (error) { + logger.error("Error writing storage snapshot", { error: String(error) }); + } +} diff --git a/wrangler-example.toml b/wrangler-example.toml index e4202ec..e4453d5 100644 --- a/wrangler-example.toml +++ b/wrangler-example.toml @@ -39,6 +39,10 @@ DOMAIN = "REPLACE_WITH_YOUR_DOMAIN" # Web domain (used for feed URLs and admin U # Optional: size-based feed trimming threshold in bytes (default: 524288 = 512 KB) # FEED_MAX_SIZE_BYTES = "524288" +# Optional: turn email attachments off even when an R2 bucket is bound. +# Unset (or any value other than "false") keeps attachments on whenever R2 is configured. +# ATTACHMENTS_ENABLED = "false" + # Optional: lock feed lifetime for all users (hours). When set, the TTL field in # the admin UI is pre-filled and read-only. Remove to allow per-feed configuration. # FEED_TTL_HOURS = "24" @@ -89,6 +93,11 @@ kv_namespaces = [ { binding = "EMAIL_STORAGE", id = "REPLACE_WITH_DEMO_KV_NAMESPACE_ID" } ] +# R2 bucket for storing email attachment enclosures (demo) +r2_buckets = [ + { binding = "ATTACHMENT_BUCKET", bucket_name = "ktn-attachment-bucket-demo" } +] + routes = [ { pattern = "demo.kill-the.news", custom_domain = true } ]