mirror of
https://github.com/juherr/kill-the-news.git
synced 2026-06-20 22:03:48 +00:00
feat(attachments): R2 toggle, storage metrics, and demo R2 config
Add an ATTACHMENTS_ENABLED switch (default on when R2 is bound) via a central getAttachmentBucket helper, surface R2 + estimated KV usage against the free tier on the status page and /api/stats (refreshed by the hourly cron), let setup.sh create and wire the R2 bucket, and bind the demo bucket so the deployed demo has attachments. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -179,7 +179,9 @@ When an incoming email contains attachments, the Worker can store them in a Clou
|
||||
|
||||
This feature is **optional**. If no R2 bucket is bound, attachments are silently ignored and nothing else changes.
|
||||
|
||||
**Setup:**
|
||||
**Setup (automated):** `setup.sh` now asks _"Enable email attachments stored in R2?"_. Answer yes and it creates the buckets (`<worker>-attachments` and `<worker>-attachments-preview`) and wires the binding into the generated `wrangler.toml` for you.
|
||||
|
||||
**Setup (manual):**
|
||||
|
||||
1. Create an R2 bucket in the Cloudflare dashboard (_R2 Object Storage → Create bucket_), or with Wrangler:
|
||||
```bash
|
||||
@@ -191,14 +193,18 @@ This feature is **optional**. If no R2 bucket is bound, attachments are silently
|
||||
{ binding = "ATTACHMENT_BUCKET", bucket_name = "your-bucket-name", preview_bucket_name = "your-bucket-name-preview" }
|
||||
]
|
||||
```
|
||||
Do the same under `[env.production]` (without `preview_bucket_name`).
|
||||
The binding is **per environment**: add it under every env you deploy (`[env.production]`, `[env.demo]`, …), each pointing at its own bucket.
|
||||
3. Redeploy:
|
||||
```bash
|
||||
npm run deploy
|
||||
```
|
||||
|
||||
**Turning it off:** set `ATTACHMENTS_ENABLED = "false"` in `[vars]` to disable attachments even while the R2 bucket stays bound (useful to cap usage on a demo). Any other value (or leaving it unset) keeps the feature on whenever R2 is configured.
|
||||
|
||||
Attachments are deleted from R2 automatically when the corresponding email is deleted from the admin UI, or when an email is dropped during feed size trimming.
|
||||
|
||||
**Monitoring storage / free tier:** the status page (`/`) and `/api/stats` report R2 space used (against the **10 GB** R2 free tier) and an estimate of KV space used (against the **1 GB** KV free tier). The figures are refreshed hourly by the cron trigger. KV usage is an estimate based on stored email sizes, so treat it as a lower bound.
|
||||
|
||||
### External auth provider (Authelia / Authentik / reverse proxy)
|
||||
|
||||
Instead of the built-in password login you can delegate admin authentication to a reverse proxy that sets a trusted user header (`Remote-User` or `X-Forwarded-User`).
|
||||
|
||||
@@ -1180,6 +1180,10 @@ bucket_name = "kill-the-news-attachments"</span></pre>
|
||||
<summary>How do I delete a feed?</summary>
|
||||
<p>From the password-protected admin UI — open the Feeds tab and delete it there. Its entries and attachments are removed along with it.</p>
|
||||
</details>
|
||||
<details class="faq-item">
|
||||
<summary>Does it handle attachments?</summary>
|
||||
<p>Yes — optionally. When an R2 bucket is configured, email attachments are stored there and exposed as RSS/Atom enclosures, downloadable from each entry. It's off by default: if no R2 bucket is bound (or you set <code>ATTACHMENTS_ENABLED = "false"</code>), attachments are simply skipped and everything else works as usual. R2 usage is shown on the status page so you can stay within the 10 GB free tier.</p>
|
||||
</details>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
||||
@@ -134,6 +134,34 @@ if [ -z "$domain" ]; then
|
||||
fi
|
||||
echo "✅ Domain: $domain"
|
||||
|
||||
ENABLE_R2=false
|
||||
R2_BUCKET=""
|
||||
R2_PREVIEW_BUCKET=""
|
||||
read -r -p "Enable email attachments stored in R2? [y/N]: " enable_r2
|
||||
if [[ "$enable_r2" =~ ^[Yy]$ ]]; then
|
||||
R2_BUCKET="${WORKER_NAME}-attachments"
|
||||
R2_PREVIEW_BUCKET="${R2_BUCKET}-preview"
|
||||
echo "🪣 Creating R2 buckets..."
|
||||
set +e
|
||||
R2_OUT="$(npx wrangler r2 bucket create "$R2_BUCKET" 2>&1)"
|
||||
R2_STATUS=$?
|
||||
R2_PREVIEW_OUT="$(npx wrangler r2 bucket create "$R2_PREVIEW_BUCKET" 2>&1)"
|
||||
R2_PREVIEW_STATUS=$?
|
||||
set -e
|
||||
# An existing bucket is fine; only treat real failures as blocking.
|
||||
echo "$R2_OUT" | grep -qi "already exists" && R2_STATUS=0
|
||||
echo "$R2_PREVIEW_OUT" | grep -qi "already exists" && R2_PREVIEW_STATUS=0
|
||||
if [ "$R2_STATUS" -eq 0 ] && [ "$R2_PREVIEW_STATUS" -eq 0 ]; then
|
||||
ENABLE_R2=true
|
||||
echo " ✅ R2 bucket: $R2_BUCKET"
|
||||
echo " ✅ R2 preview bucket: $R2_PREVIEW_BUCKET"
|
||||
else
|
||||
echo " ⚠️ Could not create R2 buckets (is R2 enabled on your account?)."
|
||||
echo " Attachments will stay disabled — see README → 'Email attachments (R2)'."
|
||||
echo "$R2_OUT"
|
||||
fi
|
||||
fi
|
||||
|
||||
escape_sed_replacement() {
|
||||
printf '%s' "$1" | sed -e 's/[\/&]/\\&/g'
|
||||
}
|
||||
@@ -158,6 +186,24 @@ else
|
||||
sed -i "s/REPLACE_WITH_COMPATIBILITY_DATE/$COMPATIBILITY_DATE_ESCAPED/g" wrangler.toml
|
||||
fi
|
||||
|
||||
if [ "$ENABLE_R2" = true ]; then
|
||||
echo "🔗 Enabling R2 attachment binding in wrangler.toml..."
|
||||
node - "wrangler.toml" "$R2_BUCKET" "$R2_PREVIEW_BUCKET" <<'NODE'
|
||||
const fs = require("node:fs");
|
||||
const [file, bucket, previewBucket] = process.argv.slice(2);
|
||||
let txt = fs.readFileSync(file, "utf8");
|
||||
txt = txt.split("REPLACE_WITH_YOUR_PREVIEW_BUCKET_NAME").join(previewBucket);
|
||||
txt = txt.split("REPLACE_WITH_YOUR_BUCKET_NAME").join(bucket);
|
||||
// Uncomment the commented r2_buckets blocks (global + [env.production]).
|
||||
txt = txt.replace(
|
||||
/# r2_buckets = \[\n#(\s+\{ binding = "ATTACHMENT_BUCKET".*\})\n# \]/g,
|
||||
'r2_buckets = [\n$1\n]',
|
||||
);
|
||||
fs.writeFileSync(file, txt);
|
||||
NODE
|
||||
echo " ✅ ATTACHMENT_BUCKET bound to $R2_BUCKET"
|
||||
fi
|
||||
|
||||
echo "✅ wrangler.toml has been created and configured successfully!"
|
||||
echo ""
|
||||
echo "✅ Setup complete! Next steps:"
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
/** Maximum total size of emails stored per feed (bytes). */
|
||||
export const FEED_MAX_BYTES = 524288; // 512 KB
|
||||
|
||||
/** Cloudflare R2 free tier storage allowance (bytes). */
|
||||
export const R2_FREE_TIER_BYTES = 10 * 1024 ** 3; // 10 GB
|
||||
|
||||
/** Cloudflare KV free tier storage allowance (bytes). */
|
||||
export const KV_FREE_TIER_BYTES = 1 * 1024 ** 3; // 1 GB
|
||||
|
||||
/** Cache TTL for ForwardEmail.net IP list (milliseconds). */
|
||||
export const FORWARD_EMAIL_IPS_CACHE_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours
|
||||
|
||||
|
||||
+26
-2
@@ -18,7 +18,13 @@ import {
|
||||
purgeExpiredFeeds,
|
||||
removeFeedsFromListBulk,
|
||||
} from "./routes/admin/helpers";
|
||||
import { bumpCounters } from "./utils/stats";
|
||||
import {
|
||||
bumpCounters,
|
||||
scanR2Usage,
|
||||
scanKvUsage,
|
||||
setStorageSnapshot,
|
||||
} from "./utils/stats";
|
||||
import { getAttachmentBucket } from "./utils/attachments";
|
||||
import { FORWARD_EMAIL_IPS_CACHE_TTL_MS } from "./config/constants";
|
||||
|
||||
type AppEnv = { Bindings: Env };
|
||||
@@ -196,6 +202,7 @@ export default {
|
||||
await handleCloudflareEmail(message, env, ctx);
|
||||
},
|
||||
async scheduled(_event: ScheduledEvent, env: Env, _ctx: ExecutionContext) {
|
||||
const attachmentBucket = getAttachmentBucket(env);
|
||||
const feeds = await listAllFeeds(env.EMAIL_STORAGE);
|
||||
const now = Date.now();
|
||||
const expiredIds = feeds
|
||||
@@ -203,7 +210,7 @@ export default {
|
||||
.map((f) => f.id);
|
||||
|
||||
for (const feedId of expiredIds) {
|
||||
await purgeExpiredFeeds(env.EMAIL_STORAGE, feedId, env.ATTACHMENT_BUCKET);
|
||||
await purgeExpiredFeeds(env.EMAIL_STORAGE, feedId, attachmentBucket);
|
||||
}
|
||||
if (expiredIds.length > 0) {
|
||||
await removeFeedsFromListBulk(env.EMAIL_STORAGE, expiredIds);
|
||||
@@ -212,5 +219,22 @@ export default {
|
||||
});
|
||||
logger.info("Feed TTL cleanup", { deleted: expiredIds.length });
|
||||
}
|
||||
|
||||
// Refresh the cached storage-usage snapshot for the status page / /api/stats.
|
||||
try {
|
||||
const r2 = attachmentBucket
|
||||
? await scanR2Usage(attachmentBucket)
|
||||
: { bytes: 0, count: 0 };
|
||||
const kv = await scanKvUsage(env.EMAIL_STORAGE);
|
||||
await setStorageSnapshot(env.EMAIL_STORAGE, {
|
||||
attachments_bytes: r2.bytes,
|
||||
attachments_count: r2.count,
|
||||
kv_bytes_estimated: kv.bytes,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error("Error refreshing storage snapshot", {
|
||||
error: String(error),
|
||||
});
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
@@ -363,6 +363,32 @@ describe("processEmail — attachments", () => {
|
||||
expect(emailData.attachments).toBeUndefined();
|
||||
});
|
||||
|
||||
it("skips R2 upload when ATTACHMENTS_ENABLED is 'false' even with R2 bound", async () => {
|
||||
const env = createMockEnv({ withR2: true });
|
||||
(env as any).ATTACHMENTS_ENABLED = "false";
|
||||
const mockR2 = (env as any).ATTACHMENT_BUCKET as unknown as MockR2;
|
||||
await env.EMAIL_STORAGE.put(
|
||||
`feed:${VALID_FEED_ID}:config`,
|
||||
JSON.stringify({}),
|
||||
);
|
||||
const res = await processEmail(
|
||||
makeInput({ attachments: [pdfAttachment] }),
|
||||
env as any,
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
|
||||
const metadata = await env.EMAIL_STORAGE.get(
|
||||
`feed:${VALID_FEED_ID}:metadata`,
|
||||
"json",
|
||||
);
|
||||
const emailData = await env.EMAIL_STORAGE.get(
|
||||
metadata.emails[0].key,
|
||||
"json",
|
||||
);
|
||||
expect(emailData.attachments).toBeUndefined();
|
||||
expect((await mockR2.list()).objects).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("uploads attachments to R2 and stores AttachmentData in emailData", async () => {
|
||||
const env = createMockEnv({ withR2: true });
|
||||
const mockR2 = (env as any).ATTACHMENT_BUCKET as unknown as MockR2;
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
extractEmailDomain,
|
||||
} from "../utils/favicon-fetcher";
|
||||
import { parseOneClickUnsubscribe } from "../utils/unsubscribe";
|
||||
import { getAttachmentBucket } from "../utils/attachments";
|
||||
import { logger } from "./logger";
|
||||
import { FEED_MAX_BYTES } from "../config/constants";
|
||||
|
||||
@@ -170,9 +171,10 @@ export async function storeEmail(
|
||||
env: Env,
|
||||
ctx?: ExecutionContext,
|
||||
): Promise<void> {
|
||||
const attachmentBucket = getAttachmentBucket(env);
|
||||
const storedAttachments: AttachmentData[] =
|
||||
env.ATTACHMENT_BUCKET && input.attachments?.length
|
||||
? await uploadAttachments(input.attachments, env.ATTACHMENT_BUCKET)
|
||||
attachmentBucket && input.attachments?.length
|
||||
? await uploadAttachments(input.attachments, attachmentBucket)
|
||||
: [];
|
||||
|
||||
const emailData = {
|
||||
@@ -249,10 +251,10 @@ export async function storeEmail(
|
||||
}
|
||||
|
||||
const r2Deletions =
|
||||
env.ATTACHMENT_BUCKET && toDelete.length > 0
|
||||
attachmentBucket && toDelete.length > 0
|
||||
? toDelete
|
||||
.flatMap((e) => e.attachmentIds ?? [])
|
||||
.map((id) => env.ATTACHMENT_BUCKET!.delete(id))
|
||||
.map((id) => attachmentBucket.delete(id))
|
||||
: [];
|
||||
|
||||
await Promise.all([
|
||||
|
||||
@@ -10,6 +10,7 @@ import { logger } from "../../lib/logger";
|
||||
import { Layout, clampText } from "./ui";
|
||||
import { deleteKeysWithConcurrency } from "./helpers";
|
||||
import { feedRssUrl, feedAtomUrl, feedEmailAddress } from "../../utils/urls";
|
||||
import { getAttachmentBucket } from "../../utils/attachments";
|
||||
import { emailsPageScript } from "../../scripts/generated/emails-page";
|
||||
|
||||
type AppEnv = { Bindings: Env };
|
||||
@@ -643,9 +644,10 @@ emailsRouter.post("/emails/:emailKey/delete", async (c) => {
|
||||
await emailStorage.put(feedMetadataKey, JSON.stringify(feedMetadata));
|
||||
}
|
||||
|
||||
if (env.ATTACHMENT_BUCKET && attachmentIds.length > 0) {
|
||||
const attachmentBucket = getAttachmentBucket(env);
|
||||
if (attachmentBucket && attachmentIds.length > 0) {
|
||||
await Promise.allSettled(
|
||||
attachmentIds.map((id) => env.ATTACHMENT_BUCKET!.delete(id)),
|
||||
attachmentIds.map((id) => attachmentBucket.delete(id)),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -726,9 +728,10 @@ emailsRouter.post("/feeds/:feedId/emails/bulk-delete", async (c) => {
|
||||
);
|
||||
await emailStorage.put(feedMetadataKey, JSON.stringify(feedMetadata));
|
||||
|
||||
if (env.ATTACHMENT_BUCKET && r2AttachmentIds.length > 0) {
|
||||
const attachmentBucket = getAttachmentBucket(env);
|
||||
if (attachmentBucket && r2AttachmentIds.length > 0) {
|
||||
await Promise.allSettled(
|
||||
r2AttachmentIds.map((id) => env.ATTACHMENT_BUCKET!.delete(id)),
|
||||
r2AttachmentIds.map((id) => attachmentBucket.delete(id)),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import { waitUntilSafe } from "../../utils/worker";
|
||||
import { feedRssUrl, feedEmailAddress } from "../../utils/urls";
|
||||
import { logger } from "../../lib/logger";
|
||||
import { sendUnsubscribes } from "../../utils/unsubscribe";
|
||||
import { getAttachmentBucket } from "../../utils/attachments";
|
||||
import { Layout } from "./ui";
|
||||
import {
|
||||
addFeedToList,
|
||||
@@ -555,7 +556,7 @@ feedsRouter.post("/:feedId/delete", async (c) => {
|
||||
waitUntilSafe(
|
||||
c,
|
||||
purgeFeedKeysStep(emailStorage, feedId, {
|
||||
bucket: env.ATTACHMENT_BUCKET,
|
||||
bucket: getAttachmentBucket(env),
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -594,7 +595,7 @@ feedsRouter.post("/:feedId/purge", async (c) => {
|
||||
const step = await purgeFeedKeysStep(emailStorage, feedId, {
|
||||
cursor,
|
||||
limit,
|
||||
bucket: env.ATTACHMENT_BUCKET,
|
||||
bucket: getAttachmentBucket(env),
|
||||
});
|
||||
|
||||
return c.json({
|
||||
|
||||
@@ -2,12 +2,7 @@ import { Context } from "hono";
|
||||
import { html, raw } from "hono/html";
|
||||
import { Env, FeedConfig, FeedMetadata, EmailData } from "../types";
|
||||
import { processEmailContent } from "../utils/html-processor";
|
||||
|
||||
function formatBytes(bytes: number): string {
|
||||
if (bytes < 1024) return `${bytes} B`;
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
|
||||
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
||||
}
|
||||
import { formatBytes } from "../utils/format";
|
||||
|
||||
export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
|
||||
const feedId = c.req.param("feedId");
|
||||
|
||||
+4
-2
@@ -1,8 +1,10 @@
|
||||
import { Context } from "hono";
|
||||
import { Env } from "../types";
|
||||
import { getAttachmentBucket } from "../utils/attachments";
|
||||
|
||||
export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
|
||||
if (!c.env.ATTACHMENT_BUCKET) {
|
||||
const bucket = getAttachmentBucket(c.env);
|
||||
if (!bucket) {
|
||||
return new Response("Attachment storage not configured", { status: 404 });
|
||||
}
|
||||
|
||||
@@ -13,7 +15,7 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
|
||||
return new Response("Not found", { status: 404 });
|
||||
}
|
||||
|
||||
const object = await c.env.ATTACHMENT_BUCKET.get(attachmentId);
|
||||
const object = await bucket.get(attachmentId);
|
||||
|
||||
if (!object) {
|
||||
return new Response("Not found", { status: 404 });
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { Context } from "hono";
|
||||
import { Env } from "../types";
|
||||
import { getStats } from "../utils/stats";
|
||||
import { formatBytes } from "../utils/format";
|
||||
import { R2_FREE_TIER_BYTES, KV_FREE_TIER_BYTES } from "../config/constants";
|
||||
import { Layout } from "./admin/ui";
|
||||
|
||||
function formatDateTime(iso?: string): string {
|
||||
@@ -43,6 +45,11 @@ function formatUptime(iso?: string): string {
|
||||
return `${days} ${days === 1 ? "day" : "days"}`;
|
||||
}
|
||||
|
||||
function tierPercent(used: number, total: number): number {
|
||||
if (total <= 0) return 0;
|
||||
return Math.round((used / total) * 100);
|
||||
}
|
||||
|
||||
type Tone = "success" | "danger";
|
||||
|
||||
type StatProps = {
|
||||
@@ -83,6 +90,11 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
|
||||
? (stats.emails_received / stats.feeds_created).toFixed(1)
|
||||
: "—";
|
||||
|
||||
const kvBytes = stats.kv_bytes_estimated;
|
||||
const kvPercent = tierPercent(kvBytes ?? 0, KV_FREE_TIER_BYTES);
|
||||
const r2Bytes = stats.attachments_bytes;
|
||||
const r2Percent = tierPercent(r2Bytes ?? 0, R2_FREE_TIER_BYTES);
|
||||
|
||||
return c.html(
|
||||
<Layout title="Status" label="status">
|
||||
<div class="container fade-in">
|
||||
@@ -167,6 +179,34 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="stat-section">
|
||||
<h2 class="stat-section-title">Storage</h2>
|
||||
<div class="stats-grid">
|
||||
<Stat
|
||||
label="KV space used (est.)"
|
||||
value={kvBytes === undefined ? "—" : formatBytes(kvBytes)}
|
||||
title={`${kvPercent}% of 1 GB free tier — estimate`}
|
||||
tone={kvPercent >= 80 ? "danger" : undefined}
|
||||
/>
|
||||
{stats.attachments_enabled ? (
|
||||
<>
|
||||
<Stat
|
||||
label="Attachments stored"
|
||||
value={stats.attachments_count ?? "—"}
|
||||
/>
|
||||
<Stat
|
||||
label="R2 space used"
|
||||
value={r2Bytes === undefined ? "—" : formatBytes(r2Bytes)}
|
||||
title={`${r2Percent}% of 10 GB free tier`}
|
||||
tone={r2Percent >= 80 ? "danger" : undefined}
|
||||
/>
|
||||
</>
|
||||
) : (
|
||||
<Stat label="Attachments (R2)" value="Off" />
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="stat-section">
|
||||
<h2 class="stat-section-title">Instance</h2>
|
||||
<div class="stats-grid">
|
||||
|
||||
@@ -237,6 +237,14 @@ export class MockR2 {
|
||||
}
|
||||
}
|
||||
|
||||
async list(_options?: { cursor?: string }) {
|
||||
const objects = Array.from(this.store.entries()).map(([key, entry]) => ({
|
||||
key,
|
||||
size: entry.body.byteLength,
|
||||
}));
|
||||
return { objects, truncated: false as const, cursor: undefined };
|
||||
}
|
||||
|
||||
_has(key: string) {
|
||||
return this.store.has(key);
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ export interface Env {
|
||||
DOMAIN: string;
|
||||
EMAIL_DOMAIN?: string;
|
||||
ATTACHMENT_BUCKET?: R2Bucket;
|
||||
ATTACHMENTS_ENABLED?: string; // "false" disables attachments even when R2 is bound
|
||||
FEED_MAX_SIZE_BYTES?: string;
|
||||
PROXY_TRUSTED_IPS?: string;
|
||||
PROXY_AUTH_SECRET?: string;
|
||||
@@ -83,12 +84,18 @@ export interface Counters {
|
||||
last_email_at?: string; // ISO 8601
|
||||
last_feed_created_at?: string; // ISO 8601
|
||||
first_seen?: string; // ISO 8601 — first time counters were written (instance start)
|
||||
// Storage usage snapshot, refreshed by the hourly cron (overwritten, not incremented).
|
||||
attachments_bytes?: number; // Total R2 bytes used by attachments
|
||||
attachments_count?: number; // Number of R2 objects
|
||||
kv_bytes_estimated?: number; // Estimated KV bytes (sum of stored email sizes)
|
||||
storage_scanned_at?: string; // ISO 8601 — last storage scan
|
||||
}
|
||||
|
||||
// Monitoring API response: persisted counters + live-computed values
|
||||
export interface StatsResponse extends Counters {
|
||||
active_feeds: number;
|
||||
websub_subscriptions_active: number;
|
||||
attachments_enabled: boolean;
|
||||
}
|
||||
|
||||
// WebSub (PubSubHubbub) subscription configuration
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
import { Env } from "../types";
|
||||
|
||||
// Returns the attachment bucket only when the feature is enabled, so callers can
|
||||
// narrow cleanly. Attachments are on whenever R2 is bound, unless explicitly
|
||||
// turned off with ATTACHMENTS_ENABLED="false".
|
||||
export function getAttachmentBucket(env: Env): R2Bucket | undefined {
|
||||
if (env.ATTACHMENTS_ENABLED === "false") return undefined;
|
||||
return env.ATTACHMENT_BUCKET;
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
/** Human-readable byte size (B / KB / MB / GB). */
|
||||
export function formatBytes(bytes: number): string {
|
||||
if (bytes < 1024) return `${bytes} B`;
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
|
||||
if (bytes < 1024 * 1024 * 1024)
|
||||
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
||||
return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
|
||||
}
|
||||
+96
-1
@@ -1,11 +1,15 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { createMockEnv } from "../test/setup";
|
||||
import { createMockEnv, MockR2 } from "../test/setup";
|
||||
import {
|
||||
getCounters,
|
||||
bumpCounters,
|
||||
countKeysByPrefix,
|
||||
getStats,
|
||||
scanR2Usage,
|
||||
scanKvUsage,
|
||||
setStorageSnapshot,
|
||||
} from "./stats";
|
||||
import { getAttachmentBucket } from "./attachments";
|
||||
import { STATS_KEY, FEEDS_LIST_KEY } from "../config/constants";
|
||||
import { Env } from "../types";
|
||||
|
||||
@@ -119,4 +123,95 @@ describe("stats helper", () => {
|
||||
};
|
||||
expect(raw.feeds_created).toBe(1);
|
||||
});
|
||||
|
||||
it("getStats reports attachments_enabled based on the toggle", async () => {
|
||||
const off = createMockEnv() as unknown as Env;
|
||||
expect((await getStats(off)).attachments_enabled).toBe(false);
|
||||
|
||||
const on = createMockEnv({ withR2: true }) as unknown as Env;
|
||||
expect((await getStats(on)).attachments_enabled).toBe(true);
|
||||
|
||||
const disabled = createMockEnv({ withR2: true }) as unknown as Env;
|
||||
(disabled as any).ATTACHMENTS_ENABLED = "false";
|
||||
expect((await getStats(disabled)).attachments_enabled).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("getAttachmentBucket", () => {
|
||||
it("returns the bucket when bound and not disabled", () => {
|
||||
const env = createMockEnv({ withR2: true }) as unknown as Env;
|
||||
expect(getAttachmentBucket(env)).toBeDefined();
|
||||
});
|
||||
|
||||
it("returns undefined when no bucket is bound", () => {
|
||||
const env = createMockEnv() as unknown as Env;
|
||||
expect(getAttachmentBucket(env)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("returns undefined when explicitly disabled", () => {
|
||||
const env = createMockEnv({ withR2: true }) as unknown as Env;
|
||||
(env as any).ATTACHMENTS_ENABLED = "false";
|
||||
expect(getAttachmentBucket(env)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("storage usage scans", () => {
|
||||
it("scanR2Usage sums object sizes and counts", async () => {
|
||||
const bucket = new MockR2();
|
||||
await bucket.put("a", new Uint8Array(100));
|
||||
await bucket.put("b", new Uint8Array(250));
|
||||
const usage = await scanR2Usage(bucket as unknown as R2Bucket);
|
||||
expect(usage.count).toBe(2);
|
||||
expect(usage.bytes).toBe(350);
|
||||
});
|
||||
|
||||
it("scanR2Usage returns zeros for an empty bucket", async () => {
|
||||
const usage = await scanR2Usage(new MockR2() as unknown as R2Bucket);
|
||||
expect(usage).toEqual({ bytes: 0, count: 0 });
|
||||
});
|
||||
|
||||
it("scanKvUsage estimates KV bytes from stored email sizes", async () => {
|
||||
const env = createMockEnv() as unknown as Env;
|
||||
const kv = env.EMAIL_STORAGE;
|
||||
await kv.put(
|
||||
FEEDS_LIST_KEY,
|
||||
JSON.stringify({
|
||||
feeds: [
|
||||
{ id: "a", title: "A" },
|
||||
{ id: "b", title: "B" },
|
||||
],
|
||||
}),
|
||||
);
|
||||
await kv.put(
|
||||
"feed:a:metadata",
|
||||
JSON.stringify({
|
||||
emails: [
|
||||
{ key: "k1", size: 100 },
|
||||
{ key: "k2", size: 50 },
|
||||
],
|
||||
}),
|
||||
);
|
||||
await kv.put(
|
||||
"feed:b:metadata",
|
||||
JSON.stringify({ emails: [{ key: "k3", size: 25 }] }),
|
||||
);
|
||||
|
||||
const usage = await scanKvUsage(kv);
|
||||
expect(usage.bytes).toBe(175);
|
||||
});
|
||||
|
||||
it("setStorageSnapshot writes the snapshot fields", async () => {
|
||||
const env = createMockEnv() as unknown as Env;
|
||||
const kv = env.EMAIL_STORAGE;
|
||||
await setStorageSnapshot(kv, {
|
||||
attachments_bytes: 1234,
|
||||
attachments_count: 5,
|
||||
kv_bytes_estimated: 678,
|
||||
});
|
||||
const counters = await getCounters(kv);
|
||||
expect(counters.attachments_bytes).toBe(1234);
|
||||
expect(counters.attachments_count).toBe(5);
|
||||
expect(counters.kv_bytes_estimated).toBe(678);
|
||||
expect(counters.storage_scanned_at).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -2,6 +2,8 @@ import { Counters, Env, StatsResponse } from "../types";
|
||||
import { STATS_KEY } from "../config/constants";
|
||||
import { logger } from "../lib/logger";
|
||||
import { listAllFeeds } from "../routes/admin/helpers";
|
||||
import { getFeedMetadata } from "./storage";
|
||||
import { getAttachmentBucket } from "./attachments";
|
||||
|
||||
const EMPTY_COUNTERS: Counters = {
|
||||
feeds_created: 0,
|
||||
@@ -82,5 +84,76 @@ export async function getStats(env: Env): Promise<StatsResponse> {
|
||||
...counters,
|
||||
active_feeds: feeds.length,
|
||||
websub_subscriptions_active: websubCount,
|
||||
attachments_enabled: !!getAttachmentBucket(env),
|
||||
};
|
||||
}
|
||||
|
||||
/** Sum the byte size and object count of every attachment stored in R2. */
|
||||
export async function scanR2Usage(
|
||||
bucket: R2Bucket,
|
||||
): Promise<{ bytes: number; count: number }> {
|
||||
let bytes = 0;
|
||||
let count = 0;
|
||||
let cursor: string | undefined;
|
||||
try {
|
||||
do {
|
||||
const listed = await bucket.list({ cursor });
|
||||
for (const obj of listed.objects) {
|
||||
bytes += obj.size;
|
||||
count += 1;
|
||||
}
|
||||
cursor = listed.truncated ? listed.cursor : undefined;
|
||||
} while (cursor);
|
||||
} catch (error) {
|
||||
logger.error("Error scanning R2 usage", { error: String(error) });
|
||||
}
|
||||
return { bytes, count };
|
||||
}
|
||||
|
||||
/**
|
||||
* Estimate KV storage used. KV exposes no size API, so we sum the per-email
|
||||
* sizes already recorded in each feed's metadata — email bodies dominate KV
|
||||
* usage. Feed config/websub/stats keys are excluded, so this is a lower-bound
|
||||
* estimate.
|
||||
*/
|
||||
export async function scanKvUsage(kv: KVNamespace): Promise<{ bytes: number }> {
|
||||
let bytes = 0;
|
||||
try {
|
||||
const feeds = await listAllFeeds(kv);
|
||||
for (const feed of feeds) {
|
||||
const metadata = await getFeedMetadata(kv, feed.id);
|
||||
if (!metadata) continue;
|
||||
for (const email of metadata.emails) {
|
||||
bytes += email.size ?? 0;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error("Error estimating KV usage", { error: String(error) });
|
||||
}
|
||||
return { bytes };
|
||||
}
|
||||
|
||||
/**
|
||||
* Overwrite the storage-usage snapshot fields on the counters singleton.
|
||||
* Unlike bumpCounters these are set (not incremented). Never throws.
|
||||
*/
|
||||
export async function setStorageSnapshot(
|
||||
kv: KVNamespace,
|
||||
snapshot: {
|
||||
attachments_bytes: number;
|
||||
attachments_count: number;
|
||||
kv_bytes_estimated: number;
|
||||
},
|
||||
): Promise<void> {
|
||||
try {
|
||||
const current = await getCounters(kv);
|
||||
current.attachments_bytes = snapshot.attachments_bytes;
|
||||
current.attachments_count = snapshot.attachments_count;
|
||||
current.kv_bytes_estimated = snapshot.kv_bytes_estimated;
|
||||
current.storage_scanned_at = new Date().toISOString();
|
||||
if (!current.first_seen) current.first_seen = new Date().toISOString();
|
||||
await kv.put(STATS_KEY, JSON.stringify(current));
|
||||
} catch (error) {
|
||||
logger.error("Error writing storage snapshot", { error: String(error) });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,6 +39,10 @@ DOMAIN = "REPLACE_WITH_YOUR_DOMAIN" # Web domain (used for feed URLs and admin U
|
||||
# Optional: size-based feed trimming threshold in bytes (default: 524288 = 512 KB)
|
||||
# FEED_MAX_SIZE_BYTES = "524288"
|
||||
|
||||
# Optional: turn email attachments off even when an R2 bucket is bound.
|
||||
# Unset (or any value other than "false") keeps attachments on whenever R2 is configured.
|
||||
# ATTACHMENTS_ENABLED = "false"
|
||||
|
||||
# Optional: lock feed lifetime for all users (hours). When set, the TTL field in
|
||||
# the admin UI is pre-filled and read-only. Remove to allow per-feed configuration.
|
||||
# FEED_TTL_HOURS = "24"
|
||||
@@ -89,6 +93,11 @@ kv_namespaces = [
|
||||
{ binding = "EMAIL_STORAGE", id = "REPLACE_WITH_DEMO_KV_NAMESPACE_ID" }
|
||||
]
|
||||
|
||||
# R2 bucket for storing email attachment enclosures (demo)
|
||||
r2_buckets = [
|
||||
{ binding = "ATTACHMENT_BUCKET", bucket_name = "ktn-attachment-bucket-demo" }
|
||||
]
|
||||
|
||||
routes = [
|
||||
{ pattern = "demo.kill-the.news", custom_domain = true }
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user