From 2b3f00f7e31d9bf2d4a444308fb669394d9198b9 Mon Sep 17 00:00:00 2001 From: Julien Herr Date: Sat, 23 May 2026 23:56:44 +0200 Subject: [PATCH] refactor(domain): introduce FeedRepository as the single KV access layer Centralise the KV key schema and all get/put access behind a FeedRepository class under src/domain/. Every feed/email/list/icon/websub/counter key was previously inlined across ~12 modules with two divergent storeEmail and addFeedToList implementations; the dead src/utils/storage.ts write path is removed and the email key convention unified on feed::. Behaviour-preserving: existing tests pass unchanged in logic, plus a new feed-repository.test.ts covering CRUD, key builders, list ops and counters. Co-Authored-By: Claude Opus 4.7 --- src/domain/feed-repository.test.ts | 171 +++++++++++++++++ src/domain/feed-repository.ts | 296 +++++++++++++++++++++++++++++ src/index.ts | 12 +- src/lib/email-processor.test.ts | 3 +- src/lib/email-processor.ts | 30 +-- src/lib/feed-service.ts | 47 ++--- src/routes/admin.tsx | 5 +- src/routes/admin/emails.tsx | 48 ++--- src/routes/admin/feeds.tsx | 43 ++--- src/routes/admin/helpers.ts | 143 ++------------ src/routes/api/index.ts | 32 ++-- src/routes/entries.ts | 20 +- src/routes/favicon.test.ts | 3 +- src/routes/favicon.ts | 4 +- src/routes/hub.ts | 6 +- src/utils/favicon-fetcher.test.ts | 3 +- src/utils/favicon-fetcher.ts | 15 +- src/utils/feed-fetcher.ts | 18 +- src/utils/stats.ts | 38 ++-- src/utils/storage.ts | 169 ---------------- src/utils/websub.test.ts | 9 +- src/utils/websub.ts | 40 +--- 22 files changed, 616 insertions(+), 539 deletions(-) create mode 100644 src/domain/feed-repository.test.ts create mode 100644 src/domain/feed-repository.ts delete mode 100644 src/utils/storage.ts diff --git a/src/domain/feed-repository.test.ts b/src/domain/feed-repository.test.ts new file mode 100644 index 0000000..aa4ba46 --- /dev/null +++ b/src/domain/feed-repository.test.ts @@ -0,0 +1,171 @@ +import { describe, it, expect } from "vitest"; +import { createMockEnv } from "../test/setup"; +import { FeedRepository } from "./feed-repository"; +import type { + Env, + FeedConfig, + FeedMetadata, + EmailData, + WebSubSubscription, +} from "../types"; + +const mockEnv = () => createMockEnv() as unknown as Env; + +const sampleConfig = (overrides: Partial = {}): FeedConfig => ({ + title: "Test Feed", + language: "en", + created_at: 1000, + ...overrides, +}); + +const sampleEmail = (overrides: Partial = {}): EmailData => ({ + subject: "Hello", + from: "news@example.com", + content: "

hi

", + receivedAt: 1234, + headers: {}, + ...overrides, +}); + +describe("FeedRepository key schema", () => { + it("builds the canonical KV keys via the public API", () => { + const repo = new FeedRepository(mockEnv().EMAIL_STORAGE); + expect(repo.feedKeyPrefix("a.b.42")).toBe("feed:a.b.42:"); + expect(repo.iconKey("example.com")).toBe("icon:example.com"); + expect(repo.newEmailKey("a.b.42")).toMatch(/^feed:a\.b\.42:\d+$/); + }); + + it("recognises email keys vs config/metadata keys", () => { + const repo = new FeedRepository(mockEnv().EMAIL_STORAGE); + expect(repo.isEmailKey("a.b.42", "feed:a.b.42:config")).toBe(false); + expect(repo.isEmailKey("a.b.42", "feed:a.b.42:metadata")).toBe(false); + expect(repo.isEmailKey("a.b.42", "feed:a.b.42:1700000000000")).toBe(true); + }); + + it("recovers the feed id from an email key", () => { + const repo = new FeedRepository(mockEnv().EMAIL_STORAGE); + expect(repo.feedIdFromEmailKey("feed:a.b.42:1700000000000")).toBe("a.b.42"); + }); +}); + +describe("FeedRepository config & metadata", () => { + it("round-trips and deletes a feed config", async () => { + const repo = new FeedRepository(mockEnv().EMAIL_STORAGE); + expect(await repo.getConfig("a.b.42")).toBeNull(); + await repo.putConfig("a.b.42", sampleConfig()); + expect(await repo.getConfig("a.b.42")).toMatchObject({ + title: "Test Feed", + }); + await repo.deleteConfig("a.b.42"); + expect(await repo.getConfig("a.b.42")).toBeNull(); + }); + + it("round-trips and deletes feed metadata", async () => { + const repo = new FeedRepository(mockEnv().EMAIL_STORAGE); + const meta: FeedMetadata = { emails: [] }; + await repo.putMetadata("a.b.42", meta); + expect(await repo.getMetadata("a.b.42")).toEqual(meta); + await repo.deleteMetadata("a.b.42"); + expect(await repo.getMetadata("a.b.42")).toBeNull(); + }); +}); + +describe("FeedRepository emails", () => { + it("stores and reads an email under a minted key", async () => { + const repo = new FeedRepository(mockEnv().EMAIL_STORAGE); + const key = repo.newEmailKey("a.b.42"); + await repo.putEmail(key, sampleEmail()); + expect(await repo.getEmail(key)).toMatchObject({ subject: "Hello" }); + await repo.deleteEmail(key); + expect(await repo.getEmail(key)).toBeNull(); + }); + + it("lists every key under a feed prefix", async () => { + const repo = new FeedRepository(mockEnv().EMAIL_STORAGE); + await repo.putConfig("a.b.42", sampleConfig()); + await repo.putMetadata("a.b.42", { emails: [] }); + const emailKey = repo.newEmailKey("a.b.42"); + await repo.putEmail(emailKey, sampleEmail()); + + const listed = await repo.listFeedKeys("a.b.42"); + expect(listed.names).toContain("feed:a.b.42:config"); + expect(listed.names).toContain("feed:a.b.42:metadata"); + expect(listed.names).toContain(emailKey); + expect(listed.names.filter((k) => repo.isEmailKey("a.b.42", k))).toEqual([ + emailKey, + ]); + }); +}); + +describe("FeedRepository feed list", () => { + it("adds, updates, lists and removes feeds with expiry", async () => { + const repo = new FeedRepository(mockEnv().EMAIL_STORAGE); + await repo.addToList("a.b.42", "One", "desc", 5000); + await repo.addToList("c.d.99", "Two"); + + let feeds = await repo.listFeeds(); + expect(feeds).toHaveLength(2); + expect(feeds.find((f) => f.id === "a.b.42")).toMatchObject({ + title: "One", + expires_at: 5000, + }); + + await repo.updateInList("a.b.42", "One-updated", undefined, undefined); + feeds = await repo.listFeeds(); + const updated = feeds.find((f) => f.id === "a.b.42"); + expect(updated).toMatchObject({ title: "One-updated" }); + expect(updated?.expires_at).toBeUndefined(); + + expect(await repo.removeFromList("a.b.42")).toBe(true); + expect(await repo.removeFromList("missing")).toBe(false); + feeds = await repo.listFeeds(); + expect(feeds.map((f) => f.id)).toEqual(["c.d.99"]); + }); + + it("bulk-removes only the matching ids", async () => { + const repo = new FeedRepository(mockEnv().EMAIL_STORAGE); + await repo.addToList("a.b.42", "One"); + await repo.addToList("c.d.99", "Two"); + await repo.addToList("e.f.10", "Three"); + + const removed = await repo.removeFromListBulk(["a.b.42", "e.f.10", "nope"]); + expect(removed.sort()).toEqual(["a.b.42", "e.f.10"]); + expect((await repo.listFeeds()).map((f) => f.id)).toEqual(["c.d.99"]); + }); +}); + +describe("FeedRepository counters, icons, websub", () => { + it("round-trips raw counters", async () => { + const repo = new FeedRepository(mockEnv().EMAIL_STORAGE); + expect(await repo.getCountersRaw()).toBeNull(); + await repo.putCounters({ + feeds_created: 1, + feeds_deleted: 0, + emails_received: 2, + emails_rejected: 0, + unsubscribes_sent: 0, + }); + expect(await repo.getCountersRaw()).toMatchObject({ emails_received: 2 }); + }); + + it("stores and reads favicons as text or json", async () => { + const repo = new FeedRepository(mockEnv().EMAIL_STORAGE); + expect(await repo.getIconText("example.com")).toBeNull(); + await repo.putIcon("example.com", JSON.stringify({ data: null }), 60); + expect(await repo.getIconText("example.com")).toBe('{"data":null}'); + expect(await repo.getIconJson<{ data: null }>("example.com")).toEqual({ + data: null, + }); + }); + + it("round-trips websub subscriptions and counts them", async () => { + const repo = new FeedRepository(mockEnv().EMAIL_STORAGE); + expect(await repo.getSubscriptions("a.b.42")).toEqual([]); + const subs: WebSubSubscription[] = [ + { callbackUrl: "https://r.example/cb", expiresAt: 9999 }, + ]; + await repo.saveSubscriptions("a.b.42", subs); + expect(await repo.getSubscriptions("a.b.42")).toEqual(subs); + expect(await repo.countSubscriptionKeys()).toBe(1); + }); +}); diff --git a/src/domain/feed-repository.ts b/src/domain/feed-repository.ts new file mode 100644 index 0000000..a80a8cc --- /dev/null +++ b/src/domain/feed-repository.ts @@ -0,0 +1,296 @@ +import { + Counters, + EmailData, + Env, + FeedConfig, + FeedList, + FeedListItem, + FeedMetadata, + WebSubSubscription, +} from "../types"; +import { FEEDS_LIST_KEY, STATS_KEY } from "../config/constants"; +import { logger } from "../lib/logger"; + +const WEBSUB_PREFIX = "websub:subs:"; + +/** + * Single source of truth for the KV key schema and all KV access. No other + * module should build a `feed:`/`feeds:list`/`websub:`/`icon:`/`stats:counters` + * key string — go through a repository method instead. + * + * Wraps one `KVNamespace`; construct per request via `FeedRepository.from(env)`. + */ +export class FeedRepository { + constructor(private readonly kv: KVNamespace) {} + + static from(env: Env): FeedRepository { + return new FeedRepository(env.EMAIL_STORAGE); + } + + // ── Key schema ──────────────────────────────────────────────────────────── + + private configKey(feedId: string): string { + return `feed:${feedId}:config`; + } + + private metadataKey(feedId: string): string { + return `feed:${feedId}:metadata`; + } + + /** KV key for a domain's cached favicon (shared across feeds). */ + iconKey(domain: string): string { + return `icon:${domain}`; + } + + private websubKey(feedId: string): string { + return `${WEBSUB_PREFIX}${feedId}`; + } + + /** Prefix covering every key owned by a feed (config, metadata, emails). */ + feedKeyPrefix(feedId: string): string { + return `feed:${feedId}:`; + } + + /** Mint a fresh, time-ordered email key. Call once and reuse the result. */ + newEmailKey(feedId: string): string { + return `feed:${feedId}:${Date.now()}`; + } + + /** True when `key` is an email entry (not the feed's config/metadata key). */ + isEmailKey(feedId: string, key: string): boolean { + const suffix = key.slice(this.feedKeyPrefix(feedId).length); + return suffix !== "config" && suffix !== "metadata"; + } + + /** Recover the feed id embedded in an email key (`feed::`). */ + feedIdFromEmailKey(key: string): string { + return key.split(":")[1]; + } + + // ── Feed config ─────────────────────────────────────────────────────────── + + async getConfig(feedId: string): Promise { + return (await this.kv.get(this.configKey(feedId), { + type: "json", + })) as FeedConfig | null; + } + + async putConfig(feedId: string, config: FeedConfig): Promise { + await this.kv.put(this.configKey(feedId), JSON.stringify(config)); + } + + async deleteConfig(feedId: string): Promise { + await this.kv.delete(this.configKey(feedId)); + } + + // ── Feed metadata ───────────────────────────────────────────────────────── + + async getMetadata(feedId: string): Promise { + return (await this.kv.get(this.metadataKey(feedId), { + type: "json", + })) as FeedMetadata | null; + } + + async putMetadata(feedId: string, metadata: FeedMetadata): Promise { + await this.kv.put(this.metadataKey(feedId), JSON.stringify(metadata)); + } + + async deleteMetadata(feedId: string): Promise { + await this.kv.delete(this.metadataKey(feedId)); + } + + // ── Emails ──────────────────────────────────────────────────────────────── + + async putEmail(key: string, data: EmailData): Promise { + await this.kv.put(key, JSON.stringify(data)); + } + + async getEmail(key: string): Promise { + return (await this.kv.get(key, { type: "json" })) as EmailData | null; + } + + async deleteEmail(key: string): Promise { + await this.kv.delete(key); + } + + // ── Global feed list ────────────────────────────────────────────────────── + + async listFeeds(): Promise { + try { + const feedList = (await this.kv.get(FEEDS_LIST_KEY, { + type: "json", + })) as FeedList | null; + return feedList?.feeds || []; + } catch (error) { + logger.error("Error listing feeds", { error: String(error) }); + return []; + } + } + + async addToList( + feedId: string, + title: string, + description?: string, + expires_at?: number, + ): Promise { + try { + const feedList = ((await this.kv.get(FEEDS_LIST_KEY, { + type: "json", + })) as FeedList | null) || { feeds: [] }; + + feedList.feeds.push({ id: feedId, title, description, expires_at }); + await this.kv.put(FEEDS_LIST_KEY, JSON.stringify(feedList)); + } catch (error) { + logger.error("Error adding feed to list", { + feedId, + error: String(error), + }); + } + } + + async updateInList( + feedId: string, + title: string, + description?: string, + expires_at?: number, + ): Promise { + try { + const feedList = ((await this.kv.get(FEEDS_LIST_KEY, { + type: "json", + })) as FeedList | null) || { feeds: [] }; + + const feedIndex = feedList.feeds.findIndex((feed) => feed.id === feedId); + if (feedIndex !== -1) { + feedList.feeds[feedIndex].title = title; + feedList.feeds[feedIndex].description = description; + feedList.feeds[feedIndex].expires_at = expires_at; + await this.kv.put(FEEDS_LIST_KEY, JSON.stringify(feedList)); + } + } catch (error) { + logger.error("Error updating feed in list", { + feedId, + error: String(error), + }); + } + } + + async removeFromListBulk(feedIds: string[]): Promise { + try { + const feedList = ((await this.kv.get(FEEDS_LIST_KEY, { + type: "json", + })) as FeedList | null) || { feeds: [] }; + + const toRemove = new Set(feedIds.filter(Boolean)); + if (toRemove.size === 0) return []; + + const removed: string[] = []; + const nextFeeds: FeedListItem[] = []; + + for (const feed of feedList.feeds) { + if (toRemove.has(feed.id)) { + removed.push(feed.id); + continue; + } + nextFeeds.push(feed); + } + + if (removed.length === 0) return []; + + feedList.feeds = nextFeeds; + await this.kv.put(FEEDS_LIST_KEY, JSON.stringify(feedList)); + return removed; + } catch (error) { + logger.error("Error removing feeds from list", { error: String(error) }); + return []; + } + } + + async removeFromList(feedId: string): Promise { + const removed = await this.removeFromListBulk([feedId]); + return removed.includes(feedId); + } + + // ── Key listing / counting ──────────────────────────────────────────────── + + async listFeedKeys( + feedId: string, + options: { cursor?: string; limit?: number } = {}, + ): Promise<{ names: string[]; cursor: string; listComplete: boolean }> { + const prefix = this.feedKeyPrefix(feedId); + const limit = Math.min(1000, Math.max(1, Math.floor(options.limit || 100))); + const cursor = options.cursor || undefined; + + const listed = await this.kv.list({ prefix, cursor, limit }); + return { + names: (listed.keys || []).map((k) => k.name), + cursor: listed.cursor || "", + listComplete: !!listed.list_complete, + }; + } + + async countKeysByPrefix(prefix: string): Promise { + let total = 0; + let cursor: string | undefined; + try { + do { + const listed = await this.kv.list({ prefix, cursor, limit: 1000 }); + total += listed.keys.length; + cursor = listed.list_complete ? undefined : listed.cursor; + } while (cursor); + } catch (error) { + logger.error("Error counting keys", { prefix, error: String(error) }); + } + return total; + } + + /** Number of feeds that currently hold at least one WebSub subscription. */ + countSubscriptionKeys(): Promise { + return this.countKeysByPrefix("websub:"); + } + + // ── Monitoring counters ─────────────────────────────────────────────────── + + async getCountersRaw(): Promise { + return (await this.kv.get(STATS_KEY, { type: "json" })) as Counters | null; + } + + async putCounters(counters: Counters): Promise { + await this.kv.put(STATS_KEY, JSON.stringify(counters)); + } + + // ── Favicons ────────────────────────────────────────────────────────────── + + async getIconText(domain: string): Promise { + return this.kv.get(this.iconKey(domain), "text"); + } + + async getIconJson(domain: string): Promise { + return (await this.kv.get(this.iconKey(domain), { + type: "json", + })) as T | null; + } + + async putIcon( + domain: string, + value: string, + ttlSeconds: number, + ): Promise { + await this.kv.put(this.iconKey(domain), value, { + expirationTtl: ttlSeconds, + }); + } + + // ── WebSub subscriptions ────────────────────────────────────────────────── + + async getSubscriptions(feedId: string): Promise { + const raw = await this.kv.get(this.websubKey(feedId), "json"); + return (raw as WebSubSubscription[] | null) ?? []; + } + + async saveSubscriptions( + feedId: string, + subscriptions: WebSubSubscription[], + ): Promise { + await this.kv.put(this.websubKey(feedId), JSON.stringify(subscriptions)); + } +} diff --git a/src/index.ts b/src/index.ts index 41ca3be..9e71e03 100644 --- a/src/index.ts +++ b/src/index.ts @@ -13,11 +13,8 @@ import { apiApp } from "./routes/api"; import { handleCloudflareEmail } from "./lib/cloudflare-email"; import { Env } from "./types"; import { logger } from "./lib/logger"; -import { - listAllFeeds, - purgeExpiredFeeds, - removeFeedsFromListBulk, -} from "./routes/admin/helpers"; +import { FeedRepository } from "./domain/feed-repository"; +import { purgeExpiredFeeds } from "./routes/admin/helpers"; import { bumpCounters, scanR2Usage, @@ -201,7 +198,8 @@ export default { }, async scheduled(_event: ScheduledEvent, env: Env, _ctx: ExecutionContext) { const attachmentBucket = getAttachmentBucket(env); - const feeds = await listAllFeeds(env.EMAIL_STORAGE); + const repo = FeedRepository.from(env); + const feeds = await repo.listFeeds(); const now = Date.now(); const expiredIds = feeds .filter((f) => f.expires_at !== undefined && f.expires_at <= now) @@ -211,7 +209,7 @@ export default { await purgeExpiredFeeds(env.EMAIL_STORAGE, feedId, attachmentBucket); } if (expiredIds.length > 0) { - await removeFeedsFromListBulk(env.EMAIL_STORAGE, expiredIds); + await repo.removeFromListBulk(expiredIds); await bumpCounters(env.EMAIL_STORAGE, { feeds_deleted: expiredIds.length, }); diff --git a/src/lib/email-processor.test.ts b/src/lib/email-processor.test.ts index f5a78ad..5dfcc8d 100644 --- a/src/lib/email-processor.test.ts +++ b/src/lib/email-processor.test.ts @@ -7,7 +7,8 @@ import { RawAttachment, } from "./email-processor"; import { getCounters } from "../utils/stats"; -import { iconKey } from "../utils/storage"; + +const iconKey = (domain: string) => `icon:${domain}`; const VALID_FEED_ID = "apple.mountain.42"; const VALID_TO = `${VALID_FEED_ID}@test.getmynews.app`; diff --git a/src/lib/email-processor.ts b/src/lib/email-processor.ts index 3e9d787..2ac0850 100644 --- a/src/lib/email-processor.ts +++ b/src/lib/email-processor.ts @@ -1,11 +1,5 @@ import { EmailParser } from "../utils/email-parser"; -import { - AttachmentData, - EmailMetadata, - Env, - FeedConfig, - FeedMetadata, -} from "../types"; +import { AttachmentData, EmailMetadata, Env, FeedConfig } from "../types"; import { notifySubscribers } from "../utils/websub"; import { bumpCounters } from "../utils/stats"; import { @@ -14,6 +8,7 @@ import { } from "../utils/favicon-fetcher"; import { parseOneClickUnsubscribe } from "../utils/unsubscribe"; import { getAttachmentBucket } from "../utils/attachments"; +import { FeedRepository } from "../domain/feed-repository"; import { logger } from "./logger"; import { FEED_MAX_BYTES } from "../config/constants"; @@ -110,10 +105,7 @@ export async function validateEmail( }; } - const feedConfig = (await env.EMAIL_STORAGE.get( - `feed:${feedId}:config`, - "json", - )) as FeedConfig | null; + const feedConfig = await FeedRepository.from(env).getConfig(feedId); if (!feedConfig) { logger.error("Feed not found", { feedId }); return { @@ -188,12 +180,12 @@ export async function storeEmail( ...(storedAttachments.length > 0 ? { attachments: storedAttachments } : {}), }; - const emailKey = `feed:${feedId}:${Date.now()}`; - const feedMetadataKey = `feed:${feedId}:metadata`; + const repo = FeedRepository.from(env); + const emailKey = repo.newEmailKey(feedId); const [, rawMetadata] = await Promise.all([ - env.EMAIL_STORAGE.put(emailKey, JSON.stringify(emailData)), - env.EMAIL_STORAGE.get(feedMetadataKey, "json"), + repo.putEmail(emailKey, emailData), + repo.getMetadata(feedId), ]); // Note: KV has no atomic compare-and-swap. Concurrent invocations for the @@ -202,9 +194,7 @@ export async function storeEmail( // KV's eventual-consistency model. // TODO: Migrate feed metadata writes to Cloudflare Durable Objects to serialise // concurrent writes and eliminate this race condition. - const feedMetadata = ((rawMetadata as FeedMetadata | null) || { - emails: [], - }) as FeedMetadata; + const feedMetadata = rawMetadata || { emails: [] }; const maxBytes = parseInt(env.FEED_MAX_SIZE_BYTES ?? "", 10) || FEED_MAX_BYTES; @@ -260,8 +250,8 @@ export async function storeEmail( : []; await Promise.all([ - env.EMAIL_STORAGE.put(feedMetadataKey, JSON.stringify(feedMetadata)), - ...toDelete.map((e) => env.EMAIL_STORAGE.delete(e.key)), + repo.putMetadata(feedId, feedMetadata), + ...toDelete.map((e) => repo.deleteEmail(e.key)), ...r2Deletions, ]); diff --git a/src/lib/feed-service.ts b/src/lib/feed-service.ts index b6de352..a7f7984 100644 --- a/src/lib/feed-service.ts +++ b/src/lib/feed-service.ts @@ -5,10 +5,8 @@ import { bumpCounters } from "../utils/stats"; import { waitUntilSafe } from "../utils/worker"; import { sendUnsubscribes } from "../utils/unsubscribe"; import { getAttachmentBucket } from "../utils/attachments"; +import { FeedRepository } from "../domain/feed-repository"; import { - addFeedToList, - updateFeedInList, - removeFeedFromList, purgeFeedKeysStep, collectUnsubscribeUrls, } from "../routes/admin/helpers"; @@ -49,7 +47,7 @@ export async function createFeedRecord( env: Env, input: CreateFeedInput, ): Promise<{ feedId: string; config: FeedConfig }> { - const emailStorage = env.EMAIL_STORAGE; + const repo = FeedRepository.from(env); const expiresAt = resolveExpiresAt(env, input.lifetimeHours); const feedId = generateFeedId(); @@ -67,19 +65,13 @@ export async function createFeedRecord( const metadata: FeedMetadata = { emails: [] }; await Promise.all([ - emailStorage.put(`feed:${feedId}:config`, JSON.stringify(config)), - emailStorage.put(`feed:${feedId}:metadata`, JSON.stringify(metadata)), + repo.putConfig(feedId, config), + repo.putMetadata(feedId, metadata), ]); - await addFeedToList( - emailStorage, - feedId, - input.title, - input.description, - expiresAt, - ); + await repo.addToList(feedId, input.title, input.description, expiresAt); - await bumpCounters(emailStorage, { + await bumpCounters(env.EMAIL_STORAGE, { feeds_created: 1, last_feed_created_at: new Date().toISOString(), }); @@ -115,12 +107,9 @@ export async function updateFeedRecord( input: UpdateFeedInput, options: { inPlace?: boolean } = {}, ): Promise { - const emailStorage = env.EMAIL_STORAGE; - const feedConfigKey = `feed:${feedId}:config`; + const repo = FeedRepository.from(env); - const existing = (await emailStorage.get(feedConfigKey, { - type: "json", - })) as FeedConfig | null; + const existing = await repo.getConfig(feedId); if (!existing) return { status: "not_found" }; @@ -157,14 +146,8 @@ export async function updateFeedRecord( expires_at: expiresAt, }; - await emailStorage.put(feedConfigKey, JSON.stringify(config)); - await updateFeedInList( - emailStorage, - feedId, - config.title, - config.description, - expiresAt, - ); + await repo.putConfig(feedId, config); + await repo.updateInList(feedId, config.title, config.description, expiresAt); return { status: "ok", config }; } @@ -184,22 +167,21 @@ export async function deleteFeedFastDetailed( emailStorage: KVNamespace, feedId: string, ): Promise { - const feedConfigKey = `feed:${feedId}:config`; - const feedMetadataKey = `feed:${feedId}:metadata`; + const repo = new FeedRepository(emailStorage); const errors: string[] = []; let configDeleted = false; let metadataDeleted = false; try { - await emailStorage.delete(feedConfigKey); + await repo.deleteConfig(feedId); configDeleted = true; } catch (error) { errors.push(`config delete failed: ${String(error)}`); } try { - await emailStorage.delete(feedMetadataKey); + await repo.deleteMetadata(feedId); metadataDeleted = true; } catch (error) { errors.push(`metadata delete failed: ${String(error)}`); @@ -220,12 +202,13 @@ export async function deleteFeedRecord( feedId: string, ): Promise { const emailStorage = env.EMAIL_STORAGE; + const repo = new FeedRepository(emailStorage); // Read unsubscribe URLs before the metadata is deleted below. const unsubscribeUrls = await collectUnsubscribeUrls(emailStorage, feedId); await deleteFeedFastDetailed(emailStorage, feedId); - const removed = await removeFeedFromList(emailStorage, feedId); + const removed = await repo.removeFromList(feedId); if (removed) { await bumpCounters(emailStorage, { feeds_deleted: 1 }); } diff --git a/src/routes/admin.tsx b/src/routes/admin.tsx index bb7f672..031f24a 100644 --- a/src/routes/admin.tsx +++ b/src/routes/admin.tsx @@ -8,7 +8,7 @@ import { ADMIN_COOKIE_MAX_AGE } from "../config/constants"; import { logger } from "../lib/logger"; import { timingSafeEqual, checkProxyAuth } from "../lib/auth"; import { Layout, clampText } from "./admin/ui"; -import { listAllFeeds } from "./admin/helpers"; +import { FeedRepository } from "../domain/feed-repository"; import { updateFeedRecord } from "../lib/feed-service"; import { feedRssUrl, feedAtomUrl, feedEmailAddress } from "../utils/urls"; import { feedsRouter } from "./admin/feeds"; @@ -282,14 +282,13 @@ const ExpiryBadge = ({ expiresAt }: { expiresAt: number }) => { app.get("/", async (c) => { // Type assertion for environment variables const env = c.env; - const emailStorage = env.EMAIL_STORAGE; const url = new URL(c.req.url); const view = url.searchParams.get("view") === "table" ? "table" : "list"; const message = url.searchParams.get("message"); const count = Number(url.searchParams.get("count") || "0"); // List all feeds - const feedList = await listAllFeeds(emailStorage); + const feedList = await FeedRepository.from(env).listFeeds(); // Keep the dashboard fast: avoid N KV reads for N feeds. // We store title/description in `feeds:list` (description is optional for older data). diff --git a/src/routes/admin/emails.tsx b/src/routes/admin/emails.tsx index 76d69b7..d977b5b 100644 --- a/src/routes/admin/emails.tsx +++ b/src/routes/admin/emails.tsx @@ -1,17 +1,12 @@ import { Hono } from "hono"; -import { - Env, - FeedConfig, - FeedMetadata, - EmailData, - EmailMetadata, -} from "../../types"; +import { Env, EmailMetadata } from "../../types"; import { logger } from "../../lib/logger"; import { Layout, clampText } from "./ui"; import { deleteAttachmentsForEmails, deleteKeysWithConcurrency, } from "./helpers"; +import { FeedRepository } from "../../domain/feed-repository"; import { feedRssUrl, feedAtomUrl, feedEmailAddress } from "../../utils/urls"; import { formatBytes } from "../../utils/format"; import { emailsPageScript } from "../../scripts/generated/emails-page"; @@ -156,17 +151,13 @@ const SenderField = ({ from, feedId }: SenderFieldProps) => { emailsRouter.get("/feeds/:feedId/emails", async (c) => { const env = c.env; - const emailStorage = env.EMAIL_STORAGE; + const repo = FeedRepository.from(env); const feedId = c.req.param("feedId"); const message = c.req.query("message"); const count = Number(c.req.query("count") || "0"); - const feedConfig = (await emailStorage.get(`feed:${feedId}:config`, { - type: "json", - })) as FeedConfig | null; - const feedMetadata = (await emailStorage.get(`feed:${feedId}:metadata`, { - type: "json", - })) as FeedMetadata | null; + const feedConfig = await repo.getConfig(feedId); + const feedMetadata = await repo.getMetadata(feedId); if (!feedConfig || !feedMetadata) { return c.text("Feed not found", 404); @@ -461,16 +452,14 @@ emailsRouter.get("/feeds/:feedId/emails", async (c) => { emailsRouter.get("/emails/:emailKey", async (c) => { const env = c.env; - const emailStorage = env.EMAIL_STORAGE; + const repo = FeedRepository.from(env); const emailKey = c.req.param("emailKey"); - const emailData = (await emailStorage.get(emailKey, { - type: "json", - })) as EmailData | null; + const emailData = await repo.getEmail(emailKey); if (!emailData) return c.text("Email not found", 404); - const feedId = emailKey.split(":")[1]; + const feedId = repo.feedIdFromEmailKey(emailKey); const attachments = emailData.attachments ?? []; const htmlContent = `${emailData.content}`; @@ -652,7 +641,7 @@ emailsRouter.get("/emails/:emailKey", async (c) => { emailsRouter.post("/emails/:emailKey/delete", async (c) => { const env = c.env; - const emailStorage = env.EMAIL_STORAGE; + const repo = FeedRepository.from(env); const emailKey = c.req.param("emailKey"); const wantsJson = (c.req.header("Accept") || "").includes("application/json"); @@ -664,12 +653,9 @@ emailsRouter.post("/emails/:emailKey/delete", async (c) => { return c.text("Feed ID is required", 400); } - const feedMetadataKey = `feed:${feedId}:metadata`; - const feedMetadata = (await emailStorage.get(feedMetadataKey, { - type: "json", - })) as FeedMetadata | null; + const feedMetadata = await repo.getMetadata(feedId); - await emailStorage.delete(emailKey); + await repo.deleteEmail(emailKey); await deleteAttachmentsForEmails(env, feedMetadata?.emails ?? [], [ emailKey, ]); @@ -678,7 +664,7 @@ emailsRouter.post("/emails/:emailKey/delete", async (c) => { feedMetadata.emails = feedMetadata.emails.filter( (email) => email.key !== emailKey, ); - await emailStorage.put(feedMetadataKey, JSON.stringify(feedMetadata)); + await repo.putMetadata(feedId, feedMetadata); } if (wantsJson) return c.json({ ok: true, emailKey, feedId }); @@ -699,6 +685,7 @@ emailsRouter.post("/emails/:emailKey/delete", async (c) => { emailsRouter.post("/feeds/:feedId/emails/bulk-delete", async (c) => { const env = c.env; const emailStorage = env.EMAIL_STORAGE; + const repo = new FeedRepository(emailStorage); const feedId = c.req.param("feedId"); const contentType = c.req.header("Content-Type") || ""; const wantsJson = @@ -706,10 +693,7 @@ emailsRouter.post("/feeds/:feedId/emails/bulk-delete", async (c) => { (c.req.header("Accept") || "").includes("application/json"); try { - const feedMetadataKey = `feed:${feedId}:metadata`; - const feedMetadata = (await emailStorage.get(feedMetadataKey, { - type: "json", - })) as FeedMetadata | null; + const feedMetadata = await repo.getMetadata(feedId); if (!feedMetadata) { return wantsJson @@ -753,7 +737,7 @@ emailsRouter.post("/feeds/:feedId/emails/bulk-delete", async (c) => { feedMetadata.emails = feedMetadata.emails.filter( (email) => !deletedSet.has(email.key), ); - await emailStorage.put(feedMetadataKey, JSON.stringify(feedMetadata)); + await repo.putMetadata(feedId, feedMetadata); return c.json({ ok: failedEmailKeys.length === 0, @@ -784,7 +768,7 @@ emailsRouter.post("/feeds/:feedId/emails/bulk-delete", async (c) => { feedMetadata.emails = feedMetadata.emails.filter( (email) => !deletedSet.has(email.key), ); - await emailStorage.put(feedMetadataKey, JSON.stringify(feedMetadata)); + await repo.putMetadata(feedId, feedMetadata); return c.redirect( `/admin/feeds/${feedId}/emails?message=bulkDeleted&count=${deletedOk.length}`, diff --git a/src/routes/admin/feeds.tsx b/src/routes/admin/feeds.tsx index f92d4e1..a91d9b3 100644 --- a/src/routes/admin/feeds.tsx +++ b/src/routes/admin/feeds.tsx @@ -1,6 +1,6 @@ import { Hono } from "hono"; import { z } from "zod"; -import { Env, FeedConfig } from "../../types"; +import { Env } from "../../types"; import { bumpCounters } from "../../utils/stats"; import { waitUntilSafe } from "../../utils/worker"; import { feedRssUrl, feedEmailAddress } from "../../utils/urls"; @@ -8,11 +8,8 @@ import { logger } from "../../lib/logger"; import { sendUnsubscribes } from "../../utils/unsubscribe"; import { getAttachmentBucket } from "../../utils/attachments"; import { Layout } from "./ui"; -import { - removeFeedsFromListBulk, - purgeFeedKeysStep, - collectUnsubscribeUrls, -} from "./helpers"; +import { purgeFeedKeysStep, collectUnsubscribeUrls } from "./helpers"; +import { FeedRepository } from "../../domain/feed-repository"; import { createFeedRecord, updateFeedRecord, @@ -149,12 +146,9 @@ feedsRouter.post("/create", async (c) => { feedsRouter.get("/:feedId/edit", async (c) => { const env = c.env; - const emailStorage = env.EMAIL_STORAGE; const feedId = c.req.param("feedId"); - const feedConfig = (await emailStorage.get(`feed:${feedId}:config`, { - type: "json", - })) as FeedConfig | null; + const feedConfig = await FeedRepository.from(env).getConfig(feedId); if (!feedConfig) { return c.text("Feed not found", 404); @@ -365,7 +359,7 @@ feedsRouter.post("/:feedId/edit", async (c) => { feedsRouter.post("/:feedId/sender-filter", async (c) => { const env = c.env; const feedId = c.req.param("feedId"); - const feedConfigKey = `feed:${feedId}:config`; + const repo = FeedRepository.from(env); const body = await c.req.json().catch(() => null); const parsed = senderFilterSchema.safeParse(body); @@ -376,9 +370,7 @@ feedsRouter.post("/:feedId/sender-filter", async (c) => { const { action, value } = parsed.data; const normalized = value.trim().toLowerCase(); - const feedConfig = (await env.EMAIL_STORAGE.get(feedConfigKey, { - type: "json", - })) as FeedConfig | null; + const feedConfig = await repo.getConfig(feedId); if (!feedConfig) return c.json({ ok: false, error: "Feed not found" }, 404); const allowedSenders = (feedConfig.allowed_senders || []).map((s) => @@ -405,15 +397,12 @@ feedsRouter.post("/:feedId/sender-filter", async (c) => { if (!targetList.includes(normalized)) { targetList.push(normalized); - await env.EMAIL_STORAGE.put( - feedConfigKey, - JSON.stringify({ - ...feedConfig, - allowed_senders: allowedSenders, - blocked_senders: blockedSenders, - updated_at: Date.now(), - }), - ); + await repo.putConfig(feedId, { + ...feedConfig, + allowed_senders: allowedSenders, + blocked_senders: blockedSenders, + updated_at: Date.now(), + }); } return c.json({ ok: true }); @@ -552,7 +541,9 @@ feedsRouter.post("/bulk-delete", async (c) => { } } - const deletedFeedIds = await removeFeedsFromListBulk(emailStorage, okIds); + const deletedFeedIds = await new FeedRepository( + emailStorage, + ).removeFromListBulk(okIds); if (deletedFeedIds.length > 0) { await bumpCounters(emailStorage, { feeds_deleted: deletedFeedIds.length, @@ -616,7 +607,9 @@ feedsRouter.post("/bulk-delete", async (c) => { } } - const deletedFeedIds = await removeFeedsFromListBulk(emailStorage, okIds); + const deletedFeedIds = await new FeedRepository( + emailStorage, + ).removeFromListBulk(okIds); if (deletedFeedIds.length > 0) { await bumpCounters(emailStorage, { feeds_deleted: deletedFeedIds.length, diff --git a/src/routes/admin/helpers.ts b/src/routes/admin/helpers.ts index 7119596..0cf9296 100644 --- a/src/routes/admin/helpers.ts +++ b/src/routes/admin/helpers.ts @@ -1,14 +1,7 @@ -import { - EmailData, - EmailMetadata, - Env, - FeedList, - FeedListItem, - FeedMetadata, -} from "../../types"; -import { FEEDS_LIST_KEY } from "../../config/constants"; +import { EmailData, EmailMetadata, Env } from "../../types"; import { logger } from "../../lib/logger"; import { getAttachmentBucket } from "../../utils/attachments"; +import { FeedRepository } from "../../domain/feed-repository"; // Delete the R2 attachments belonging to the given email keys. Call before the // emails are removed from feed metadata, while `emails` still carries their @@ -58,108 +51,6 @@ export async function deleteKeysWithConcurrency( return { ok, failed }; } -export async function listAllFeeds( - emailStorage: KVNamespace, -): Promise { - try { - const feedList = (await emailStorage.get(FEEDS_LIST_KEY, { - type: "json", - })) as FeedList | null; - return feedList?.feeds || []; - } catch (error) { - logger.error("Error listing feeds", { error: String(error) }); - return []; - } -} - -export async function addFeedToList( - emailStorage: KVNamespace, - feedId: string, - title: string, - description?: string, - expires_at?: number, -): Promise { - try { - const feedList = ((await emailStorage.get(FEEDS_LIST_KEY, { - type: "json", - })) as FeedList | null) || { feeds: [] }; - - feedList.feeds.push({ id: feedId, title, description, expires_at }); - await emailStorage.put(FEEDS_LIST_KEY, JSON.stringify(feedList)); - } catch (error) { - logger.error("Error adding feed to list", { feedId, error: String(error) }); - } -} - -export async function updateFeedInList( - emailStorage: KVNamespace, - feedId: string, - title: string, - description?: string, - expires_at?: number, -): Promise { - try { - const feedList = ((await emailStorage.get(FEEDS_LIST_KEY, { - type: "json", - })) as FeedList | null) || { feeds: [] }; - - const feedIndex = feedList.feeds.findIndex((feed) => feed.id === feedId); - if (feedIndex !== -1) { - feedList.feeds[feedIndex].title = title; - feedList.feeds[feedIndex].description = description; - feedList.feeds[feedIndex].expires_at = expires_at; - await emailStorage.put(FEEDS_LIST_KEY, JSON.stringify(feedList)); - } - } catch (error) { - logger.error("Error updating feed in list", { - feedId, - error: String(error), - }); - } -} - -export async function removeFeedsFromListBulk( - emailStorage: KVNamespace, - feedIds: string[], -): Promise { - try { - const feedList = ((await emailStorage.get(FEEDS_LIST_KEY, { - type: "json", - })) as FeedList | null) || { feeds: [] }; - - const toRemove = new Set(feedIds.filter(Boolean)); - if (toRemove.size === 0) return []; - - const removed: string[] = []; - const nextFeeds: FeedListItem[] = []; - - for (const feed of feedList.feeds) { - if (toRemove.has(feed.id)) { - removed.push(feed.id); - continue; - } - nextFeeds.push(feed); - } - - if (removed.length === 0) return []; - - feedList.feeds = nextFeeds; - await emailStorage.put(FEEDS_LIST_KEY, JSON.stringify(feedList)); - return removed; - } catch (error) { - logger.error("Error removing feeds from list", { error: String(error) }); - return []; - } -} - -export async function removeFeedFromList( - emailStorage: KVNamespace, - feedId: string, -): Promise { - const removed = await removeFeedsFromListBulk(emailStorage, [feedId]); - return removed.includes(feedId); -} - /** * Read a feed's stored RFC 8058 one-click unsubscribe URLs (one per sender). * Must be called before the feed metadata is deleted. Never throws. @@ -169,9 +60,7 @@ export async function collectUnsubscribeUrls( feedId: string, ): Promise { try { - const metadata = (await emailStorage.get(`feed:${feedId}:metadata`, { - type: "json", - })) as FeedMetadata | null; + const metadata = await new FeedRepository(emailStorage).getMetadata(feedId); return Object.values(metadata?.unsubscribe ?? {}); } catch (error) { logger.error("Error reading unsubscribe URLs", { @@ -192,24 +81,18 @@ export async function purgeFeedKeysStep( cursor: string; listComplete: boolean; }> { - const prefix = `feed:${feedId}:`; - const limit = Math.min(1000, Math.max(1, Math.floor(options.limit || 100))); - const cursor = options.cursor || undefined; - - const listed = await emailStorage.list({ prefix, cursor, limit }); - const keys = (listed.keys || []).map((k) => k.name); + const repo = new FeedRepository(emailStorage); + const listed = await repo.listFeedKeys(feedId, { + cursor: options.cursor, + limit: options.limit, + }); + const keys = listed.names; if (options.bucket && keys.length > 0) { - const emailKeys = keys.filter((k) => { - const suffix = k.slice(prefix.length); - return suffix !== "config" && suffix !== "metadata"; - }); + const emailKeys = keys.filter((k) => repo.isEmailKey(feedId, k)); if (emailKeys.length > 0) { const emailDataResults = await Promise.allSettled( - emailKeys.map( - (k) => - emailStorage.get(k, { type: "json" }) as Promise, - ), + emailKeys.map((k) => repo.getEmail(k)), ); const attachmentIds = emailDataResults .filter( @@ -234,8 +117,8 @@ export async function purgeFeedKeysStep( return { deletedKeys: ok, failedKeys: failed, - cursor: listed.cursor || "", - listComplete: !!listed.list_complete, + cursor: listed.cursor, + listComplete: listed.listComplete, }; } diff --git a/src/routes/api/index.ts b/src/routes/api/index.ts index b384a00..a333faf 100644 --- a/src/routes/api/index.ts +++ b/src/routes/api/index.ts @@ -8,12 +8,8 @@ import { updateFeedRecord, deleteFeedRecord, } from "../../lib/feed-service"; -import { listAllFeeds, deleteAttachmentsForEmails } from "../admin/helpers"; -import { - getFeedConfig, - getFeedMetadata, - getEmailData, -} from "../../utils/storage"; +import { deleteAttachmentsForEmails } from "../admin/helpers"; +import { FeedRepository } from "../../domain/feed-repository"; import { getStats } from "../../utils/stats"; import { feedEmailAddress, feedRssUrl, feedAtomUrl } from "../../utils/urls"; import { @@ -107,7 +103,7 @@ apiApp.openapi( }), async (c) => { const env = c.env; - const feeds = await listAllFeeds(env.EMAIL_STORAGE); + const feeds = await FeedRepository.from(env).listFeeds(); return c.json( { feeds: feeds.map((f) => ({ @@ -173,9 +169,10 @@ apiApp.openapi( async (c) => { const env = c.env; const { feedId } = c.req.valid("param"); - const config = await getFeedConfig(env.EMAIL_STORAGE, feedId); + const repo = FeedRepository.from(env); + const config = await repo.getConfig(feedId); if (!config) return c.json({ error: "Feed not found" }, 404); - const metadata = await getFeedMetadata(env.EMAIL_STORAGE, feedId); + const metadata = await repo.getMetadata(feedId); return c.json( toFeed(feedId, config, metadata?.emails.length ?? 0, env), 200, @@ -218,7 +215,7 @@ apiApp.openapi( return c.json({ error: "Feed not found" }, 404); if (result.status === "expired") return c.json({ error: "Feed has expired and cannot be modified" }, 409); - const metadata = await getFeedMetadata(env.EMAIL_STORAGE, feedId); + const metadata = await FeedRepository.from(env).getMetadata(feedId); return c.json( toFeed(feedId, result.config, metadata?.emails.length ?? 0, env), 200, @@ -268,7 +265,7 @@ apiApp.openapi( async (c) => { const env = c.env; const { feedId } = c.req.valid("param"); - const metadata = await getFeedMetadata(env.EMAIL_STORAGE, feedId); + const metadata = await FeedRepository.from(env).getMetadata(feedId); if (!metadata) return c.json({ error: "Feed not found" }, 404); return c.json( { @@ -303,10 +300,11 @@ apiApp.openapi( const env = c.env; const { feedId, entryId } = c.req.valid("param"); const receivedAt = parseInt(entryId, 10); - const metadata = await getFeedMetadata(env.EMAIL_STORAGE, feedId); + const repo = FeedRepository.from(env); + const metadata = await repo.getMetadata(feedId); const metaEntry = metadata?.emails.find((e) => e.receivedAt === receivedAt); if (!metaEntry) return c.json({ error: "Email not found" }, 404); - const data = await getEmailData(env.EMAIL_STORAGE, metaEntry.key); + const data = await repo.getEmail(metaEntry.key); if (!data) return c.json({ error: "Email not found" }, 404); return c.json( { @@ -344,18 +342,18 @@ apiApp.openapi( }), async (c) => { const env = c.env; - const emailStorage = env.EMAIL_STORAGE; + const repo = FeedRepository.from(env); const { feedId, entryId } = c.req.valid("param"); const receivedAt = parseInt(entryId, 10); - const metadata = await getFeedMetadata(emailStorage, feedId); + const metadata = await repo.getMetadata(feedId); const metaEntry = metadata?.emails.find((e) => e.receivedAt === receivedAt); if (!metadata || !metaEntry) return c.json({ error: "Email not found" }, 404); - await emailStorage.delete(metaEntry.key); + await repo.deleteEmail(metaEntry.key); await deleteAttachmentsForEmails(env, metadata.emails, [metaEntry.key]); metadata.emails = metadata.emails.filter((e) => e.key !== metaEntry.key); - await emailStorage.put(`feed:${feedId}:metadata`, JSON.stringify(metadata)); + await repo.putMetadata(feedId, metadata); return c.json({ ok: true }, 200); }, diff --git a/src/routes/entries.ts b/src/routes/entries.ts index e9fe890..032f305 100644 --- a/src/routes/entries.ts +++ b/src/routes/entries.ts @@ -1,8 +1,9 @@ import { Context } from "hono"; import { html, raw } from "hono/html"; -import { Env, FeedConfig, FeedMetadata, EmailData } from "../types"; +import { Env } from "../types"; import { processEmailContent } from "../utils/html-processor"; import { formatBytes } from "../utils/format"; +import { FeedRepository } from "../domain/feed-repository"; export async function handle(c: Context<{ Bindings: Env }>): Promise { const feedId = c.req.param("feedId"); @@ -12,17 +13,11 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise { return new Response("Not Found", { status: 404 }); } - const emailStorage = c.env.EMAIL_STORAGE; + const repo = FeedRepository.from(c.env); const [feedMetadata, feedConfig] = await Promise.all([ - emailStorage.get( - `feed:${feedId}:metadata`, - "json", - ) as Promise, - emailStorage.get( - `feed:${feedId}:config`, - "json", - ) as Promise, + repo.getMetadata(feedId), + repo.getConfig(feedId), ]); if (!feedMetadata) { return new Response("Feed not found", { status: 404 }); @@ -41,10 +36,7 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise { return new Response("Entry not found", { status: 404 }); } - const emailData = (await emailStorage.get( - metaEntry.key, - "json", - )) as EmailData | null; + const emailData = await repo.getEmail(metaEntry.key); if (!emailData) { return new Response("Entry not found", { status: 404 }); } diff --git a/src/routes/favicon.test.ts b/src/routes/favicon.test.ts index 5149e58..6834ff1 100644 --- a/src/routes/favicon.test.ts +++ b/src/routes/favicon.test.ts @@ -1,9 +1,10 @@ import { describe, it, expect } from "vitest"; import worker from "../index"; import { createMockEnv } from "../test/setup"; -import { iconKey } from "../utils/storage"; import type { Env } from "../types"; +const iconKey = (domain: string) => `icon:${domain}`; + function req(path: string): Request { return new Request(`https://test.getmynews.app${path}`); } diff --git a/src/routes/favicon.ts b/src/routes/favicon.ts index 6fbe820..2357468 100644 --- a/src/routes/favicon.ts +++ b/src/routes/favicon.ts @@ -1,6 +1,6 @@ import { Context } from "hono"; import { Env } from "../types"; -import { getFeedMetadata } from "../utils/storage"; +import { FeedRepository } from "../domain/feed-repository"; import { cacheFaviconForDomain, getCachedIcon } from "../utils/favicon-fetcher"; export const FAVICON_PATH = "/favicon.svg"; @@ -40,7 +40,7 @@ export async function handleFeedFavicon( const feedId = c.req.param("feedId"); if (!feedId) return projectFavicon(); - const metadata = await getFeedMetadata(env.EMAIL_STORAGE, feedId); + const metadata = await FeedRepository.from(env).getMetadata(feedId); const domain = metadata?.iconDomain; if (!domain) return projectFavicon(); diff --git a/src/routes/hub.ts b/src/routes/hub.ts index ac7bcc3..76c0e1f 100644 --- a/src/routes/hub.ts +++ b/src/routes/hub.ts @@ -7,6 +7,7 @@ import { import { waitUntilSafe } from "../utils/worker"; import { DEFAULT_LEASE_SECONDS, MAX_LEASE_SECONDS } from "../config/constants"; import { feedTopicPattern } from "../utils/urls"; +import { FeedRepository } from "../domain/feed-repository"; type AppEnv = { Bindings: Env }; @@ -73,10 +74,7 @@ hubRouter.post("/", async (c) => { const feedId = match[2]; // Verify the feed exists before accepting any subscription - const feedConfig = await env.EMAIL_STORAGE.get( - `feed:${feedId}:config`, - "json", - ); + const feedConfig = await FeedRepository.from(env).getConfig(feedId); if (!feedConfig) { return c.text("Not Found: feed does not exist", 404); } diff --git a/src/utils/favicon-fetcher.test.ts b/src/utils/favicon-fetcher.test.ts index 0b62100..478a9c1 100644 --- a/src/utils/favicon-fetcher.test.ts +++ b/src/utils/favicon-fetcher.test.ts @@ -6,8 +6,9 @@ import { extractEmailDomain, getCachedIcon, } from "./favicon-fetcher"; -import { iconKey } from "./storage"; import { MAX_ICON_BYTES } from "../config/constants"; + +const iconKey = (domain: string) => `icon:${domain}`; import type { Env } from "../types"; const PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 1, 2, 3, 4]); diff --git a/src/utils/favicon-fetcher.ts b/src/utils/favicon-fetcher.ts index a425512..9cb1e62 100644 --- a/src/utils/favicon-fetcher.ts +++ b/src/utils/favicon-fetcher.ts @@ -4,7 +4,7 @@ import { ICON_TTL_SECONDS, MAX_ICON_BYTES, } from "../config/constants"; -import { iconKey } from "./storage"; +import { FeedRepository } from "../domain/feed-repository"; import { logger } from "../lib/logger"; interface IconRecord { @@ -92,8 +92,8 @@ export async function cacheFaviconForDomain( env: Env, ): Promise { try { - const key = iconKey(domain); - const existing = await env.EMAIL_STORAGE.get(key, "text"); + const repo = FeedRepository.from(env); + const existing = await repo.getIconText(domain); if (existing !== null) return; // present (incl. negative) → nothing to do const icon = await resolveIcon(domain); @@ -104,9 +104,7 @@ export async function cacheFaviconForDomain( } : { data: null, contentType: "" }; - await env.EMAIL_STORAGE.put(key, JSON.stringify(record), { - expirationTtl: ICON_TTL_SECONDS, - }); + await repo.putIcon(domain, JSON.stringify(record), ICON_TTL_SECONDS); } catch (error) { logger.warn("Favicon cache failed", { domain, error: String(error) }); } @@ -119,10 +117,7 @@ export async function getCachedIcon( domain: string, env: Env, ): Promise<{ bytes: ArrayBuffer; contentType: string } | null> { - const record = (await env.EMAIL_STORAGE.get( - iconKey(domain), - "json", - )) as IconRecord | null; + const record = await FeedRepository.from(env).getIconJson(domain); if (!record || record.data === null) return null; return { bytes: base64ToArrayBuffer(record.data), diff --git a/src/utils/feed-fetcher.ts b/src/utils/feed-fetcher.ts index dddbf32..2494401 100644 --- a/src/utils/feed-fetcher.ts +++ b/src/utils/feed-fetcher.ts @@ -1,5 +1,6 @@ -import { Env, FeedConfig, FeedMetadata, EmailData } from "../types"; +import { Env, FeedConfig, EmailData } from "../types"; import { MAX_FEED_ITEMS } from "../config/constants"; +import { FeedRepository } from "../domain/feed-repository"; export interface FeedData { feedConfig: FeedConfig; @@ -10,19 +11,12 @@ export async function fetchFeedData( feedId: string, env: Env, ): Promise { - const storage = env.EMAIL_STORAGE; - - const feedMetadata = (await storage.get( - `feed:${feedId}:metadata`, - "json", - )) as FeedMetadata | null; + const repo = FeedRepository.from(env); + const feedMetadata = await repo.getMetadata(feedId); if (!feedMetadata) return null; - const feedConfig = ((await storage.get( - `feed:${feedId}:config`, - "json", - )) as FeedConfig | null) ?? { + const feedConfig = (await repo.getConfig(feedId)) ?? { title: `Newsletter Feed ${feedId}`, description: "Converted email newsletter", language: "en", @@ -32,7 +26,7 @@ export async function fetchFeedData( const emailRefs = feedMetadata.emails.slice(0, MAX_FEED_ITEMS); const emails: EmailData[] = []; for (const ref of emailRefs) { - const data = (await storage.get(ref.key, "json")) as EmailData | null; + const data = await repo.getEmail(ref.key); if (data) emails.push(data); } diff --git a/src/utils/stats.ts b/src/utils/stats.ts index 67e508f..b084869 100644 --- a/src/utils/stats.ts +++ b/src/utils/stats.ts @@ -1,8 +1,6 @@ import { Counters, Env, StatsResponse } from "../types"; -import { STATS_KEY } from "../config/constants"; import { logger } from "../lib/logger"; -import { listAllFeeds } from "../routes/admin/helpers"; -import { getFeedMetadata } from "./storage"; +import { FeedRepository } from "../domain/feed-repository"; import { getAttachmentBucket } from "./attachments"; const EMPTY_COUNTERS: Counters = { @@ -15,9 +13,7 @@ const EMPTY_COUNTERS: Counters = { export async function getCounters(kv: KVNamespace): Promise { try { - const stored = (await kv.get(STATS_KEY, { - type: "json", - })) as Counters | null; + const stored = await new FeedRepository(kv).getCountersRaw(); return { ...EMPTY_COUNTERS, ...(stored || {}) }; } catch (error) { logger.error("Error reading counters", { error: String(error) }); @@ -48,7 +44,7 @@ export async function bumpCounters( current.last_feed_created_at = changes.last_feed_created_at; if (!current.first_seen) current.first_seen = new Date().toISOString(); - await kv.put(STATS_KEY, JSON.stringify(current)); + await new FeedRepository(kv).putCounters(current); } catch (error) { logger.error("Error updating counters", { error: String(error) }); } @@ -58,26 +54,15 @@ export async function countKeysByPrefix( kv: KVNamespace, prefix: string, ): Promise { - let total = 0; - let cursor: string | undefined; - try { - do { - const listed = await kv.list({ prefix, cursor, limit: 1000 }); - total += listed.keys.length; - cursor = listed.list_complete ? undefined : listed.cursor; - } while (cursor); - } catch (error) { - logger.error("Error counting keys", { prefix, error: String(error) }); - } - return total; + return new FeedRepository(kv).countKeysByPrefix(prefix); } export async function getStats(env: Env): Promise { - const kv = env.EMAIL_STORAGE; + const repo = FeedRepository.from(env); const [counters, feeds, websubCount] = await Promise.all([ - getCounters(kv), - listAllFeeds(kv), - countKeysByPrefix(kv, "websub:"), + getCounters(env.EMAIL_STORAGE), + repo.listFeeds(), + repo.countSubscriptionKeys(), ]); return { @@ -119,9 +104,10 @@ export async function scanR2Usage( export async function scanKvUsage(kv: KVNamespace): Promise<{ bytes: number }> { let bytes = 0; try { - const feeds = await listAllFeeds(kv); + const repo = new FeedRepository(kv); + const feeds = await repo.listFeeds(); for (const feed of feeds) { - const metadata = await getFeedMetadata(kv, feed.id); + const metadata = await repo.getMetadata(feed.id); if (!metadata) continue; for (const email of metadata.emails) { bytes += email.size ?? 0; @@ -152,7 +138,7 @@ export async function setStorageSnapshot( current.kv_bytes_estimated = snapshot.kv_bytes_estimated; current.storage_scanned_at = new Date().toISOString(); if (!current.first_seen) current.first_seen = new Date().toISOString(); - await kv.put(STATS_KEY, JSON.stringify(current)); + await new FeedRepository(kv).putCounters(current); } catch (error) { logger.error("Error writing storage snapshot", { error: String(error) }); } diff --git a/src/utils/storage.ts b/src/utils/storage.ts deleted file mode 100644 index 3819d12..0000000 --- a/src/utils/storage.ts +++ /dev/null @@ -1,169 +0,0 @@ -import { - EmailData, - FeedConfig, - FeedMetadata, - FeedList, - EmailMetadata, -} from "../types"; -import { MAX_METADATA_EMAILS } from "../config/constants"; - -/** - * KV key for a domain's cached favicon (shared across feeds from the same sender). - */ -export function iconKey(domain: string): string { - return `icon:${domain}`; -} - -/** - * Store email data in KV - */ -export async function storeEmail( - kv: KVNamespace, - feedId: string, - emailData: EmailData, -): Promise { - // Generate a unique key for this email - const timestamp = Date.now(); - const key = `feed:${feedId}:email:${timestamp}`; - - // Store the email content - await kv.put(key, JSON.stringify(emailData)); - - // Update the feed's metadata (list of emails) - await updateFeedMetadata(kv, feedId, { - key, - subject: emailData.subject, - receivedAt: timestamp, - }); - - return key; -} - -/** - * Update feed metadata with a new email - */ -async function updateFeedMetadata( - kv: KVNamespace, - feedId: string, - emailMetadata: EmailMetadata, -): Promise { - const feedMetadataKey = `feed:${feedId}:metadata`; - const existingMetadata = (await kv.get(feedMetadataKey, { - type: "json", - })) as FeedMetadata | null; - - const metadata: FeedMetadata = existingMetadata || { emails: [] }; - - // Add new email to the beginning of the list - metadata.emails.unshift(emailMetadata); - - // Keep only the last MAX_METADATA_EMAILS in the metadata; delete orphaned KV entries - const toDelete = - metadata.emails.length > MAX_METADATA_EMAILS - ? metadata.emails.slice(MAX_METADATA_EMAILS) - : []; - if (toDelete.length > 0) { - metadata.emails = metadata.emails.slice(0, MAX_METADATA_EMAILS); - } - - await Promise.all([ - kv.put(feedMetadataKey, JSON.stringify(metadata)), - ...toDelete.map((e) => kv.delete(e.key)), - ]); -} - -/** - * Get feed metadata - */ -export async function getFeedMetadata( - kv: KVNamespace, - feedId: string, -): Promise { - const feedMetadataKey = `feed:${feedId}:metadata`; - return (await kv.get(feedMetadataKey, { - type: "json", - })) as FeedMetadata | null; -} - -/** - * Get feed configuration - */ -export async function getFeedConfig( - kv: KVNamespace, - feedId: string, -): Promise { - const feedConfigKey = `feed:${feedId}:config`; - return (await kv.get(feedConfigKey, { type: "json" })) as FeedConfig | null; -} - -/** - * Get email data - */ -export async function getEmailData( - kv: KVNamespace, - key: string, -): Promise { - return (await kv.get(key, { type: "json" })) as EmailData | null; -} - -/** - * Create a new feed - */ -export async function createFeed( - kv: KVNamespace, - feedId: string, - feedConfig: FeedConfig, -): Promise { - // Store feed configuration - const feedConfigKey = `feed:${feedId}:config`; - await kv.put(feedConfigKey, JSON.stringify(feedConfig)); - - // Create empty metadata for the feed - const feedMetadataKey = `feed:${feedId}:metadata`; - await kv.put( - feedMetadataKey, - JSON.stringify({ - emails: [], - }), - ); - - // Add feed to the list of all feeds - await addFeedToList(kv, feedId, feedConfig.title, feedConfig.description); -} - -/** - * Add a feed to the global list - */ -export async function addFeedToList( - kv: KVNamespace, - feedId: string, - title: string, - description?: string, -): Promise { - const feedListKey = "feeds:list"; - const existingList = (await kv.get(feedListKey, { - type: "json", - })) as FeedList | null; - - const feedList: FeedList = existingList || { feeds: [] }; - - feedList.feeds.push({ - id: feedId, - title, - description, - }); - - await kv.put(feedListKey, JSON.stringify(feedList)); -} - -/** - * Get all feeds - */ -export async function getAllFeeds(kv: KVNamespace): Promise { - const feedListKey = "feeds:list"; - const feedList = (await kv.get(feedListKey, { - type: "json", - })) as FeedList | null; - - return feedList || { feeds: [] }; -} diff --git a/src/utils/websub.test.ts b/src/utils/websub.test.ts index abf8717..7d3c929 100644 --- a/src/utils/websub.test.ts +++ b/src/utils/websub.test.ts @@ -8,7 +8,6 @@ import { notifySubscribers, verifyAndStoreSubscription, verifyAndDeleteSubscription, - subscriptionKey, } from "./websub"; import type { Env, WebSubSubscription } from "../types"; @@ -51,8 +50,12 @@ describe("getSubscriptions / saveSubscriptions", () => { expect(await getSubscriptions("feed1", env)).toEqual(subs); }); - it("uses the correct KV key", () => { - expect(subscriptionKey("abc")).toBe("websub:subs:abc"); + it("uses the correct KV key", async () => { + const env = mockEnv(); + await saveSubscriptions("abc", [], env); + expect( + await env.EMAIL_STORAGE.get("websub:subs:abc", { type: "json" }), + ).toEqual([]); }); }); diff --git a/src/utils/websub.ts b/src/utils/websub.ts index 274d200..8f76189 100644 --- a/src/utils/websub.ts +++ b/src/utils/websub.ts @@ -1,25 +1,13 @@ -import { - Env, - FeedConfig, - FeedMetadata, - EmailData, - WebSubSubscription, -} from "../types"; +import { Env, FeedConfig, EmailData, WebSubSubscription } from "../types"; import { generateRssFeed, generateAtomFeed } from "./feed-generator"; import { baseUrl, feedRssUrl, feedAtomUrl, feedUrl } from "./urls"; - -const KV_PREFIX = "websub:subs:"; - -export function subscriptionKey(feedId: string): string { - return `${KV_PREFIX}${feedId}`; -} +import { FeedRepository } from "../domain/feed-repository"; export async function getSubscriptions( feedId: string, env: Env, ): Promise { - const raw = await env.EMAIL_STORAGE.get(subscriptionKey(feedId), "json"); - return (raw as WebSubSubscription[] | null) ?? []; + return FeedRepository.from(env).getSubscriptions(feedId); } export async function saveSubscriptions( @@ -27,10 +15,7 @@ export async function saveSubscriptions( subscriptions: WebSubSubscription[], env: Env, ): Promise { - await env.EMAIL_STORAGE.put( - subscriptionKey(feedId), - JSON.stringify(subscriptions), - ); + await FeedRepository.from(env).saveSubscriptions(feedId, subscriptions); } export async function buildHmacSignature( @@ -60,16 +45,16 @@ async function buildFeedXml( env: Env, format: "rss" | "atom" = "rss", ): Promise { - const [rawMetadata, rawConfig] = await Promise.all([ - env.EMAIL_STORAGE.get(`feed:${feedId}:metadata`, "json"), - env.EMAIL_STORAGE.get(`feed:${feedId}:config`, "json"), + const repo = FeedRepository.from(env); + const [feedMetadata, rawConfig] = await Promise.all([ + repo.getMetadata(feedId), + repo.getConfig(feedId), ]); - const feedMetadata = rawMetadata as FeedMetadata | null; if (!feedMetadata) return null; const base = baseUrl(env); - const feedConfig = (rawConfig as FeedConfig | null) ?? { + const feedConfig: FeedConfig = rawConfig ?? { title: `Newsletter Feed ${feedId}`, description: "Converted email newsletter", language: "en", @@ -78,12 +63,7 @@ async function buildFeedXml( const emails = feedMetadata.emails.slice(0, 20); const emailsData = ( - await Promise.all( - emails.map( - (m) => - env.EMAIL_STORAGE.get(m.key, "json") as Promise, - ), - ) + await Promise.all(emails.map((m) => repo.getEmail(m.key))) ).filter((d): d is EmailData => d !== null); if (format === "atom") {