refactor(domain): introduce FeedRepository as the single KV access layer

Centralise the KV key schema and all get/put access behind a FeedRepository
class under src/domain/. Every feed/email/list/icon/websub/counter key was
previously inlined across ~12 modules with two divergent storeEmail and
addFeedToList implementations; the dead src/utils/storage.ts write path is
removed and the email key convention unified on feed:<id>:<ts>.

Behaviour-preserving: existing tests pass unchanged in logic, plus a new
feed-repository.test.ts covering CRUD, key builders, list ops and counters.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Julien Herr
2026-05-23 23:56:44 +02:00
parent a0eaebe749
commit 2b3f00f7e3
22 changed files with 616 additions and 539 deletions
+171
View File
@@ -0,0 +1,171 @@
import { describe, it, expect } from "vitest";
import { createMockEnv } from "../test/setup";
import { FeedRepository } from "./feed-repository";
import type {
Env,
FeedConfig,
FeedMetadata,
EmailData,
WebSubSubscription,
} from "../types";
const mockEnv = () => createMockEnv() as unknown as Env;
const sampleConfig = (overrides: Partial<FeedConfig> = {}): FeedConfig => ({
title: "Test Feed",
language: "en",
created_at: 1000,
...overrides,
});
const sampleEmail = (overrides: Partial<EmailData> = {}): EmailData => ({
subject: "Hello",
from: "news@example.com",
content: "<p>hi</p>",
receivedAt: 1234,
headers: {},
...overrides,
});
describe("FeedRepository key schema", () => {
it("builds the canonical KV keys via the public API", () => {
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
expect(repo.feedKeyPrefix("a.b.42")).toBe("feed:a.b.42:");
expect(repo.iconKey("example.com")).toBe("icon:example.com");
expect(repo.newEmailKey("a.b.42")).toMatch(/^feed:a\.b\.42:\d+$/);
});
it("recognises email keys vs config/metadata keys", () => {
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
expect(repo.isEmailKey("a.b.42", "feed:a.b.42:config")).toBe(false);
expect(repo.isEmailKey("a.b.42", "feed:a.b.42:metadata")).toBe(false);
expect(repo.isEmailKey("a.b.42", "feed:a.b.42:1700000000000")).toBe(true);
});
it("recovers the feed id from an email key", () => {
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
expect(repo.feedIdFromEmailKey("feed:a.b.42:1700000000000")).toBe("a.b.42");
});
});
describe("FeedRepository config & metadata", () => {
it("round-trips and deletes a feed config", async () => {
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
expect(await repo.getConfig("a.b.42")).toBeNull();
await repo.putConfig("a.b.42", sampleConfig());
expect(await repo.getConfig("a.b.42")).toMatchObject({
title: "Test Feed",
});
await repo.deleteConfig("a.b.42");
expect(await repo.getConfig("a.b.42")).toBeNull();
});
it("round-trips and deletes feed metadata", async () => {
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
const meta: FeedMetadata = { emails: [] };
await repo.putMetadata("a.b.42", meta);
expect(await repo.getMetadata("a.b.42")).toEqual(meta);
await repo.deleteMetadata("a.b.42");
expect(await repo.getMetadata("a.b.42")).toBeNull();
});
});
describe("FeedRepository emails", () => {
it("stores and reads an email under a minted key", async () => {
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
const key = repo.newEmailKey("a.b.42");
await repo.putEmail(key, sampleEmail());
expect(await repo.getEmail(key)).toMatchObject({ subject: "Hello" });
await repo.deleteEmail(key);
expect(await repo.getEmail(key)).toBeNull();
});
it("lists every key under a feed prefix", async () => {
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
await repo.putConfig("a.b.42", sampleConfig());
await repo.putMetadata("a.b.42", { emails: [] });
const emailKey = repo.newEmailKey("a.b.42");
await repo.putEmail(emailKey, sampleEmail());
const listed = await repo.listFeedKeys("a.b.42");
expect(listed.names).toContain("feed:a.b.42:config");
expect(listed.names).toContain("feed:a.b.42:metadata");
expect(listed.names).toContain(emailKey);
expect(listed.names.filter((k) => repo.isEmailKey("a.b.42", k))).toEqual([
emailKey,
]);
});
});
describe("FeedRepository feed list", () => {
it("adds, updates, lists and removes feeds with expiry", async () => {
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
await repo.addToList("a.b.42", "One", "desc", 5000);
await repo.addToList("c.d.99", "Two");
let feeds = await repo.listFeeds();
expect(feeds).toHaveLength(2);
expect(feeds.find((f) => f.id === "a.b.42")).toMatchObject({
title: "One",
expires_at: 5000,
});
await repo.updateInList("a.b.42", "One-updated", undefined, undefined);
feeds = await repo.listFeeds();
const updated = feeds.find((f) => f.id === "a.b.42");
expect(updated).toMatchObject({ title: "One-updated" });
expect(updated?.expires_at).toBeUndefined();
expect(await repo.removeFromList("a.b.42")).toBe(true);
expect(await repo.removeFromList("missing")).toBe(false);
feeds = await repo.listFeeds();
expect(feeds.map((f) => f.id)).toEqual(["c.d.99"]);
});
it("bulk-removes only the matching ids", async () => {
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
await repo.addToList("a.b.42", "One");
await repo.addToList("c.d.99", "Two");
await repo.addToList("e.f.10", "Three");
const removed = await repo.removeFromListBulk(["a.b.42", "e.f.10", "nope"]);
expect(removed.sort()).toEqual(["a.b.42", "e.f.10"]);
expect((await repo.listFeeds()).map((f) => f.id)).toEqual(["c.d.99"]);
});
});
describe("FeedRepository counters, icons, websub", () => {
it("round-trips raw counters", async () => {
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
expect(await repo.getCountersRaw()).toBeNull();
await repo.putCounters({
feeds_created: 1,
feeds_deleted: 0,
emails_received: 2,
emails_rejected: 0,
unsubscribes_sent: 0,
});
expect(await repo.getCountersRaw()).toMatchObject({ emails_received: 2 });
});
it("stores and reads favicons as text or json", async () => {
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
expect(await repo.getIconText("example.com")).toBeNull();
await repo.putIcon("example.com", JSON.stringify({ data: null }), 60);
expect(await repo.getIconText("example.com")).toBe('{"data":null}');
expect(await repo.getIconJson<{ data: null }>("example.com")).toEqual({
data: null,
});
});
it("round-trips websub subscriptions and counts them", async () => {
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
expect(await repo.getSubscriptions("a.b.42")).toEqual([]);
const subs: WebSubSubscription[] = [
{ callbackUrl: "https://r.example/cb", expiresAt: 9999 },
];
await repo.saveSubscriptions("a.b.42", subs);
expect(await repo.getSubscriptions("a.b.42")).toEqual(subs);
expect(await repo.countSubscriptionKeys()).toBe(1);
});
});
+296
View File
@@ -0,0 +1,296 @@
import {
Counters,
EmailData,
Env,
FeedConfig,
FeedList,
FeedListItem,
FeedMetadata,
WebSubSubscription,
} from "../types";
import { FEEDS_LIST_KEY, STATS_KEY } from "../config/constants";
import { logger } from "../lib/logger";
const WEBSUB_PREFIX = "websub:subs:";
/**
* Single source of truth for the KV key schema and all KV access. No other
* module should build a `feed:`/`feeds:list`/`websub:`/`icon:`/`stats:counters`
* key string — go through a repository method instead.
*
* Wraps one `KVNamespace`; construct per request via `FeedRepository.from(env)`.
*/
export class FeedRepository {
constructor(private readonly kv: KVNamespace) {}
static from(env: Env): FeedRepository {
return new FeedRepository(env.EMAIL_STORAGE);
}
// ── Key schema ────────────────────────────────────────────────────────────
private configKey(feedId: string): string {
return `feed:${feedId}:config`;
}
private metadataKey(feedId: string): string {
return `feed:${feedId}:metadata`;
}
/** KV key for a domain's cached favicon (shared across feeds). */
iconKey(domain: string): string {
return `icon:${domain}`;
}
private websubKey(feedId: string): string {
return `${WEBSUB_PREFIX}${feedId}`;
}
/** Prefix covering every key owned by a feed (config, metadata, emails). */
feedKeyPrefix(feedId: string): string {
return `feed:${feedId}:`;
}
/** Mint a fresh, time-ordered email key. Call once and reuse the result. */
newEmailKey(feedId: string): string {
return `feed:${feedId}:${Date.now()}`;
}
/** True when `key` is an email entry (not the feed's config/metadata key). */
isEmailKey(feedId: string, key: string): boolean {
const suffix = key.slice(this.feedKeyPrefix(feedId).length);
return suffix !== "config" && suffix !== "metadata";
}
/** Recover the feed id embedded in an email key (`feed:<id>:<ts>`). */
feedIdFromEmailKey(key: string): string {
return key.split(":")[1];
}
// ── Feed config ───────────────────────────────────────────────────────────
async getConfig(feedId: string): Promise<FeedConfig | null> {
return (await this.kv.get(this.configKey(feedId), {
type: "json",
})) as FeedConfig | null;
}
async putConfig(feedId: string, config: FeedConfig): Promise<void> {
await this.kv.put(this.configKey(feedId), JSON.stringify(config));
}
async deleteConfig(feedId: string): Promise<void> {
await this.kv.delete(this.configKey(feedId));
}
// ── Feed metadata ─────────────────────────────────────────────────────────
async getMetadata(feedId: string): Promise<FeedMetadata | null> {
return (await this.kv.get(this.metadataKey(feedId), {
type: "json",
})) as FeedMetadata | null;
}
async putMetadata(feedId: string, metadata: FeedMetadata): Promise<void> {
await this.kv.put(this.metadataKey(feedId), JSON.stringify(metadata));
}
async deleteMetadata(feedId: string): Promise<void> {
await this.kv.delete(this.metadataKey(feedId));
}
// ── Emails ────────────────────────────────────────────────────────────────
async putEmail(key: string, data: EmailData): Promise<void> {
await this.kv.put(key, JSON.stringify(data));
}
async getEmail(key: string): Promise<EmailData | null> {
return (await this.kv.get(key, { type: "json" })) as EmailData | null;
}
async deleteEmail(key: string): Promise<void> {
await this.kv.delete(key);
}
// ── Global feed list ──────────────────────────────────────────────────────
async listFeeds(): Promise<FeedListItem[]> {
try {
const feedList = (await this.kv.get(FEEDS_LIST_KEY, {
type: "json",
})) as FeedList | null;
return feedList?.feeds || [];
} catch (error) {
logger.error("Error listing feeds", { error: String(error) });
return [];
}
}
async addToList(
feedId: string,
title: string,
description?: string,
expires_at?: number,
): Promise<void> {
try {
const feedList = ((await this.kv.get(FEEDS_LIST_KEY, {
type: "json",
})) as FeedList | null) || { feeds: [] };
feedList.feeds.push({ id: feedId, title, description, expires_at });
await this.kv.put(FEEDS_LIST_KEY, JSON.stringify(feedList));
} catch (error) {
logger.error("Error adding feed to list", {
feedId,
error: String(error),
});
}
}
async updateInList(
feedId: string,
title: string,
description?: string,
expires_at?: number,
): Promise<void> {
try {
const feedList = ((await this.kv.get(FEEDS_LIST_KEY, {
type: "json",
})) as FeedList | null) || { feeds: [] };
const feedIndex = feedList.feeds.findIndex((feed) => feed.id === feedId);
if (feedIndex !== -1) {
feedList.feeds[feedIndex].title = title;
feedList.feeds[feedIndex].description = description;
feedList.feeds[feedIndex].expires_at = expires_at;
await this.kv.put(FEEDS_LIST_KEY, JSON.stringify(feedList));
}
} catch (error) {
logger.error("Error updating feed in list", {
feedId,
error: String(error),
});
}
}
async removeFromListBulk(feedIds: string[]): Promise<string[]> {
try {
const feedList = ((await this.kv.get(FEEDS_LIST_KEY, {
type: "json",
})) as FeedList | null) || { feeds: [] };
const toRemove = new Set(feedIds.filter(Boolean));
if (toRemove.size === 0) return [];
const removed: string[] = [];
const nextFeeds: FeedListItem[] = [];
for (const feed of feedList.feeds) {
if (toRemove.has(feed.id)) {
removed.push(feed.id);
continue;
}
nextFeeds.push(feed);
}
if (removed.length === 0) return [];
feedList.feeds = nextFeeds;
await this.kv.put(FEEDS_LIST_KEY, JSON.stringify(feedList));
return removed;
} catch (error) {
logger.error("Error removing feeds from list", { error: String(error) });
return [];
}
}
async removeFromList(feedId: string): Promise<boolean> {
const removed = await this.removeFromListBulk([feedId]);
return removed.includes(feedId);
}
// ── Key listing / counting ────────────────────────────────────────────────
async listFeedKeys(
feedId: string,
options: { cursor?: string; limit?: number } = {},
): Promise<{ names: string[]; cursor: string; listComplete: boolean }> {
const prefix = this.feedKeyPrefix(feedId);
const limit = Math.min(1000, Math.max(1, Math.floor(options.limit || 100)));
const cursor = options.cursor || undefined;
const listed = await this.kv.list({ prefix, cursor, limit });
return {
names: (listed.keys || []).map((k) => k.name),
cursor: listed.cursor || "",
listComplete: !!listed.list_complete,
};
}
async countKeysByPrefix(prefix: string): Promise<number> {
let total = 0;
let cursor: string | undefined;
try {
do {
const listed = await this.kv.list({ prefix, cursor, limit: 1000 });
total += listed.keys.length;
cursor = listed.list_complete ? undefined : listed.cursor;
} while (cursor);
} catch (error) {
logger.error("Error counting keys", { prefix, error: String(error) });
}
return total;
}
/** Number of feeds that currently hold at least one WebSub subscription. */
countSubscriptionKeys(): Promise<number> {
return this.countKeysByPrefix("websub:");
}
// ── Monitoring counters ───────────────────────────────────────────────────
async getCountersRaw(): Promise<Counters | null> {
return (await this.kv.get(STATS_KEY, { type: "json" })) as Counters | null;
}
async putCounters(counters: Counters): Promise<void> {
await this.kv.put(STATS_KEY, JSON.stringify(counters));
}
// ── Favicons ──────────────────────────────────────────────────────────────
async getIconText(domain: string): Promise<string | null> {
return this.kv.get(this.iconKey(domain), "text");
}
async getIconJson<T>(domain: string): Promise<T | null> {
return (await this.kv.get(this.iconKey(domain), {
type: "json",
})) as T | null;
}
async putIcon(
domain: string,
value: string,
ttlSeconds: number,
): Promise<void> {
await this.kv.put(this.iconKey(domain), value, {
expirationTtl: ttlSeconds,
});
}
// ── WebSub subscriptions ──────────────────────────────────────────────────
async getSubscriptions(feedId: string): Promise<WebSubSubscription[]> {
const raw = await this.kv.get(this.websubKey(feedId), "json");
return (raw as WebSubSubscription[] | null) ?? [];
}
async saveSubscriptions(
feedId: string,
subscriptions: WebSubSubscription[],
): Promise<void> {
await this.kv.put(this.websubKey(feedId), JSON.stringify(subscriptions));
}
}
+5 -7
View File
@@ -13,11 +13,8 @@ import { apiApp } from "./routes/api";
import { handleCloudflareEmail } from "./lib/cloudflare-email";
import { Env } from "./types";
import { logger } from "./lib/logger";
import {
listAllFeeds,
purgeExpiredFeeds,
removeFeedsFromListBulk,
} from "./routes/admin/helpers";
import { FeedRepository } from "./domain/feed-repository";
import { purgeExpiredFeeds } from "./routes/admin/helpers";
import {
bumpCounters,
scanR2Usage,
@@ -201,7 +198,8 @@ export default {
},
async scheduled(_event: ScheduledEvent, env: Env, _ctx: ExecutionContext) {
const attachmentBucket = getAttachmentBucket(env);
const feeds = await listAllFeeds(env.EMAIL_STORAGE);
const repo = FeedRepository.from(env);
const feeds = await repo.listFeeds();
const now = Date.now();
const expiredIds = feeds
.filter((f) => f.expires_at !== undefined && f.expires_at <= now)
@@ -211,7 +209,7 @@ export default {
await purgeExpiredFeeds(env.EMAIL_STORAGE, feedId, attachmentBucket);
}
if (expiredIds.length > 0) {
await removeFeedsFromListBulk(env.EMAIL_STORAGE, expiredIds);
await repo.removeFromListBulk(expiredIds);
await bumpCounters(env.EMAIL_STORAGE, {
feeds_deleted: expiredIds.length,
});
+2 -1
View File
@@ -7,7 +7,8 @@ import {
RawAttachment,
} from "./email-processor";
import { getCounters } from "../utils/stats";
import { iconKey } from "../utils/storage";
const iconKey = (domain: string) => `icon:${domain}`;
const VALID_FEED_ID = "apple.mountain.42";
const VALID_TO = `${VALID_FEED_ID}@test.getmynews.app`;
+10 -20
View File
@@ -1,11 +1,5 @@
import { EmailParser } from "../utils/email-parser";
import {
AttachmentData,
EmailMetadata,
Env,
FeedConfig,
FeedMetadata,
} from "../types";
import { AttachmentData, EmailMetadata, Env, FeedConfig } from "../types";
import { notifySubscribers } from "../utils/websub";
import { bumpCounters } from "../utils/stats";
import {
@@ -14,6 +8,7 @@ import {
} from "../utils/favicon-fetcher";
import { parseOneClickUnsubscribe } from "../utils/unsubscribe";
import { getAttachmentBucket } from "../utils/attachments";
import { FeedRepository } from "../domain/feed-repository";
import { logger } from "./logger";
import { FEED_MAX_BYTES } from "../config/constants";
@@ -110,10 +105,7 @@ export async function validateEmail(
};
}
const feedConfig = (await env.EMAIL_STORAGE.get(
`feed:${feedId}:config`,
"json",
)) as FeedConfig | null;
const feedConfig = await FeedRepository.from(env).getConfig(feedId);
if (!feedConfig) {
logger.error("Feed not found", { feedId });
return {
@@ -188,12 +180,12 @@ export async function storeEmail(
...(storedAttachments.length > 0 ? { attachments: storedAttachments } : {}),
};
const emailKey = `feed:${feedId}:${Date.now()}`;
const feedMetadataKey = `feed:${feedId}:metadata`;
const repo = FeedRepository.from(env);
const emailKey = repo.newEmailKey(feedId);
const [, rawMetadata] = await Promise.all([
env.EMAIL_STORAGE.put(emailKey, JSON.stringify(emailData)),
env.EMAIL_STORAGE.get(feedMetadataKey, "json"),
repo.putEmail(emailKey, emailData),
repo.getMetadata(feedId),
]);
// Note: KV has no atomic compare-and-swap. Concurrent invocations for the
@@ -202,9 +194,7 @@ export async function storeEmail(
// KV's eventual-consistency model.
// TODO: Migrate feed metadata writes to Cloudflare Durable Objects to serialise
// concurrent writes and eliminate this race condition.
const feedMetadata = ((rawMetadata as FeedMetadata | null) || {
emails: [],
}) as FeedMetadata;
const feedMetadata = rawMetadata || { emails: [] };
const maxBytes =
parseInt(env.FEED_MAX_SIZE_BYTES ?? "", 10) || FEED_MAX_BYTES;
@@ -260,8 +250,8 @@ export async function storeEmail(
: [];
await Promise.all([
env.EMAIL_STORAGE.put(feedMetadataKey, JSON.stringify(feedMetadata)),
...toDelete.map((e) => env.EMAIL_STORAGE.delete(e.key)),
repo.putMetadata(feedId, feedMetadata),
...toDelete.map((e) => repo.deleteEmail(e.key)),
...r2Deletions,
]);
+15 -32
View File
@@ -5,10 +5,8 @@ import { bumpCounters } from "../utils/stats";
import { waitUntilSafe } from "../utils/worker";
import { sendUnsubscribes } from "../utils/unsubscribe";
import { getAttachmentBucket } from "../utils/attachments";
import { FeedRepository } from "../domain/feed-repository";
import {
addFeedToList,
updateFeedInList,
removeFeedFromList,
purgeFeedKeysStep,
collectUnsubscribeUrls,
} from "../routes/admin/helpers";
@@ -49,7 +47,7 @@ export async function createFeedRecord(
env: Env,
input: CreateFeedInput,
): Promise<{ feedId: string; config: FeedConfig }> {
const emailStorage = env.EMAIL_STORAGE;
const repo = FeedRepository.from(env);
const expiresAt = resolveExpiresAt(env, input.lifetimeHours);
const feedId = generateFeedId();
@@ -67,19 +65,13 @@ export async function createFeedRecord(
const metadata: FeedMetadata = { emails: [] };
await Promise.all([
emailStorage.put(`feed:${feedId}:config`, JSON.stringify(config)),
emailStorage.put(`feed:${feedId}:metadata`, JSON.stringify(metadata)),
repo.putConfig(feedId, config),
repo.putMetadata(feedId, metadata),
]);
await addFeedToList(
emailStorage,
feedId,
input.title,
input.description,
expiresAt,
);
await repo.addToList(feedId, input.title, input.description, expiresAt);
await bumpCounters(emailStorage, {
await bumpCounters(env.EMAIL_STORAGE, {
feeds_created: 1,
last_feed_created_at: new Date().toISOString(),
});
@@ -115,12 +107,9 @@ export async function updateFeedRecord(
input: UpdateFeedInput,
options: { inPlace?: boolean } = {},
): Promise<UpdateFeedResult> {
const emailStorage = env.EMAIL_STORAGE;
const feedConfigKey = `feed:${feedId}:config`;
const repo = FeedRepository.from(env);
const existing = (await emailStorage.get(feedConfigKey, {
type: "json",
})) as FeedConfig | null;
const existing = await repo.getConfig(feedId);
if (!existing) return { status: "not_found" };
@@ -157,14 +146,8 @@ export async function updateFeedRecord(
expires_at: expiresAt,
};
await emailStorage.put(feedConfigKey, JSON.stringify(config));
await updateFeedInList(
emailStorage,
feedId,
config.title,
config.description,
expiresAt,
);
await repo.putConfig(feedId, config);
await repo.updateInList(feedId, config.title, config.description, expiresAt);
return { status: "ok", config };
}
@@ -184,22 +167,21 @@ export async function deleteFeedFastDetailed(
emailStorage: KVNamespace,
feedId: string,
): Promise<DeleteFeedFastResult> {
const feedConfigKey = `feed:${feedId}:config`;
const feedMetadataKey = `feed:${feedId}:metadata`;
const repo = new FeedRepository(emailStorage);
const errors: string[] = [];
let configDeleted = false;
let metadataDeleted = false;
try {
await emailStorage.delete(feedConfigKey);
await repo.deleteConfig(feedId);
configDeleted = true;
} catch (error) {
errors.push(`config delete failed: ${String(error)}`);
}
try {
await emailStorage.delete(feedMetadataKey);
await repo.deleteMetadata(feedId);
metadataDeleted = true;
} catch (error) {
errors.push(`metadata delete failed: ${String(error)}`);
@@ -220,12 +202,13 @@ export async function deleteFeedRecord(
feedId: string,
): Promise<boolean> {
const emailStorage = env.EMAIL_STORAGE;
const repo = new FeedRepository(emailStorage);
// Read unsubscribe URLs before the metadata is deleted below.
const unsubscribeUrls = await collectUnsubscribeUrls(emailStorage, feedId);
await deleteFeedFastDetailed(emailStorage, feedId);
const removed = await removeFeedFromList(emailStorage, feedId);
const removed = await repo.removeFromList(feedId);
if (removed) {
await bumpCounters(emailStorage, { feeds_deleted: 1 });
}
+2 -3
View File
@@ -8,7 +8,7 @@ import { ADMIN_COOKIE_MAX_AGE } from "../config/constants";
import { logger } from "../lib/logger";
import { timingSafeEqual, checkProxyAuth } from "../lib/auth";
import { Layout, clampText } from "./admin/ui";
import { listAllFeeds } from "./admin/helpers";
import { FeedRepository } from "../domain/feed-repository";
import { updateFeedRecord } from "../lib/feed-service";
import { feedRssUrl, feedAtomUrl, feedEmailAddress } from "../utils/urls";
import { feedsRouter } from "./admin/feeds";
@@ -282,14 +282,13 @@ const ExpiryBadge = ({ expiresAt }: { expiresAt: number }) => {
app.get("/", async (c) => {
// Type assertion for environment variables
const env = c.env;
const emailStorage = env.EMAIL_STORAGE;
const url = new URL(c.req.url);
const view = url.searchParams.get("view") === "table" ? "table" : "list";
const message = url.searchParams.get("message");
const count = Number(url.searchParams.get("count") || "0");
// List all feeds
const feedList = await listAllFeeds(emailStorage);
const feedList = await FeedRepository.from(env).listFeeds();
// Keep the dashboard fast: avoid N KV reads for N feeds.
// We store title/description in `feeds:list` (description is optional for older data).
+16 -32
View File
@@ -1,17 +1,12 @@
import { Hono } from "hono";
import {
Env,
FeedConfig,
FeedMetadata,
EmailData,
EmailMetadata,
} from "../../types";
import { Env, EmailMetadata } from "../../types";
import { logger } from "../../lib/logger";
import { Layout, clampText } from "./ui";
import {
deleteAttachmentsForEmails,
deleteKeysWithConcurrency,
} from "./helpers";
import { FeedRepository } from "../../domain/feed-repository";
import { feedRssUrl, feedAtomUrl, feedEmailAddress } from "../../utils/urls";
import { formatBytes } from "../../utils/format";
import { emailsPageScript } from "../../scripts/generated/emails-page";
@@ -156,17 +151,13 @@ const SenderField = ({ from, feedId }: SenderFieldProps) => {
emailsRouter.get("/feeds/:feedId/emails", async (c) => {
const env = c.env;
const emailStorage = env.EMAIL_STORAGE;
const repo = FeedRepository.from(env);
const feedId = c.req.param("feedId");
const message = c.req.query("message");
const count = Number(c.req.query("count") || "0");
const feedConfig = (await emailStorage.get(`feed:${feedId}:config`, {
type: "json",
})) as FeedConfig | null;
const feedMetadata = (await emailStorage.get(`feed:${feedId}:metadata`, {
type: "json",
})) as FeedMetadata | null;
const feedConfig = await repo.getConfig(feedId);
const feedMetadata = await repo.getMetadata(feedId);
if (!feedConfig || !feedMetadata) {
return c.text("Feed not found", 404);
@@ -461,16 +452,14 @@ emailsRouter.get("/feeds/:feedId/emails", async (c) => {
emailsRouter.get("/emails/:emailKey", async (c) => {
const env = c.env;
const emailStorage = env.EMAIL_STORAGE;
const repo = FeedRepository.from(env);
const emailKey = c.req.param("emailKey");
const emailData = (await emailStorage.get(emailKey, {
type: "json",
})) as EmailData | null;
const emailData = await repo.getEmail(emailKey);
if (!emailData) return c.text("Email not found", 404);
const feedId = emailKey.split(":")[1];
const feedId = repo.feedIdFromEmailKey(emailKey);
const attachments = emailData.attachments ?? [];
const htmlContent = `<!DOCTYPE html><html><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><style>body{font-family:-apple-system,BlinkMacSystemFont,'SF Pro Text','SF Pro Display','Helvetica Neue',Arial,sans-serif;line-height:1.5;padding:16px;margin:0;color:#333;box-sizing:border-box}img{max-width:100%;height:auto}a{color:#0070f3}@media(prefers-color-scheme:dark){body{background-color:#1c1c1e;color:#ffffff}a{color:#0a84ff}}</style></head><body>${emailData.content}</body></html>`;
@@ -652,7 +641,7 @@ emailsRouter.get("/emails/:emailKey", async (c) => {
emailsRouter.post("/emails/:emailKey/delete", async (c) => {
const env = c.env;
const emailStorage = env.EMAIL_STORAGE;
const repo = FeedRepository.from(env);
const emailKey = c.req.param("emailKey");
const wantsJson = (c.req.header("Accept") || "").includes("application/json");
@@ -664,12 +653,9 @@ emailsRouter.post("/emails/:emailKey/delete", async (c) => {
return c.text("Feed ID is required", 400);
}
const feedMetadataKey = `feed:${feedId}:metadata`;
const feedMetadata = (await emailStorage.get(feedMetadataKey, {
type: "json",
})) as FeedMetadata | null;
const feedMetadata = await repo.getMetadata(feedId);
await emailStorage.delete(emailKey);
await repo.deleteEmail(emailKey);
await deleteAttachmentsForEmails(env, feedMetadata?.emails ?? [], [
emailKey,
]);
@@ -678,7 +664,7 @@ emailsRouter.post("/emails/:emailKey/delete", async (c) => {
feedMetadata.emails = feedMetadata.emails.filter(
(email) => email.key !== emailKey,
);
await emailStorage.put(feedMetadataKey, JSON.stringify(feedMetadata));
await repo.putMetadata(feedId, feedMetadata);
}
if (wantsJson) return c.json({ ok: true, emailKey, feedId });
@@ -699,6 +685,7 @@ emailsRouter.post("/emails/:emailKey/delete", async (c) => {
emailsRouter.post("/feeds/:feedId/emails/bulk-delete", async (c) => {
const env = c.env;
const emailStorage = env.EMAIL_STORAGE;
const repo = new FeedRepository(emailStorage);
const feedId = c.req.param("feedId");
const contentType = c.req.header("Content-Type") || "";
const wantsJson =
@@ -706,10 +693,7 @@ emailsRouter.post("/feeds/:feedId/emails/bulk-delete", async (c) => {
(c.req.header("Accept") || "").includes("application/json");
try {
const feedMetadataKey = `feed:${feedId}:metadata`;
const feedMetadata = (await emailStorage.get(feedMetadataKey, {
type: "json",
})) as FeedMetadata | null;
const feedMetadata = await repo.getMetadata(feedId);
if (!feedMetadata) {
return wantsJson
@@ -753,7 +737,7 @@ emailsRouter.post("/feeds/:feedId/emails/bulk-delete", async (c) => {
feedMetadata.emails = feedMetadata.emails.filter(
(email) => !deletedSet.has(email.key),
);
await emailStorage.put(feedMetadataKey, JSON.stringify(feedMetadata));
await repo.putMetadata(feedId, feedMetadata);
return c.json({
ok: failedEmailKeys.length === 0,
@@ -784,7 +768,7 @@ emailsRouter.post("/feeds/:feedId/emails/bulk-delete", async (c) => {
feedMetadata.emails = feedMetadata.emails.filter(
(email) => !deletedSet.has(email.key),
);
await emailStorage.put(feedMetadataKey, JSON.stringify(feedMetadata));
await repo.putMetadata(feedId, feedMetadata);
return c.redirect(
`/admin/feeds/${feedId}/emails?message=bulkDeleted&count=${deletedOk.length}`,
+18 -25
View File
@@ -1,6 +1,6 @@
import { Hono } from "hono";
import { z } from "zod";
import { Env, FeedConfig } from "../../types";
import { Env } from "../../types";
import { bumpCounters } from "../../utils/stats";
import { waitUntilSafe } from "../../utils/worker";
import { feedRssUrl, feedEmailAddress } from "../../utils/urls";
@@ -8,11 +8,8 @@ import { logger } from "../../lib/logger";
import { sendUnsubscribes } from "../../utils/unsubscribe";
import { getAttachmentBucket } from "../../utils/attachments";
import { Layout } from "./ui";
import {
removeFeedsFromListBulk,
purgeFeedKeysStep,
collectUnsubscribeUrls,
} from "./helpers";
import { purgeFeedKeysStep, collectUnsubscribeUrls } from "./helpers";
import { FeedRepository } from "../../domain/feed-repository";
import {
createFeedRecord,
updateFeedRecord,
@@ -149,12 +146,9 @@ feedsRouter.post("/create", async (c) => {
feedsRouter.get("/:feedId/edit", async (c) => {
const env = c.env;
const emailStorage = env.EMAIL_STORAGE;
const feedId = c.req.param("feedId");
const feedConfig = (await emailStorage.get(`feed:${feedId}:config`, {
type: "json",
})) as FeedConfig | null;
const feedConfig = await FeedRepository.from(env).getConfig(feedId);
if (!feedConfig) {
return c.text("Feed not found", 404);
@@ -365,7 +359,7 @@ feedsRouter.post("/:feedId/edit", async (c) => {
feedsRouter.post("/:feedId/sender-filter", async (c) => {
const env = c.env;
const feedId = c.req.param("feedId");
const feedConfigKey = `feed:${feedId}:config`;
const repo = FeedRepository.from(env);
const body = await c.req.json().catch(() => null);
const parsed = senderFilterSchema.safeParse(body);
@@ -376,9 +370,7 @@ feedsRouter.post("/:feedId/sender-filter", async (c) => {
const { action, value } = parsed.data;
const normalized = value.trim().toLowerCase();
const feedConfig = (await env.EMAIL_STORAGE.get(feedConfigKey, {
type: "json",
})) as FeedConfig | null;
const feedConfig = await repo.getConfig(feedId);
if (!feedConfig) return c.json({ ok: false, error: "Feed not found" }, 404);
const allowedSenders = (feedConfig.allowed_senders || []).map((s) =>
@@ -405,15 +397,12 @@ feedsRouter.post("/:feedId/sender-filter", async (c) => {
if (!targetList.includes(normalized)) {
targetList.push(normalized);
await env.EMAIL_STORAGE.put(
feedConfigKey,
JSON.stringify({
...feedConfig,
allowed_senders: allowedSenders,
blocked_senders: blockedSenders,
updated_at: Date.now(),
}),
);
await repo.putConfig(feedId, {
...feedConfig,
allowed_senders: allowedSenders,
blocked_senders: blockedSenders,
updated_at: Date.now(),
});
}
return c.json({ ok: true });
@@ -552,7 +541,9 @@ feedsRouter.post("/bulk-delete", async (c) => {
}
}
const deletedFeedIds = await removeFeedsFromListBulk(emailStorage, okIds);
const deletedFeedIds = await new FeedRepository(
emailStorage,
).removeFromListBulk(okIds);
if (deletedFeedIds.length > 0) {
await bumpCounters(emailStorage, {
feeds_deleted: deletedFeedIds.length,
@@ -616,7 +607,9 @@ feedsRouter.post("/bulk-delete", async (c) => {
}
}
const deletedFeedIds = await removeFeedsFromListBulk(emailStorage, okIds);
const deletedFeedIds = await new FeedRepository(
emailStorage,
).removeFromListBulk(okIds);
if (deletedFeedIds.length > 0) {
await bumpCounters(emailStorage, {
feeds_deleted: deletedFeedIds.length,
+13 -130
View File
@@ -1,14 +1,7 @@
import {
EmailData,
EmailMetadata,
Env,
FeedList,
FeedListItem,
FeedMetadata,
} from "../../types";
import { FEEDS_LIST_KEY } from "../../config/constants";
import { EmailData, EmailMetadata, Env } from "../../types";
import { logger } from "../../lib/logger";
import { getAttachmentBucket } from "../../utils/attachments";
import { FeedRepository } from "../../domain/feed-repository";
// Delete the R2 attachments belonging to the given email keys. Call before the
// emails are removed from feed metadata, while `emails` still carries their
@@ -58,108 +51,6 @@ export async function deleteKeysWithConcurrency(
return { ok, failed };
}
export async function listAllFeeds(
emailStorage: KVNamespace,
): Promise<FeedListItem[]> {
try {
const feedList = (await emailStorage.get(FEEDS_LIST_KEY, {
type: "json",
})) as FeedList | null;
return feedList?.feeds || [];
} catch (error) {
logger.error("Error listing feeds", { error: String(error) });
return [];
}
}
export async function addFeedToList(
emailStorage: KVNamespace,
feedId: string,
title: string,
description?: string,
expires_at?: number,
): Promise<void> {
try {
const feedList = ((await emailStorage.get(FEEDS_LIST_KEY, {
type: "json",
})) as FeedList | null) || { feeds: [] };
feedList.feeds.push({ id: feedId, title, description, expires_at });
await emailStorage.put(FEEDS_LIST_KEY, JSON.stringify(feedList));
} catch (error) {
logger.error("Error adding feed to list", { feedId, error: String(error) });
}
}
export async function updateFeedInList(
emailStorage: KVNamespace,
feedId: string,
title: string,
description?: string,
expires_at?: number,
): Promise<void> {
try {
const feedList = ((await emailStorage.get(FEEDS_LIST_KEY, {
type: "json",
})) as FeedList | null) || { feeds: [] };
const feedIndex = feedList.feeds.findIndex((feed) => feed.id === feedId);
if (feedIndex !== -1) {
feedList.feeds[feedIndex].title = title;
feedList.feeds[feedIndex].description = description;
feedList.feeds[feedIndex].expires_at = expires_at;
await emailStorage.put(FEEDS_LIST_KEY, JSON.stringify(feedList));
}
} catch (error) {
logger.error("Error updating feed in list", {
feedId,
error: String(error),
});
}
}
export async function removeFeedsFromListBulk(
emailStorage: KVNamespace,
feedIds: string[],
): Promise<string[]> {
try {
const feedList = ((await emailStorage.get(FEEDS_LIST_KEY, {
type: "json",
})) as FeedList | null) || { feeds: [] };
const toRemove = new Set(feedIds.filter(Boolean));
if (toRemove.size === 0) return [];
const removed: string[] = [];
const nextFeeds: FeedListItem[] = [];
for (const feed of feedList.feeds) {
if (toRemove.has(feed.id)) {
removed.push(feed.id);
continue;
}
nextFeeds.push(feed);
}
if (removed.length === 0) return [];
feedList.feeds = nextFeeds;
await emailStorage.put(FEEDS_LIST_KEY, JSON.stringify(feedList));
return removed;
} catch (error) {
logger.error("Error removing feeds from list", { error: String(error) });
return [];
}
}
export async function removeFeedFromList(
emailStorage: KVNamespace,
feedId: string,
): Promise<boolean> {
const removed = await removeFeedsFromListBulk(emailStorage, [feedId]);
return removed.includes(feedId);
}
/**
* Read a feed's stored RFC 8058 one-click unsubscribe URLs (one per sender).
* Must be called before the feed metadata is deleted. Never throws.
@@ -169,9 +60,7 @@ export async function collectUnsubscribeUrls(
feedId: string,
): Promise<string[]> {
try {
const metadata = (await emailStorage.get(`feed:${feedId}:metadata`, {
type: "json",
})) as FeedMetadata | null;
const metadata = await new FeedRepository(emailStorage).getMetadata(feedId);
return Object.values(metadata?.unsubscribe ?? {});
} catch (error) {
logger.error("Error reading unsubscribe URLs", {
@@ -192,24 +81,18 @@ export async function purgeFeedKeysStep(
cursor: string;
listComplete: boolean;
}> {
const prefix = `feed:${feedId}:`;
const limit = Math.min(1000, Math.max(1, Math.floor(options.limit || 100)));
const cursor = options.cursor || undefined;
const listed = await emailStorage.list({ prefix, cursor, limit });
const keys = (listed.keys || []).map((k) => k.name);
const repo = new FeedRepository(emailStorage);
const listed = await repo.listFeedKeys(feedId, {
cursor: options.cursor,
limit: options.limit,
});
const keys = listed.names;
if (options.bucket && keys.length > 0) {
const emailKeys = keys.filter((k) => {
const suffix = k.slice(prefix.length);
return suffix !== "config" && suffix !== "metadata";
});
const emailKeys = keys.filter((k) => repo.isEmailKey(feedId, k));
if (emailKeys.length > 0) {
const emailDataResults = await Promise.allSettled(
emailKeys.map(
(k) =>
emailStorage.get(k, { type: "json" }) as Promise<EmailData | null>,
),
emailKeys.map((k) => repo.getEmail(k)),
);
const attachmentIds = emailDataResults
.filter(
@@ -234,8 +117,8 @@ export async function purgeFeedKeysStep(
return {
deletedKeys: ok,
failedKeys: failed,
cursor: listed.cursor || "",
listComplete: !!listed.list_complete,
cursor: listed.cursor,
listComplete: listed.listComplete,
};
}
+15 -17
View File
@@ -8,12 +8,8 @@ import {
updateFeedRecord,
deleteFeedRecord,
} from "../../lib/feed-service";
import { listAllFeeds, deleteAttachmentsForEmails } from "../admin/helpers";
import {
getFeedConfig,
getFeedMetadata,
getEmailData,
} from "../../utils/storage";
import { deleteAttachmentsForEmails } from "../admin/helpers";
import { FeedRepository } from "../../domain/feed-repository";
import { getStats } from "../../utils/stats";
import { feedEmailAddress, feedRssUrl, feedAtomUrl } from "../../utils/urls";
import {
@@ -107,7 +103,7 @@ apiApp.openapi(
}),
async (c) => {
const env = c.env;
const feeds = await listAllFeeds(env.EMAIL_STORAGE);
const feeds = await FeedRepository.from(env).listFeeds();
return c.json(
{
feeds: feeds.map((f) => ({
@@ -173,9 +169,10 @@ apiApp.openapi(
async (c) => {
const env = c.env;
const { feedId } = c.req.valid("param");
const config = await getFeedConfig(env.EMAIL_STORAGE, feedId);
const repo = FeedRepository.from(env);
const config = await repo.getConfig(feedId);
if (!config) return c.json({ error: "Feed not found" }, 404);
const metadata = await getFeedMetadata(env.EMAIL_STORAGE, feedId);
const metadata = await repo.getMetadata(feedId);
return c.json(
toFeed(feedId, config, metadata?.emails.length ?? 0, env),
200,
@@ -218,7 +215,7 @@ apiApp.openapi(
return c.json({ error: "Feed not found" }, 404);
if (result.status === "expired")
return c.json({ error: "Feed has expired and cannot be modified" }, 409);
const metadata = await getFeedMetadata(env.EMAIL_STORAGE, feedId);
const metadata = await FeedRepository.from(env).getMetadata(feedId);
return c.json(
toFeed(feedId, result.config, metadata?.emails.length ?? 0, env),
200,
@@ -268,7 +265,7 @@ apiApp.openapi(
async (c) => {
const env = c.env;
const { feedId } = c.req.valid("param");
const metadata = await getFeedMetadata(env.EMAIL_STORAGE, feedId);
const metadata = await FeedRepository.from(env).getMetadata(feedId);
if (!metadata) return c.json({ error: "Feed not found" }, 404);
return c.json(
{
@@ -303,10 +300,11 @@ apiApp.openapi(
const env = c.env;
const { feedId, entryId } = c.req.valid("param");
const receivedAt = parseInt(entryId, 10);
const metadata = await getFeedMetadata(env.EMAIL_STORAGE, feedId);
const repo = FeedRepository.from(env);
const metadata = await repo.getMetadata(feedId);
const metaEntry = metadata?.emails.find((e) => e.receivedAt === receivedAt);
if (!metaEntry) return c.json({ error: "Email not found" }, 404);
const data = await getEmailData(env.EMAIL_STORAGE, metaEntry.key);
const data = await repo.getEmail(metaEntry.key);
if (!data) return c.json({ error: "Email not found" }, 404);
return c.json(
{
@@ -344,18 +342,18 @@ apiApp.openapi(
}),
async (c) => {
const env = c.env;
const emailStorage = env.EMAIL_STORAGE;
const repo = FeedRepository.from(env);
const { feedId, entryId } = c.req.valid("param");
const receivedAt = parseInt(entryId, 10);
const metadata = await getFeedMetadata(emailStorage, feedId);
const metadata = await repo.getMetadata(feedId);
const metaEntry = metadata?.emails.find((e) => e.receivedAt === receivedAt);
if (!metadata || !metaEntry)
return c.json({ error: "Email not found" }, 404);
await emailStorage.delete(metaEntry.key);
await repo.deleteEmail(metaEntry.key);
await deleteAttachmentsForEmails(env, metadata.emails, [metaEntry.key]);
metadata.emails = metadata.emails.filter((e) => e.key !== metaEntry.key);
await emailStorage.put(`feed:${feedId}:metadata`, JSON.stringify(metadata));
await repo.putMetadata(feedId, metadata);
return c.json({ ok: true }, 200);
},
+6 -14
View File
@@ -1,8 +1,9 @@
import { Context } from "hono";
import { html, raw } from "hono/html";
import { Env, FeedConfig, FeedMetadata, EmailData } from "../types";
import { Env } from "../types";
import { processEmailContent } from "../utils/html-processor";
import { formatBytes } from "../utils/format";
import { FeedRepository } from "../domain/feed-repository";
export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
const feedId = c.req.param("feedId");
@@ -12,17 +13,11 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
return new Response("Not Found", { status: 404 });
}
const emailStorage = c.env.EMAIL_STORAGE;
const repo = FeedRepository.from(c.env);
const [feedMetadata, feedConfig] = await Promise.all([
emailStorage.get(
`feed:${feedId}:metadata`,
"json",
) as Promise<FeedMetadata | null>,
emailStorage.get(
`feed:${feedId}:config`,
"json",
) as Promise<FeedConfig | null>,
repo.getMetadata(feedId),
repo.getConfig(feedId),
]);
if (!feedMetadata) {
return new Response("Feed not found", { status: 404 });
@@ -41,10 +36,7 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
return new Response("Entry not found", { status: 404 });
}
const emailData = (await emailStorage.get(
metaEntry.key,
"json",
)) as EmailData | null;
const emailData = await repo.getEmail(metaEntry.key);
if (!emailData) {
return new Response("Entry not found", { status: 404 });
}
+2 -1
View File
@@ -1,9 +1,10 @@
import { describe, it, expect } from "vitest";
import worker from "../index";
import { createMockEnv } from "../test/setup";
import { iconKey } from "../utils/storage";
import type { Env } from "../types";
const iconKey = (domain: string) => `icon:${domain}`;
function req(path: string): Request {
return new Request(`https://test.getmynews.app${path}`);
}
+2 -2
View File
@@ -1,6 +1,6 @@
import { Context } from "hono";
import { Env } from "../types";
import { getFeedMetadata } from "../utils/storage";
import { FeedRepository } from "../domain/feed-repository";
import { cacheFaviconForDomain, getCachedIcon } from "../utils/favicon-fetcher";
export const FAVICON_PATH = "/favicon.svg";
@@ -40,7 +40,7 @@ export async function handleFeedFavicon(
const feedId = c.req.param("feedId");
if (!feedId) return projectFavicon();
const metadata = await getFeedMetadata(env.EMAIL_STORAGE, feedId);
const metadata = await FeedRepository.from(env).getMetadata(feedId);
const domain = metadata?.iconDomain;
if (!domain) return projectFavicon();
+2 -4
View File
@@ -7,6 +7,7 @@ import {
import { waitUntilSafe } from "../utils/worker";
import { DEFAULT_LEASE_SECONDS, MAX_LEASE_SECONDS } from "../config/constants";
import { feedTopicPattern } from "../utils/urls";
import { FeedRepository } from "../domain/feed-repository";
type AppEnv = { Bindings: Env };
@@ -73,10 +74,7 @@ hubRouter.post("/", async (c) => {
const feedId = match[2];
// Verify the feed exists before accepting any subscription
const feedConfig = await env.EMAIL_STORAGE.get(
`feed:${feedId}:config`,
"json",
);
const feedConfig = await FeedRepository.from(env).getConfig(feedId);
if (!feedConfig) {
return c.text("Not Found: feed does not exist", 404);
}
+2 -1
View File
@@ -6,8 +6,9 @@ import {
extractEmailDomain,
getCachedIcon,
} from "./favicon-fetcher";
import { iconKey } from "./storage";
import { MAX_ICON_BYTES } from "../config/constants";
const iconKey = (domain: string) => `icon:${domain}`;
import type { Env } from "../types";
const PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 1, 2, 3, 4]);
+5 -10
View File
@@ -4,7 +4,7 @@ import {
ICON_TTL_SECONDS,
MAX_ICON_BYTES,
} from "../config/constants";
import { iconKey } from "./storage";
import { FeedRepository } from "../domain/feed-repository";
import { logger } from "../lib/logger";
interface IconRecord {
@@ -92,8 +92,8 @@ export async function cacheFaviconForDomain(
env: Env,
): Promise<void> {
try {
const key = iconKey(domain);
const existing = await env.EMAIL_STORAGE.get(key, "text");
const repo = FeedRepository.from(env);
const existing = await repo.getIconText(domain);
if (existing !== null) return; // present (incl. negative) → nothing to do
const icon = await resolveIcon(domain);
@@ -104,9 +104,7 @@ export async function cacheFaviconForDomain(
}
: { data: null, contentType: "" };
await env.EMAIL_STORAGE.put(key, JSON.stringify(record), {
expirationTtl: ICON_TTL_SECONDS,
});
await repo.putIcon(domain, JSON.stringify(record), ICON_TTL_SECONDS);
} catch (error) {
logger.warn("Favicon cache failed", { domain, error: String(error) });
}
@@ -119,10 +117,7 @@ export async function getCachedIcon(
domain: string,
env: Env,
): Promise<{ bytes: ArrayBuffer; contentType: string } | null> {
const record = (await env.EMAIL_STORAGE.get(
iconKey(domain),
"json",
)) as IconRecord | null;
const record = await FeedRepository.from(env).getIconJson<IconRecord>(domain);
if (!record || record.data === null) return null;
return {
bytes: base64ToArrayBuffer(record.data),
+6 -12
View File
@@ -1,5 +1,6 @@
import { Env, FeedConfig, FeedMetadata, EmailData } from "../types";
import { Env, FeedConfig, EmailData } from "../types";
import { MAX_FEED_ITEMS } from "../config/constants";
import { FeedRepository } from "../domain/feed-repository";
export interface FeedData {
feedConfig: FeedConfig;
@@ -10,19 +11,12 @@ export async function fetchFeedData(
feedId: string,
env: Env,
): Promise<FeedData | null> {
const storage = env.EMAIL_STORAGE;
const feedMetadata = (await storage.get(
`feed:${feedId}:metadata`,
"json",
)) as FeedMetadata | null;
const repo = FeedRepository.from(env);
const feedMetadata = await repo.getMetadata(feedId);
if (!feedMetadata) return null;
const feedConfig = ((await storage.get(
`feed:${feedId}:config`,
"json",
)) as FeedConfig | null) ?? {
const feedConfig = (await repo.getConfig(feedId)) ?? {
title: `Newsletter Feed ${feedId}`,
description: "Converted email newsletter",
language: "en",
@@ -32,7 +26,7 @@ export async function fetchFeedData(
const emailRefs = feedMetadata.emails.slice(0, MAX_FEED_ITEMS);
const emails: EmailData[] = [];
for (const ref of emailRefs) {
const data = (await storage.get(ref.key, "json")) as EmailData | null;
const data = await repo.getEmail(ref.key);
if (data) emails.push(data);
}
+12 -26
View File
@@ -1,8 +1,6 @@
import { Counters, Env, StatsResponse } from "../types";
import { STATS_KEY } from "../config/constants";
import { logger } from "../lib/logger";
import { listAllFeeds } from "../routes/admin/helpers";
import { getFeedMetadata } from "./storage";
import { FeedRepository } from "../domain/feed-repository";
import { getAttachmentBucket } from "./attachments";
const EMPTY_COUNTERS: Counters = {
@@ -15,9 +13,7 @@ const EMPTY_COUNTERS: Counters = {
export async function getCounters(kv: KVNamespace): Promise<Counters> {
try {
const stored = (await kv.get(STATS_KEY, {
type: "json",
})) as Counters | null;
const stored = await new FeedRepository(kv).getCountersRaw();
return { ...EMPTY_COUNTERS, ...(stored || {}) };
} catch (error) {
logger.error("Error reading counters", { error: String(error) });
@@ -48,7 +44,7 @@ export async function bumpCounters(
current.last_feed_created_at = changes.last_feed_created_at;
if (!current.first_seen) current.first_seen = new Date().toISOString();
await kv.put(STATS_KEY, JSON.stringify(current));
await new FeedRepository(kv).putCounters(current);
} catch (error) {
logger.error("Error updating counters", { error: String(error) });
}
@@ -58,26 +54,15 @@ export async function countKeysByPrefix(
kv: KVNamespace,
prefix: string,
): Promise<number> {
let total = 0;
let cursor: string | undefined;
try {
do {
const listed = await kv.list({ prefix, cursor, limit: 1000 });
total += listed.keys.length;
cursor = listed.list_complete ? undefined : listed.cursor;
} while (cursor);
} catch (error) {
logger.error("Error counting keys", { prefix, error: String(error) });
}
return total;
return new FeedRepository(kv).countKeysByPrefix(prefix);
}
export async function getStats(env: Env): Promise<StatsResponse> {
const kv = env.EMAIL_STORAGE;
const repo = FeedRepository.from(env);
const [counters, feeds, websubCount] = await Promise.all([
getCounters(kv),
listAllFeeds(kv),
countKeysByPrefix(kv, "websub:"),
getCounters(env.EMAIL_STORAGE),
repo.listFeeds(),
repo.countSubscriptionKeys(),
]);
return {
@@ -119,9 +104,10 @@ export async function scanR2Usage(
export async function scanKvUsage(kv: KVNamespace): Promise<{ bytes: number }> {
let bytes = 0;
try {
const feeds = await listAllFeeds(kv);
const repo = new FeedRepository(kv);
const feeds = await repo.listFeeds();
for (const feed of feeds) {
const metadata = await getFeedMetadata(kv, feed.id);
const metadata = await repo.getMetadata(feed.id);
if (!metadata) continue;
for (const email of metadata.emails) {
bytes += email.size ?? 0;
@@ -152,7 +138,7 @@ export async function setStorageSnapshot(
current.kv_bytes_estimated = snapshot.kv_bytes_estimated;
current.storage_scanned_at = new Date().toISOString();
if (!current.first_seen) current.first_seen = new Date().toISOString();
await kv.put(STATS_KEY, JSON.stringify(current));
await new FeedRepository(kv).putCounters(current);
} catch (error) {
logger.error("Error writing storage snapshot", { error: String(error) });
}
-169
View File
@@ -1,169 +0,0 @@
import {
EmailData,
FeedConfig,
FeedMetadata,
FeedList,
EmailMetadata,
} from "../types";
import { MAX_METADATA_EMAILS } from "../config/constants";
/**
* KV key for a domain's cached favicon (shared across feeds from the same sender).
*/
export function iconKey(domain: string): string {
return `icon:${domain}`;
}
/**
* Store email data in KV
*/
export async function storeEmail(
kv: KVNamespace,
feedId: string,
emailData: EmailData,
): Promise<string> {
// Generate a unique key for this email
const timestamp = Date.now();
const key = `feed:${feedId}:email:${timestamp}`;
// Store the email content
await kv.put(key, JSON.stringify(emailData));
// Update the feed's metadata (list of emails)
await updateFeedMetadata(kv, feedId, {
key,
subject: emailData.subject,
receivedAt: timestamp,
});
return key;
}
/**
* Update feed metadata with a new email
*/
async function updateFeedMetadata(
kv: KVNamespace,
feedId: string,
emailMetadata: EmailMetadata,
): Promise<void> {
const feedMetadataKey = `feed:${feedId}:metadata`;
const existingMetadata = (await kv.get(feedMetadataKey, {
type: "json",
})) as FeedMetadata | null;
const metadata: FeedMetadata = existingMetadata || { emails: [] };
// Add new email to the beginning of the list
metadata.emails.unshift(emailMetadata);
// Keep only the last MAX_METADATA_EMAILS in the metadata; delete orphaned KV entries
const toDelete =
metadata.emails.length > MAX_METADATA_EMAILS
? metadata.emails.slice(MAX_METADATA_EMAILS)
: [];
if (toDelete.length > 0) {
metadata.emails = metadata.emails.slice(0, MAX_METADATA_EMAILS);
}
await Promise.all([
kv.put(feedMetadataKey, JSON.stringify(metadata)),
...toDelete.map((e) => kv.delete(e.key)),
]);
}
/**
* Get feed metadata
*/
export async function getFeedMetadata(
kv: KVNamespace,
feedId: string,
): Promise<FeedMetadata | null> {
const feedMetadataKey = `feed:${feedId}:metadata`;
return (await kv.get(feedMetadataKey, {
type: "json",
})) as FeedMetadata | null;
}
/**
* Get feed configuration
*/
export async function getFeedConfig(
kv: KVNamespace,
feedId: string,
): Promise<FeedConfig | null> {
const feedConfigKey = `feed:${feedId}:config`;
return (await kv.get(feedConfigKey, { type: "json" })) as FeedConfig | null;
}
/**
* Get email data
*/
export async function getEmailData(
kv: KVNamespace,
key: string,
): Promise<EmailData | null> {
return (await kv.get(key, { type: "json" })) as EmailData | null;
}
/**
* Create a new feed
*/
export async function createFeed(
kv: KVNamespace,
feedId: string,
feedConfig: FeedConfig,
): Promise<void> {
// Store feed configuration
const feedConfigKey = `feed:${feedId}:config`;
await kv.put(feedConfigKey, JSON.stringify(feedConfig));
// Create empty metadata for the feed
const feedMetadataKey = `feed:${feedId}:metadata`;
await kv.put(
feedMetadataKey,
JSON.stringify({
emails: [],
}),
);
// Add feed to the list of all feeds
await addFeedToList(kv, feedId, feedConfig.title, feedConfig.description);
}
/**
* Add a feed to the global list
*/
export async function addFeedToList(
kv: KVNamespace,
feedId: string,
title: string,
description?: string,
): Promise<void> {
const feedListKey = "feeds:list";
const existingList = (await kv.get(feedListKey, {
type: "json",
})) as FeedList | null;
const feedList: FeedList = existingList || { feeds: [] };
feedList.feeds.push({
id: feedId,
title,
description,
});
await kv.put(feedListKey, JSON.stringify(feedList));
}
/**
* Get all feeds
*/
export async function getAllFeeds(kv: KVNamespace): Promise<FeedList> {
const feedListKey = "feeds:list";
const feedList = (await kv.get(feedListKey, {
type: "json",
})) as FeedList | null;
return feedList || { feeds: [] };
}
+6 -3
View File
@@ -8,7 +8,6 @@ import {
notifySubscribers,
verifyAndStoreSubscription,
verifyAndDeleteSubscription,
subscriptionKey,
} from "./websub";
import type { Env, WebSubSubscription } from "../types";
@@ -51,8 +50,12 @@ describe("getSubscriptions / saveSubscriptions", () => {
expect(await getSubscriptions("feed1", env)).toEqual(subs);
});
it("uses the correct KV key", () => {
expect(subscriptionKey("abc")).toBe("websub:subs:abc");
it("uses the correct KV key", async () => {
const env = mockEnv();
await saveSubscriptions("abc", [], env);
expect(
await env.EMAIL_STORAGE.get("websub:subs:abc", { type: "json" }),
).toEqual([]);
});
});
+10 -30
View File
@@ -1,25 +1,13 @@
import {
Env,
FeedConfig,
FeedMetadata,
EmailData,
WebSubSubscription,
} from "../types";
import { Env, FeedConfig, EmailData, WebSubSubscription } from "../types";
import { generateRssFeed, generateAtomFeed } from "./feed-generator";
import { baseUrl, feedRssUrl, feedAtomUrl, feedUrl } from "./urls";
const KV_PREFIX = "websub:subs:";
export function subscriptionKey(feedId: string): string {
return `${KV_PREFIX}${feedId}`;
}
import { FeedRepository } from "../domain/feed-repository";
export async function getSubscriptions(
feedId: string,
env: Env,
): Promise<WebSubSubscription[]> {
const raw = await env.EMAIL_STORAGE.get(subscriptionKey(feedId), "json");
return (raw as WebSubSubscription[] | null) ?? [];
return FeedRepository.from(env).getSubscriptions(feedId);
}
export async function saveSubscriptions(
@@ -27,10 +15,7 @@ export async function saveSubscriptions(
subscriptions: WebSubSubscription[],
env: Env,
): Promise<void> {
await env.EMAIL_STORAGE.put(
subscriptionKey(feedId),
JSON.stringify(subscriptions),
);
await FeedRepository.from(env).saveSubscriptions(feedId, subscriptions);
}
export async function buildHmacSignature(
@@ -60,16 +45,16 @@ async function buildFeedXml(
env: Env,
format: "rss" | "atom" = "rss",
): Promise<string | null> {
const [rawMetadata, rawConfig] = await Promise.all([
env.EMAIL_STORAGE.get(`feed:${feedId}:metadata`, "json"),
env.EMAIL_STORAGE.get(`feed:${feedId}:config`, "json"),
const repo = FeedRepository.from(env);
const [feedMetadata, rawConfig] = await Promise.all([
repo.getMetadata(feedId),
repo.getConfig(feedId),
]);
const feedMetadata = rawMetadata as FeedMetadata | null;
if (!feedMetadata) return null;
const base = baseUrl(env);
const feedConfig = (rawConfig as FeedConfig | null) ?? {
const feedConfig: FeedConfig = rawConfig ?? {
title: `Newsletter Feed ${feedId}`,
description: "Converted email newsletter",
language: "en",
@@ -78,12 +63,7 @@ async function buildFeedXml(
const emails = feedMetadata.emails.slice(0, 20);
const emailsData = (
await Promise.all(
emails.map(
(m) =>
env.EMAIL_STORAGE.get(m.key, "json") as Promise<EmailData | null>,
),
)
await Promise.all(emails.map((m) => repo.getEmail(m.key)))
).filter((d): d is EmailData => d !== null);
if (format === "atom") {