mirror of
https://github.com/juherr/kill-the-news.git
synced 2026-06-20 22:03:48 +00:00
refactor(domain): introduce FeedRepository as the single KV access layer
Centralise the KV key schema and all get/put access behind a FeedRepository class under src/domain/. Every feed/email/list/icon/websub/counter key was previously inlined across ~12 modules with two divergent storeEmail and addFeedToList implementations; the dead src/utils/storage.ts write path is removed and the email key convention unified on feed:<id>:<ts>. Behaviour-preserving: existing tests pass unchanged in logic, plus a new feed-repository.test.ts covering CRUD, key builders, list ops and counters. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,171 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { createMockEnv } from "../test/setup";
|
||||
import { FeedRepository } from "./feed-repository";
|
||||
import type {
|
||||
Env,
|
||||
FeedConfig,
|
||||
FeedMetadata,
|
||||
EmailData,
|
||||
WebSubSubscription,
|
||||
} from "../types";
|
||||
|
||||
const mockEnv = () => createMockEnv() as unknown as Env;
|
||||
|
||||
const sampleConfig = (overrides: Partial<FeedConfig> = {}): FeedConfig => ({
|
||||
title: "Test Feed",
|
||||
language: "en",
|
||||
created_at: 1000,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const sampleEmail = (overrides: Partial<EmailData> = {}): EmailData => ({
|
||||
subject: "Hello",
|
||||
from: "news@example.com",
|
||||
content: "<p>hi</p>",
|
||||
receivedAt: 1234,
|
||||
headers: {},
|
||||
...overrides,
|
||||
});
|
||||
|
||||
describe("FeedRepository key schema", () => {
|
||||
it("builds the canonical KV keys via the public API", () => {
|
||||
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
|
||||
expect(repo.feedKeyPrefix("a.b.42")).toBe("feed:a.b.42:");
|
||||
expect(repo.iconKey("example.com")).toBe("icon:example.com");
|
||||
expect(repo.newEmailKey("a.b.42")).toMatch(/^feed:a\.b\.42:\d+$/);
|
||||
});
|
||||
|
||||
it("recognises email keys vs config/metadata keys", () => {
|
||||
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
|
||||
expect(repo.isEmailKey("a.b.42", "feed:a.b.42:config")).toBe(false);
|
||||
expect(repo.isEmailKey("a.b.42", "feed:a.b.42:metadata")).toBe(false);
|
||||
expect(repo.isEmailKey("a.b.42", "feed:a.b.42:1700000000000")).toBe(true);
|
||||
});
|
||||
|
||||
it("recovers the feed id from an email key", () => {
|
||||
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
|
||||
expect(repo.feedIdFromEmailKey("feed:a.b.42:1700000000000")).toBe("a.b.42");
|
||||
});
|
||||
});
|
||||
|
||||
describe("FeedRepository config & metadata", () => {
|
||||
it("round-trips and deletes a feed config", async () => {
|
||||
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
|
||||
expect(await repo.getConfig("a.b.42")).toBeNull();
|
||||
await repo.putConfig("a.b.42", sampleConfig());
|
||||
expect(await repo.getConfig("a.b.42")).toMatchObject({
|
||||
title: "Test Feed",
|
||||
});
|
||||
await repo.deleteConfig("a.b.42");
|
||||
expect(await repo.getConfig("a.b.42")).toBeNull();
|
||||
});
|
||||
|
||||
it("round-trips and deletes feed metadata", async () => {
|
||||
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
|
||||
const meta: FeedMetadata = { emails: [] };
|
||||
await repo.putMetadata("a.b.42", meta);
|
||||
expect(await repo.getMetadata("a.b.42")).toEqual(meta);
|
||||
await repo.deleteMetadata("a.b.42");
|
||||
expect(await repo.getMetadata("a.b.42")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("FeedRepository emails", () => {
|
||||
it("stores and reads an email under a minted key", async () => {
|
||||
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
|
||||
const key = repo.newEmailKey("a.b.42");
|
||||
await repo.putEmail(key, sampleEmail());
|
||||
expect(await repo.getEmail(key)).toMatchObject({ subject: "Hello" });
|
||||
await repo.deleteEmail(key);
|
||||
expect(await repo.getEmail(key)).toBeNull();
|
||||
});
|
||||
|
||||
it("lists every key under a feed prefix", async () => {
|
||||
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
|
||||
await repo.putConfig("a.b.42", sampleConfig());
|
||||
await repo.putMetadata("a.b.42", { emails: [] });
|
||||
const emailKey = repo.newEmailKey("a.b.42");
|
||||
await repo.putEmail(emailKey, sampleEmail());
|
||||
|
||||
const listed = await repo.listFeedKeys("a.b.42");
|
||||
expect(listed.names).toContain("feed:a.b.42:config");
|
||||
expect(listed.names).toContain("feed:a.b.42:metadata");
|
||||
expect(listed.names).toContain(emailKey);
|
||||
expect(listed.names.filter((k) => repo.isEmailKey("a.b.42", k))).toEqual([
|
||||
emailKey,
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("FeedRepository feed list", () => {
|
||||
it("adds, updates, lists and removes feeds with expiry", async () => {
|
||||
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
|
||||
await repo.addToList("a.b.42", "One", "desc", 5000);
|
||||
await repo.addToList("c.d.99", "Two");
|
||||
|
||||
let feeds = await repo.listFeeds();
|
||||
expect(feeds).toHaveLength(2);
|
||||
expect(feeds.find((f) => f.id === "a.b.42")).toMatchObject({
|
||||
title: "One",
|
||||
expires_at: 5000,
|
||||
});
|
||||
|
||||
await repo.updateInList("a.b.42", "One-updated", undefined, undefined);
|
||||
feeds = await repo.listFeeds();
|
||||
const updated = feeds.find((f) => f.id === "a.b.42");
|
||||
expect(updated).toMatchObject({ title: "One-updated" });
|
||||
expect(updated?.expires_at).toBeUndefined();
|
||||
|
||||
expect(await repo.removeFromList("a.b.42")).toBe(true);
|
||||
expect(await repo.removeFromList("missing")).toBe(false);
|
||||
feeds = await repo.listFeeds();
|
||||
expect(feeds.map((f) => f.id)).toEqual(["c.d.99"]);
|
||||
});
|
||||
|
||||
it("bulk-removes only the matching ids", async () => {
|
||||
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
|
||||
await repo.addToList("a.b.42", "One");
|
||||
await repo.addToList("c.d.99", "Two");
|
||||
await repo.addToList("e.f.10", "Three");
|
||||
|
||||
const removed = await repo.removeFromListBulk(["a.b.42", "e.f.10", "nope"]);
|
||||
expect(removed.sort()).toEqual(["a.b.42", "e.f.10"]);
|
||||
expect((await repo.listFeeds()).map((f) => f.id)).toEqual(["c.d.99"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("FeedRepository counters, icons, websub", () => {
|
||||
it("round-trips raw counters", async () => {
|
||||
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
|
||||
expect(await repo.getCountersRaw()).toBeNull();
|
||||
await repo.putCounters({
|
||||
feeds_created: 1,
|
||||
feeds_deleted: 0,
|
||||
emails_received: 2,
|
||||
emails_rejected: 0,
|
||||
unsubscribes_sent: 0,
|
||||
});
|
||||
expect(await repo.getCountersRaw()).toMatchObject({ emails_received: 2 });
|
||||
});
|
||||
|
||||
it("stores and reads favicons as text or json", async () => {
|
||||
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
|
||||
expect(await repo.getIconText("example.com")).toBeNull();
|
||||
await repo.putIcon("example.com", JSON.stringify({ data: null }), 60);
|
||||
expect(await repo.getIconText("example.com")).toBe('{"data":null}');
|
||||
expect(await repo.getIconJson<{ data: null }>("example.com")).toEqual({
|
||||
data: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("round-trips websub subscriptions and counts them", async () => {
|
||||
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
|
||||
expect(await repo.getSubscriptions("a.b.42")).toEqual([]);
|
||||
const subs: WebSubSubscription[] = [
|
||||
{ callbackUrl: "https://r.example/cb", expiresAt: 9999 },
|
||||
];
|
||||
await repo.saveSubscriptions("a.b.42", subs);
|
||||
expect(await repo.getSubscriptions("a.b.42")).toEqual(subs);
|
||||
expect(await repo.countSubscriptionKeys()).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,296 @@
|
||||
import {
|
||||
Counters,
|
||||
EmailData,
|
||||
Env,
|
||||
FeedConfig,
|
||||
FeedList,
|
||||
FeedListItem,
|
||||
FeedMetadata,
|
||||
WebSubSubscription,
|
||||
} from "../types";
|
||||
import { FEEDS_LIST_KEY, STATS_KEY } from "../config/constants";
|
||||
import { logger } from "../lib/logger";
|
||||
|
||||
const WEBSUB_PREFIX = "websub:subs:";
|
||||
|
||||
/**
|
||||
* Single source of truth for the KV key schema and all KV access. No other
|
||||
* module should build a `feed:`/`feeds:list`/`websub:`/`icon:`/`stats:counters`
|
||||
* key string — go through a repository method instead.
|
||||
*
|
||||
* Wraps one `KVNamespace`; construct per request via `FeedRepository.from(env)`.
|
||||
*/
|
||||
export class FeedRepository {
|
||||
constructor(private readonly kv: KVNamespace) {}
|
||||
|
||||
static from(env: Env): FeedRepository {
|
||||
return new FeedRepository(env.EMAIL_STORAGE);
|
||||
}
|
||||
|
||||
// ── Key schema ────────────────────────────────────────────────────────────
|
||||
|
||||
private configKey(feedId: string): string {
|
||||
return `feed:${feedId}:config`;
|
||||
}
|
||||
|
||||
private metadataKey(feedId: string): string {
|
||||
return `feed:${feedId}:metadata`;
|
||||
}
|
||||
|
||||
/** KV key for a domain's cached favicon (shared across feeds). */
|
||||
iconKey(domain: string): string {
|
||||
return `icon:${domain}`;
|
||||
}
|
||||
|
||||
private websubKey(feedId: string): string {
|
||||
return `${WEBSUB_PREFIX}${feedId}`;
|
||||
}
|
||||
|
||||
/** Prefix covering every key owned by a feed (config, metadata, emails). */
|
||||
feedKeyPrefix(feedId: string): string {
|
||||
return `feed:${feedId}:`;
|
||||
}
|
||||
|
||||
/** Mint a fresh, time-ordered email key. Call once and reuse the result. */
|
||||
newEmailKey(feedId: string): string {
|
||||
return `feed:${feedId}:${Date.now()}`;
|
||||
}
|
||||
|
||||
/** True when `key` is an email entry (not the feed's config/metadata key). */
|
||||
isEmailKey(feedId: string, key: string): boolean {
|
||||
const suffix = key.slice(this.feedKeyPrefix(feedId).length);
|
||||
return suffix !== "config" && suffix !== "metadata";
|
||||
}
|
||||
|
||||
/** Recover the feed id embedded in an email key (`feed:<id>:<ts>`). */
|
||||
feedIdFromEmailKey(key: string): string {
|
||||
return key.split(":")[1];
|
||||
}
|
||||
|
||||
// ── Feed config ───────────────────────────────────────────────────────────
|
||||
|
||||
async getConfig(feedId: string): Promise<FeedConfig | null> {
|
||||
return (await this.kv.get(this.configKey(feedId), {
|
||||
type: "json",
|
||||
})) as FeedConfig | null;
|
||||
}
|
||||
|
||||
async putConfig(feedId: string, config: FeedConfig): Promise<void> {
|
||||
await this.kv.put(this.configKey(feedId), JSON.stringify(config));
|
||||
}
|
||||
|
||||
async deleteConfig(feedId: string): Promise<void> {
|
||||
await this.kv.delete(this.configKey(feedId));
|
||||
}
|
||||
|
||||
// ── Feed metadata ─────────────────────────────────────────────────────────
|
||||
|
||||
async getMetadata(feedId: string): Promise<FeedMetadata | null> {
|
||||
return (await this.kv.get(this.metadataKey(feedId), {
|
||||
type: "json",
|
||||
})) as FeedMetadata | null;
|
||||
}
|
||||
|
||||
async putMetadata(feedId: string, metadata: FeedMetadata): Promise<void> {
|
||||
await this.kv.put(this.metadataKey(feedId), JSON.stringify(metadata));
|
||||
}
|
||||
|
||||
async deleteMetadata(feedId: string): Promise<void> {
|
||||
await this.kv.delete(this.metadataKey(feedId));
|
||||
}
|
||||
|
||||
// ── Emails ────────────────────────────────────────────────────────────────
|
||||
|
||||
async putEmail(key: string, data: EmailData): Promise<void> {
|
||||
await this.kv.put(key, JSON.stringify(data));
|
||||
}
|
||||
|
||||
async getEmail(key: string): Promise<EmailData | null> {
|
||||
return (await this.kv.get(key, { type: "json" })) as EmailData | null;
|
||||
}
|
||||
|
||||
async deleteEmail(key: string): Promise<void> {
|
||||
await this.kv.delete(key);
|
||||
}
|
||||
|
||||
// ── Global feed list ──────────────────────────────────────────────────────
|
||||
|
||||
async listFeeds(): Promise<FeedListItem[]> {
|
||||
try {
|
||||
const feedList = (await this.kv.get(FEEDS_LIST_KEY, {
|
||||
type: "json",
|
||||
})) as FeedList | null;
|
||||
return feedList?.feeds || [];
|
||||
} catch (error) {
|
||||
logger.error("Error listing feeds", { error: String(error) });
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async addToList(
|
||||
feedId: string,
|
||||
title: string,
|
||||
description?: string,
|
||||
expires_at?: number,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const feedList = ((await this.kv.get(FEEDS_LIST_KEY, {
|
||||
type: "json",
|
||||
})) as FeedList | null) || { feeds: [] };
|
||||
|
||||
feedList.feeds.push({ id: feedId, title, description, expires_at });
|
||||
await this.kv.put(FEEDS_LIST_KEY, JSON.stringify(feedList));
|
||||
} catch (error) {
|
||||
logger.error("Error adding feed to list", {
|
||||
feedId,
|
||||
error: String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async updateInList(
|
||||
feedId: string,
|
||||
title: string,
|
||||
description?: string,
|
||||
expires_at?: number,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const feedList = ((await this.kv.get(FEEDS_LIST_KEY, {
|
||||
type: "json",
|
||||
})) as FeedList | null) || { feeds: [] };
|
||||
|
||||
const feedIndex = feedList.feeds.findIndex((feed) => feed.id === feedId);
|
||||
if (feedIndex !== -1) {
|
||||
feedList.feeds[feedIndex].title = title;
|
||||
feedList.feeds[feedIndex].description = description;
|
||||
feedList.feeds[feedIndex].expires_at = expires_at;
|
||||
await this.kv.put(FEEDS_LIST_KEY, JSON.stringify(feedList));
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error("Error updating feed in list", {
|
||||
feedId,
|
||||
error: String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async removeFromListBulk(feedIds: string[]): Promise<string[]> {
|
||||
try {
|
||||
const feedList = ((await this.kv.get(FEEDS_LIST_KEY, {
|
||||
type: "json",
|
||||
})) as FeedList | null) || { feeds: [] };
|
||||
|
||||
const toRemove = new Set(feedIds.filter(Boolean));
|
||||
if (toRemove.size === 0) return [];
|
||||
|
||||
const removed: string[] = [];
|
||||
const nextFeeds: FeedListItem[] = [];
|
||||
|
||||
for (const feed of feedList.feeds) {
|
||||
if (toRemove.has(feed.id)) {
|
||||
removed.push(feed.id);
|
||||
continue;
|
||||
}
|
||||
nextFeeds.push(feed);
|
||||
}
|
||||
|
||||
if (removed.length === 0) return [];
|
||||
|
||||
feedList.feeds = nextFeeds;
|
||||
await this.kv.put(FEEDS_LIST_KEY, JSON.stringify(feedList));
|
||||
return removed;
|
||||
} catch (error) {
|
||||
logger.error("Error removing feeds from list", { error: String(error) });
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async removeFromList(feedId: string): Promise<boolean> {
|
||||
const removed = await this.removeFromListBulk([feedId]);
|
||||
return removed.includes(feedId);
|
||||
}
|
||||
|
||||
// ── Key listing / counting ────────────────────────────────────────────────
|
||||
|
||||
async listFeedKeys(
|
||||
feedId: string,
|
||||
options: { cursor?: string; limit?: number } = {},
|
||||
): Promise<{ names: string[]; cursor: string; listComplete: boolean }> {
|
||||
const prefix = this.feedKeyPrefix(feedId);
|
||||
const limit = Math.min(1000, Math.max(1, Math.floor(options.limit || 100)));
|
||||
const cursor = options.cursor || undefined;
|
||||
|
||||
const listed = await this.kv.list({ prefix, cursor, limit });
|
||||
return {
|
||||
names: (listed.keys || []).map((k) => k.name),
|
||||
cursor: listed.cursor || "",
|
||||
listComplete: !!listed.list_complete,
|
||||
};
|
||||
}
|
||||
|
||||
async countKeysByPrefix(prefix: string): Promise<number> {
|
||||
let total = 0;
|
||||
let cursor: string | undefined;
|
||||
try {
|
||||
do {
|
||||
const listed = await this.kv.list({ prefix, cursor, limit: 1000 });
|
||||
total += listed.keys.length;
|
||||
cursor = listed.list_complete ? undefined : listed.cursor;
|
||||
} while (cursor);
|
||||
} catch (error) {
|
||||
logger.error("Error counting keys", { prefix, error: String(error) });
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
/** Number of feeds that currently hold at least one WebSub subscription. */
|
||||
countSubscriptionKeys(): Promise<number> {
|
||||
return this.countKeysByPrefix("websub:");
|
||||
}
|
||||
|
||||
// ── Monitoring counters ───────────────────────────────────────────────────
|
||||
|
||||
async getCountersRaw(): Promise<Counters | null> {
|
||||
return (await this.kv.get(STATS_KEY, { type: "json" })) as Counters | null;
|
||||
}
|
||||
|
||||
async putCounters(counters: Counters): Promise<void> {
|
||||
await this.kv.put(STATS_KEY, JSON.stringify(counters));
|
||||
}
|
||||
|
||||
// ── Favicons ──────────────────────────────────────────────────────────────
|
||||
|
||||
async getIconText(domain: string): Promise<string | null> {
|
||||
return this.kv.get(this.iconKey(domain), "text");
|
||||
}
|
||||
|
||||
async getIconJson<T>(domain: string): Promise<T | null> {
|
||||
return (await this.kv.get(this.iconKey(domain), {
|
||||
type: "json",
|
||||
})) as T | null;
|
||||
}
|
||||
|
||||
async putIcon(
|
||||
domain: string,
|
||||
value: string,
|
||||
ttlSeconds: number,
|
||||
): Promise<void> {
|
||||
await this.kv.put(this.iconKey(domain), value, {
|
||||
expirationTtl: ttlSeconds,
|
||||
});
|
||||
}
|
||||
|
||||
// ── WebSub subscriptions ──────────────────────────────────────────────────
|
||||
|
||||
async getSubscriptions(feedId: string): Promise<WebSubSubscription[]> {
|
||||
const raw = await this.kv.get(this.websubKey(feedId), "json");
|
||||
return (raw as WebSubSubscription[] | null) ?? [];
|
||||
}
|
||||
|
||||
async saveSubscriptions(
|
||||
feedId: string,
|
||||
subscriptions: WebSubSubscription[],
|
||||
): Promise<void> {
|
||||
await this.kv.put(this.websubKey(feedId), JSON.stringify(subscriptions));
|
||||
}
|
||||
}
|
||||
+5
-7
@@ -13,11 +13,8 @@ import { apiApp } from "./routes/api";
|
||||
import { handleCloudflareEmail } from "./lib/cloudflare-email";
|
||||
import { Env } from "./types";
|
||||
import { logger } from "./lib/logger";
|
||||
import {
|
||||
listAllFeeds,
|
||||
purgeExpiredFeeds,
|
||||
removeFeedsFromListBulk,
|
||||
} from "./routes/admin/helpers";
|
||||
import { FeedRepository } from "./domain/feed-repository";
|
||||
import { purgeExpiredFeeds } from "./routes/admin/helpers";
|
||||
import {
|
||||
bumpCounters,
|
||||
scanR2Usage,
|
||||
@@ -201,7 +198,8 @@ export default {
|
||||
},
|
||||
async scheduled(_event: ScheduledEvent, env: Env, _ctx: ExecutionContext) {
|
||||
const attachmentBucket = getAttachmentBucket(env);
|
||||
const feeds = await listAllFeeds(env.EMAIL_STORAGE);
|
||||
const repo = FeedRepository.from(env);
|
||||
const feeds = await repo.listFeeds();
|
||||
const now = Date.now();
|
||||
const expiredIds = feeds
|
||||
.filter((f) => f.expires_at !== undefined && f.expires_at <= now)
|
||||
@@ -211,7 +209,7 @@ export default {
|
||||
await purgeExpiredFeeds(env.EMAIL_STORAGE, feedId, attachmentBucket);
|
||||
}
|
||||
if (expiredIds.length > 0) {
|
||||
await removeFeedsFromListBulk(env.EMAIL_STORAGE, expiredIds);
|
||||
await repo.removeFromListBulk(expiredIds);
|
||||
await bumpCounters(env.EMAIL_STORAGE, {
|
||||
feeds_deleted: expiredIds.length,
|
||||
});
|
||||
|
||||
@@ -7,7 +7,8 @@ import {
|
||||
RawAttachment,
|
||||
} from "./email-processor";
|
||||
import { getCounters } from "../utils/stats";
|
||||
import { iconKey } from "../utils/storage";
|
||||
|
||||
const iconKey = (domain: string) => `icon:${domain}`;
|
||||
|
||||
const VALID_FEED_ID = "apple.mountain.42";
|
||||
const VALID_TO = `${VALID_FEED_ID}@test.getmynews.app`;
|
||||
|
||||
+10
-20
@@ -1,11 +1,5 @@
|
||||
import { EmailParser } from "../utils/email-parser";
|
||||
import {
|
||||
AttachmentData,
|
||||
EmailMetadata,
|
||||
Env,
|
||||
FeedConfig,
|
||||
FeedMetadata,
|
||||
} from "../types";
|
||||
import { AttachmentData, EmailMetadata, Env, FeedConfig } from "../types";
|
||||
import { notifySubscribers } from "../utils/websub";
|
||||
import { bumpCounters } from "../utils/stats";
|
||||
import {
|
||||
@@ -14,6 +8,7 @@ import {
|
||||
} from "../utils/favicon-fetcher";
|
||||
import { parseOneClickUnsubscribe } from "../utils/unsubscribe";
|
||||
import { getAttachmentBucket } from "../utils/attachments";
|
||||
import { FeedRepository } from "../domain/feed-repository";
|
||||
import { logger } from "./logger";
|
||||
import { FEED_MAX_BYTES } from "../config/constants";
|
||||
|
||||
@@ -110,10 +105,7 @@ export async function validateEmail(
|
||||
};
|
||||
}
|
||||
|
||||
const feedConfig = (await env.EMAIL_STORAGE.get(
|
||||
`feed:${feedId}:config`,
|
||||
"json",
|
||||
)) as FeedConfig | null;
|
||||
const feedConfig = await FeedRepository.from(env).getConfig(feedId);
|
||||
if (!feedConfig) {
|
||||
logger.error("Feed not found", { feedId });
|
||||
return {
|
||||
@@ -188,12 +180,12 @@ export async function storeEmail(
|
||||
...(storedAttachments.length > 0 ? { attachments: storedAttachments } : {}),
|
||||
};
|
||||
|
||||
const emailKey = `feed:${feedId}:${Date.now()}`;
|
||||
const feedMetadataKey = `feed:${feedId}:metadata`;
|
||||
const repo = FeedRepository.from(env);
|
||||
const emailKey = repo.newEmailKey(feedId);
|
||||
|
||||
const [, rawMetadata] = await Promise.all([
|
||||
env.EMAIL_STORAGE.put(emailKey, JSON.stringify(emailData)),
|
||||
env.EMAIL_STORAGE.get(feedMetadataKey, "json"),
|
||||
repo.putEmail(emailKey, emailData),
|
||||
repo.getMetadata(feedId),
|
||||
]);
|
||||
|
||||
// Note: KV has no atomic compare-and-swap. Concurrent invocations for the
|
||||
@@ -202,9 +194,7 @@ export async function storeEmail(
|
||||
// KV's eventual-consistency model.
|
||||
// TODO: Migrate feed metadata writes to Cloudflare Durable Objects to serialise
|
||||
// concurrent writes and eliminate this race condition.
|
||||
const feedMetadata = ((rawMetadata as FeedMetadata | null) || {
|
||||
emails: [],
|
||||
}) as FeedMetadata;
|
||||
const feedMetadata = rawMetadata || { emails: [] };
|
||||
|
||||
const maxBytes =
|
||||
parseInt(env.FEED_MAX_SIZE_BYTES ?? "", 10) || FEED_MAX_BYTES;
|
||||
@@ -260,8 +250,8 @@ export async function storeEmail(
|
||||
: [];
|
||||
|
||||
await Promise.all([
|
||||
env.EMAIL_STORAGE.put(feedMetadataKey, JSON.stringify(feedMetadata)),
|
||||
...toDelete.map((e) => env.EMAIL_STORAGE.delete(e.key)),
|
||||
repo.putMetadata(feedId, feedMetadata),
|
||||
...toDelete.map((e) => repo.deleteEmail(e.key)),
|
||||
...r2Deletions,
|
||||
]);
|
||||
|
||||
|
||||
+15
-32
@@ -5,10 +5,8 @@ import { bumpCounters } from "../utils/stats";
|
||||
import { waitUntilSafe } from "../utils/worker";
|
||||
import { sendUnsubscribes } from "../utils/unsubscribe";
|
||||
import { getAttachmentBucket } from "../utils/attachments";
|
||||
import { FeedRepository } from "../domain/feed-repository";
|
||||
import {
|
||||
addFeedToList,
|
||||
updateFeedInList,
|
||||
removeFeedFromList,
|
||||
purgeFeedKeysStep,
|
||||
collectUnsubscribeUrls,
|
||||
} from "../routes/admin/helpers";
|
||||
@@ -49,7 +47,7 @@ export async function createFeedRecord(
|
||||
env: Env,
|
||||
input: CreateFeedInput,
|
||||
): Promise<{ feedId: string; config: FeedConfig }> {
|
||||
const emailStorage = env.EMAIL_STORAGE;
|
||||
const repo = FeedRepository.from(env);
|
||||
const expiresAt = resolveExpiresAt(env, input.lifetimeHours);
|
||||
const feedId = generateFeedId();
|
||||
|
||||
@@ -67,19 +65,13 @@ export async function createFeedRecord(
|
||||
const metadata: FeedMetadata = { emails: [] };
|
||||
|
||||
await Promise.all([
|
||||
emailStorage.put(`feed:${feedId}:config`, JSON.stringify(config)),
|
||||
emailStorage.put(`feed:${feedId}:metadata`, JSON.stringify(metadata)),
|
||||
repo.putConfig(feedId, config),
|
||||
repo.putMetadata(feedId, metadata),
|
||||
]);
|
||||
|
||||
await addFeedToList(
|
||||
emailStorage,
|
||||
feedId,
|
||||
input.title,
|
||||
input.description,
|
||||
expiresAt,
|
||||
);
|
||||
await repo.addToList(feedId, input.title, input.description, expiresAt);
|
||||
|
||||
await bumpCounters(emailStorage, {
|
||||
await bumpCounters(env.EMAIL_STORAGE, {
|
||||
feeds_created: 1,
|
||||
last_feed_created_at: new Date().toISOString(),
|
||||
});
|
||||
@@ -115,12 +107,9 @@ export async function updateFeedRecord(
|
||||
input: UpdateFeedInput,
|
||||
options: { inPlace?: boolean } = {},
|
||||
): Promise<UpdateFeedResult> {
|
||||
const emailStorage = env.EMAIL_STORAGE;
|
||||
const feedConfigKey = `feed:${feedId}:config`;
|
||||
const repo = FeedRepository.from(env);
|
||||
|
||||
const existing = (await emailStorage.get(feedConfigKey, {
|
||||
type: "json",
|
||||
})) as FeedConfig | null;
|
||||
const existing = await repo.getConfig(feedId);
|
||||
|
||||
if (!existing) return { status: "not_found" };
|
||||
|
||||
@@ -157,14 +146,8 @@ export async function updateFeedRecord(
|
||||
expires_at: expiresAt,
|
||||
};
|
||||
|
||||
await emailStorage.put(feedConfigKey, JSON.stringify(config));
|
||||
await updateFeedInList(
|
||||
emailStorage,
|
||||
feedId,
|
||||
config.title,
|
||||
config.description,
|
||||
expiresAt,
|
||||
);
|
||||
await repo.putConfig(feedId, config);
|
||||
await repo.updateInList(feedId, config.title, config.description, expiresAt);
|
||||
|
||||
return { status: "ok", config };
|
||||
}
|
||||
@@ -184,22 +167,21 @@ export async function deleteFeedFastDetailed(
|
||||
emailStorage: KVNamespace,
|
||||
feedId: string,
|
||||
): Promise<DeleteFeedFastResult> {
|
||||
const feedConfigKey = `feed:${feedId}:config`;
|
||||
const feedMetadataKey = `feed:${feedId}:metadata`;
|
||||
const repo = new FeedRepository(emailStorage);
|
||||
|
||||
const errors: string[] = [];
|
||||
let configDeleted = false;
|
||||
let metadataDeleted = false;
|
||||
|
||||
try {
|
||||
await emailStorage.delete(feedConfigKey);
|
||||
await repo.deleteConfig(feedId);
|
||||
configDeleted = true;
|
||||
} catch (error) {
|
||||
errors.push(`config delete failed: ${String(error)}`);
|
||||
}
|
||||
|
||||
try {
|
||||
await emailStorage.delete(feedMetadataKey);
|
||||
await repo.deleteMetadata(feedId);
|
||||
metadataDeleted = true;
|
||||
} catch (error) {
|
||||
errors.push(`metadata delete failed: ${String(error)}`);
|
||||
@@ -220,12 +202,13 @@ export async function deleteFeedRecord(
|
||||
feedId: string,
|
||||
): Promise<boolean> {
|
||||
const emailStorage = env.EMAIL_STORAGE;
|
||||
const repo = new FeedRepository(emailStorage);
|
||||
|
||||
// Read unsubscribe URLs before the metadata is deleted below.
|
||||
const unsubscribeUrls = await collectUnsubscribeUrls(emailStorage, feedId);
|
||||
|
||||
await deleteFeedFastDetailed(emailStorage, feedId);
|
||||
const removed = await removeFeedFromList(emailStorage, feedId);
|
||||
const removed = await repo.removeFromList(feedId);
|
||||
if (removed) {
|
||||
await bumpCounters(emailStorage, { feeds_deleted: 1 });
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ import { ADMIN_COOKIE_MAX_AGE } from "../config/constants";
|
||||
import { logger } from "../lib/logger";
|
||||
import { timingSafeEqual, checkProxyAuth } from "../lib/auth";
|
||||
import { Layout, clampText } from "./admin/ui";
|
||||
import { listAllFeeds } from "./admin/helpers";
|
||||
import { FeedRepository } from "../domain/feed-repository";
|
||||
import { updateFeedRecord } from "../lib/feed-service";
|
||||
import { feedRssUrl, feedAtomUrl, feedEmailAddress } from "../utils/urls";
|
||||
import { feedsRouter } from "./admin/feeds";
|
||||
@@ -282,14 +282,13 @@ const ExpiryBadge = ({ expiresAt }: { expiresAt: number }) => {
|
||||
app.get("/", async (c) => {
|
||||
// Type assertion for environment variables
|
||||
const env = c.env;
|
||||
const emailStorage = env.EMAIL_STORAGE;
|
||||
const url = new URL(c.req.url);
|
||||
const view = url.searchParams.get("view") === "table" ? "table" : "list";
|
||||
const message = url.searchParams.get("message");
|
||||
const count = Number(url.searchParams.get("count") || "0");
|
||||
|
||||
// List all feeds
|
||||
const feedList = await listAllFeeds(emailStorage);
|
||||
const feedList = await FeedRepository.from(env).listFeeds();
|
||||
|
||||
// Keep the dashboard fast: avoid N KV reads for N feeds.
|
||||
// We store title/description in `feeds:list` (description is optional for older data).
|
||||
|
||||
+16
-32
@@ -1,17 +1,12 @@
|
||||
import { Hono } from "hono";
|
||||
import {
|
||||
Env,
|
||||
FeedConfig,
|
||||
FeedMetadata,
|
||||
EmailData,
|
||||
EmailMetadata,
|
||||
} from "../../types";
|
||||
import { Env, EmailMetadata } from "../../types";
|
||||
import { logger } from "../../lib/logger";
|
||||
import { Layout, clampText } from "./ui";
|
||||
import {
|
||||
deleteAttachmentsForEmails,
|
||||
deleteKeysWithConcurrency,
|
||||
} from "./helpers";
|
||||
import { FeedRepository } from "../../domain/feed-repository";
|
||||
import { feedRssUrl, feedAtomUrl, feedEmailAddress } from "../../utils/urls";
|
||||
import { formatBytes } from "../../utils/format";
|
||||
import { emailsPageScript } from "../../scripts/generated/emails-page";
|
||||
@@ -156,17 +151,13 @@ const SenderField = ({ from, feedId }: SenderFieldProps) => {
|
||||
|
||||
emailsRouter.get("/feeds/:feedId/emails", async (c) => {
|
||||
const env = c.env;
|
||||
const emailStorage = env.EMAIL_STORAGE;
|
||||
const repo = FeedRepository.from(env);
|
||||
const feedId = c.req.param("feedId");
|
||||
const message = c.req.query("message");
|
||||
const count = Number(c.req.query("count") || "0");
|
||||
|
||||
const feedConfig = (await emailStorage.get(`feed:${feedId}:config`, {
|
||||
type: "json",
|
||||
})) as FeedConfig | null;
|
||||
const feedMetadata = (await emailStorage.get(`feed:${feedId}:metadata`, {
|
||||
type: "json",
|
||||
})) as FeedMetadata | null;
|
||||
const feedConfig = await repo.getConfig(feedId);
|
||||
const feedMetadata = await repo.getMetadata(feedId);
|
||||
|
||||
if (!feedConfig || !feedMetadata) {
|
||||
return c.text("Feed not found", 404);
|
||||
@@ -461,16 +452,14 @@ emailsRouter.get("/feeds/:feedId/emails", async (c) => {
|
||||
|
||||
emailsRouter.get("/emails/:emailKey", async (c) => {
|
||||
const env = c.env;
|
||||
const emailStorage = env.EMAIL_STORAGE;
|
||||
const repo = FeedRepository.from(env);
|
||||
const emailKey = c.req.param("emailKey");
|
||||
|
||||
const emailData = (await emailStorage.get(emailKey, {
|
||||
type: "json",
|
||||
})) as EmailData | null;
|
||||
const emailData = await repo.getEmail(emailKey);
|
||||
|
||||
if (!emailData) return c.text("Email not found", 404);
|
||||
|
||||
const feedId = emailKey.split(":")[1];
|
||||
const feedId = repo.feedIdFromEmailKey(emailKey);
|
||||
const attachments = emailData.attachments ?? [];
|
||||
|
||||
const htmlContent = `<!DOCTYPE html><html><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><style>body{font-family:-apple-system,BlinkMacSystemFont,'SF Pro Text','SF Pro Display','Helvetica Neue',Arial,sans-serif;line-height:1.5;padding:16px;margin:0;color:#333;box-sizing:border-box}img{max-width:100%;height:auto}a{color:#0070f3}@media(prefers-color-scheme:dark){body{background-color:#1c1c1e;color:#ffffff}a{color:#0a84ff}}</style></head><body>${emailData.content}</body></html>`;
|
||||
@@ -652,7 +641,7 @@ emailsRouter.get("/emails/:emailKey", async (c) => {
|
||||
|
||||
emailsRouter.post("/emails/:emailKey/delete", async (c) => {
|
||||
const env = c.env;
|
||||
const emailStorage = env.EMAIL_STORAGE;
|
||||
const repo = FeedRepository.from(env);
|
||||
const emailKey = c.req.param("emailKey");
|
||||
const wantsJson = (c.req.header("Accept") || "").includes("application/json");
|
||||
|
||||
@@ -664,12 +653,9 @@ emailsRouter.post("/emails/:emailKey/delete", async (c) => {
|
||||
return c.text("Feed ID is required", 400);
|
||||
}
|
||||
|
||||
const feedMetadataKey = `feed:${feedId}:metadata`;
|
||||
const feedMetadata = (await emailStorage.get(feedMetadataKey, {
|
||||
type: "json",
|
||||
})) as FeedMetadata | null;
|
||||
const feedMetadata = await repo.getMetadata(feedId);
|
||||
|
||||
await emailStorage.delete(emailKey);
|
||||
await repo.deleteEmail(emailKey);
|
||||
await deleteAttachmentsForEmails(env, feedMetadata?.emails ?? [], [
|
||||
emailKey,
|
||||
]);
|
||||
@@ -678,7 +664,7 @@ emailsRouter.post("/emails/:emailKey/delete", async (c) => {
|
||||
feedMetadata.emails = feedMetadata.emails.filter(
|
||||
(email) => email.key !== emailKey,
|
||||
);
|
||||
await emailStorage.put(feedMetadataKey, JSON.stringify(feedMetadata));
|
||||
await repo.putMetadata(feedId, feedMetadata);
|
||||
}
|
||||
|
||||
if (wantsJson) return c.json({ ok: true, emailKey, feedId });
|
||||
@@ -699,6 +685,7 @@ emailsRouter.post("/emails/:emailKey/delete", async (c) => {
|
||||
emailsRouter.post("/feeds/:feedId/emails/bulk-delete", async (c) => {
|
||||
const env = c.env;
|
||||
const emailStorage = env.EMAIL_STORAGE;
|
||||
const repo = new FeedRepository(emailStorage);
|
||||
const feedId = c.req.param("feedId");
|
||||
const contentType = c.req.header("Content-Type") || "";
|
||||
const wantsJson =
|
||||
@@ -706,10 +693,7 @@ emailsRouter.post("/feeds/:feedId/emails/bulk-delete", async (c) => {
|
||||
(c.req.header("Accept") || "").includes("application/json");
|
||||
|
||||
try {
|
||||
const feedMetadataKey = `feed:${feedId}:metadata`;
|
||||
const feedMetadata = (await emailStorage.get(feedMetadataKey, {
|
||||
type: "json",
|
||||
})) as FeedMetadata | null;
|
||||
const feedMetadata = await repo.getMetadata(feedId);
|
||||
|
||||
if (!feedMetadata) {
|
||||
return wantsJson
|
||||
@@ -753,7 +737,7 @@ emailsRouter.post("/feeds/:feedId/emails/bulk-delete", async (c) => {
|
||||
feedMetadata.emails = feedMetadata.emails.filter(
|
||||
(email) => !deletedSet.has(email.key),
|
||||
);
|
||||
await emailStorage.put(feedMetadataKey, JSON.stringify(feedMetadata));
|
||||
await repo.putMetadata(feedId, feedMetadata);
|
||||
|
||||
return c.json({
|
||||
ok: failedEmailKeys.length === 0,
|
||||
@@ -784,7 +768,7 @@ emailsRouter.post("/feeds/:feedId/emails/bulk-delete", async (c) => {
|
||||
feedMetadata.emails = feedMetadata.emails.filter(
|
||||
(email) => !deletedSet.has(email.key),
|
||||
);
|
||||
await emailStorage.put(feedMetadataKey, JSON.stringify(feedMetadata));
|
||||
await repo.putMetadata(feedId, feedMetadata);
|
||||
|
||||
return c.redirect(
|
||||
`/admin/feeds/${feedId}/emails?message=bulkDeleted&count=${deletedOk.length}`,
|
||||
|
||||
+18
-25
@@ -1,6 +1,6 @@
|
||||
import { Hono } from "hono";
|
||||
import { z } from "zod";
|
||||
import { Env, FeedConfig } from "../../types";
|
||||
import { Env } from "../../types";
|
||||
import { bumpCounters } from "../../utils/stats";
|
||||
import { waitUntilSafe } from "../../utils/worker";
|
||||
import { feedRssUrl, feedEmailAddress } from "../../utils/urls";
|
||||
@@ -8,11 +8,8 @@ import { logger } from "../../lib/logger";
|
||||
import { sendUnsubscribes } from "../../utils/unsubscribe";
|
||||
import { getAttachmentBucket } from "../../utils/attachments";
|
||||
import { Layout } from "./ui";
|
||||
import {
|
||||
removeFeedsFromListBulk,
|
||||
purgeFeedKeysStep,
|
||||
collectUnsubscribeUrls,
|
||||
} from "./helpers";
|
||||
import { purgeFeedKeysStep, collectUnsubscribeUrls } from "./helpers";
|
||||
import { FeedRepository } from "../../domain/feed-repository";
|
||||
import {
|
||||
createFeedRecord,
|
||||
updateFeedRecord,
|
||||
@@ -149,12 +146,9 @@ feedsRouter.post("/create", async (c) => {
|
||||
|
||||
feedsRouter.get("/:feedId/edit", async (c) => {
|
||||
const env = c.env;
|
||||
const emailStorage = env.EMAIL_STORAGE;
|
||||
const feedId = c.req.param("feedId");
|
||||
|
||||
const feedConfig = (await emailStorage.get(`feed:${feedId}:config`, {
|
||||
type: "json",
|
||||
})) as FeedConfig | null;
|
||||
const feedConfig = await FeedRepository.from(env).getConfig(feedId);
|
||||
|
||||
if (!feedConfig) {
|
||||
return c.text("Feed not found", 404);
|
||||
@@ -365,7 +359,7 @@ feedsRouter.post("/:feedId/edit", async (c) => {
|
||||
feedsRouter.post("/:feedId/sender-filter", async (c) => {
|
||||
const env = c.env;
|
||||
const feedId = c.req.param("feedId");
|
||||
const feedConfigKey = `feed:${feedId}:config`;
|
||||
const repo = FeedRepository.from(env);
|
||||
|
||||
const body = await c.req.json().catch(() => null);
|
||||
const parsed = senderFilterSchema.safeParse(body);
|
||||
@@ -376,9 +370,7 @@ feedsRouter.post("/:feedId/sender-filter", async (c) => {
|
||||
const { action, value } = parsed.data;
|
||||
const normalized = value.trim().toLowerCase();
|
||||
|
||||
const feedConfig = (await env.EMAIL_STORAGE.get(feedConfigKey, {
|
||||
type: "json",
|
||||
})) as FeedConfig | null;
|
||||
const feedConfig = await repo.getConfig(feedId);
|
||||
if (!feedConfig) return c.json({ ok: false, error: "Feed not found" }, 404);
|
||||
|
||||
const allowedSenders = (feedConfig.allowed_senders || []).map((s) =>
|
||||
@@ -405,15 +397,12 @@ feedsRouter.post("/:feedId/sender-filter", async (c) => {
|
||||
|
||||
if (!targetList.includes(normalized)) {
|
||||
targetList.push(normalized);
|
||||
await env.EMAIL_STORAGE.put(
|
||||
feedConfigKey,
|
||||
JSON.stringify({
|
||||
...feedConfig,
|
||||
allowed_senders: allowedSenders,
|
||||
blocked_senders: blockedSenders,
|
||||
updated_at: Date.now(),
|
||||
}),
|
||||
);
|
||||
await repo.putConfig(feedId, {
|
||||
...feedConfig,
|
||||
allowed_senders: allowedSenders,
|
||||
blocked_senders: blockedSenders,
|
||||
updated_at: Date.now(),
|
||||
});
|
||||
}
|
||||
|
||||
return c.json({ ok: true });
|
||||
@@ -552,7 +541,9 @@ feedsRouter.post("/bulk-delete", async (c) => {
|
||||
}
|
||||
}
|
||||
|
||||
const deletedFeedIds = await removeFeedsFromListBulk(emailStorage, okIds);
|
||||
const deletedFeedIds = await new FeedRepository(
|
||||
emailStorage,
|
||||
).removeFromListBulk(okIds);
|
||||
if (deletedFeedIds.length > 0) {
|
||||
await bumpCounters(emailStorage, {
|
||||
feeds_deleted: deletedFeedIds.length,
|
||||
@@ -616,7 +607,9 @@ feedsRouter.post("/bulk-delete", async (c) => {
|
||||
}
|
||||
}
|
||||
|
||||
const deletedFeedIds = await removeFeedsFromListBulk(emailStorage, okIds);
|
||||
const deletedFeedIds = await new FeedRepository(
|
||||
emailStorage,
|
||||
).removeFromListBulk(okIds);
|
||||
if (deletedFeedIds.length > 0) {
|
||||
await bumpCounters(emailStorage, {
|
||||
feeds_deleted: deletedFeedIds.length,
|
||||
|
||||
+13
-130
@@ -1,14 +1,7 @@
|
||||
import {
|
||||
EmailData,
|
||||
EmailMetadata,
|
||||
Env,
|
||||
FeedList,
|
||||
FeedListItem,
|
||||
FeedMetadata,
|
||||
} from "../../types";
|
||||
import { FEEDS_LIST_KEY } from "../../config/constants";
|
||||
import { EmailData, EmailMetadata, Env } from "../../types";
|
||||
import { logger } from "../../lib/logger";
|
||||
import { getAttachmentBucket } from "../../utils/attachments";
|
||||
import { FeedRepository } from "../../domain/feed-repository";
|
||||
|
||||
// Delete the R2 attachments belonging to the given email keys. Call before the
|
||||
// emails are removed from feed metadata, while `emails` still carries their
|
||||
@@ -58,108 +51,6 @@ export async function deleteKeysWithConcurrency(
|
||||
return { ok, failed };
|
||||
}
|
||||
|
||||
export async function listAllFeeds(
|
||||
emailStorage: KVNamespace,
|
||||
): Promise<FeedListItem[]> {
|
||||
try {
|
||||
const feedList = (await emailStorage.get(FEEDS_LIST_KEY, {
|
||||
type: "json",
|
||||
})) as FeedList | null;
|
||||
return feedList?.feeds || [];
|
||||
} catch (error) {
|
||||
logger.error("Error listing feeds", { error: String(error) });
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export async function addFeedToList(
|
||||
emailStorage: KVNamespace,
|
||||
feedId: string,
|
||||
title: string,
|
||||
description?: string,
|
||||
expires_at?: number,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const feedList = ((await emailStorage.get(FEEDS_LIST_KEY, {
|
||||
type: "json",
|
||||
})) as FeedList | null) || { feeds: [] };
|
||||
|
||||
feedList.feeds.push({ id: feedId, title, description, expires_at });
|
||||
await emailStorage.put(FEEDS_LIST_KEY, JSON.stringify(feedList));
|
||||
} catch (error) {
|
||||
logger.error("Error adding feed to list", { feedId, error: String(error) });
|
||||
}
|
||||
}
|
||||
|
||||
export async function updateFeedInList(
|
||||
emailStorage: KVNamespace,
|
||||
feedId: string,
|
||||
title: string,
|
||||
description?: string,
|
||||
expires_at?: number,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const feedList = ((await emailStorage.get(FEEDS_LIST_KEY, {
|
||||
type: "json",
|
||||
})) as FeedList | null) || { feeds: [] };
|
||||
|
||||
const feedIndex = feedList.feeds.findIndex((feed) => feed.id === feedId);
|
||||
if (feedIndex !== -1) {
|
||||
feedList.feeds[feedIndex].title = title;
|
||||
feedList.feeds[feedIndex].description = description;
|
||||
feedList.feeds[feedIndex].expires_at = expires_at;
|
||||
await emailStorage.put(FEEDS_LIST_KEY, JSON.stringify(feedList));
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error("Error updating feed in list", {
|
||||
feedId,
|
||||
error: String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function removeFeedsFromListBulk(
|
||||
emailStorage: KVNamespace,
|
||||
feedIds: string[],
|
||||
): Promise<string[]> {
|
||||
try {
|
||||
const feedList = ((await emailStorage.get(FEEDS_LIST_KEY, {
|
||||
type: "json",
|
||||
})) as FeedList | null) || { feeds: [] };
|
||||
|
||||
const toRemove = new Set(feedIds.filter(Boolean));
|
||||
if (toRemove.size === 0) return [];
|
||||
|
||||
const removed: string[] = [];
|
||||
const nextFeeds: FeedListItem[] = [];
|
||||
|
||||
for (const feed of feedList.feeds) {
|
||||
if (toRemove.has(feed.id)) {
|
||||
removed.push(feed.id);
|
||||
continue;
|
||||
}
|
||||
nextFeeds.push(feed);
|
||||
}
|
||||
|
||||
if (removed.length === 0) return [];
|
||||
|
||||
feedList.feeds = nextFeeds;
|
||||
await emailStorage.put(FEEDS_LIST_KEY, JSON.stringify(feedList));
|
||||
return removed;
|
||||
} catch (error) {
|
||||
logger.error("Error removing feeds from list", { error: String(error) });
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export async function removeFeedFromList(
|
||||
emailStorage: KVNamespace,
|
||||
feedId: string,
|
||||
): Promise<boolean> {
|
||||
const removed = await removeFeedsFromListBulk(emailStorage, [feedId]);
|
||||
return removed.includes(feedId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a feed's stored RFC 8058 one-click unsubscribe URLs (one per sender).
|
||||
* Must be called before the feed metadata is deleted. Never throws.
|
||||
@@ -169,9 +60,7 @@ export async function collectUnsubscribeUrls(
|
||||
feedId: string,
|
||||
): Promise<string[]> {
|
||||
try {
|
||||
const metadata = (await emailStorage.get(`feed:${feedId}:metadata`, {
|
||||
type: "json",
|
||||
})) as FeedMetadata | null;
|
||||
const metadata = await new FeedRepository(emailStorage).getMetadata(feedId);
|
||||
return Object.values(metadata?.unsubscribe ?? {});
|
||||
} catch (error) {
|
||||
logger.error("Error reading unsubscribe URLs", {
|
||||
@@ -192,24 +81,18 @@ export async function purgeFeedKeysStep(
|
||||
cursor: string;
|
||||
listComplete: boolean;
|
||||
}> {
|
||||
const prefix = `feed:${feedId}:`;
|
||||
const limit = Math.min(1000, Math.max(1, Math.floor(options.limit || 100)));
|
||||
const cursor = options.cursor || undefined;
|
||||
|
||||
const listed = await emailStorage.list({ prefix, cursor, limit });
|
||||
const keys = (listed.keys || []).map((k) => k.name);
|
||||
const repo = new FeedRepository(emailStorage);
|
||||
const listed = await repo.listFeedKeys(feedId, {
|
||||
cursor: options.cursor,
|
||||
limit: options.limit,
|
||||
});
|
||||
const keys = listed.names;
|
||||
|
||||
if (options.bucket && keys.length > 0) {
|
||||
const emailKeys = keys.filter((k) => {
|
||||
const suffix = k.slice(prefix.length);
|
||||
return suffix !== "config" && suffix !== "metadata";
|
||||
});
|
||||
const emailKeys = keys.filter((k) => repo.isEmailKey(feedId, k));
|
||||
if (emailKeys.length > 0) {
|
||||
const emailDataResults = await Promise.allSettled(
|
||||
emailKeys.map(
|
||||
(k) =>
|
||||
emailStorage.get(k, { type: "json" }) as Promise<EmailData | null>,
|
||||
),
|
||||
emailKeys.map((k) => repo.getEmail(k)),
|
||||
);
|
||||
const attachmentIds = emailDataResults
|
||||
.filter(
|
||||
@@ -234,8 +117,8 @@ export async function purgeFeedKeysStep(
|
||||
return {
|
||||
deletedKeys: ok,
|
||||
failedKeys: failed,
|
||||
cursor: listed.cursor || "",
|
||||
listComplete: !!listed.list_complete,
|
||||
cursor: listed.cursor,
|
||||
listComplete: listed.listComplete,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
+15
-17
@@ -8,12 +8,8 @@ import {
|
||||
updateFeedRecord,
|
||||
deleteFeedRecord,
|
||||
} from "../../lib/feed-service";
|
||||
import { listAllFeeds, deleteAttachmentsForEmails } from "../admin/helpers";
|
||||
import {
|
||||
getFeedConfig,
|
||||
getFeedMetadata,
|
||||
getEmailData,
|
||||
} from "../../utils/storage";
|
||||
import { deleteAttachmentsForEmails } from "../admin/helpers";
|
||||
import { FeedRepository } from "../../domain/feed-repository";
|
||||
import { getStats } from "../../utils/stats";
|
||||
import { feedEmailAddress, feedRssUrl, feedAtomUrl } from "../../utils/urls";
|
||||
import {
|
||||
@@ -107,7 +103,7 @@ apiApp.openapi(
|
||||
}),
|
||||
async (c) => {
|
||||
const env = c.env;
|
||||
const feeds = await listAllFeeds(env.EMAIL_STORAGE);
|
||||
const feeds = await FeedRepository.from(env).listFeeds();
|
||||
return c.json(
|
||||
{
|
||||
feeds: feeds.map((f) => ({
|
||||
@@ -173,9 +169,10 @@ apiApp.openapi(
|
||||
async (c) => {
|
||||
const env = c.env;
|
||||
const { feedId } = c.req.valid("param");
|
||||
const config = await getFeedConfig(env.EMAIL_STORAGE, feedId);
|
||||
const repo = FeedRepository.from(env);
|
||||
const config = await repo.getConfig(feedId);
|
||||
if (!config) return c.json({ error: "Feed not found" }, 404);
|
||||
const metadata = await getFeedMetadata(env.EMAIL_STORAGE, feedId);
|
||||
const metadata = await repo.getMetadata(feedId);
|
||||
return c.json(
|
||||
toFeed(feedId, config, metadata?.emails.length ?? 0, env),
|
||||
200,
|
||||
@@ -218,7 +215,7 @@ apiApp.openapi(
|
||||
return c.json({ error: "Feed not found" }, 404);
|
||||
if (result.status === "expired")
|
||||
return c.json({ error: "Feed has expired and cannot be modified" }, 409);
|
||||
const metadata = await getFeedMetadata(env.EMAIL_STORAGE, feedId);
|
||||
const metadata = await FeedRepository.from(env).getMetadata(feedId);
|
||||
return c.json(
|
||||
toFeed(feedId, result.config, metadata?.emails.length ?? 0, env),
|
||||
200,
|
||||
@@ -268,7 +265,7 @@ apiApp.openapi(
|
||||
async (c) => {
|
||||
const env = c.env;
|
||||
const { feedId } = c.req.valid("param");
|
||||
const metadata = await getFeedMetadata(env.EMAIL_STORAGE, feedId);
|
||||
const metadata = await FeedRepository.from(env).getMetadata(feedId);
|
||||
if (!metadata) return c.json({ error: "Feed not found" }, 404);
|
||||
return c.json(
|
||||
{
|
||||
@@ -303,10 +300,11 @@ apiApp.openapi(
|
||||
const env = c.env;
|
||||
const { feedId, entryId } = c.req.valid("param");
|
||||
const receivedAt = parseInt(entryId, 10);
|
||||
const metadata = await getFeedMetadata(env.EMAIL_STORAGE, feedId);
|
||||
const repo = FeedRepository.from(env);
|
||||
const metadata = await repo.getMetadata(feedId);
|
||||
const metaEntry = metadata?.emails.find((e) => e.receivedAt === receivedAt);
|
||||
if (!metaEntry) return c.json({ error: "Email not found" }, 404);
|
||||
const data = await getEmailData(env.EMAIL_STORAGE, metaEntry.key);
|
||||
const data = await repo.getEmail(metaEntry.key);
|
||||
if (!data) return c.json({ error: "Email not found" }, 404);
|
||||
return c.json(
|
||||
{
|
||||
@@ -344,18 +342,18 @@ apiApp.openapi(
|
||||
}),
|
||||
async (c) => {
|
||||
const env = c.env;
|
||||
const emailStorage = env.EMAIL_STORAGE;
|
||||
const repo = FeedRepository.from(env);
|
||||
const { feedId, entryId } = c.req.valid("param");
|
||||
const receivedAt = parseInt(entryId, 10);
|
||||
const metadata = await getFeedMetadata(emailStorage, feedId);
|
||||
const metadata = await repo.getMetadata(feedId);
|
||||
const metaEntry = metadata?.emails.find((e) => e.receivedAt === receivedAt);
|
||||
if (!metadata || !metaEntry)
|
||||
return c.json({ error: "Email not found" }, 404);
|
||||
|
||||
await emailStorage.delete(metaEntry.key);
|
||||
await repo.deleteEmail(metaEntry.key);
|
||||
await deleteAttachmentsForEmails(env, metadata.emails, [metaEntry.key]);
|
||||
metadata.emails = metadata.emails.filter((e) => e.key !== metaEntry.key);
|
||||
await emailStorage.put(`feed:${feedId}:metadata`, JSON.stringify(metadata));
|
||||
await repo.putMetadata(feedId, metadata);
|
||||
|
||||
return c.json({ ok: true }, 200);
|
||||
},
|
||||
|
||||
+6
-14
@@ -1,8 +1,9 @@
|
||||
import { Context } from "hono";
|
||||
import { html, raw } from "hono/html";
|
||||
import { Env, FeedConfig, FeedMetadata, EmailData } from "../types";
|
||||
import { Env } from "../types";
|
||||
import { processEmailContent } from "../utils/html-processor";
|
||||
import { formatBytes } from "../utils/format";
|
||||
import { FeedRepository } from "../domain/feed-repository";
|
||||
|
||||
export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
|
||||
const feedId = c.req.param("feedId");
|
||||
@@ -12,17 +13,11 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
|
||||
return new Response("Not Found", { status: 404 });
|
||||
}
|
||||
|
||||
const emailStorage = c.env.EMAIL_STORAGE;
|
||||
const repo = FeedRepository.from(c.env);
|
||||
|
||||
const [feedMetadata, feedConfig] = await Promise.all([
|
||||
emailStorage.get(
|
||||
`feed:${feedId}:metadata`,
|
||||
"json",
|
||||
) as Promise<FeedMetadata | null>,
|
||||
emailStorage.get(
|
||||
`feed:${feedId}:config`,
|
||||
"json",
|
||||
) as Promise<FeedConfig | null>,
|
||||
repo.getMetadata(feedId),
|
||||
repo.getConfig(feedId),
|
||||
]);
|
||||
if (!feedMetadata) {
|
||||
return new Response("Feed not found", { status: 404 });
|
||||
@@ -41,10 +36,7 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
|
||||
return new Response("Entry not found", { status: 404 });
|
||||
}
|
||||
|
||||
const emailData = (await emailStorage.get(
|
||||
metaEntry.key,
|
||||
"json",
|
||||
)) as EmailData | null;
|
||||
const emailData = await repo.getEmail(metaEntry.key);
|
||||
if (!emailData) {
|
||||
return new Response("Entry not found", { status: 404 });
|
||||
}
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import worker from "../index";
|
||||
import { createMockEnv } from "../test/setup";
|
||||
import { iconKey } from "../utils/storage";
|
||||
import type { Env } from "../types";
|
||||
|
||||
const iconKey = (domain: string) => `icon:${domain}`;
|
||||
|
||||
function req(path: string): Request {
|
||||
return new Request(`https://test.getmynews.app${path}`);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { Context } from "hono";
|
||||
import { Env } from "../types";
|
||||
import { getFeedMetadata } from "../utils/storage";
|
||||
import { FeedRepository } from "../domain/feed-repository";
|
||||
import { cacheFaviconForDomain, getCachedIcon } from "../utils/favicon-fetcher";
|
||||
|
||||
export const FAVICON_PATH = "/favicon.svg";
|
||||
@@ -40,7 +40,7 @@ export async function handleFeedFavicon(
|
||||
const feedId = c.req.param("feedId");
|
||||
if (!feedId) return projectFavicon();
|
||||
|
||||
const metadata = await getFeedMetadata(env.EMAIL_STORAGE, feedId);
|
||||
const metadata = await FeedRepository.from(env).getMetadata(feedId);
|
||||
const domain = metadata?.iconDomain;
|
||||
if (!domain) return projectFavicon();
|
||||
|
||||
|
||||
+2
-4
@@ -7,6 +7,7 @@ import {
|
||||
import { waitUntilSafe } from "../utils/worker";
|
||||
import { DEFAULT_LEASE_SECONDS, MAX_LEASE_SECONDS } from "../config/constants";
|
||||
import { feedTopicPattern } from "../utils/urls";
|
||||
import { FeedRepository } from "../domain/feed-repository";
|
||||
|
||||
type AppEnv = { Bindings: Env };
|
||||
|
||||
@@ -73,10 +74,7 @@ hubRouter.post("/", async (c) => {
|
||||
const feedId = match[2];
|
||||
|
||||
// Verify the feed exists before accepting any subscription
|
||||
const feedConfig = await env.EMAIL_STORAGE.get(
|
||||
`feed:${feedId}:config`,
|
||||
"json",
|
||||
);
|
||||
const feedConfig = await FeedRepository.from(env).getConfig(feedId);
|
||||
if (!feedConfig) {
|
||||
return c.text("Not Found: feed does not exist", 404);
|
||||
}
|
||||
|
||||
@@ -6,8 +6,9 @@ import {
|
||||
extractEmailDomain,
|
||||
getCachedIcon,
|
||||
} from "./favicon-fetcher";
|
||||
import { iconKey } from "./storage";
|
||||
import { MAX_ICON_BYTES } from "../config/constants";
|
||||
|
||||
const iconKey = (domain: string) => `icon:${domain}`;
|
||||
import type { Env } from "../types";
|
||||
|
||||
const PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 1, 2, 3, 4]);
|
||||
|
||||
@@ -4,7 +4,7 @@ import {
|
||||
ICON_TTL_SECONDS,
|
||||
MAX_ICON_BYTES,
|
||||
} from "../config/constants";
|
||||
import { iconKey } from "./storage";
|
||||
import { FeedRepository } from "../domain/feed-repository";
|
||||
import { logger } from "../lib/logger";
|
||||
|
||||
interface IconRecord {
|
||||
@@ -92,8 +92,8 @@ export async function cacheFaviconForDomain(
|
||||
env: Env,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const key = iconKey(domain);
|
||||
const existing = await env.EMAIL_STORAGE.get(key, "text");
|
||||
const repo = FeedRepository.from(env);
|
||||
const existing = await repo.getIconText(domain);
|
||||
if (existing !== null) return; // present (incl. negative) → nothing to do
|
||||
|
||||
const icon = await resolveIcon(domain);
|
||||
@@ -104,9 +104,7 @@ export async function cacheFaviconForDomain(
|
||||
}
|
||||
: { data: null, contentType: "" };
|
||||
|
||||
await env.EMAIL_STORAGE.put(key, JSON.stringify(record), {
|
||||
expirationTtl: ICON_TTL_SECONDS,
|
||||
});
|
||||
await repo.putIcon(domain, JSON.stringify(record), ICON_TTL_SECONDS);
|
||||
} catch (error) {
|
||||
logger.warn("Favicon cache failed", { domain, error: String(error) });
|
||||
}
|
||||
@@ -119,10 +117,7 @@ export async function getCachedIcon(
|
||||
domain: string,
|
||||
env: Env,
|
||||
): Promise<{ bytes: ArrayBuffer; contentType: string } | null> {
|
||||
const record = (await env.EMAIL_STORAGE.get(
|
||||
iconKey(domain),
|
||||
"json",
|
||||
)) as IconRecord | null;
|
||||
const record = await FeedRepository.from(env).getIconJson<IconRecord>(domain);
|
||||
if (!record || record.data === null) return null;
|
||||
return {
|
||||
bytes: base64ToArrayBuffer(record.data),
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Env, FeedConfig, FeedMetadata, EmailData } from "../types";
|
||||
import { Env, FeedConfig, EmailData } from "../types";
|
||||
import { MAX_FEED_ITEMS } from "../config/constants";
|
||||
import { FeedRepository } from "../domain/feed-repository";
|
||||
|
||||
export interface FeedData {
|
||||
feedConfig: FeedConfig;
|
||||
@@ -10,19 +11,12 @@ export async function fetchFeedData(
|
||||
feedId: string,
|
||||
env: Env,
|
||||
): Promise<FeedData | null> {
|
||||
const storage = env.EMAIL_STORAGE;
|
||||
|
||||
const feedMetadata = (await storage.get(
|
||||
`feed:${feedId}:metadata`,
|
||||
"json",
|
||||
)) as FeedMetadata | null;
|
||||
const repo = FeedRepository.from(env);
|
||||
|
||||
const feedMetadata = await repo.getMetadata(feedId);
|
||||
if (!feedMetadata) return null;
|
||||
|
||||
const feedConfig = ((await storage.get(
|
||||
`feed:${feedId}:config`,
|
||||
"json",
|
||||
)) as FeedConfig | null) ?? {
|
||||
const feedConfig = (await repo.getConfig(feedId)) ?? {
|
||||
title: `Newsletter Feed ${feedId}`,
|
||||
description: "Converted email newsletter",
|
||||
language: "en",
|
||||
@@ -32,7 +26,7 @@ export async function fetchFeedData(
|
||||
const emailRefs = feedMetadata.emails.slice(0, MAX_FEED_ITEMS);
|
||||
const emails: EmailData[] = [];
|
||||
for (const ref of emailRefs) {
|
||||
const data = (await storage.get(ref.key, "json")) as EmailData | null;
|
||||
const data = await repo.getEmail(ref.key);
|
||||
if (data) emails.push(data);
|
||||
}
|
||||
|
||||
|
||||
+12
-26
@@ -1,8 +1,6 @@
|
||||
import { Counters, Env, StatsResponse } from "../types";
|
||||
import { STATS_KEY } from "../config/constants";
|
||||
import { logger } from "../lib/logger";
|
||||
import { listAllFeeds } from "../routes/admin/helpers";
|
||||
import { getFeedMetadata } from "./storage";
|
||||
import { FeedRepository } from "../domain/feed-repository";
|
||||
import { getAttachmentBucket } from "./attachments";
|
||||
|
||||
const EMPTY_COUNTERS: Counters = {
|
||||
@@ -15,9 +13,7 @@ const EMPTY_COUNTERS: Counters = {
|
||||
|
||||
export async function getCounters(kv: KVNamespace): Promise<Counters> {
|
||||
try {
|
||||
const stored = (await kv.get(STATS_KEY, {
|
||||
type: "json",
|
||||
})) as Counters | null;
|
||||
const stored = await new FeedRepository(kv).getCountersRaw();
|
||||
return { ...EMPTY_COUNTERS, ...(stored || {}) };
|
||||
} catch (error) {
|
||||
logger.error("Error reading counters", { error: String(error) });
|
||||
@@ -48,7 +44,7 @@ export async function bumpCounters(
|
||||
current.last_feed_created_at = changes.last_feed_created_at;
|
||||
if (!current.first_seen) current.first_seen = new Date().toISOString();
|
||||
|
||||
await kv.put(STATS_KEY, JSON.stringify(current));
|
||||
await new FeedRepository(kv).putCounters(current);
|
||||
} catch (error) {
|
||||
logger.error("Error updating counters", { error: String(error) });
|
||||
}
|
||||
@@ -58,26 +54,15 @@ export async function countKeysByPrefix(
|
||||
kv: KVNamespace,
|
||||
prefix: string,
|
||||
): Promise<number> {
|
||||
let total = 0;
|
||||
let cursor: string | undefined;
|
||||
try {
|
||||
do {
|
||||
const listed = await kv.list({ prefix, cursor, limit: 1000 });
|
||||
total += listed.keys.length;
|
||||
cursor = listed.list_complete ? undefined : listed.cursor;
|
||||
} while (cursor);
|
||||
} catch (error) {
|
||||
logger.error("Error counting keys", { prefix, error: String(error) });
|
||||
}
|
||||
return total;
|
||||
return new FeedRepository(kv).countKeysByPrefix(prefix);
|
||||
}
|
||||
|
||||
export async function getStats(env: Env): Promise<StatsResponse> {
|
||||
const kv = env.EMAIL_STORAGE;
|
||||
const repo = FeedRepository.from(env);
|
||||
const [counters, feeds, websubCount] = await Promise.all([
|
||||
getCounters(kv),
|
||||
listAllFeeds(kv),
|
||||
countKeysByPrefix(kv, "websub:"),
|
||||
getCounters(env.EMAIL_STORAGE),
|
||||
repo.listFeeds(),
|
||||
repo.countSubscriptionKeys(),
|
||||
]);
|
||||
|
||||
return {
|
||||
@@ -119,9 +104,10 @@ export async function scanR2Usage(
|
||||
export async function scanKvUsage(kv: KVNamespace): Promise<{ bytes: number }> {
|
||||
let bytes = 0;
|
||||
try {
|
||||
const feeds = await listAllFeeds(kv);
|
||||
const repo = new FeedRepository(kv);
|
||||
const feeds = await repo.listFeeds();
|
||||
for (const feed of feeds) {
|
||||
const metadata = await getFeedMetadata(kv, feed.id);
|
||||
const metadata = await repo.getMetadata(feed.id);
|
||||
if (!metadata) continue;
|
||||
for (const email of metadata.emails) {
|
||||
bytes += email.size ?? 0;
|
||||
@@ -152,7 +138,7 @@ export async function setStorageSnapshot(
|
||||
current.kv_bytes_estimated = snapshot.kv_bytes_estimated;
|
||||
current.storage_scanned_at = new Date().toISOString();
|
||||
if (!current.first_seen) current.first_seen = new Date().toISOString();
|
||||
await kv.put(STATS_KEY, JSON.stringify(current));
|
||||
await new FeedRepository(kv).putCounters(current);
|
||||
} catch (error) {
|
||||
logger.error("Error writing storage snapshot", { error: String(error) });
|
||||
}
|
||||
|
||||
@@ -1,169 +0,0 @@
|
||||
import {
|
||||
EmailData,
|
||||
FeedConfig,
|
||||
FeedMetadata,
|
||||
FeedList,
|
||||
EmailMetadata,
|
||||
} from "../types";
|
||||
import { MAX_METADATA_EMAILS } from "../config/constants";
|
||||
|
||||
/**
|
||||
* KV key for a domain's cached favicon (shared across feeds from the same sender).
|
||||
*/
|
||||
export function iconKey(domain: string): string {
|
||||
return `icon:${domain}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store email data in KV
|
||||
*/
|
||||
export async function storeEmail(
|
||||
kv: KVNamespace,
|
||||
feedId: string,
|
||||
emailData: EmailData,
|
||||
): Promise<string> {
|
||||
// Generate a unique key for this email
|
||||
const timestamp = Date.now();
|
||||
const key = `feed:${feedId}:email:${timestamp}`;
|
||||
|
||||
// Store the email content
|
||||
await kv.put(key, JSON.stringify(emailData));
|
||||
|
||||
// Update the feed's metadata (list of emails)
|
||||
await updateFeedMetadata(kv, feedId, {
|
||||
key,
|
||||
subject: emailData.subject,
|
||||
receivedAt: timestamp,
|
||||
});
|
||||
|
||||
return key;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update feed metadata with a new email
|
||||
*/
|
||||
async function updateFeedMetadata(
|
||||
kv: KVNamespace,
|
||||
feedId: string,
|
||||
emailMetadata: EmailMetadata,
|
||||
): Promise<void> {
|
||||
const feedMetadataKey = `feed:${feedId}:metadata`;
|
||||
const existingMetadata = (await kv.get(feedMetadataKey, {
|
||||
type: "json",
|
||||
})) as FeedMetadata | null;
|
||||
|
||||
const metadata: FeedMetadata = existingMetadata || { emails: [] };
|
||||
|
||||
// Add new email to the beginning of the list
|
||||
metadata.emails.unshift(emailMetadata);
|
||||
|
||||
// Keep only the last MAX_METADATA_EMAILS in the metadata; delete orphaned KV entries
|
||||
const toDelete =
|
||||
metadata.emails.length > MAX_METADATA_EMAILS
|
||||
? metadata.emails.slice(MAX_METADATA_EMAILS)
|
||||
: [];
|
||||
if (toDelete.length > 0) {
|
||||
metadata.emails = metadata.emails.slice(0, MAX_METADATA_EMAILS);
|
||||
}
|
||||
|
||||
await Promise.all([
|
||||
kv.put(feedMetadataKey, JSON.stringify(metadata)),
|
||||
...toDelete.map((e) => kv.delete(e.key)),
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get feed metadata
|
||||
*/
|
||||
export async function getFeedMetadata(
|
||||
kv: KVNamespace,
|
||||
feedId: string,
|
||||
): Promise<FeedMetadata | null> {
|
||||
const feedMetadataKey = `feed:${feedId}:metadata`;
|
||||
return (await kv.get(feedMetadataKey, {
|
||||
type: "json",
|
||||
})) as FeedMetadata | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get feed configuration
|
||||
*/
|
||||
export async function getFeedConfig(
|
||||
kv: KVNamespace,
|
||||
feedId: string,
|
||||
): Promise<FeedConfig | null> {
|
||||
const feedConfigKey = `feed:${feedId}:config`;
|
||||
return (await kv.get(feedConfigKey, { type: "json" })) as FeedConfig | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get email data
|
||||
*/
|
||||
export async function getEmailData(
|
||||
kv: KVNamespace,
|
||||
key: string,
|
||||
): Promise<EmailData | null> {
|
||||
return (await kv.get(key, { type: "json" })) as EmailData | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new feed
|
||||
*/
|
||||
export async function createFeed(
|
||||
kv: KVNamespace,
|
||||
feedId: string,
|
||||
feedConfig: FeedConfig,
|
||||
): Promise<void> {
|
||||
// Store feed configuration
|
||||
const feedConfigKey = `feed:${feedId}:config`;
|
||||
await kv.put(feedConfigKey, JSON.stringify(feedConfig));
|
||||
|
||||
// Create empty metadata for the feed
|
||||
const feedMetadataKey = `feed:${feedId}:metadata`;
|
||||
await kv.put(
|
||||
feedMetadataKey,
|
||||
JSON.stringify({
|
||||
emails: [],
|
||||
}),
|
||||
);
|
||||
|
||||
// Add feed to the list of all feeds
|
||||
await addFeedToList(kv, feedId, feedConfig.title, feedConfig.description);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a feed to the global list
|
||||
*/
|
||||
export async function addFeedToList(
|
||||
kv: KVNamespace,
|
||||
feedId: string,
|
||||
title: string,
|
||||
description?: string,
|
||||
): Promise<void> {
|
||||
const feedListKey = "feeds:list";
|
||||
const existingList = (await kv.get(feedListKey, {
|
||||
type: "json",
|
||||
})) as FeedList | null;
|
||||
|
||||
const feedList: FeedList = existingList || { feeds: [] };
|
||||
|
||||
feedList.feeds.push({
|
||||
id: feedId,
|
||||
title,
|
||||
description,
|
||||
});
|
||||
|
||||
await kv.put(feedListKey, JSON.stringify(feedList));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all feeds
|
||||
*/
|
||||
export async function getAllFeeds(kv: KVNamespace): Promise<FeedList> {
|
||||
const feedListKey = "feeds:list";
|
||||
const feedList = (await kv.get(feedListKey, {
|
||||
type: "json",
|
||||
})) as FeedList | null;
|
||||
|
||||
return feedList || { feeds: [] };
|
||||
}
|
||||
@@ -8,7 +8,6 @@ import {
|
||||
notifySubscribers,
|
||||
verifyAndStoreSubscription,
|
||||
verifyAndDeleteSubscription,
|
||||
subscriptionKey,
|
||||
} from "./websub";
|
||||
import type { Env, WebSubSubscription } from "../types";
|
||||
|
||||
@@ -51,8 +50,12 @@ describe("getSubscriptions / saveSubscriptions", () => {
|
||||
expect(await getSubscriptions("feed1", env)).toEqual(subs);
|
||||
});
|
||||
|
||||
it("uses the correct KV key", () => {
|
||||
expect(subscriptionKey("abc")).toBe("websub:subs:abc");
|
||||
it("uses the correct KV key", async () => {
|
||||
const env = mockEnv();
|
||||
await saveSubscriptions("abc", [], env);
|
||||
expect(
|
||||
await env.EMAIL_STORAGE.get("websub:subs:abc", { type: "json" }),
|
||||
).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
+10
-30
@@ -1,25 +1,13 @@
|
||||
import {
|
||||
Env,
|
||||
FeedConfig,
|
||||
FeedMetadata,
|
||||
EmailData,
|
||||
WebSubSubscription,
|
||||
} from "../types";
|
||||
import { Env, FeedConfig, EmailData, WebSubSubscription } from "../types";
|
||||
import { generateRssFeed, generateAtomFeed } from "./feed-generator";
|
||||
import { baseUrl, feedRssUrl, feedAtomUrl, feedUrl } from "./urls";
|
||||
|
||||
const KV_PREFIX = "websub:subs:";
|
||||
|
||||
export function subscriptionKey(feedId: string): string {
|
||||
return `${KV_PREFIX}${feedId}`;
|
||||
}
|
||||
import { FeedRepository } from "../domain/feed-repository";
|
||||
|
||||
export async function getSubscriptions(
|
||||
feedId: string,
|
||||
env: Env,
|
||||
): Promise<WebSubSubscription[]> {
|
||||
const raw = await env.EMAIL_STORAGE.get(subscriptionKey(feedId), "json");
|
||||
return (raw as WebSubSubscription[] | null) ?? [];
|
||||
return FeedRepository.from(env).getSubscriptions(feedId);
|
||||
}
|
||||
|
||||
export async function saveSubscriptions(
|
||||
@@ -27,10 +15,7 @@ export async function saveSubscriptions(
|
||||
subscriptions: WebSubSubscription[],
|
||||
env: Env,
|
||||
): Promise<void> {
|
||||
await env.EMAIL_STORAGE.put(
|
||||
subscriptionKey(feedId),
|
||||
JSON.stringify(subscriptions),
|
||||
);
|
||||
await FeedRepository.from(env).saveSubscriptions(feedId, subscriptions);
|
||||
}
|
||||
|
||||
export async function buildHmacSignature(
|
||||
@@ -60,16 +45,16 @@ async function buildFeedXml(
|
||||
env: Env,
|
||||
format: "rss" | "atom" = "rss",
|
||||
): Promise<string | null> {
|
||||
const [rawMetadata, rawConfig] = await Promise.all([
|
||||
env.EMAIL_STORAGE.get(`feed:${feedId}:metadata`, "json"),
|
||||
env.EMAIL_STORAGE.get(`feed:${feedId}:config`, "json"),
|
||||
const repo = FeedRepository.from(env);
|
||||
const [feedMetadata, rawConfig] = await Promise.all([
|
||||
repo.getMetadata(feedId),
|
||||
repo.getConfig(feedId),
|
||||
]);
|
||||
|
||||
const feedMetadata = rawMetadata as FeedMetadata | null;
|
||||
if (!feedMetadata) return null;
|
||||
|
||||
const base = baseUrl(env);
|
||||
const feedConfig = (rawConfig as FeedConfig | null) ?? {
|
||||
const feedConfig: FeedConfig = rawConfig ?? {
|
||||
title: `Newsletter Feed ${feedId}`,
|
||||
description: "Converted email newsletter",
|
||||
language: "en",
|
||||
@@ -78,12 +63,7 @@ async function buildFeedXml(
|
||||
|
||||
const emails = feedMetadata.emails.slice(0, 20);
|
||||
const emailsData = (
|
||||
await Promise.all(
|
||||
emails.map(
|
||||
(m) =>
|
||||
env.EMAIL_STORAGE.get(m.key, "json") as Promise<EmailData | null>,
|
||||
),
|
||||
)
|
||||
await Promise.all(emails.map((m) => repo.getEmail(m.key)))
|
||||
).filter((d): d is EmailData => d !== null);
|
||||
|
||||
if (format === "atom") {
|
||||
|
||||
Reference in New Issue
Block a user