mirror of
https://github.com/juherr/kill-the-news.git
synced 2026-06-20 22:03:48 +00:00
refactor(domain): introduce FeedRepository as the single KV access layer
Centralise the KV key schema and all get/put access behind a FeedRepository class under src/domain/. Every feed/email/list/icon/websub/counter key was previously inlined across ~12 modules with two divergent storeEmail and addFeedToList implementations; the dead src/utils/storage.ts write path is removed and the email key convention unified on feed:<id>:<ts>. Behaviour-preserving: existing tests pass unchanged in logic, plus a new feed-repository.test.ts covering CRUD, key builders, list ops and counters. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,171 @@
|
|||||||
|
import { describe, it, expect } from "vitest";
|
||||||
|
import { createMockEnv } from "../test/setup";
|
||||||
|
import { FeedRepository } from "./feed-repository";
|
||||||
|
import type {
|
||||||
|
Env,
|
||||||
|
FeedConfig,
|
||||||
|
FeedMetadata,
|
||||||
|
EmailData,
|
||||||
|
WebSubSubscription,
|
||||||
|
} from "../types";
|
||||||
|
|
||||||
|
const mockEnv = () => createMockEnv() as unknown as Env;
|
||||||
|
|
||||||
|
const sampleConfig = (overrides: Partial<FeedConfig> = {}): FeedConfig => ({
|
||||||
|
title: "Test Feed",
|
||||||
|
language: "en",
|
||||||
|
created_at: 1000,
|
||||||
|
...overrides,
|
||||||
|
});
|
||||||
|
|
||||||
|
const sampleEmail = (overrides: Partial<EmailData> = {}): EmailData => ({
|
||||||
|
subject: "Hello",
|
||||||
|
from: "news@example.com",
|
||||||
|
content: "<p>hi</p>",
|
||||||
|
receivedAt: 1234,
|
||||||
|
headers: {},
|
||||||
|
...overrides,
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("FeedRepository key schema", () => {
|
||||||
|
it("builds the canonical KV keys via the public API", () => {
|
||||||
|
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
|
||||||
|
expect(repo.feedKeyPrefix("a.b.42")).toBe("feed:a.b.42:");
|
||||||
|
expect(repo.iconKey("example.com")).toBe("icon:example.com");
|
||||||
|
expect(repo.newEmailKey("a.b.42")).toMatch(/^feed:a\.b\.42:\d+$/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("recognises email keys vs config/metadata keys", () => {
|
||||||
|
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
|
||||||
|
expect(repo.isEmailKey("a.b.42", "feed:a.b.42:config")).toBe(false);
|
||||||
|
expect(repo.isEmailKey("a.b.42", "feed:a.b.42:metadata")).toBe(false);
|
||||||
|
expect(repo.isEmailKey("a.b.42", "feed:a.b.42:1700000000000")).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("recovers the feed id from an email key", () => {
|
||||||
|
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
|
||||||
|
expect(repo.feedIdFromEmailKey("feed:a.b.42:1700000000000")).toBe("a.b.42");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("FeedRepository config & metadata", () => {
|
||||||
|
it("round-trips and deletes a feed config", async () => {
|
||||||
|
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
|
||||||
|
expect(await repo.getConfig("a.b.42")).toBeNull();
|
||||||
|
await repo.putConfig("a.b.42", sampleConfig());
|
||||||
|
expect(await repo.getConfig("a.b.42")).toMatchObject({
|
||||||
|
title: "Test Feed",
|
||||||
|
});
|
||||||
|
await repo.deleteConfig("a.b.42");
|
||||||
|
expect(await repo.getConfig("a.b.42")).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("round-trips and deletes feed metadata", async () => {
|
||||||
|
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
|
||||||
|
const meta: FeedMetadata = { emails: [] };
|
||||||
|
await repo.putMetadata("a.b.42", meta);
|
||||||
|
expect(await repo.getMetadata("a.b.42")).toEqual(meta);
|
||||||
|
await repo.deleteMetadata("a.b.42");
|
||||||
|
expect(await repo.getMetadata("a.b.42")).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("FeedRepository emails", () => {
|
||||||
|
it("stores and reads an email under a minted key", async () => {
|
||||||
|
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
|
||||||
|
const key = repo.newEmailKey("a.b.42");
|
||||||
|
await repo.putEmail(key, sampleEmail());
|
||||||
|
expect(await repo.getEmail(key)).toMatchObject({ subject: "Hello" });
|
||||||
|
await repo.deleteEmail(key);
|
||||||
|
expect(await repo.getEmail(key)).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("lists every key under a feed prefix", async () => {
|
||||||
|
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
|
||||||
|
await repo.putConfig("a.b.42", sampleConfig());
|
||||||
|
await repo.putMetadata("a.b.42", { emails: [] });
|
||||||
|
const emailKey = repo.newEmailKey("a.b.42");
|
||||||
|
await repo.putEmail(emailKey, sampleEmail());
|
||||||
|
|
||||||
|
const listed = await repo.listFeedKeys("a.b.42");
|
||||||
|
expect(listed.names).toContain("feed:a.b.42:config");
|
||||||
|
expect(listed.names).toContain("feed:a.b.42:metadata");
|
||||||
|
expect(listed.names).toContain(emailKey);
|
||||||
|
expect(listed.names.filter((k) => repo.isEmailKey("a.b.42", k))).toEqual([
|
||||||
|
emailKey,
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("FeedRepository feed list", () => {
|
||||||
|
it("adds, updates, lists and removes feeds with expiry", async () => {
|
||||||
|
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
|
||||||
|
await repo.addToList("a.b.42", "One", "desc", 5000);
|
||||||
|
await repo.addToList("c.d.99", "Two");
|
||||||
|
|
||||||
|
let feeds = await repo.listFeeds();
|
||||||
|
expect(feeds).toHaveLength(2);
|
||||||
|
expect(feeds.find((f) => f.id === "a.b.42")).toMatchObject({
|
||||||
|
title: "One",
|
||||||
|
expires_at: 5000,
|
||||||
|
});
|
||||||
|
|
||||||
|
await repo.updateInList("a.b.42", "One-updated", undefined, undefined);
|
||||||
|
feeds = await repo.listFeeds();
|
||||||
|
const updated = feeds.find((f) => f.id === "a.b.42");
|
||||||
|
expect(updated).toMatchObject({ title: "One-updated" });
|
||||||
|
expect(updated?.expires_at).toBeUndefined();
|
||||||
|
|
||||||
|
expect(await repo.removeFromList("a.b.42")).toBe(true);
|
||||||
|
expect(await repo.removeFromList("missing")).toBe(false);
|
||||||
|
feeds = await repo.listFeeds();
|
||||||
|
expect(feeds.map((f) => f.id)).toEqual(["c.d.99"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("bulk-removes only the matching ids", async () => {
|
||||||
|
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
|
||||||
|
await repo.addToList("a.b.42", "One");
|
||||||
|
await repo.addToList("c.d.99", "Two");
|
||||||
|
await repo.addToList("e.f.10", "Three");
|
||||||
|
|
||||||
|
const removed = await repo.removeFromListBulk(["a.b.42", "e.f.10", "nope"]);
|
||||||
|
expect(removed.sort()).toEqual(["a.b.42", "e.f.10"]);
|
||||||
|
expect((await repo.listFeeds()).map((f) => f.id)).toEqual(["c.d.99"]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("FeedRepository counters, icons, websub", () => {
|
||||||
|
it("round-trips raw counters", async () => {
|
||||||
|
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
|
||||||
|
expect(await repo.getCountersRaw()).toBeNull();
|
||||||
|
await repo.putCounters({
|
||||||
|
feeds_created: 1,
|
||||||
|
feeds_deleted: 0,
|
||||||
|
emails_received: 2,
|
||||||
|
emails_rejected: 0,
|
||||||
|
unsubscribes_sent: 0,
|
||||||
|
});
|
||||||
|
expect(await repo.getCountersRaw()).toMatchObject({ emails_received: 2 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("stores and reads favicons as text or json", async () => {
|
||||||
|
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
|
||||||
|
expect(await repo.getIconText("example.com")).toBeNull();
|
||||||
|
await repo.putIcon("example.com", JSON.stringify({ data: null }), 60);
|
||||||
|
expect(await repo.getIconText("example.com")).toBe('{"data":null}');
|
||||||
|
expect(await repo.getIconJson<{ data: null }>("example.com")).toEqual({
|
||||||
|
data: null,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("round-trips websub subscriptions and counts them", async () => {
|
||||||
|
const repo = new FeedRepository(mockEnv().EMAIL_STORAGE);
|
||||||
|
expect(await repo.getSubscriptions("a.b.42")).toEqual([]);
|
||||||
|
const subs: WebSubSubscription[] = [
|
||||||
|
{ callbackUrl: "https://r.example/cb", expiresAt: 9999 },
|
||||||
|
];
|
||||||
|
await repo.saveSubscriptions("a.b.42", subs);
|
||||||
|
expect(await repo.getSubscriptions("a.b.42")).toEqual(subs);
|
||||||
|
expect(await repo.countSubscriptionKeys()).toBe(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,296 @@
|
|||||||
|
import {
|
||||||
|
Counters,
|
||||||
|
EmailData,
|
||||||
|
Env,
|
||||||
|
FeedConfig,
|
||||||
|
FeedList,
|
||||||
|
FeedListItem,
|
||||||
|
FeedMetadata,
|
||||||
|
WebSubSubscription,
|
||||||
|
} from "../types";
|
||||||
|
import { FEEDS_LIST_KEY, STATS_KEY } from "../config/constants";
|
||||||
|
import { logger } from "../lib/logger";
|
||||||
|
|
||||||
|
const WEBSUB_PREFIX = "websub:subs:";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Single source of truth for the KV key schema and all KV access. No other
|
||||||
|
* module should build a `feed:`/`feeds:list`/`websub:`/`icon:`/`stats:counters`
|
||||||
|
* key string — go through a repository method instead.
|
||||||
|
*
|
||||||
|
* Wraps one `KVNamespace`; construct per request via `FeedRepository.from(env)`.
|
||||||
|
*/
|
||||||
|
export class FeedRepository {
|
||||||
|
constructor(private readonly kv: KVNamespace) {}
|
||||||
|
|
||||||
|
static from(env: Env): FeedRepository {
|
||||||
|
return new FeedRepository(env.EMAIL_STORAGE);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Key schema ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
private configKey(feedId: string): string {
|
||||||
|
return `feed:${feedId}:config`;
|
||||||
|
}
|
||||||
|
|
||||||
|
private metadataKey(feedId: string): string {
|
||||||
|
return `feed:${feedId}:metadata`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** KV key for a domain's cached favicon (shared across feeds). */
|
||||||
|
iconKey(domain: string): string {
|
||||||
|
return `icon:${domain}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
private websubKey(feedId: string): string {
|
||||||
|
return `${WEBSUB_PREFIX}${feedId}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Prefix covering every key owned by a feed (config, metadata, emails). */
|
||||||
|
feedKeyPrefix(feedId: string): string {
|
||||||
|
return `feed:${feedId}:`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Mint a fresh, time-ordered email key. Call once and reuse the result. */
|
||||||
|
newEmailKey(feedId: string): string {
|
||||||
|
return `feed:${feedId}:${Date.now()}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** True when `key` is an email entry (not the feed's config/metadata key). */
|
||||||
|
isEmailKey(feedId: string, key: string): boolean {
|
||||||
|
const suffix = key.slice(this.feedKeyPrefix(feedId).length);
|
||||||
|
return suffix !== "config" && suffix !== "metadata";
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Recover the feed id embedded in an email key (`feed:<id>:<ts>`). */
|
||||||
|
feedIdFromEmailKey(key: string): string {
|
||||||
|
return key.split(":")[1];
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Feed config ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
async getConfig(feedId: string): Promise<FeedConfig | null> {
|
||||||
|
return (await this.kv.get(this.configKey(feedId), {
|
||||||
|
type: "json",
|
||||||
|
})) as FeedConfig | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async putConfig(feedId: string, config: FeedConfig): Promise<void> {
|
||||||
|
await this.kv.put(this.configKey(feedId), JSON.stringify(config));
|
||||||
|
}
|
||||||
|
|
||||||
|
async deleteConfig(feedId: string): Promise<void> {
|
||||||
|
await this.kv.delete(this.configKey(feedId));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Feed metadata ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
async getMetadata(feedId: string): Promise<FeedMetadata | null> {
|
||||||
|
return (await this.kv.get(this.metadataKey(feedId), {
|
||||||
|
type: "json",
|
||||||
|
})) as FeedMetadata | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async putMetadata(feedId: string, metadata: FeedMetadata): Promise<void> {
|
||||||
|
await this.kv.put(this.metadataKey(feedId), JSON.stringify(metadata));
|
||||||
|
}
|
||||||
|
|
||||||
|
async deleteMetadata(feedId: string): Promise<void> {
|
||||||
|
await this.kv.delete(this.metadataKey(feedId));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Emails ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
async putEmail(key: string, data: EmailData): Promise<void> {
|
||||||
|
await this.kv.put(key, JSON.stringify(data));
|
||||||
|
}
|
||||||
|
|
||||||
|
async getEmail(key: string): Promise<EmailData | null> {
|
||||||
|
return (await this.kv.get(key, { type: "json" })) as EmailData | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async deleteEmail(key: string): Promise<void> {
|
||||||
|
await this.kv.delete(key);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Global feed list ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
async listFeeds(): Promise<FeedListItem[]> {
|
||||||
|
try {
|
||||||
|
const feedList = (await this.kv.get(FEEDS_LIST_KEY, {
|
||||||
|
type: "json",
|
||||||
|
})) as FeedList | null;
|
||||||
|
return feedList?.feeds || [];
|
||||||
|
} catch (error) {
|
||||||
|
logger.error("Error listing feeds", { error: String(error) });
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async addToList(
|
||||||
|
feedId: string,
|
||||||
|
title: string,
|
||||||
|
description?: string,
|
||||||
|
expires_at?: number,
|
||||||
|
): Promise<void> {
|
||||||
|
try {
|
||||||
|
const feedList = ((await this.kv.get(FEEDS_LIST_KEY, {
|
||||||
|
type: "json",
|
||||||
|
})) as FeedList | null) || { feeds: [] };
|
||||||
|
|
||||||
|
feedList.feeds.push({ id: feedId, title, description, expires_at });
|
||||||
|
await this.kv.put(FEEDS_LIST_KEY, JSON.stringify(feedList));
|
||||||
|
} catch (error) {
|
||||||
|
logger.error("Error adding feed to list", {
|
||||||
|
feedId,
|
||||||
|
error: String(error),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async updateInList(
|
||||||
|
feedId: string,
|
||||||
|
title: string,
|
||||||
|
description?: string,
|
||||||
|
expires_at?: number,
|
||||||
|
): Promise<void> {
|
||||||
|
try {
|
||||||
|
const feedList = ((await this.kv.get(FEEDS_LIST_KEY, {
|
||||||
|
type: "json",
|
||||||
|
})) as FeedList | null) || { feeds: [] };
|
||||||
|
|
||||||
|
const feedIndex = feedList.feeds.findIndex((feed) => feed.id === feedId);
|
||||||
|
if (feedIndex !== -1) {
|
||||||
|
feedList.feeds[feedIndex].title = title;
|
||||||
|
feedList.feeds[feedIndex].description = description;
|
||||||
|
feedList.feeds[feedIndex].expires_at = expires_at;
|
||||||
|
await this.kv.put(FEEDS_LIST_KEY, JSON.stringify(feedList));
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
logger.error("Error updating feed in list", {
|
||||||
|
feedId,
|
||||||
|
error: String(error),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async removeFromListBulk(feedIds: string[]): Promise<string[]> {
|
||||||
|
try {
|
||||||
|
const feedList = ((await this.kv.get(FEEDS_LIST_KEY, {
|
||||||
|
type: "json",
|
||||||
|
})) as FeedList | null) || { feeds: [] };
|
||||||
|
|
||||||
|
const toRemove = new Set(feedIds.filter(Boolean));
|
||||||
|
if (toRemove.size === 0) return [];
|
||||||
|
|
||||||
|
const removed: string[] = [];
|
||||||
|
const nextFeeds: FeedListItem[] = [];
|
||||||
|
|
||||||
|
for (const feed of feedList.feeds) {
|
||||||
|
if (toRemove.has(feed.id)) {
|
||||||
|
removed.push(feed.id);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
nextFeeds.push(feed);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (removed.length === 0) return [];
|
||||||
|
|
||||||
|
feedList.feeds = nextFeeds;
|
||||||
|
await this.kv.put(FEEDS_LIST_KEY, JSON.stringify(feedList));
|
||||||
|
return removed;
|
||||||
|
} catch (error) {
|
||||||
|
logger.error("Error removing feeds from list", { error: String(error) });
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async removeFromList(feedId: string): Promise<boolean> {
|
||||||
|
const removed = await this.removeFromListBulk([feedId]);
|
||||||
|
return removed.includes(feedId);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Key listing / counting ────────────────────────────────────────────────
|
||||||
|
|
||||||
|
async listFeedKeys(
|
||||||
|
feedId: string,
|
||||||
|
options: { cursor?: string; limit?: number } = {},
|
||||||
|
): Promise<{ names: string[]; cursor: string; listComplete: boolean }> {
|
||||||
|
const prefix = this.feedKeyPrefix(feedId);
|
||||||
|
const limit = Math.min(1000, Math.max(1, Math.floor(options.limit || 100)));
|
||||||
|
const cursor = options.cursor || undefined;
|
||||||
|
|
||||||
|
const listed = await this.kv.list({ prefix, cursor, limit });
|
||||||
|
return {
|
||||||
|
names: (listed.keys || []).map((k) => k.name),
|
||||||
|
cursor: listed.cursor || "",
|
||||||
|
listComplete: !!listed.list_complete,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async countKeysByPrefix(prefix: string): Promise<number> {
|
||||||
|
let total = 0;
|
||||||
|
let cursor: string | undefined;
|
||||||
|
try {
|
||||||
|
do {
|
||||||
|
const listed = await this.kv.list({ prefix, cursor, limit: 1000 });
|
||||||
|
total += listed.keys.length;
|
||||||
|
cursor = listed.list_complete ? undefined : listed.cursor;
|
||||||
|
} while (cursor);
|
||||||
|
} catch (error) {
|
||||||
|
logger.error("Error counting keys", { prefix, error: String(error) });
|
||||||
|
}
|
||||||
|
return total;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Number of feeds that currently hold at least one WebSub subscription. */
|
||||||
|
countSubscriptionKeys(): Promise<number> {
|
||||||
|
return this.countKeysByPrefix("websub:");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Monitoring counters ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
async getCountersRaw(): Promise<Counters | null> {
|
||||||
|
return (await this.kv.get(STATS_KEY, { type: "json" })) as Counters | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async putCounters(counters: Counters): Promise<void> {
|
||||||
|
await this.kv.put(STATS_KEY, JSON.stringify(counters));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Favicons ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
async getIconText(domain: string): Promise<string | null> {
|
||||||
|
return this.kv.get(this.iconKey(domain), "text");
|
||||||
|
}
|
||||||
|
|
||||||
|
async getIconJson<T>(domain: string): Promise<T | null> {
|
||||||
|
return (await this.kv.get(this.iconKey(domain), {
|
||||||
|
type: "json",
|
||||||
|
})) as T | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async putIcon(
|
||||||
|
domain: string,
|
||||||
|
value: string,
|
||||||
|
ttlSeconds: number,
|
||||||
|
): Promise<void> {
|
||||||
|
await this.kv.put(this.iconKey(domain), value, {
|
||||||
|
expirationTtl: ttlSeconds,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── WebSub subscriptions ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
async getSubscriptions(feedId: string): Promise<WebSubSubscription[]> {
|
||||||
|
const raw = await this.kv.get(this.websubKey(feedId), "json");
|
||||||
|
return (raw as WebSubSubscription[] | null) ?? [];
|
||||||
|
}
|
||||||
|
|
||||||
|
async saveSubscriptions(
|
||||||
|
feedId: string,
|
||||||
|
subscriptions: WebSubSubscription[],
|
||||||
|
): Promise<void> {
|
||||||
|
await this.kv.put(this.websubKey(feedId), JSON.stringify(subscriptions));
|
||||||
|
}
|
||||||
|
}
|
||||||
+5
-7
@@ -13,11 +13,8 @@ import { apiApp } from "./routes/api";
|
|||||||
import { handleCloudflareEmail } from "./lib/cloudflare-email";
|
import { handleCloudflareEmail } from "./lib/cloudflare-email";
|
||||||
import { Env } from "./types";
|
import { Env } from "./types";
|
||||||
import { logger } from "./lib/logger";
|
import { logger } from "./lib/logger";
|
||||||
import {
|
import { FeedRepository } from "./domain/feed-repository";
|
||||||
listAllFeeds,
|
import { purgeExpiredFeeds } from "./routes/admin/helpers";
|
||||||
purgeExpiredFeeds,
|
|
||||||
removeFeedsFromListBulk,
|
|
||||||
} from "./routes/admin/helpers";
|
|
||||||
import {
|
import {
|
||||||
bumpCounters,
|
bumpCounters,
|
||||||
scanR2Usage,
|
scanR2Usage,
|
||||||
@@ -201,7 +198,8 @@ export default {
|
|||||||
},
|
},
|
||||||
async scheduled(_event: ScheduledEvent, env: Env, _ctx: ExecutionContext) {
|
async scheduled(_event: ScheduledEvent, env: Env, _ctx: ExecutionContext) {
|
||||||
const attachmentBucket = getAttachmentBucket(env);
|
const attachmentBucket = getAttachmentBucket(env);
|
||||||
const feeds = await listAllFeeds(env.EMAIL_STORAGE);
|
const repo = FeedRepository.from(env);
|
||||||
|
const feeds = await repo.listFeeds();
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
const expiredIds = feeds
|
const expiredIds = feeds
|
||||||
.filter((f) => f.expires_at !== undefined && f.expires_at <= now)
|
.filter((f) => f.expires_at !== undefined && f.expires_at <= now)
|
||||||
@@ -211,7 +209,7 @@ export default {
|
|||||||
await purgeExpiredFeeds(env.EMAIL_STORAGE, feedId, attachmentBucket);
|
await purgeExpiredFeeds(env.EMAIL_STORAGE, feedId, attachmentBucket);
|
||||||
}
|
}
|
||||||
if (expiredIds.length > 0) {
|
if (expiredIds.length > 0) {
|
||||||
await removeFeedsFromListBulk(env.EMAIL_STORAGE, expiredIds);
|
await repo.removeFromListBulk(expiredIds);
|
||||||
await bumpCounters(env.EMAIL_STORAGE, {
|
await bumpCounters(env.EMAIL_STORAGE, {
|
||||||
feeds_deleted: expiredIds.length,
|
feeds_deleted: expiredIds.length,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -7,7 +7,8 @@ import {
|
|||||||
RawAttachment,
|
RawAttachment,
|
||||||
} from "./email-processor";
|
} from "./email-processor";
|
||||||
import { getCounters } from "../utils/stats";
|
import { getCounters } from "../utils/stats";
|
||||||
import { iconKey } from "../utils/storage";
|
|
||||||
|
const iconKey = (domain: string) => `icon:${domain}`;
|
||||||
|
|
||||||
const VALID_FEED_ID = "apple.mountain.42";
|
const VALID_FEED_ID = "apple.mountain.42";
|
||||||
const VALID_TO = `${VALID_FEED_ID}@test.getmynews.app`;
|
const VALID_TO = `${VALID_FEED_ID}@test.getmynews.app`;
|
||||||
|
|||||||
+10
-20
@@ -1,11 +1,5 @@
|
|||||||
import { EmailParser } from "../utils/email-parser";
|
import { EmailParser } from "../utils/email-parser";
|
||||||
import {
|
import { AttachmentData, EmailMetadata, Env, FeedConfig } from "../types";
|
||||||
AttachmentData,
|
|
||||||
EmailMetadata,
|
|
||||||
Env,
|
|
||||||
FeedConfig,
|
|
||||||
FeedMetadata,
|
|
||||||
} from "../types";
|
|
||||||
import { notifySubscribers } from "../utils/websub";
|
import { notifySubscribers } from "../utils/websub";
|
||||||
import { bumpCounters } from "../utils/stats";
|
import { bumpCounters } from "../utils/stats";
|
||||||
import {
|
import {
|
||||||
@@ -14,6 +8,7 @@ import {
|
|||||||
} from "../utils/favicon-fetcher";
|
} from "../utils/favicon-fetcher";
|
||||||
import { parseOneClickUnsubscribe } from "../utils/unsubscribe";
|
import { parseOneClickUnsubscribe } from "../utils/unsubscribe";
|
||||||
import { getAttachmentBucket } from "../utils/attachments";
|
import { getAttachmentBucket } from "../utils/attachments";
|
||||||
|
import { FeedRepository } from "../domain/feed-repository";
|
||||||
import { logger } from "./logger";
|
import { logger } from "./logger";
|
||||||
import { FEED_MAX_BYTES } from "../config/constants";
|
import { FEED_MAX_BYTES } from "../config/constants";
|
||||||
|
|
||||||
@@ -110,10 +105,7 @@ export async function validateEmail(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const feedConfig = (await env.EMAIL_STORAGE.get(
|
const feedConfig = await FeedRepository.from(env).getConfig(feedId);
|
||||||
`feed:${feedId}:config`,
|
|
||||||
"json",
|
|
||||||
)) as FeedConfig | null;
|
|
||||||
if (!feedConfig) {
|
if (!feedConfig) {
|
||||||
logger.error("Feed not found", { feedId });
|
logger.error("Feed not found", { feedId });
|
||||||
return {
|
return {
|
||||||
@@ -188,12 +180,12 @@ export async function storeEmail(
|
|||||||
...(storedAttachments.length > 0 ? { attachments: storedAttachments } : {}),
|
...(storedAttachments.length > 0 ? { attachments: storedAttachments } : {}),
|
||||||
};
|
};
|
||||||
|
|
||||||
const emailKey = `feed:${feedId}:${Date.now()}`;
|
const repo = FeedRepository.from(env);
|
||||||
const feedMetadataKey = `feed:${feedId}:metadata`;
|
const emailKey = repo.newEmailKey(feedId);
|
||||||
|
|
||||||
const [, rawMetadata] = await Promise.all([
|
const [, rawMetadata] = await Promise.all([
|
||||||
env.EMAIL_STORAGE.put(emailKey, JSON.stringify(emailData)),
|
repo.putEmail(emailKey, emailData),
|
||||||
env.EMAIL_STORAGE.get(feedMetadataKey, "json"),
|
repo.getMetadata(feedId),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
// Note: KV has no atomic compare-and-swap. Concurrent invocations for the
|
// Note: KV has no atomic compare-and-swap. Concurrent invocations for the
|
||||||
@@ -202,9 +194,7 @@ export async function storeEmail(
|
|||||||
// KV's eventual-consistency model.
|
// KV's eventual-consistency model.
|
||||||
// TODO: Migrate feed metadata writes to Cloudflare Durable Objects to serialise
|
// TODO: Migrate feed metadata writes to Cloudflare Durable Objects to serialise
|
||||||
// concurrent writes and eliminate this race condition.
|
// concurrent writes and eliminate this race condition.
|
||||||
const feedMetadata = ((rawMetadata as FeedMetadata | null) || {
|
const feedMetadata = rawMetadata || { emails: [] };
|
||||||
emails: [],
|
|
||||||
}) as FeedMetadata;
|
|
||||||
|
|
||||||
const maxBytes =
|
const maxBytes =
|
||||||
parseInt(env.FEED_MAX_SIZE_BYTES ?? "", 10) || FEED_MAX_BYTES;
|
parseInt(env.FEED_MAX_SIZE_BYTES ?? "", 10) || FEED_MAX_BYTES;
|
||||||
@@ -260,8 +250,8 @@ export async function storeEmail(
|
|||||||
: [];
|
: [];
|
||||||
|
|
||||||
await Promise.all([
|
await Promise.all([
|
||||||
env.EMAIL_STORAGE.put(feedMetadataKey, JSON.stringify(feedMetadata)),
|
repo.putMetadata(feedId, feedMetadata),
|
||||||
...toDelete.map((e) => env.EMAIL_STORAGE.delete(e.key)),
|
...toDelete.map((e) => repo.deleteEmail(e.key)),
|
||||||
...r2Deletions,
|
...r2Deletions,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
|||||||
+15
-32
@@ -5,10 +5,8 @@ import { bumpCounters } from "../utils/stats";
|
|||||||
import { waitUntilSafe } from "../utils/worker";
|
import { waitUntilSafe } from "../utils/worker";
|
||||||
import { sendUnsubscribes } from "../utils/unsubscribe";
|
import { sendUnsubscribes } from "../utils/unsubscribe";
|
||||||
import { getAttachmentBucket } from "../utils/attachments";
|
import { getAttachmentBucket } from "../utils/attachments";
|
||||||
|
import { FeedRepository } from "../domain/feed-repository";
|
||||||
import {
|
import {
|
||||||
addFeedToList,
|
|
||||||
updateFeedInList,
|
|
||||||
removeFeedFromList,
|
|
||||||
purgeFeedKeysStep,
|
purgeFeedKeysStep,
|
||||||
collectUnsubscribeUrls,
|
collectUnsubscribeUrls,
|
||||||
} from "../routes/admin/helpers";
|
} from "../routes/admin/helpers";
|
||||||
@@ -49,7 +47,7 @@ export async function createFeedRecord(
|
|||||||
env: Env,
|
env: Env,
|
||||||
input: CreateFeedInput,
|
input: CreateFeedInput,
|
||||||
): Promise<{ feedId: string; config: FeedConfig }> {
|
): Promise<{ feedId: string; config: FeedConfig }> {
|
||||||
const emailStorage = env.EMAIL_STORAGE;
|
const repo = FeedRepository.from(env);
|
||||||
const expiresAt = resolveExpiresAt(env, input.lifetimeHours);
|
const expiresAt = resolveExpiresAt(env, input.lifetimeHours);
|
||||||
const feedId = generateFeedId();
|
const feedId = generateFeedId();
|
||||||
|
|
||||||
@@ -67,19 +65,13 @@ export async function createFeedRecord(
|
|||||||
const metadata: FeedMetadata = { emails: [] };
|
const metadata: FeedMetadata = { emails: [] };
|
||||||
|
|
||||||
await Promise.all([
|
await Promise.all([
|
||||||
emailStorage.put(`feed:${feedId}:config`, JSON.stringify(config)),
|
repo.putConfig(feedId, config),
|
||||||
emailStorage.put(`feed:${feedId}:metadata`, JSON.stringify(metadata)),
|
repo.putMetadata(feedId, metadata),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
await addFeedToList(
|
await repo.addToList(feedId, input.title, input.description, expiresAt);
|
||||||
emailStorage,
|
|
||||||
feedId,
|
|
||||||
input.title,
|
|
||||||
input.description,
|
|
||||||
expiresAt,
|
|
||||||
);
|
|
||||||
|
|
||||||
await bumpCounters(emailStorage, {
|
await bumpCounters(env.EMAIL_STORAGE, {
|
||||||
feeds_created: 1,
|
feeds_created: 1,
|
||||||
last_feed_created_at: new Date().toISOString(),
|
last_feed_created_at: new Date().toISOString(),
|
||||||
});
|
});
|
||||||
@@ -115,12 +107,9 @@ export async function updateFeedRecord(
|
|||||||
input: UpdateFeedInput,
|
input: UpdateFeedInput,
|
||||||
options: { inPlace?: boolean } = {},
|
options: { inPlace?: boolean } = {},
|
||||||
): Promise<UpdateFeedResult> {
|
): Promise<UpdateFeedResult> {
|
||||||
const emailStorage = env.EMAIL_STORAGE;
|
const repo = FeedRepository.from(env);
|
||||||
const feedConfigKey = `feed:${feedId}:config`;
|
|
||||||
|
|
||||||
const existing = (await emailStorage.get(feedConfigKey, {
|
const existing = await repo.getConfig(feedId);
|
||||||
type: "json",
|
|
||||||
})) as FeedConfig | null;
|
|
||||||
|
|
||||||
if (!existing) return { status: "not_found" };
|
if (!existing) return { status: "not_found" };
|
||||||
|
|
||||||
@@ -157,14 +146,8 @@ export async function updateFeedRecord(
|
|||||||
expires_at: expiresAt,
|
expires_at: expiresAt,
|
||||||
};
|
};
|
||||||
|
|
||||||
await emailStorage.put(feedConfigKey, JSON.stringify(config));
|
await repo.putConfig(feedId, config);
|
||||||
await updateFeedInList(
|
await repo.updateInList(feedId, config.title, config.description, expiresAt);
|
||||||
emailStorage,
|
|
||||||
feedId,
|
|
||||||
config.title,
|
|
||||||
config.description,
|
|
||||||
expiresAt,
|
|
||||||
);
|
|
||||||
|
|
||||||
return { status: "ok", config };
|
return { status: "ok", config };
|
||||||
}
|
}
|
||||||
@@ -184,22 +167,21 @@ export async function deleteFeedFastDetailed(
|
|||||||
emailStorage: KVNamespace,
|
emailStorage: KVNamespace,
|
||||||
feedId: string,
|
feedId: string,
|
||||||
): Promise<DeleteFeedFastResult> {
|
): Promise<DeleteFeedFastResult> {
|
||||||
const feedConfigKey = `feed:${feedId}:config`;
|
const repo = new FeedRepository(emailStorage);
|
||||||
const feedMetadataKey = `feed:${feedId}:metadata`;
|
|
||||||
|
|
||||||
const errors: string[] = [];
|
const errors: string[] = [];
|
||||||
let configDeleted = false;
|
let configDeleted = false;
|
||||||
let metadataDeleted = false;
|
let metadataDeleted = false;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await emailStorage.delete(feedConfigKey);
|
await repo.deleteConfig(feedId);
|
||||||
configDeleted = true;
|
configDeleted = true;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
errors.push(`config delete failed: ${String(error)}`);
|
errors.push(`config delete failed: ${String(error)}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await emailStorage.delete(feedMetadataKey);
|
await repo.deleteMetadata(feedId);
|
||||||
metadataDeleted = true;
|
metadataDeleted = true;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
errors.push(`metadata delete failed: ${String(error)}`);
|
errors.push(`metadata delete failed: ${String(error)}`);
|
||||||
@@ -220,12 +202,13 @@ export async function deleteFeedRecord(
|
|||||||
feedId: string,
|
feedId: string,
|
||||||
): Promise<boolean> {
|
): Promise<boolean> {
|
||||||
const emailStorage = env.EMAIL_STORAGE;
|
const emailStorage = env.EMAIL_STORAGE;
|
||||||
|
const repo = new FeedRepository(emailStorage);
|
||||||
|
|
||||||
// Read unsubscribe URLs before the metadata is deleted below.
|
// Read unsubscribe URLs before the metadata is deleted below.
|
||||||
const unsubscribeUrls = await collectUnsubscribeUrls(emailStorage, feedId);
|
const unsubscribeUrls = await collectUnsubscribeUrls(emailStorage, feedId);
|
||||||
|
|
||||||
await deleteFeedFastDetailed(emailStorage, feedId);
|
await deleteFeedFastDetailed(emailStorage, feedId);
|
||||||
const removed = await removeFeedFromList(emailStorage, feedId);
|
const removed = await repo.removeFromList(feedId);
|
||||||
if (removed) {
|
if (removed) {
|
||||||
await bumpCounters(emailStorage, { feeds_deleted: 1 });
|
await bumpCounters(emailStorage, { feeds_deleted: 1 });
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import { ADMIN_COOKIE_MAX_AGE } from "../config/constants";
|
|||||||
import { logger } from "../lib/logger";
|
import { logger } from "../lib/logger";
|
||||||
import { timingSafeEqual, checkProxyAuth } from "../lib/auth";
|
import { timingSafeEqual, checkProxyAuth } from "../lib/auth";
|
||||||
import { Layout, clampText } from "./admin/ui";
|
import { Layout, clampText } from "./admin/ui";
|
||||||
import { listAllFeeds } from "./admin/helpers";
|
import { FeedRepository } from "../domain/feed-repository";
|
||||||
import { updateFeedRecord } from "../lib/feed-service";
|
import { updateFeedRecord } from "../lib/feed-service";
|
||||||
import { feedRssUrl, feedAtomUrl, feedEmailAddress } from "../utils/urls";
|
import { feedRssUrl, feedAtomUrl, feedEmailAddress } from "../utils/urls";
|
||||||
import { feedsRouter } from "./admin/feeds";
|
import { feedsRouter } from "./admin/feeds";
|
||||||
@@ -282,14 +282,13 @@ const ExpiryBadge = ({ expiresAt }: { expiresAt: number }) => {
|
|||||||
app.get("/", async (c) => {
|
app.get("/", async (c) => {
|
||||||
// Type assertion for environment variables
|
// Type assertion for environment variables
|
||||||
const env = c.env;
|
const env = c.env;
|
||||||
const emailStorage = env.EMAIL_STORAGE;
|
|
||||||
const url = new URL(c.req.url);
|
const url = new URL(c.req.url);
|
||||||
const view = url.searchParams.get("view") === "table" ? "table" : "list";
|
const view = url.searchParams.get("view") === "table" ? "table" : "list";
|
||||||
const message = url.searchParams.get("message");
|
const message = url.searchParams.get("message");
|
||||||
const count = Number(url.searchParams.get("count") || "0");
|
const count = Number(url.searchParams.get("count") || "0");
|
||||||
|
|
||||||
// List all feeds
|
// List all feeds
|
||||||
const feedList = await listAllFeeds(emailStorage);
|
const feedList = await FeedRepository.from(env).listFeeds();
|
||||||
|
|
||||||
// Keep the dashboard fast: avoid N KV reads for N feeds.
|
// Keep the dashboard fast: avoid N KV reads for N feeds.
|
||||||
// We store title/description in `feeds:list` (description is optional for older data).
|
// We store title/description in `feeds:list` (description is optional for older data).
|
||||||
|
|||||||
+16
-32
@@ -1,17 +1,12 @@
|
|||||||
import { Hono } from "hono";
|
import { Hono } from "hono";
|
||||||
import {
|
import { Env, EmailMetadata } from "../../types";
|
||||||
Env,
|
|
||||||
FeedConfig,
|
|
||||||
FeedMetadata,
|
|
||||||
EmailData,
|
|
||||||
EmailMetadata,
|
|
||||||
} from "../../types";
|
|
||||||
import { logger } from "../../lib/logger";
|
import { logger } from "../../lib/logger";
|
||||||
import { Layout, clampText } from "./ui";
|
import { Layout, clampText } from "./ui";
|
||||||
import {
|
import {
|
||||||
deleteAttachmentsForEmails,
|
deleteAttachmentsForEmails,
|
||||||
deleteKeysWithConcurrency,
|
deleteKeysWithConcurrency,
|
||||||
} from "./helpers";
|
} from "./helpers";
|
||||||
|
import { FeedRepository } from "../../domain/feed-repository";
|
||||||
import { feedRssUrl, feedAtomUrl, feedEmailAddress } from "../../utils/urls";
|
import { feedRssUrl, feedAtomUrl, feedEmailAddress } from "../../utils/urls";
|
||||||
import { formatBytes } from "../../utils/format";
|
import { formatBytes } from "../../utils/format";
|
||||||
import { emailsPageScript } from "../../scripts/generated/emails-page";
|
import { emailsPageScript } from "../../scripts/generated/emails-page";
|
||||||
@@ -156,17 +151,13 @@ const SenderField = ({ from, feedId }: SenderFieldProps) => {
|
|||||||
|
|
||||||
emailsRouter.get("/feeds/:feedId/emails", async (c) => {
|
emailsRouter.get("/feeds/:feedId/emails", async (c) => {
|
||||||
const env = c.env;
|
const env = c.env;
|
||||||
const emailStorage = env.EMAIL_STORAGE;
|
const repo = FeedRepository.from(env);
|
||||||
const feedId = c.req.param("feedId");
|
const feedId = c.req.param("feedId");
|
||||||
const message = c.req.query("message");
|
const message = c.req.query("message");
|
||||||
const count = Number(c.req.query("count") || "0");
|
const count = Number(c.req.query("count") || "0");
|
||||||
|
|
||||||
const feedConfig = (await emailStorage.get(`feed:${feedId}:config`, {
|
const feedConfig = await repo.getConfig(feedId);
|
||||||
type: "json",
|
const feedMetadata = await repo.getMetadata(feedId);
|
||||||
})) as FeedConfig | null;
|
|
||||||
const feedMetadata = (await emailStorage.get(`feed:${feedId}:metadata`, {
|
|
||||||
type: "json",
|
|
||||||
})) as FeedMetadata | null;
|
|
||||||
|
|
||||||
if (!feedConfig || !feedMetadata) {
|
if (!feedConfig || !feedMetadata) {
|
||||||
return c.text("Feed not found", 404);
|
return c.text("Feed not found", 404);
|
||||||
@@ -461,16 +452,14 @@ emailsRouter.get("/feeds/:feedId/emails", async (c) => {
|
|||||||
|
|
||||||
emailsRouter.get("/emails/:emailKey", async (c) => {
|
emailsRouter.get("/emails/:emailKey", async (c) => {
|
||||||
const env = c.env;
|
const env = c.env;
|
||||||
const emailStorage = env.EMAIL_STORAGE;
|
const repo = FeedRepository.from(env);
|
||||||
const emailKey = c.req.param("emailKey");
|
const emailKey = c.req.param("emailKey");
|
||||||
|
|
||||||
const emailData = (await emailStorage.get(emailKey, {
|
const emailData = await repo.getEmail(emailKey);
|
||||||
type: "json",
|
|
||||||
})) as EmailData | null;
|
|
||||||
|
|
||||||
if (!emailData) return c.text("Email not found", 404);
|
if (!emailData) return c.text("Email not found", 404);
|
||||||
|
|
||||||
const feedId = emailKey.split(":")[1];
|
const feedId = repo.feedIdFromEmailKey(emailKey);
|
||||||
const attachments = emailData.attachments ?? [];
|
const attachments = emailData.attachments ?? [];
|
||||||
|
|
||||||
const htmlContent = `<!DOCTYPE html><html><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><style>body{font-family:-apple-system,BlinkMacSystemFont,'SF Pro Text','SF Pro Display','Helvetica Neue',Arial,sans-serif;line-height:1.5;padding:16px;margin:0;color:#333;box-sizing:border-box}img{max-width:100%;height:auto}a{color:#0070f3}@media(prefers-color-scheme:dark){body{background-color:#1c1c1e;color:#ffffff}a{color:#0a84ff}}</style></head><body>${emailData.content}</body></html>`;
|
const htmlContent = `<!DOCTYPE html><html><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><style>body{font-family:-apple-system,BlinkMacSystemFont,'SF Pro Text','SF Pro Display','Helvetica Neue',Arial,sans-serif;line-height:1.5;padding:16px;margin:0;color:#333;box-sizing:border-box}img{max-width:100%;height:auto}a{color:#0070f3}@media(prefers-color-scheme:dark){body{background-color:#1c1c1e;color:#ffffff}a{color:#0a84ff}}</style></head><body>${emailData.content}</body></html>`;
|
||||||
@@ -652,7 +641,7 @@ emailsRouter.get("/emails/:emailKey", async (c) => {
|
|||||||
|
|
||||||
emailsRouter.post("/emails/:emailKey/delete", async (c) => {
|
emailsRouter.post("/emails/:emailKey/delete", async (c) => {
|
||||||
const env = c.env;
|
const env = c.env;
|
||||||
const emailStorage = env.EMAIL_STORAGE;
|
const repo = FeedRepository.from(env);
|
||||||
const emailKey = c.req.param("emailKey");
|
const emailKey = c.req.param("emailKey");
|
||||||
const wantsJson = (c.req.header("Accept") || "").includes("application/json");
|
const wantsJson = (c.req.header("Accept") || "").includes("application/json");
|
||||||
|
|
||||||
@@ -664,12 +653,9 @@ emailsRouter.post("/emails/:emailKey/delete", async (c) => {
|
|||||||
return c.text("Feed ID is required", 400);
|
return c.text("Feed ID is required", 400);
|
||||||
}
|
}
|
||||||
|
|
||||||
const feedMetadataKey = `feed:${feedId}:metadata`;
|
const feedMetadata = await repo.getMetadata(feedId);
|
||||||
const feedMetadata = (await emailStorage.get(feedMetadataKey, {
|
|
||||||
type: "json",
|
|
||||||
})) as FeedMetadata | null;
|
|
||||||
|
|
||||||
await emailStorage.delete(emailKey);
|
await repo.deleteEmail(emailKey);
|
||||||
await deleteAttachmentsForEmails(env, feedMetadata?.emails ?? [], [
|
await deleteAttachmentsForEmails(env, feedMetadata?.emails ?? [], [
|
||||||
emailKey,
|
emailKey,
|
||||||
]);
|
]);
|
||||||
@@ -678,7 +664,7 @@ emailsRouter.post("/emails/:emailKey/delete", async (c) => {
|
|||||||
feedMetadata.emails = feedMetadata.emails.filter(
|
feedMetadata.emails = feedMetadata.emails.filter(
|
||||||
(email) => email.key !== emailKey,
|
(email) => email.key !== emailKey,
|
||||||
);
|
);
|
||||||
await emailStorage.put(feedMetadataKey, JSON.stringify(feedMetadata));
|
await repo.putMetadata(feedId, feedMetadata);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (wantsJson) return c.json({ ok: true, emailKey, feedId });
|
if (wantsJson) return c.json({ ok: true, emailKey, feedId });
|
||||||
@@ -699,6 +685,7 @@ emailsRouter.post("/emails/:emailKey/delete", async (c) => {
|
|||||||
emailsRouter.post("/feeds/:feedId/emails/bulk-delete", async (c) => {
|
emailsRouter.post("/feeds/:feedId/emails/bulk-delete", async (c) => {
|
||||||
const env = c.env;
|
const env = c.env;
|
||||||
const emailStorage = env.EMAIL_STORAGE;
|
const emailStorage = env.EMAIL_STORAGE;
|
||||||
|
const repo = new FeedRepository(emailStorage);
|
||||||
const feedId = c.req.param("feedId");
|
const feedId = c.req.param("feedId");
|
||||||
const contentType = c.req.header("Content-Type") || "";
|
const contentType = c.req.header("Content-Type") || "";
|
||||||
const wantsJson =
|
const wantsJson =
|
||||||
@@ -706,10 +693,7 @@ emailsRouter.post("/feeds/:feedId/emails/bulk-delete", async (c) => {
|
|||||||
(c.req.header("Accept") || "").includes("application/json");
|
(c.req.header("Accept") || "").includes("application/json");
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const feedMetadataKey = `feed:${feedId}:metadata`;
|
const feedMetadata = await repo.getMetadata(feedId);
|
||||||
const feedMetadata = (await emailStorage.get(feedMetadataKey, {
|
|
||||||
type: "json",
|
|
||||||
})) as FeedMetadata | null;
|
|
||||||
|
|
||||||
if (!feedMetadata) {
|
if (!feedMetadata) {
|
||||||
return wantsJson
|
return wantsJson
|
||||||
@@ -753,7 +737,7 @@ emailsRouter.post("/feeds/:feedId/emails/bulk-delete", async (c) => {
|
|||||||
feedMetadata.emails = feedMetadata.emails.filter(
|
feedMetadata.emails = feedMetadata.emails.filter(
|
||||||
(email) => !deletedSet.has(email.key),
|
(email) => !deletedSet.has(email.key),
|
||||||
);
|
);
|
||||||
await emailStorage.put(feedMetadataKey, JSON.stringify(feedMetadata));
|
await repo.putMetadata(feedId, feedMetadata);
|
||||||
|
|
||||||
return c.json({
|
return c.json({
|
||||||
ok: failedEmailKeys.length === 0,
|
ok: failedEmailKeys.length === 0,
|
||||||
@@ -784,7 +768,7 @@ emailsRouter.post("/feeds/:feedId/emails/bulk-delete", async (c) => {
|
|||||||
feedMetadata.emails = feedMetadata.emails.filter(
|
feedMetadata.emails = feedMetadata.emails.filter(
|
||||||
(email) => !deletedSet.has(email.key),
|
(email) => !deletedSet.has(email.key),
|
||||||
);
|
);
|
||||||
await emailStorage.put(feedMetadataKey, JSON.stringify(feedMetadata));
|
await repo.putMetadata(feedId, feedMetadata);
|
||||||
|
|
||||||
return c.redirect(
|
return c.redirect(
|
||||||
`/admin/feeds/${feedId}/emails?message=bulkDeleted&count=${deletedOk.length}`,
|
`/admin/feeds/${feedId}/emails?message=bulkDeleted&count=${deletedOk.length}`,
|
||||||
|
|||||||
+18
-25
@@ -1,6 +1,6 @@
|
|||||||
import { Hono } from "hono";
|
import { Hono } from "hono";
|
||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
import { Env, FeedConfig } from "../../types";
|
import { Env } from "../../types";
|
||||||
import { bumpCounters } from "../../utils/stats";
|
import { bumpCounters } from "../../utils/stats";
|
||||||
import { waitUntilSafe } from "../../utils/worker";
|
import { waitUntilSafe } from "../../utils/worker";
|
||||||
import { feedRssUrl, feedEmailAddress } from "../../utils/urls";
|
import { feedRssUrl, feedEmailAddress } from "../../utils/urls";
|
||||||
@@ -8,11 +8,8 @@ import { logger } from "../../lib/logger";
|
|||||||
import { sendUnsubscribes } from "../../utils/unsubscribe";
|
import { sendUnsubscribes } from "../../utils/unsubscribe";
|
||||||
import { getAttachmentBucket } from "../../utils/attachments";
|
import { getAttachmentBucket } from "../../utils/attachments";
|
||||||
import { Layout } from "./ui";
|
import { Layout } from "./ui";
|
||||||
import {
|
import { purgeFeedKeysStep, collectUnsubscribeUrls } from "./helpers";
|
||||||
removeFeedsFromListBulk,
|
import { FeedRepository } from "../../domain/feed-repository";
|
||||||
purgeFeedKeysStep,
|
|
||||||
collectUnsubscribeUrls,
|
|
||||||
} from "./helpers";
|
|
||||||
import {
|
import {
|
||||||
createFeedRecord,
|
createFeedRecord,
|
||||||
updateFeedRecord,
|
updateFeedRecord,
|
||||||
@@ -149,12 +146,9 @@ feedsRouter.post("/create", async (c) => {
|
|||||||
|
|
||||||
feedsRouter.get("/:feedId/edit", async (c) => {
|
feedsRouter.get("/:feedId/edit", async (c) => {
|
||||||
const env = c.env;
|
const env = c.env;
|
||||||
const emailStorage = env.EMAIL_STORAGE;
|
|
||||||
const feedId = c.req.param("feedId");
|
const feedId = c.req.param("feedId");
|
||||||
|
|
||||||
const feedConfig = (await emailStorage.get(`feed:${feedId}:config`, {
|
const feedConfig = await FeedRepository.from(env).getConfig(feedId);
|
||||||
type: "json",
|
|
||||||
})) as FeedConfig | null;
|
|
||||||
|
|
||||||
if (!feedConfig) {
|
if (!feedConfig) {
|
||||||
return c.text("Feed not found", 404);
|
return c.text("Feed not found", 404);
|
||||||
@@ -365,7 +359,7 @@ feedsRouter.post("/:feedId/edit", async (c) => {
|
|||||||
feedsRouter.post("/:feedId/sender-filter", async (c) => {
|
feedsRouter.post("/:feedId/sender-filter", async (c) => {
|
||||||
const env = c.env;
|
const env = c.env;
|
||||||
const feedId = c.req.param("feedId");
|
const feedId = c.req.param("feedId");
|
||||||
const feedConfigKey = `feed:${feedId}:config`;
|
const repo = FeedRepository.from(env);
|
||||||
|
|
||||||
const body = await c.req.json().catch(() => null);
|
const body = await c.req.json().catch(() => null);
|
||||||
const parsed = senderFilterSchema.safeParse(body);
|
const parsed = senderFilterSchema.safeParse(body);
|
||||||
@@ -376,9 +370,7 @@ feedsRouter.post("/:feedId/sender-filter", async (c) => {
|
|||||||
const { action, value } = parsed.data;
|
const { action, value } = parsed.data;
|
||||||
const normalized = value.trim().toLowerCase();
|
const normalized = value.trim().toLowerCase();
|
||||||
|
|
||||||
const feedConfig = (await env.EMAIL_STORAGE.get(feedConfigKey, {
|
const feedConfig = await repo.getConfig(feedId);
|
||||||
type: "json",
|
|
||||||
})) as FeedConfig | null;
|
|
||||||
if (!feedConfig) return c.json({ ok: false, error: "Feed not found" }, 404);
|
if (!feedConfig) return c.json({ ok: false, error: "Feed not found" }, 404);
|
||||||
|
|
||||||
const allowedSenders = (feedConfig.allowed_senders || []).map((s) =>
|
const allowedSenders = (feedConfig.allowed_senders || []).map((s) =>
|
||||||
@@ -405,15 +397,12 @@ feedsRouter.post("/:feedId/sender-filter", async (c) => {
|
|||||||
|
|
||||||
if (!targetList.includes(normalized)) {
|
if (!targetList.includes(normalized)) {
|
||||||
targetList.push(normalized);
|
targetList.push(normalized);
|
||||||
await env.EMAIL_STORAGE.put(
|
await repo.putConfig(feedId, {
|
||||||
feedConfigKey,
|
...feedConfig,
|
||||||
JSON.stringify({
|
allowed_senders: allowedSenders,
|
||||||
...feedConfig,
|
blocked_senders: blockedSenders,
|
||||||
allowed_senders: allowedSenders,
|
updated_at: Date.now(),
|
||||||
blocked_senders: blockedSenders,
|
});
|
||||||
updated_at: Date.now(),
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return c.json({ ok: true });
|
return c.json({ ok: true });
|
||||||
@@ -552,7 +541,9 @@ feedsRouter.post("/bulk-delete", async (c) => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const deletedFeedIds = await removeFeedsFromListBulk(emailStorage, okIds);
|
const deletedFeedIds = await new FeedRepository(
|
||||||
|
emailStorage,
|
||||||
|
).removeFromListBulk(okIds);
|
||||||
if (deletedFeedIds.length > 0) {
|
if (deletedFeedIds.length > 0) {
|
||||||
await bumpCounters(emailStorage, {
|
await bumpCounters(emailStorage, {
|
||||||
feeds_deleted: deletedFeedIds.length,
|
feeds_deleted: deletedFeedIds.length,
|
||||||
@@ -616,7 +607,9 @@ feedsRouter.post("/bulk-delete", async (c) => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const deletedFeedIds = await removeFeedsFromListBulk(emailStorage, okIds);
|
const deletedFeedIds = await new FeedRepository(
|
||||||
|
emailStorage,
|
||||||
|
).removeFromListBulk(okIds);
|
||||||
if (deletedFeedIds.length > 0) {
|
if (deletedFeedIds.length > 0) {
|
||||||
await bumpCounters(emailStorage, {
|
await bumpCounters(emailStorage, {
|
||||||
feeds_deleted: deletedFeedIds.length,
|
feeds_deleted: deletedFeedIds.length,
|
||||||
|
|||||||
+13
-130
@@ -1,14 +1,7 @@
|
|||||||
import {
|
import { EmailData, EmailMetadata, Env } from "../../types";
|
||||||
EmailData,
|
|
||||||
EmailMetadata,
|
|
||||||
Env,
|
|
||||||
FeedList,
|
|
||||||
FeedListItem,
|
|
||||||
FeedMetadata,
|
|
||||||
} from "../../types";
|
|
||||||
import { FEEDS_LIST_KEY } from "../../config/constants";
|
|
||||||
import { logger } from "../../lib/logger";
|
import { logger } from "../../lib/logger";
|
||||||
import { getAttachmentBucket } from "../../utils/attachments";
|
import { getAttachmentBucket } from "../../utils/attachments";
|
||||||
|
import { FeedRepository } from "../../domain/feed-repository";
|
||||||
|
|
||||||
// Delete the R2 attachments belonging to the given email keys. Call before the
|
// Delete the R2 attachments belonging to the given email keys. Call before the
|
||||||
// emails are removed from feed metadata, while `emails` still carries their
|
// emails are removed from feed metadata, while `emails` still carries their
|
||||||
@@ -58,108 +51,6 @@ export async function deleteKeysWithConcurrency(
|
|||||||
return { ok, failed };
|
return { ok, failed };
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function listAllFeeds(
|
|
||||||
emailStorage: KVNamespace,
|
|
||||||
): Promise<FeedListItem[]> {
|
|
||||||
try {
|
|
||||||
const feedList = (await emailStorage.get(FEEDS_LIST_KEY, {
|
|
||||||
type: "json",
|
|
||||||
})) as FeedList | null;
|
|
||||||
return feedList?.feeds || [];
|
|
||||||
} catch (error) {
|
|
||||||
logger.error("Error listing feeds", { error: String(error) });
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function addFeedToList(
|
|
||||||
emailStorage: KVNamespace,
|
|
||||||
feedId: string,
|
|
||||||
title: string,
|
|
||||||
description?: string,
|
|
||||||
expires_at?: number,
|
|
||||||
): Promise<void> {
|
|
||||||
try {
|
|
||||||
const feedList = ((await emailStorage.get(FEEDS_LIST_KEY, {
|
|
||||||
type: "json",
|
|
||||||
})) as FeedList | null) || { feeds: [] };
|
|
||||||
|
|
||||||
feedList.feeds.push({ id: feedId, title, description, expires_at });
|
|
||||||
await emailStorage.put(FEEDS_LIST_KEY, JSON.stringify(feedList));
|
|
||||||
} catch (error) {
|
|
||||||
logger.error("Error adding feed to list", { feedId, error: String(error) });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function updateFeedInList(
|
|
||||||
emailStorage: KVNamespace,
|
|
||||||
feedId: string,
|
|
||||||
title: string,
|
|
||||||
description?: string,
|
|
||||||
expires_at?: number,
|
|
||||||
): Promise<void> {
|
|
||||||
try {
|
|
||||||
const feedList = ((await emailStorage.get(FEEDS_LIST_KEY, {
|
|
||||||
type: "json",
|
|
||||||
})) as FeedList | null) || { feeds: [] };
|
|
||||||
|
|
||||||
const feedIndex = feedList.feeds.findIndex((feed) => feed.id === feedId);
|
|
||||||
if (feedIndex !== -1) {
|
|
||||||
feedList.feeds[feedIndex].title = title;
|
|
||||||
feedList.feeds[feedIndex].description = description;
|
|
||||||
feedList.feeds[feedIndex].expires_at = expires_at;
|
|
||||||
await emailStorage.put(FEEDS_LIST_KEY, JSON.stringify(feedList));
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
logger.error("Error updating feed in list", {
|
|
||||||
feedId,
|
|
||||||
error: String(error),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function removeFeedsFromListBulk(
|
|
||||||
emailStorage: KVNamespace,
|
|
||||||
feedIds: string[],
|
|
||||||
): Promise<string[]> {
|
|
||||||
try {
|
|
||||||
const feedList = ((await emailStorage.get(FEEDS_LIST_KEY, {
|
|
||||||
type: "json",
|
|
||||||
})) as FeedList | null) || { feeds: [] };
|
|
||||||
|
|
||||||
const toRemove = new Set(feedIds.filter(Boolean));
|
|
||||||
if (toRemove.size === 0) return [];
|
|
||||||
|
|
||||||
const removed: string[] = [];
|
|
||||||
const nextFeeds: FeedListItem[] = [];
|
|
||||||
|
|
||||||
for (const feed of feedList.feeds) {
|
|
||||||
if (toRemove.has(feed.id)) {
|
|
||||||
removed.push(feed.id);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
nextFeeds.push(feed);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (removed.length === 0) return [];
|
|
||||||
|
|
||||||
feedList.feeds = nextFeeds;
|
|
||||||
await emailStorage.put(FEEDS_LIST_KEY, JSON.stringify(feedList));
|
|
||||||
return removed;
|
|
||||||
} catch (error) {
|
|
||||||
logger.error("Error removing feeds from list", { error: String(error) });
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function removeFeedFromList(
|
|
||||||
emailStorage: KVNamespace,
|
|
||||||
feedId: string,
|
|
||||||
): Promise<boolean> {
|
|
||||||
const removed = await removeFeedsFromListBulk(emailStorage, [feedId]);
|
|
||||||
return removed.includes(feedId);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Read a feed's stored RFC 8058 one-click unsubscribe URLs (one per sender).
|
* Read a feed's stored RFC 8058 one-click unsubscribe URLs (one per sender).
|
||||||
* Must be called before the feed metadata is deleted. Never throws.
|
* Must be called before the feed metadata is deleted. Never throws.
|
||||||
@@ -169,9 +60,7 @@ export async function collectUnsubscribeUrls(
|
|||||||
feedId: string,
|
feedId: string,
|
||||||
): Promise<string[]> {
|
): Promise<string[]> {
|
||||||
try {
|
try {
|
||||||
const metadata = (await emailStorage.get(`feed:${feedId}:metadata`, {
|
const metadata = await new FeedRepository(emailStorage).getMetadata(feedId);
|
||||||
type: "json",
|
|
||||||
})) as FeedMetadata | null;
|
|
||||||
return Object.values(metadata?.unsubscribe ?? {});
|
return Object.values(metadata?.unsubscribe ?? {});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error("Error reading unsubscribe URLs", {
|
logger.error("Error reading unsubscribe URLs", {
|
||||||
@@ -192,24 +81,18 @@ export async function purgeFeedKeysStep(
|
|||||||
cursor: string;
|
cursor: string;
|
||||||
listComplete: boolean;
|
listComplete: boolean;
|
||||||
}> {
|
}> {
|
||||||
const prefix = `feed:${feedId}:`;
|
const repo = new FeedRepository(emailStorage);
|
||||||
const limit = Math.min(1000, Math.max(1, Math.floor(options.limit || 100)));
|
const listed = await repo.listFeedKeys(feedId, {
|
||||||
const cursor = options.cursor || undefined;
|
cursor: options.cursor,
|
||||||
|
limit: options.limit,
|
||||||
const listed = await emailStorage.list({ prefix, cursor, limit });
|
});
|
||||||
const keys = (listed.keys || []).map((k) => k.name);
|
const keys = listed.names;
|
||||||
|
|
||||||
if (options.bucket && keys.length > 0) {
|
if (options.bucket && keys.length > 0) {
|
||||||
const emailKeys = keys.filter((k) => {
|
const emailKeys = keys.filter((k) => repo.isEmailKey(feedId, k));
|
||||||
const suffix = k.slice(prefix.length);
|
|
||||||
return suffix !== "config" && suffix !== "metadata";
|
|
||||||
});
|
|
||||||
if (emailKeys.length > 0) {
|
if (emailKeys.length > 0) {
|
||||||
const emailDataResults = await Promise.allSettled(
|
const emailDataResults = await Promise.allSettled(
|
||||||
emailKeys.map(
|
emailKeys.map((k) => repo.getEmail(k)),
|
||||||
(k) =>
|
|
||||||
emailStorage.get(k, { type: "json" }) as Promise<EmailData | null>,
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
const attachmentIds = emailDataResults
|
const attachmentIds = emailDataResults
|
||||||
.filter(
|
.filter(
|
||||||
@@ -234,8 +117,8 @@ export async function purgeFeedKeysStep(
|
|||||||
return {
|
return {
|
||||||
deletedKeys: ok,
|
deletedKeys: ok,
|
||||||
failedKeys: failed,
|
failedKeys: failed,
|
||||||
cursor: listed.cursor || "",
|
cursor: listed.cursor,
|
||||||
listComplete: !!listed.list_complete,
|
listComplete: listed.listComplete,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+15
-17
@@ -8,12 +8,8 @@ import {
|
|||||||
updateFeedRecord,
|
updateFeedRecord,
|
||||||
deleteFeedRecord,
|
deleteFeedRecord,
|
||||||
} from "../../lib/feed-service";
|
} from "../../lib/feed-service";
|
||||||
import { listAllFeeds, deleteAttachmentsForEmails } from "../admin/helpers";
|
import { deleteAttachmentsForEmails } from "../admin/helpers";
|
||||||
import {
|
import { FeedRepository } from "../../domain/feed-repository";
|
||||||
getFeedConfig,
|
|
||||||
getFeedMetadata,
|
|
||||||
getEmailData,
|
|
||||||
} from "../../utils/storage";
|
|
||||||
import { getStats } from "../../utils/stats";
|
import { getStats } from "../../utils/stats";
|
||||||
import { feedEmailAddress, feedRssUrl, feedAtomUrl } from "../../utils/urls";
|
import { feedEmailAddress, feedRssUrl, feedAtomUrl } from "../../utils/urls";
|
||||||
import {
|
import {
|
||||||
@@ -107,7 +103,7 @@ apiApp.openapi(
|
|||||||
}),
|
}),
|
||||||
async (c) => {
|
async (c) => {
|
||||||
const env = c.env;
|
const env = c.env;
|
||||||
const feeds = await listAllFeeds(env.EMAIL_STORAGE);
|
const feeds = await FeedRepository.from(env).listFeeds();
|
||||||
return c.json(
|
return c.json(
|
||||||
{
|
{
|
||||||
feeds: feeds.map((f) => ({
|
feeds: feeds.map((f) => ({
|
||||||
@@ -173,9 +169,10 @@ apiApp.openapi(
|
|||||||
async (c) => {
|
async (c) => {
|
||||||
const env = c.env;
|
const env = c.env;
|
||||||
const { feedId } = c.req.valid("param");
|
const { feedId } = c.req.valid("param");
|
||||||
const config = await getFeedConfig(env.EMAIL_STORAGE, feedId);
|
const repo = FeedRepository.from(env);
|
||||||
|
const config = await repo.getConfig(feedId);
|
||||||
if (!config) return c.json({ error: "Feed not found" }, 404);
|
if (!config) return c.json({ error: "Feed not found" }, 404);
|
||||||
const metadata = await getFeedMetadata(env.EMAIL_STORAGE, feedId);
|
const metadata = await repo.getMetadata(feedId);
|
||||||
return c.json(
|
return c.json(
|
||||||
toFeed(feedId, config, metadata?.emails.length ?? 0, env),
|
toFeed(feedId, config, metadata?.emails.length ?? 0, env),
|
||||||
200,
|
200,
|
||||||
@@ -218,7 +215,7 @@ apiApp.openapi(
|
|||||||
return c.json({ error: "Feed not found" }, 404);
|
return c.json({ error: "Feed not found" }, 404);
|
||||||
if (result.status === "expired")
|
if (result.status === "expired")
|
||||||
return c.json({ error: "Feed has expired and cannot be modified" }, 409);
|
return c.json({ error: "Feed has expired and cannot be modified" }, 409);
|
||||||
const metadata = await getFeedMetadata(env.EMAIL_STORAGE, feedId);
|
const metadata = await FeedRepository.from(env).getMetadata(feedId);
|
||||||
return c.json(
|
return c.json(
|
||||||
toFeed(feedId, result.config, metadata?.emails.length ?? 0, env),
|
toFeed(feedId, result.config, metadata?.emails.length ?? 0, env),
|
||||||
200,
|
200,
|
||||||
@@ -268,7 +265,7 @@ apiApp.openapi(
|
|||||||
async (c) => {
|
async (c) => {
|
||||||
const env = c.env;
|
const env = c.env;
|
||||||
const { feedId } = c.req.valid("param");
|
const { feedId } = c.req.valid("param");
|
||||||
const metadata = await getFeedMetadata(env.EMAIL_STORAGE, feedId);
|
const metadata = await FeedRepository.from(env).getMetadata(feedId);
|
||||||
if (!metadata) return c.json({ error: "Feed not found" }, 404);
|
if (!metadata) return c.json({ error: "Feed not found" }, 404);
|
||||||
return c.json(
|
return c.json(
|
||||||
{
|
{
|
||||||
@@ -303,10 +300,11 @@ apiApp.openapi(
|
|||||||
const env = c.env;
|
const env = c.env;
|
||||||
const { feedId, entryId } = c.req.valid("param");
|
const { feedId, entryId } = c.req.valid("param");
|
||||||
const receivedAt = parseInt(entryId, 10);
|
const receivedAt = parseInt(entryId, 10);
|
||||||
const metadata = await getFeedMetadata(env.EMAIL_STORAGE, feedId);
|
const repo = FeedRepository.from(env);
|
||||||
|
const metadata = await repo.getMetadata(feedId);
|
||||||
const metaEntry = metadata?.emails.find((e) => e.receivedAt === receivedAt);
|
const metaEntry = metadata?.emails.find((e) => e.receivedAt === receivedAt);
|
||||||
if (!metaEntry) return c.json({ error: "Email not found" }, 404);
|
if (!metaEntry) return c.json({ error: "Email not found" }, 404);
|
||||||
const data = await getEmailData(env.EMAIL_STORAGE, metaEntry.key);
|
const data = await repo.getEmail(metaEntry.key);
|
||||||
if (!data) return c.json({ error: "Email not found" }, 404);
|
if (!data) return c.json({ error: "Email not found" }, 404);
|
||||||
return c.json(
|
return c.json(
|
||||||
{
|
{
|
||||||
@@ -344,18 +342,18 @@ apiApp.openapi(
|
|||||||
}),
|
}),
|
||||||
async (c) => {
|
async (c) => {
|
||||||
const env = c.env;
|
const env = c.env;
|
||||||
const emailStorage = env.EMAIL_STORAGE;
|
const repo = FeedRepository.from(env);
|
||||||
const { feedId, entryId } = c.req.valid("param");
|
const { feedId, entryId } = c.req.valid("param");
|
||||||
const receivedAt = parseInt(entryId, 10);
|
const receivedAt = parseInt(entryId, 10);
|
||||||
const metadata = await getFeedMetadata(emailStorage, feedId);
|
const metadata = await repo.getMetadata(feedId);
|
||||||
const metaEntry = metadata?.emails.find((e) => e.receivedAt === receivedAt);
|
const metaEntry = metadata?.emails.find((e) => e.receivedAt === receivedAt);
|
||||||
if (!metadata || !metaEntry)
|
if (!metadata || !metaEntry)
|
||||||
return c.json({ error: "Email not found" }, 404);
|
return c.json({ error: "Email not found" }, 404);
|
||||||
|
|
||||||
await emailStorage.delete(metaEntry.key);
|
await repo.deleteEmail(metaEntry.key);
|
||||||
await deleteAttachmentsForEmails(env, metadata.emails, [metaEntry.key]);
|
await deleteAttachmentsForEmails(env, metadata.emails, [metaEntry.key]);
|
||||||
metadata.emails = metadata.emails.filter((e) => e.key !== metaEntry.key);
|
metadata.emails = metadata.emails.filter((e) => e.key !== metaEntry.key);
|
||||||
await emailStorage.put(`feed:${feedId}:metadata`, JSON.stringify(metadata));
|
await repo.putMetadata(feedId, metadata);
|
||||||
|
|
||||||
return c.json({ ok: true }, 200);
|
return c.json({ ok: true }, 200);
|
||||||
},
|
},
|
||||||
|
|||||||
+6
-14
@@ -1,8 +1,9 @@
|
|||||||
import { Context } from "hono";
|
import { Context } from "hono";
|
||||||
import { html, raw } from "hono/html";
|
import { html, raw } from "hono/html";
|
||||||
import { Env, FeedConfig, FeedMetadata, EmailData } from "../types";
|
import { Env } from "../types";
|
||||||
import { processEmailContent } from "../utils/html-processor";
|
import { processEmailContent } from "../utils/html-processor";
|
||||||
import { formatBytes } from "../utils/format";
|
import { formatBytes } from "../utils/format";
|
||||||
|
import { FeedRepository } from "../domain/feed-repository";
|
||||||
|
|
||||||
export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
|
export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
|
||||||
const feedId = c.req.param("feedId");
|
const feedId = c.req.param("feedId");
|
||||||
@@ -12,17 +13,11 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
|
|||||||
return new Response("Not Found", { status: 404 });
|
return new Response("Not Found", { status: 404 });
|
||||||
}
|
}
|
||||||
|
|
||||||
const emailStorage = c.env.EMAIL_STORAGE;
|
const repo = FeedRepository.from(c.env);
|
||||||
|
|
||||||
const [feedMetadata, feedConfig] = await Promise.all([
|
const [feedMetadata, feedConfig] = await Promise.all([
|
||||||
emailStorage.get(
|
repo.getMetadata(feedId),
|
||||||
`feed:${feedId}:metadata`,
|
repo.getConfig(feedId),
|
||||||
"json",
|
|
||||||
) as Promise<FeedMetadata | null>,
|
|
||||||
emailStorage.get(
|
|
||||||
`feed:${feedId}:config`,
|
|
||||||
"json",
|
|
||||||
) as Promise<FeedConfig | null>,
|
|
||||||
]);
|
]);
|
||||||
if (!feedMetadata) {
|
if (!feedMetadata) {
|
||||||
return new Response("Feed not found", { status: 404 });
|
return new Response("Feed not found", { status: 404 });
|
||||||
@@ -41,10 +36,7 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
|
|||||||
return new Response("Entry not found", { status: 404 });
|
return new Response("Entry not found", { status: 404 });
|
||||||
}
|
}
|
||||||
|
|
||||||
const emailData = (await emailStorage.get(
|
const emailData = await repo.getEmail(metaEntry.key);
|
||||||
metaEntry.key,
|
|
||||||
"json",
|
|
||||||
)) as EmailData | null;
|
|
||||||
if (!emailData) {
|
if (!emailData) {
|
||||||
return new Response("Entry not found", { status: 404 });
|
return new Response("Entry not found", { status: 404 });
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,9 +1,10 @@
|
|||||||
import { describe, it, expect } from "vitest";
|
import { describe, it, expect } from "vitest";
|
||||||
import worker from "../index";
|
import worker from "../index";
|
||||||
import { createMockEnv } from "../test/setup";
|
import { createMockEnv } from "../test/setup";
|
||||||
import { iconKey } from "../utils/storage";
|
|
||||||
import type { Env } from "../types";
|
import type { Env } from "../types";
|
||||||
|
|
||||||
|
const iconKey = (domain: string) => `icon:${domain}`;
|
||||||
|
|
||||||
function req(path: string): Request {
|
function req(path: string): Request {
|
||||||
return new Request(`https://test.getmynews.app${path}`);
|
return new Request(`https://test.getmynews.app${path}`);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { Context } from "hono";
|
import { Context } from "hono";
|
||||||
import { Env } from "../types";
|
import { Env } from "../types";
|
||||||
import { getFeedMetadata } from "../utils/storage";
|
import { FeedRepository } from "../domain/feed-repository";
|
||||||
import { cacheFaviconForDomain, getCachedIcon } from "../utils/favicon-fetcher";
|
import { cacheFaviconForDomain, getCachedIcon } from "../utils/favicon-fetcher";
|
||||||
|
|
||||||
export const FAVICON_PATH = "/favicon.svg";
|
export const FAVICON_PATH = "/favicon.svg";
|
||||||
@@ -40,7 +40,7 @@ export async function handleFeedFavicon(
|
|||||||
const feedId = c.req.param("feedId");
|
const feedId = c.req.param("feedId");
|
||||||
if (!feedId) return projectFavicon();
|
if (!feedId) return projectFavicon();
|
||||||
|
|
||||||
const metadata = await getFeedMetadata(env.EMAIL_STORAGE, feedId);
|
const metadata = await FeedRepository.from(env).getMetadata(feedId);
|
||||||
const domain = metadata?.iconDomain;
|
const domain = metadata?.iconDomain;
|
||||||
if (!domain) return projectFavicon();
|
if (!domain) return projectFavicon();
|
||||||
|
|
||||||
|
|||||||
+2
-4
@@ -7,6 +7,7 @@ import {
|
|||||||
import { waitUntilSafe } from "../utils/worker";
|
import { waitUntilSafe } from "../utils/worker";
|
||||||
import { DEFAULT_LEASE_SECONDS, MAX_LEASE_SECONDS } from "../config/constants";
|
import { DEFAULT_LEASE_SECONDS, MAX_LEASE_SECONDS } from "../config/constants";
|
||||||
import { feedTopicPattern } from "../utils/urls";
|
import { feedTopicPattern } from "../utils/urls";
|
||||||
|
import { FeedRepository } from "../domain/feed-repository";
|
||||||
|
|
||||||
type AppEnv = { Bindings: Env };
|
type AppEnv = { Bindings: Env };
|
||||||
|
|
||||||
@@ -73,10 +74,7 @@ hubRouter.post("/", async (c) => {
|
|||||||
const feedId = match[2];
|
const feedId = match[2];
|
||||||
|
|
||||||
// Verify the feed exists before accepting any subscription
|
// Verify the feed exists before accepting any subscription
|
||||||
const feedConfig = await env.EMAIL_STORAGE.get(
|
const feedConfig = await FeedRepository.from(env).getConfig(feedId);
|
||||||
`feed:${feedId}:config`,
|
|
||||||
"json",
|
|
||||||
);
|
|
||||||
if (!feedConfig) {
|
if (!feedConfig) {
|
||||||
return c.text("Not Found: feed does not exist", 404);
|
return c.text("Not Found: feed does not exist", 404);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,8 +6,9 @@ import {
|
|||||||
extractEmailDomain,
|
extractEmailDomain,
|
||||||
getCachedIcon,
|
getCachedIcon,
|
||||||
} from "./favicon-fetcher";
|
} from "./favicon-fetcher";
|
||||||
import { iconKey } from "./storage";
|
|
||||||
import { MAX_ICON_BYTES } from "../config/constants";
|
import { MAX_ICON_BYTES } from "../config/constants";
|
||||||
|
|
||||||
|
const iconKey = (domain: string) => `icon:${domain}`;
|
||||||
import type { Env } from "../types";
|
import type { Env } from "../types";
|
||||||
|
|
||||||
const PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 1, 2, 3, 4]);
|
const PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 1, 2, 3, 4]);
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import {
|
|||||||
ICON_TTL_SECONDS,
|
ICON_TTL_SECONDS,
|
||||||
MAX_ICON_BYTES,
|
MAX_ICON_BYTES,
|
||||||
} from "../config/constants";
|
} from "../config/constants";
|
||||||
import { iconKey } from "./storage";
|
import { FeedRepository } from "../domain/feed-repository";
|
||||||
import { logger } from "../lib/logger";
|
import { logger } from "../lib/logger";
|
||||||
|
|
||||||
interface IconRecord {
|
interface IconRecord {
|
||||||
@@ -92,8 +92,8 @@ export async function cacheFaviconForDomain(
|
|||||||
env: Env,
|
env: Env,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
try {
|
try {
|
||||||
const key = iconKey(domain);
|
const repo = FeedRepository.from(env);
|
||||||
const existing = await env.EMAIL_STORAGE.get(key, "text");
|
const existing = await repo.getIconText(domain);
|
||||||
if (existing !== null) return; // present (incl. negative) → nothing to do
|
if (existing !== null) return; // present (incl. negative) → nothing to do
|
||||||
|
|
||||||
const icon = await resolveIcon(domain);
|
const icon = await resolveIcon(domain);
|
||||||
@@ -104,9 +104,7 @@ export async function cacheFaviconForDomain(
|
|||||||
}
|
}
|
||||||
: { data: null, contentType: "" };
|
: { data: null, contentType: "" };
|
||||||
|
|
||||||
await env.EMAIL_STORAGE.put(key, JSON.stringify(record), {
|
await repo.putIcon(domain, JSON.stringify(record), ICON_TTL_SECONDS);
|
||||||
expirationTtl: ICON_TTL_SECONDS,
|
|
||||||
});
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.warn("Favicon cache failed", { domain, error: String(error) });
|
logger.warn("Favicon cache failed", { domain, error: String(error) });
|
||||||
}
|
}
|
||||||
@@ -119,10 +117,7 @@ export async function getCachedIcon(
|
|||||||
domain: string,
|
domain: string,
|
||||||
env: Env,
|
env: Env,
|
||||||
): Promise<{ bytes: ArrayBuffer; contentType: string } | null> {
|
): Promise<{ bytes: ArrayBuffer; contentType: string } | null> {
|
||||||
const record = (await env.EMAIL_STORAGE.get(
|
const record = await FeedRepository.from(env).getIconJson<IconRecord>(domain);
|
||||||
iconKey(domain),
|
|
||||||
"json",
|
|
||||||
)) as IconRecord | null;
|
|
||||||
if (!record || record.data === null) return null;
|
if (!record || record.data === null) return null;
|
||||||
return {
|
return {
|
||||||
bytes: base64ToArrayBuffer(record.data),
|
bytes: base64ToArrayBuffer(record.data),
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import { Env, FeedConfig, FeedMetadata, EmailData } from "../types";
|
import { Env, FeedConfig, EmailData } from "../types";
|
||||||
import { MAX_FEED_ITEMS } from "../config/constants";
|
import { MAX_FEED_ITEMS } from "../config/constants";
|
||||||
|
import { FeedRepository } from "../domain/feed-repository";
|
||||||
|
|
||||||
export interface FeedData {
|
export interface FeedData {
|
||||||
feedConfig: FeedConfig;
|
feedConfig: FeedConfig;
|
||||||
@@ -10,19 +11,12 @@ export async function fetchFeedData(
|
|||||||
feedId: string,
|
feedId: string,
|
||||||
env: Env,
|
env: Env,
|
||||||
): Promise<FeedData | null> {
|
): Promise<FeedData | null> {
|
||||||
const storage = env.EMAIL_STORAGE;
|
const repo = FeedRepository.from(env);
|
||||||
|
|
||||||
const feedMetadata = (await storage.get(
|
|
||||||
`feed:${feedId}:metadata`,
|
|
||||||
"json",
|
|
||||||
)) as FeedMetadata | null;
|
|
||||||
|
|
||||||
|
const feedMetadata = await repo.getMetadata(feedId);
|
||||||
if (!feedMetadata) return null;
|
if (!feedMetadata) return null;
|
||||||
|
|
||||||
const feedConfig = ((await storage.get(
|
const feedConfig = (await repo.getConfig(feedId)) ?? {
|
||||||
`feed:${feedId}:config`,
|
|
||||||
"json",
|
|
||||||
)) as FeedConfig | null) ?? {
|
|
||||||
title: `Newsletter Feed ${feedId}`,
|
title: `Newsletter Feed ${feedId}`,
|
||||||
description: "Converted email newsletter",
|
description: "Converted email newsletter",
|
||||||
language: "en",
|
language: "en",
|
||||||
@@ -32,7 +26,7 @@ export async function fetchFeedData(
|
|||||||
const emailRefs = feedMetadata.emails.slice(0, MAX_FEED_ITEMS);
|
const emailRefs = feedMetadata.emails.slice(0, MAX_FEED_ITEMS);
|
||||||
const emails: EmailData[] = [];
|
const emails: EmailData[] = [];
|
||||||
for (const ref of emailRefs) {
|
for (const ref of emailRefs) {
|
||||||
const data = (await storage.get(ref.key, "json")) as EmailData | null;
|
const data = await repo.getEmail(ref.key);
|
||||||
if (data) emails.push(data);
|
if (data) emails.push(data);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+12
-26
@@ -1,8 +1,6 @@
|
|||||||
import { Counters, Env, StatsResponse } from "../types";
|
import { Counters, Env, StatsResponse } from "../types";
|
||||||
import { STATS_KEY } from "../config/constants";
|
|
||||||
import { logger } from "../lib/logger";
|
import { logger } from "../lib/logger";
|
||||||
import { listAllFeeds } from "../routes/admin/helpers";
|
import { FeedRepository } from "../domain/feed-repository";
|
||||||
import { getFeedMetadata } from "./storage";
|
|
||||||
import { getAttachmentBucket } from "./attachments";
|
import { getAttachmentBucket } from "./attachments";
|
||||||
|
|
||||||
const EMPTY_COUNTERS: Counters = {
|
const EMPTY_COUNTERS: Counters = {
|
||||||
@@ -15,9 +13,7 @@ const EMPTY_COUNTERS: Counters = {
|
|||||||
|
|
||||||
export async function getCounters(kv: KVNamespace): Promise<Counters> {
|
export async function getCounters(kv: KVNamespace): Promise<Counters> {
|
||||||
try {
|
try {
|
||||||
const stored = (await kv.get(STATS_KEY, {
|
const stored = await new FeedRepository(kv).getCountersRaw();
|
||||||
type: "json",
|
|
||||||
})) as Counters | null;
|
|
||||||
return { ...EMPTY_COUNTERS, ...(stored || {}) };
|
return { ...EMPTY_COUNTERS, ...(stored || {}) };
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error("Error reading counters", { error: String(error) });
|
logger.error("Error reading counters", { error: String(error) });
|
||||||
@@ -48,7 +44,7 @@ export async function bumpCounters(
|
|||||||
current.last_feed_created_at = changes.last_feed_created_at;
|
current.last_feed_created_at = changes.last_feed_created_at;
|
||||||
if (!current.first_seen) current.first_seen = new Date().toISOString();
|
if (!current.first_seen) current.first_seen = new Date().toISOString();
|
||||||
|
|
||||||
await kv.put(STATS_KEY, JSON.stringify(current));
|
await new FeedRepository(kv).putCounters(current);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error("Error updating counters", { error: String(error) });
|
logger.error("Error updating counters", { error: String(error) });
|
||||||
}
|
}
|
||||||
@@ -58,26 +54,15 @@ export async function countKeysByPrefix(
|
|||||||
kv: KVNamespace,
|
kv: KVNamespace,
|
||||||
prefix: string,
|
prefix: string,
|
||||||
): Promise<number> {
|
): Promise<number> {
|
||||||
let total = 0;
|
return new FeedRepository(kv).countKeysByPrefix(prefix);
|
||||||
let cursor: string | undefined;
|
|
||||||
try {
|
|
||||||
do {
|
|
||||||
const listed = await kv.list({ prefix, cursor, limit: 1000 });
|
|
||||||
total += listed.keys.length;
|
|
||||||
cursor = listed.list_complete ? undefined : listed.cursor;
|
|
||||||
} while (cursor);
|
|
||||||
} catch (error) {
|
|
||||||
logger.error("Error counting keys", { prefix, error: String(error) });
|
|
||||||
}
|
|
||||||
return total;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function getStats(env: Env): Promise<StatsResponse> {
|
export async function getStats(env: Env): Promise<StatsResponse> {
|
||||||
const kv = env.EMAIL_STORAGE;
|
const repo = FeedRepository.from(env);
|
||||||
const [counters, feeds, websubCount] = await Promise.all([
|
const [counters, feeds, websubCount] = await Promise.all([
|
||||||
getCounters(kv),
|
getCounters(env.EMAIL_STORAGE),
|
||||||
listAllFeeds(kv),
|
repo.listFeeds(),
|
||||||
countKeysByPrefix(kv, "websub:"),
|
repo.countSubscriptionKeys(),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -119,9 +104,10 @@ export async function scanR2Usage(
|
|||||||
export async function scanKvUsage(kv: KVNamespace): Promise<{ bytes: number }> {
|
export async function scanKvUsage(kv: KVNamespace): Promise<{ bytes: number }> {
|
||||||
let bytes = 0;
|
let bytes = 0;
|
||||||
try {
|
try {
|
||||||
const feeds = await listAllFeeds(kv);
|
const repo = new FeedRepository(kv);
|
||||||
|
const feeds = await repo.listFeeds();
|
||||||
for (const feed of feeds) {
|
for (const feed of feeds) {
|
||||||
const metadata = await getFeedMetadata(kv, feed.id);
|
const metadata = await repo.getMetadata(feed.id);
|
||||||
if (!metadata) continue;
|
if (!metadata) continue;
|
||||||
for (const email of metadata.emails) {
|
for (const email of metadata.emails) {
|
||||||
bytes += email.size ?? 0;
|
bytes += email.size ?? 0;
|
||||||
@@ -152,7 +138,7 @@ export async function setStorageSnapshot(
|
|||||||
current.kv_bytes_estimated = snapshot.kv_bytes_estimated;
|
current.kv_bytes_estimated = snapshot.kv_bytes_estimated;
|
||||||
current.storage_scanned_at = new Date().toISOString();
|
current.storage_scanned_at = new Date().toISOString();
|
||||||
if (!current.first_seen) current.first_seen = new Date().toISOString();
|
if (!current.first_seen) current.first_seen = new Date().toISOString();
|
||||||
await kv.put(STATS_KEY, JSON.stringify(current));
|
await new FeedRepository(kv).putCounters(current);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error("Error writing storage snapshot", { error: String(error) });
|
logger.error("Error writing storage snapshot", { error: String(error) });
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,169 +0,0 @@
|
|||||||
import {
|
|
||||||
EmailData,
|
|
||||||
FeedConfig,
|
|
||||||
FeedMetadata,
|
|
||||||
FeedList,
|
|
||||||
EmailMetadata,
|
|
||||||
} from "../types";
|
|
||||||
import { MAX_METADATA_EMAILS } from "../config/constants";
|
|
||||||
|
|
||||||
/**
|
|
||||||
* KV key for a domain's cached favicon (shared across feeds from the same sender).
|
|
||||||
*/
|
|
||||||
export function iconKey(domain: string): string {
|
|
||||||
return `icon:${domain}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Store email data in KV
|
|
||||||
*/
|
|
||||||
export async function storeEmail(
|
|
||||||
kv: KVNamespace,
|
|
||||||
feedId: string,
|
|
||||||
emailData: EmailData,
|
|
||||||
): Promise<string> {
|
|
||||||
// Generate a unique key for this email
|
|
||||||
const timestamp = Date.now();
|
|
||||||
const key = `feed:${feedId}:email:${timestamp}`;
|
|
||||||
|
|
||||||
// Store the email content
|
|
||||||
await kv.put(key, JSON.stringify(emailData));
|
|
||||||
|
|
||||||
// Update the feed's metadata (list of emails)
|
|
||||||
await updateFeedMetadata(kv, feedId, {
|
|
||||||
key,
|
|
||||||
subject: emailData.subject,
|
|
||||||
receivedAt: timestamp,
|
|
||||||
});
|
|
||||||
|
|
||||||
return key;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Update feed metadata with a new email
|
|
||||||
*/
|
|
||||||
async function updateFeedMetadata(
|
|
||||||
kv: KVNamespace,
|
|
||||||
feedId: string,
|
|
||||||
emailMetadata: EmailMetadata,
|
|
||||||
): Promise<void> {
|
|
||||||
const feedMetadataKey = `feed:${feedId}:metadata`;
|
|
||||||
const existingMetadata = (await kv.get(feedMetadataKey, {
|
|
||||||
type: "json",
|
|
||||||
})) as FeedMetadata | null;
|
|
||||||
|
|
||||||
const metadata: FeedMetadata = existingMetadata || { emails: [] };
|
|
||||||
|
|
||||||
// Add new email to the beginning of the list
|
|
||||||
metadata.emails.unshift(emailMetadata);
|
|
||||||
|
|
||||||
// Keep only the last MAX_METADATA_EMAILS in the metadata; delete orphaned KV entries
|
|
||||||
const toDelete =
|
|
||||||
metadata.emails.length > MAX_METADATA_EMAILS
|
|
||||||
? metadata.emails.slice(MAX_METADATA_EMAILS)
|
|
||||||
: [];
|
|
||||||
if (toDelete.length > 0) {
|
|
||||||
metadata.emails = metadata.emails.slice(0, MAX_METADATA_EMAILS);
|
|
||||||
}
|
|
||||||
|
|
||||||
await Promise.all([
|
|
||||||
kv.put(feedMetadataKey, JSON.stringify(metadata)),
|
|
||||||
...toDelete.map((e) => kv.delete(e.key)),
|
|
||||||
]);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get feed metadata
|
|
||||||
*/
|
|
||||||
export async function getFeedMetadata(
|
|
||||||
kv: KVNamespace,
|
|
||||||
feedId: string,
|
|
||||||
): Promise<FeedMetadata | null> {
|
|
||||||
const feedMetadataKey = `feed:${feedId}:metadata`;
|
|
||||||
return (await kv.get(feedMetadataKey, {
|
|
||||||
type: "json",
|
|
||||||
})) as FeedMetadata | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get feed configuration
|
|
||||||
*/
|
|
||||||
export async function getFeedConfig(
|
|
||||||
kv: KVNamespace,
|
|
||||||
feedId: string,
|
|
||||||
): Promise<FeedConfig | null> {
|
|
||||||
const feedConfigKey = `feed:${feedId}:config`;
|
|
||||||
return (await kv.get(feedConfigKey, { type: "json" })) as FeedConfig | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get email data
|
|
||||||
*/
|
|
||||||
export async function getEmailData(
|
|
||||||
kv: KVNamespace,
|
|
||||||
key: string,
|
|
||||||
): Promise<EmailData | null> {
|
|
||||||
return (await kv.get(key, { type: "json" })) as EmailData | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Create a new feed
|
|
||||||
*/
|
|
||||||
export async function createFeed(
|
|
||||||
kv: KVNamespace,
|
|
||||||
feedId: string,
|
|
||||||
feedConfig: FeedConfig,
|
|
||||||
): Promise<void> {
|
|
||||||
// Store feed configuration
|
|
||||||
const feedConfigKey = `feed:${feedId}:config`;
|
|
||||||
await kv.put(feedConfigKey, JSON.stringify(feedConfig));
|
|
||||||
|
|
||||||
// Create empty metadata for the feed
|
|
||||||
const feedMetadataKey = `feed:${feedId}:metadata`;
|
|
||||||
await kv.put(
|
|
||||||
feedMetadataKey,
|
|
||||||
JSON.stringify({
|
|
||||||
emails: [],
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
// Add feed to the list of all feeds
|
|
||||||
await addFeedToList(kv, feedId, feedConfig.title, feedConfig.description);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Add a feed to the global list
|
|
||||||
*/
|
|
||||||
export async function addFeedToList(
|
|
||||||
kv: KVNamespace,
|
|
||||||
feedId: string,
|
|
||||||
title: string,
|
|
||||||
description?: string,
|
|
||||||
): Promise<void> {
|
|
||||||
const feedListKey = "feeds:list";
|
|
||||||
const existingList = (await kv.get(feedListKey, {
|
|
||||||
type: "json",
|
|
||||||
})) as FeedList | null;
|
|
||||||
|
|
||||||
const feedList: FeedList = existingList || { feeds: [] };
|
|
||||||
|
|
||||||
feedList.feeds.push({
|
|
||||||
id: feedId,
|
|
||||||
title,
|
|
||||||
description,
|
|
||||||
});
|
|
||||||
|
|
||||||
await kv.put(feedListKey, JSON.stringify(feedList));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get all feeds
|
|
||||||
*/
|
|
||||||
export async function getAllFeeds(kv: KVNamespace): Promise<FeedList> {
|
|
||||||
const feedListKey = "feeds:list";
|
|
||||||
const feedList = (await kv.get(feedListKey, {
|
|
||||||
type: "json",
|
|
||||||
})) as FeedList | null;
|
|
||||||
|
|
||||||
return feedList || { feeds: [] };
|
|
||||||
}
|
|
||||||
@@ -8,7 +8,6 @@ import {
|
|||||||
notifySubscribers,
|
notifySubscribers,
|
||||||
verifyAndStoreSubscription,
|
verifyAndStoreSubscription,
|
||||||
verifyAndDeleteSubscription,
|
verifyAndDeleteSubscription,
|
||||||
subscriptionKey,
|
|
||||||
} from "./websub";
|
} from "./websub";
|
||||||
import type { Env, WebSubSubscription } from "../types";
|
import type { Env, WebSubSubscription } from "../types";
|
||||||
|
|
||||||
@@ -51,8 +50,12 @@ describe("getSubscriptions / saveSubscriptions", () => {
|
|||||||
expect(await getSubscriptions("feed1", env)).toEqual(subs);
|
expect(await getSubscriptions("feed1", env)).toEqual(subs);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("uses the correct KV key", () => {
|
it("uses the correct KV key", async () => {
|
||||||
expect(subscriptionKey("abc")).toBe("websub:subs:abc");
|
const env = mockEnv();
|
||||||
|
await saveSubscriptions("abc", [], env);
|
||||||
|
expect(
|
||||||
|
await env.EMAIL_STORAGE.get("websub:subs:abc", { type: "json" }),
|
||||||
|
).toEqual([]);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
+10
-30
@@ -1,25 +1,13 @@
|
|||||||
import {
|
import { Env, FeedConfig, EmailData, WebSubSubscription } from "../types";
|
||||||
Env,
|
|
||||||
FeedConfig,
|
|
||||||
FeedMetadata,
|
|
||||||
EmailData,
|
|
||||||
WebSubSubscription,
|
|
||||||
} from "../types";
|
|
||||||
import { generateRssFeed, generateAtomFeed } from "./feed-generator";
|
import { generateRssFeed, generateAtomFeed } from "./feed-generator";
|
||||||
import { baseUrl, feedRssUrl, feedAtomUrl, feedUrl } from "./urls";
|
import { baseUrl, feedRssUrl, feedAtomUrl, feedUrl } from "./urls";
|
||||||
|
import { FeedRepository } from "../domain/feed-repository";
|
||||||
const KV_PREFIX = "websub:subs:";
|
|
||||||
|
|
||||||
export function subscriptionKey(feedId: string): string {
|
|
||||||
return `${KV_PREFIX}${feedId}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function getSubscriptions(
|
export async function getSubscriptions(
|
||||||
feedId: string,
|
feedId: string,
|
||||||
env: Env,
|
env: Env,
|
||||||
): Promise<WebSubSubscription[]> {
|
): Promise<WebSubSubscription[]> {
|
||||||
const raw = await env.EMAIL_STORAGE.get(subscriptionKey(feedId), "json");
|
return FeedRepository.from(env).getSubscriptions(feedId);
|
||||||
return (raw as WebSubSubscription[] | null) ?? [];
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function saveSubscriptions(
|
export async function saveSubscriptions(
|
||||||
@@ -27,10 +15,7 @@ export async function saveSubscriptions(
|
|||||||
subscriptions: WebSubSubscription[],
|
subscriptions: WebSubSubscription[],
|
||||||
env: Env,
|
env: Env,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
await env.EMAIL_STORAGE.put(
|
await FeedRepository.from(env).saveSubscriptions(feedId, subscriptions);
|
||||||
subscriptionKey(feedId),
|
|
||||||
JSON.stringify(subscriptions),
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function buildHmacSignature(
|
export async function buildHmacSignature(
|
||||||
@@ -60,16 +45,16 @@ async function buildFeedXml(
|
|||||||
env: Env,
|
env: Env,
|
||||||
format: "rss" | "atom" = "rss",
|
format: "rss" | "atom" = "rss",
|
||||||
): Promise<string | null> {
|
): Promise<string | null> {
|
||||||
const [rawMetadata, rawConfig] = await Promise.all([
|
const repo = FeedRepository.from(env);
|
||||||
env.EMAIL_STORAGE.get(`feed:${feedId}:metadata`, "json"),
|
const [feedMetadata, rawConfig] = await Promise.all([
|
||||||
env.EMAIL_STORAGE.get(`feed:${feedId}:config`, "json"),
|
repo.getMetadata(feedId),
|
||||||
|
repo.getConfig(feedId),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const feedMetadata = rawMetadata as FeedMetadata | null;
|
|
||||||
if (!feedMetadata) return null;
|
if (!feedMetadata) return null;
|
||||||
|
|
||||||
const base = baseUrl(env);
|
const base = baseUrl(env);
|
||||||
const feedConfig = (rawConfig as FeedConfig | null) ?? {
|
const feedConfig: FeedConfig = rawConfig ?? {
|
||||||
title: `Newsletter Feed ${feedId}`,
|
title: `Newsletter Feed ${feedId}`,
|
||||||
description: "Converted email newsletter",
|
description: "Converted email newsletter",
|
||||||
language: "en",
|
language: "en",
|
||||||
@@ -78,12 +63,7 @@ async function buildFeedXml(
|
|||||||
|
|
||||||
const emails = feedMetadata.emails.slice(0, 20);
|
const emails = feedMetadata.emails.slice(0, 20);
|
||||||
const emailsData = (
|
const emailsData = (
|
||||||
await Promise.all(
|
await Promise.all(emails.map((m) => repo.getEmail(m.key)))
|
||||||
emails.map(
|
|
||||||
(m) =>
|
|
||||||
env.EMAIL_STORAGE.get(m.key, "json") as Promise<EmailData | null>,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
).filter((d): d is EmailData => d !== null);
|
).filter((d): d is EmailData => d !== null);
|
||||||
|
|
||||||
if (format === "atom") {
|
if (format === "atom") {
|
||||||
|
|||||||
Reference in New Issue
Block a user