feat: reader-compat batch — JSON Feed, OPML export, conditional GET, dedup

Batch of four reader-facing improvements (TODO "Compat lecteurs + dedup"):

- JSON Feed at /json/:feedId (feed lib .json1()); all formats cross-link
- OPML export at /admin/opml (admin-protected; the registry lists every
  feed URL, so it must not be public)
- Conditional GET on /rss + /atom: strong ETag + Last-Modified, 304 on
  If-None-Match/If-Modified-Since, validators shared via http-cache.ts
- Duplicate-send dedup in ingestion: match by Message-ID, fall back to a
  SHA-256 of normalized subject+content; a duplicate is a no-op and bumps
  the new emails_deduplicated counter (status page + /api/v1/stats)

429 tests green, tsc clean, build dry-run OK. Docs (README/CLAUDE/TODO +
landing cards) updated.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Julien Herr
2026-05-24 20:47:54 +02:00
parent 334713fbd9
commit 0abd5f306c
23 changed files with 1015 additions and 11 deletions
+131
View File
@@ -595,6 +595,134 @@ describe("processEmail — attachments", () => {
});
});
describe("processEmail — deduplication", () => {
let env: ReturnType<typeof createMockEnv>;
beforeEach(async () => {
env = createMockEnv();
await env.EMAIL_STORAGE.put(
`feed:${VALID_FEED_ID}:config`,
JSON.stringify({}),
);
});
it("stores only one email when the same Message-ID is delivered twice", async () => {
const headers = { "Message-ID": "<abc123@example.com>" };
await processEmail(makeInput({ headers }), env as any);
await processEmail(makeInput({ headers }), env as any);
const metadata = await env.EMAIL_STORAGE.get(
`feed:${VALID_FEED_ID}:metadata`,
"json",
);
expect(metadata.emails).toHaveLength(1);
});
it("increments emails_deduplicated counter on the second delivery", async () => {
const headers = { "Message-ID": "<dup42@example.com>" };
await processEmail(makeInput({ headers }), env as any);
await processEmail(makeInput({ headers }), env as any);
const counters = await getCounters(env.EMAIL_STORAGE as any);
expect(counters.emails_deduplicated).toBe(1);
});
it("deduplicates by hash when no Message-ID header is present", async () => {
const input = makeInput({
subject: "Weekly Digest",
content: "<p>Same content</p>",
});
await processEmail(input, env as any);
await processEmail(input, env as any);
const metadata = await env.EMAIL_STORAGE.get(
`feed:${VALID_FEED_ID}:metadata`,
"json",
);
expect(metadata.emails).toHaveLength(1);
const counters = await getCounters(env.EMAIL_STORAGE as any);
expect(counters.emails_deduplicated).toBe(1);
});
it("does not deduplicate emails with different subjects (no Message-ID)", async () => {
await processEmail(
makeInput({ subject: "First", content: "<p>body</p>" }),
env as any,
);
await processEmail(
makeInput({ subject: "Second", content: "<p>body</p>" }),
env as any,
);
const metadata = await env.EMAIL_STORAGE.get(
`feed:${VALID_FEED_ID}:metadata`,
"json",
);
expect(metadata.emails).toHaveLength(2);
const counters = await getCounters(env.EMAIL_STORAGE as any);
expect(counters.emails_deduplicated).toBe(0);
});
it("does not false-positive against pre-feature entries lacking messageId/dedupHash", async () => {
// Seed a legacy metadata entry with no messageId or dedupHash
await env.EMAIL_STORAGE.put(
`feed:${VALID_FEED_ID}:metadata`,
JSON.stringify({
emails: [
{
key: `feed:${VALID_FEED_ID}:999`,
subject: "Old Subject",
receivedAt: 999,
size: 50,
// intentionally no messageId, no dedupHash
},
],
}),
);
// A new, distinct email should be stored without triggering false dedup
const res = await processEmail(
makeInput({ subject: "New Distinct Email", content: "<p>fresh</p>" }),
env as any,
);
expect(res.ok).toBe(true);
const metadata = await env.EMAIL_STORAGE.get(
`feed:${VALID_FEED_ID}:metadata`,
"json",
);
expect(metadata.emails).toHaveLength(2);
const counters = await getCounters(env.EMAIL_STORAGE as any);
expect(counters.emails_deduplicated).toBe(0);
});
it("returns { ok: true } for a genuine duplicate (not a rejection)", async () => {
const headers = { "Message-ID": "<nodrop@example.com>" };
await processEmail(makeInput({ headers }), env as any);
const res = await processEmail(makeInput({ headers }), env as any);
expect(res).toMatchObject({ ok: true });
});
it("stores messageId and dedupHash in the email metadata entry", async () => {
const headers = { "Message-ID": "<stored@example.com>" };
await processEmail(
makeInput({ subject: "Sub", content: "<p>c</p>", headers }),
env as any,
);
const metadata = await env.EMAIL_STORAGE.get(
`feed:${VALID_FEED_ID}:metadata`,
"json",
);
expect(metadata.emails[0].messageId).toBe("<stored@example.com>");
expect(typeof metadata.emails[0].dedupHash).toBe("string");
expect(metadata.emails[0].dedupHash).toHaveLength(64); // SHA-256 hex
});
});
describe("processEmail — monitoring counters", () => {
it("increments emails_received and sets last_email_at on success", async () => {
const env = createMockEnv();
@@ -709,6 +837,7 @@ describe("processEmail — unsubscribe capture", () => {
it("keeps one entry per sender and overwrites with the latest URL", async () => {
await processEmail(
makeInput({
subject: "Issue 1 from A",
senders: ["a@one.com"],
headers: {
"list-unsubscribe": "<https://one.com/u/1>",
@@ -719,6 +848,7 @@ describe("processEmail — unsubscribe capture", () => {
);
await processEmail(
makeInput({
subject: "Issue 1 from B",
senders: ["b@two.com"],
headers: {
"list-unsubscribe": "<https://two.com/u/1>",
@@ -729,6 +859,7 @@ describe("processEmail — unsubscribe capture", () => {
);
await processEmail(
makeInput({
subject: "Issue 2 from A",
senders: ["a@one.com"],
headers: {
"list-unsubscribe": "<https://one.com/u/2>",
+54 -1
View File
@@ -109,12 +109,62 @@ async function loadAcceptingFeed(
return { ok: true, feed };
}
/**
* Compute a SHA-256 hex digest of a normalised string combining subject and
* content. Used as a dedup fallback when no Message-ID header is present.
* "Normalised" means lower-cased and all whitespace runs collapsed to a single
* space — so minor whitespace differences in re-sent mails still match.
*/
async function computeDedupHash(
subject: string,
content: string,
): Promise<string> {
const normalize = (s: string) => s.toLowerCase().replace(/\s+/g, " ").trim();
const raw = `${normalize(subject)}\n${normalize(content)}`;
const buf = await crypto.subtle.digest(
"SHA-256",
new TextEncoder().encode(raw),
);
return Array.from(new Uint8Array(buf))
.map((b) => b.toString(16).padStart(2, "0"))
.join("");
}
/**
* Extract the Message-ID from request headers (case-insensitive key lookup).
* Returns undefined when absent or empty.
*/
function extractMessageId(
headers: Record<string, string> | undefined,
): string | undefined {
if (!headers) return undefined;
const value = Object.entries(headers).find(
([k]) => k.toLowerCase() === "message-id",
)?.[1];
const trimmed = value?.trim();
return trimmed || undefined;
}
async function storeEmail(
feed: Feed,
input: ProcessEmailInput,
env: Env,
ctx?: ExecutionContext,
): Promise<void> {
): Promise<boolean> {
// ── Dedup check ──────────────────────────────────────────────────────────
// Compute both dedup signals up-front (hash is async) so we only do it once.
const messageId = extractMessageId(input.headers);
const dedupHash = await computeDedupHash(input.subject, input.content);
if (feed.hasDuplicate(messageId, dedupHash)) {
logger.info("Duplicate email skipped", {
feedId: feed.id.value,
...(messageId ? { messageId } : { dedupHash }),
});
await bumpCounters(env.EMAIL_STORAGE, { emails_deduplicated: 1 });
return false; // signal: skipped (not stored)
}
const attachmentBucket = getAttachmentBucket(env);
const inlineCids = extractInlineCids(input.content);
const storedAttachments: AttachmentData[] =
@@ -149,6 +199,8 @@ async function storeEmail(
size: serialisedSize,
...(downloadableIds.length > 0 ? { attachmentIds: downloadableIds } : {}),
...(inlineIds.length > 0 ? { inlineAttachmentIds: inlineIds } : {}),
...(messageId ? { messageId } : {}),
dedupHash,
};
// Track the latest sender's domain (feed icon) and capture the RFC 8058
@@ -198,6 +250,7 @@ async function storeEmail(
? (p) => ctx.waitUntil(p)
: () => {};
await dispatchFeedEvents(feed, env, schedule);
return true; // signal: stored
}
export async function processEmail(
+2
View File
@@ -12,6 +12,7 @@ const EMPTY_COUNTERS: Counters = {
emails_received: 0,
emails_rejected: 0,
emails_forwarded: 0,
emails_deduplicated: 0,
unsubscribes_sent: 0,
};
@@ -43,6 +44,7 @@ export async function bumpCounters(
current.emails_received += changes.emails_received ?? 0;
current.emails_rejected += changes.emails_rejected ?? 0;
current.emails_forwarded += changes.emails_forwarded ?? 0;
current.emails_deduplicated += changes.emails_deduplicated ?? 0;
current.unsubscribes_sent += changes.unsubscribes_sent ?? 0;
if (changes.last_email_at) current.last_email_at = changes.last_email_at;
if (changes.last_feed_created_at)
+24
View File
@@ -203,6 +203,30 @@ export class Feed {
).decide(senders);
}
/**
* Check whether the email index already contains a duplicate of the incoming
* email. Dedup uses `messageId` as the primary key (when both sides have one)
* and falls back to `dedupHash` (SHA-256 of normalised subject+content).
* Old entries that predate the feature and carry neither field are never
* matched — they cannot cause false positives.
*/
hasDuplicate(messageId?: string, dedupHash?: string): boolean {
for (const entry of this._metadata.emails) {
if (messageId && entry.messageId && entry.messageId === messageId) {
return true;
}
if (
!messageId &&
dedupHash &&
entry.dedupHash &&
entry.dedupHash === dedupHash
) {
return true;
}
}
return false;
}
/**
* Add an email to the front of the index, refresh the icon domain and the
* per-sender unsubscribe link, then trim the oldest entries back under the
+6
View File
@@ -3,6 +3,7 @@ import { cors } from "hono/cors";
import { handle as handleInbound } from "./routes/inbound";
import { handle as handleRSS } from "./routes/rss";
import { handle as handleAtom } from "./routes/atom";
import { handle as handleJSON } from "./routes/json";
import { handle as handleAdmin } from "./routes/admin";
import { handle as handleEntry } from "./routes/entries";
import { handle as handleFiles } from "./routes/files";
@@ -116,6 +117,7 @@ app.use(
const api = new Hono<AppEnv>();
const rss = new Hono<AppEnv>();
const atom = new Hono<AppEnv>();
const json = new Hono<AppEnv>();
const entries = new Hono<AppEnv>();
const files = new Hono<AppEnv>();
const admin = new Hono<AppEnv>();
@@ -151,6 +153,9 @@ rss.get("/:feedId", handleRSS);
// Atom feed routes (public)
atom.get("/:feedId", handleAtom);
// JSON Feed routes (public)
json.get("/:feedId", handleJSON);
// Email entry HTML view (public)
entries.get("/:feedId/:entryId", handleEntry);
@@ -166,6 +171,7 @@ app.route("/api", api);
app.route("/api", apiApp);
app.route("/rss", rss);
app.route("/atom", atom);
app.route("/json", json);
app.route("/entries", entries);
app.route("/files", files);
app.route("/admin", admin);
@@ -15,6 +15,7 @@ describe("CountersRepository", () => {
emails_received: 2,
emails_rejected: 0,
emails_forwarded: 0,
emails_deduplicated: 0,
unsubscribes_sent: 0,
});
expect(await repo.getRaw()).toMatchObject({ emails_received: 2 });
+18 -1
View File
@@ -30,7 +30,7 @@ function buildFeed(
emails: EmailData[],
baseUrl: string,
feedId: string,
selfUrl?: { rss?: string; atom?: string },
selfUrl?: { rss?: string; atom?: string; json?: string },
): Feed {
const iconUrl = `${baseUrl}/favicon/${feedId}`;
const feed = new Feed({
@@ -52,6 +52,7 @@ function buildFeed(
feedLinks: {
rss: selfUrl?.rss ?? `${baseUrl}/rss/${feedId}`,
atom: selfUrl?.atom ?? `${baseUrl}/atom/${feedId}`,
json: selfUrl?.json ?? `${baseUrl}/json/${feedId}`,
},
author: feedConfig.author
? {
@@ -127,3 +128,19 @@ export function generateAtomFeed(
).atom1(),
);
}
export function generateJsonFeed(
feedConfig: FeedConfig,
emails: EmailData[],
baseUrl: string,
feedId: string,
selfUrl?: string,
): string {
return buildFeed(
feedConfig,
emails,
baseUrl,
feedId,
selfUrl ? { json: selfUrl } : undefined,
).json1();
}
+68
View File
@@ -0,0 +1,68 @@
import { FeedConfig, EmailData } from "../types";
export interface FeedValidators {
etag: string;
lastModified: string;
maxReceivedAt: number;
}
/**
* Compute HTTP cache validators (ETag + Last-Modified) for a feed.
* The ETag is derived from the feed format prefix, feedId, email count, and max
* receivedAt, making it a strong deterministic validator that changes whenever
* the feed content changes.
*/
export function computeFeedValidators(
format: "rss" | "atom",
feedId: string,
feedConfig: FeedConfig,
emails: EmailData[],
): FeedValidators {
const maxReceivedAt =
emails.length > 0
? Math.max(...emails.map((e) => e.receivedAt))
: (feedConfig.created_at ?? 0);
const hash = `${format}-${feedId}-${emails.length}-${maxReceivedAt}`;
const etag = `"${hash}"`;
const lastModified = new Date(maxReceivedAt).toUTCString();
return { etag, lastModified, maxReceivedAt };
}
/**
* Returns true if the request carries a matching conditional GET header,
* meaning a 304 Not Modified response is appropriate.
*/
export function isNotModified(
req: Request,
validators: FeedValidators,
): boolean {
const ifNoneMatch = req.headers.get("If-None-Match");
if (ifNoneMatch !== null) {
return ifNoneMatch === validators.etag;
}
const ifModifiedSince = req.headers.get("If-Modified-Since");
if (ifModifiedSince !== null) {
const clientTime = new Date(ifModifiedSince).getTime();
return !isNaN(clientTime) && clientTime >= validators.maxReceivedAt;
}
return false;
}
/**
* Build a 304 Not Modified response with the standard cache validator headers.
*/
export function notModifiedResponse(validators: FeedValidators): Response {
return new Response(null, {
status: 304,
headers: {
ETag: validators.etag,
"Last-Modified": validators.lastModified,
"Cache-Control": "max-age=1800",
"X-Robots-Tag": "noindex",
},
});
}
+4
View File
@@ -18,6 +18,7 @@ import {
} from "../infrastructure/urls";
import { feedsRouter } from "./admin/feeds";
import { emailsRouter } from "./admin/emails";
import { handleOpml } from "./opml";
import { dashboardScript } from "../scripts/generated/dashboard";
type AppEnv = { Bindings: Env };
@@ -975,6 +976,9 @@ app.get("/", async (c) => {
);
});
// OPML export (admin-protected)
app.get("/opml", handleOpml);
// Mount sub-routers
app.route("/feeds", feedsRouter);
app.route("/", emailsRouter);
+113
View File
@@ -144,4 +144,117 @@ describe("Atom Feed Route", () => {
expect(body).toContain('xmlns="http://www.w3.org/2005/Atom"');
});
});
describe("conditional GET (ETag + Last-Modified)", () => {
const FEED_ID = "test-feed-atom-cget";
const EMAIL_RECEIVED_AT = 1700000001000;
beforeEach(async () => {
const emailKey = `feed:${FEED_ID}:${EMAIL_RECEIVED_AT}`;
await mockEnv.EMAIL_STORAGE.put(
emailKey,
JSON.stringify({
subject: "Atom Subject",
from: "Sender <sender@example.com>",
content: "<p>Body</p>",
receivedAt: EMAIL_RECEIVED_AT,
headers: {},
}),
);
await mockEnv.EMAIL_STORAGE.put(
`feed:${FEED_ID}:metadata`,
JSON.stringify({
emails: [
{
key: emailKey,
subject: "Atom Subject",
receivedAt: EMAIL_RECEIVED_AT,
},
],
}),
);
await mockEnv.EMAIL_STORAGE.put(
`feed:${FEED_ID}:config`,
JSON.stringify({
title: "Atom Cget Feed",
language: "en",
created_at: 1700000000000,
}),
);
});
it("first GET returns 200 with ETag and Last-Modified headers", async () => {
const res = await testApp.request(`/${FEED_ID}`, {}, mockEnv);
expect(res.status).toBe(200);
expect(res.headers.get("ETag")).toBeTruthy();
expect(res.headers.get("Last-Modified")).toBeTruthy();
});
it("GET with matching If-None-Match returns 304 with empty body", async () => {
const first = await testApp.request(`/${FEED_ID}`, {}, mockEnv);
const etag = first.headers.get("ETag")!;
const res = await testApp.request(
`/${FEED_ID}`,
{ headers: { "If-None-Match": etag } },
mockEnv,
);
expect(res.status).toBe(304);
expect(await res.text()).toBe("");
});
it("GET with If-Modified-Since in the future returns 304", async () => {
const future = new Date(EMAIL_RECEIVED_AT + 1000).toUTCString();
const res = await testApp.request(
`/${FEED_ID}`,
{ headers: { "If-Modified-Since": future } },
mockEnv,
);
expect(res.status).toBe(304);
});
it("stale If-None-Match after new email results in 200", async () => {
const first = await testApp.request(`/${FEED_ID}`, {}, mockEnv);
const oldEtag = first.headers.get("ETag")!;
const newReceivedAt = EMAIL_RECEIVED_AT + 5000;
const newEmailKey = `feed:${FEED_ID}:${newReceivedAt}`;
await mockEnv.EMAIL_STORAGE.put(
newEmailKey,
JSON.stringify({
subject: "Newer Atom Email",
from: "Sender <sender@example.com>",
content: "<p>New body</p>",
receivedAt: newReceivedAt,
headers: {},
}),
);
await mockEnv.EMAIL_STORAGE.put(
`feed:${FEED_ID}:metadata`,
JSON.stringify({
emails: [
{
key: newEmailKey,
subject: "Newer Atom Email",
receivedAt: newReceivedAt,
},
{
key: `feed:${FEED_ID}:${EMAIL_RECEIVED_AT}`,
subject: "Atom Subject",
receivedAt: EMAIL_RECEIVED_AT,
},
],
}),
);
const res = await testApp.request(
`/${FEED_ID}`,
{ headers: { "If-None-Match": oldEtag } },
mockEnv,
);
expect(res.status).toBe(200);
const newEtag = res.headers.get("ETag");
expect(newEtag).not.toBe(oldEtag);
});
});
});
+18
View File
@@ -5,6 +5,11 @@ import { fetchFeedData } from "../application/feed-fetcher";
import { baseUrl, feedAtomUrl } from "../infrastructure/urls";
import { isExpired } from "../domain/feed";
import { FeedId } from "../domain/value-objects/feed-id";
import {
computeFeedValidators,
isNotModified,
notModifiedResponse,
} from "../infrastructure/http-cache";
export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
try {
@@ -21,6 +26,17 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
return new Response("Feed has expired", { status: 410 });
}
const validators = computeFeedValidators(
"atom",
feedId,
feedData.feedConfig,
feedData.emails,
);
if (isNotModified(c.req.raw, validators)) {
return notModifiedResponse(validators);
}
const base = baseUrl(c.env);
const selfUrl = new URL(c.req.url).origin + `/atom/${feedId}`;
const atomXml = generateAtomFeed(
@@ -42,6 +58,8 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
"Cache-Control": "max-age=1800",
"X-Robots-Tag": "noindex",
Link: linkHeader,
ETag: validators.etag,
"Last-Modified": validators.lastModified,
},
});
} catch (error) {
+1
View File
@@ -166,6 +166,7 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
label="Forwarded (catch-all)"
value={stats.emails_forwarded}
/>
<Stat label="Deduplicated" value={stats.emails_deduplicated} />
<Stat
label="Acceptance rate"
value={acceptanceRate}
+143
View File
@@ -0,0 +1,143 @@
import { describe, it, expect, beforeEach } from "vitest";
import { Hono } from "hono";
import { handle } from "./json";
import { createMockEnv } from "../test/setup";
import { Env } from "../types";
describe("JSON Feed Route", () => {
let testApp: Hono;
let mockEnv: Env;
beforeEach(() => {
mockEnv = createMockEnv() as unknown as Env;
testApp = new Hono();
testApp.get("/:feedId", handle);
});
describe("unknown feed", () => {
it("returns 404 when no metadata exists in KV", async () => {
const res = await testApp.request("/nonexistent-feed", {}, mockEnv);
expect(res.status).toBe(404);
expect(await res.text()).toBe("Feed not found");
});
});
describe("valid feed with no emails", () => {
beforeEach(async () => {
await mockEnv.EMAIL_STORAGE.put(
"feed:empty-feed:metadata",
JSON.stringify({ emails: [] }),
);
});
it("returns 200 with application/feed+json content type", async () => {
const res = await testApp.request("/empty-feed", {}, mockEnv);
expect(res.status).toBe(200);
expect(res.headers.get("Content-Type")).toContain(
"application/feed+json",
);
});
it("includes Cache-Control header", async () => {
const res = await testApp.request("/empty-feed", {}, mockEnv);
expect(res.headers.get("Cache-Control")).toBe("max-age=1800");
});
it("sets X-Robots-Tag: noindex", async () => {
const res = await testApp.request("/empty-feed", {}, mockEnv);
expect(res.headers.get("X-Robots-Tag")).toBe("noindex");
});
it("Link header advertises hub and self", async () => {
const res = await testApp.request("/empty-feed", {}, mockEnv);
const link = res.headers.get("Link") ?? "";
expect(link).toContain(`rel="hub"`);
expect(link).toContain(`rel="self"`);
});
it("body parses as JSON with jsonfeed version 1.1", async () => {
const res = await testApp.request("/empty-feed", {}, mockEnv);
const body = (await res.json()) as { version: string; items: unknown[] };
expect(body.version).toBe("https://jsonfeed.org/version/1");
expect(Array.isArray(body.items)).toBe(true);
expect(body.items).toHaveLength(0);
});
});
describe("valid feed with emails", () => {
const FEED_ID = "test-feed-json";
const EMAIL_RECEIVED_AT = 1700000001000;
beforeEach(async () => {
const emailKey = `feed:${FEED_ID}:${EMAIL_RECEIVED_AT}`;
await mockEnv.EMAIL_STORAGE.put(
emailKey,
JSON.stringify({
subject: "JSON Feed Subject",
from: "Sender <sender@example.com>",
content: "<p>Body content</p>",
receivedAt: EMAIL_RECEIVED_AT,
headers: {},
}),
);
await mockEnv.EMAIL_STORAGE.put(
`feed:${FEED_ID}:metadata`,
JSON.stringify({
emails: [
{
key: emailKey,
subject: "JSON Feed Subject",
receivedAt: EMAIL_RECEIVED_AT,
},
],
}),
);
await mockEnv.EMAIL_STORAGE.put(
`feed:${FEED_ID}:config`,
JSON.stringify({
title: "My JSON Feed",
language: "en",
created_at: 1700000000000,
}),
);
});
it("returns 200 with items containing the seeded email", async () => {
const res = await testApp.request(`/${FEED_ID}`, {}, mockEnv);
expect(res.status).toBe(200);
const body = (await res.json()) as {
version: string;
items: Array<{ title: string }>;
};
expect(body.version).toBe("https://jsonfeed.org/version/1");
expect(Array.isArray(body.items)).toBe(true);
expect(body.items).toHaveLength(1);
expect(body.items[0].title).toBe("JSON Feed Subject");
});
});
describe("expired feed", () => {
beforeEach(async () => {
const pastTimestamp = Date.now() - 1000 * 60 * 60 * 24; // 1 day ago
await mockEnv.EMAIL_STORAGE.put(
"feed:expired-feed:metadata",
JSON.stringify({ emails: [] }),
);
await mockEnv.EMAIL_STORAGE.put(
"feed:expired-feed:config",
JSON.stringify({
title: "Expired Feed",
language: "en",
created_at: pastTimestamp,
expires_at: pastTimestamp,
}),
);
});
it("returns 410 for expired feed", async () => {
const res = await testApp.request("/expired-feed", {}, mockEnv);
expect(res.status).toBe(410);
expect(await res.text()).toBe("Feed has expired");
});
});
});
+51
View File
@@ -0,0 +1,51 @@
import { Context } from "hono";
import { Env } from "../types";
import { generateJsonFeed } from "../infrastructure/feed-generator";
import { fetchFeedData } from "../application/feed-fetcher";
import { baseUrl } from "../infrastructure/urls";
import { isExpired } from "../domain/feed";
import { FeedId } from "../domain/value-objects/feed-id";
export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
try {
const feedId = c.req.param("feedId");
if (!feedId) {
return new Response("Feed ID is required", { status: 400 });
}
const feedData = await fetchFeedData(FeedId.unchecked(feedId), c.env);
if (!feedData) {
return new Response("Feed not found", { status: 404 });
}
if (isExpired(feedData.feedConfig)) {
return new Response("Feed has expired", { status: 410 });
}
const base = baseUrl(c.env);
const selfUrl = new URL(c.req.url).origin + `/json/${feedId}`;
const jsonFeed = generateJsonFeed(
feedData.feedConfig,
feedData.emails,
base,
feedId,
selfUrl,
);
const linkHeader = [
`<${base}/hub>; rel="hub"`,
`<${selfUrl}>; rel="self"`,
].join(", ");
return new Response(jsonFeed, {
status: 200,
headers: {
"Content-Type": "application/feed+json",
"Cache-Control": "max-age=1800",
"X-Robots-Tag": "noindex",
Link: linkHeader,
},
});
} catch (error) {
console.error("Error generating JSON feed:", error);
return new Response("Internal Server Error", { status: 500 });
}
}
+139
View File
@@ -0,0 +1,139 @@
import { describe, it, expect, beforeEach } from "vitest";
import { Hono } from "hono";
import app from "./admin";
import { createMockEnv } from "../test/setup";
import { Env } from "../types";
describe("OPML export — GET /admin/opml", () => {
let testApp: Hono;
let mockEnv: Env;
let request: (path: string, init?: RequestInit) => Promise<Response>;
let loginAndGetCookie: () => Promise<string>;
beforeEach(() => {
mockEnv = createMockEnv() as unknown as Env;
testApp = new Hono();
testApp.route("/admin", app);
request = (path, init = {}) =>
Promise.resolve(testApp.request(path, init, mockEnv));
loginAndGetCookie = async () => {
const formData = new FormData();
formData.append("password", "test-password");
const response = await request("/admin/login", {
method: "POST",
body: formData,
});
expect(response.status).toBe(302);
const setCookie = response.headers.get("Set-Cookie");
expect(setCookie).toBeTruthy();
return (setCookie as string).split(";")[0];
};
});
it("should return 302 redirect to login when not authenticated", async () => {
const res = await request("/admin/opml");
expect(res.status).toBe(302);
expect(res.headers.get("Location")).toBe("/admin/login");
});
it("should return 200 with OPML content when authenticated", async () => {
// Seed two feeds in the registry
await mockEnv.EMAIL_STORAGE.put(
"feeds:list",
JSON.stringify({
feeds: [
{ id: "feed-abc", title: "My Newsletter", description: "Daily news" },
{ id: "feed-xyz", title: "Tech Digest" },
],
}),
);
const authCookie = await loginAndGetCookie();
const res = await request("/admin/opml", {
headers: {
Cookie: authCookie,
Origin: "https://test.getmynews.app",
},
});
expect(res.status).toBe(200);
const contentType = res.headers.get("Content-Type") ?? "";
expect(contentType).toContain("text/x-opml");
expect(res.headers.get("Content-Disposition")).toBe(
'attachment; filename="feeds.opml"',
);
expect(res.headers.get("X-Robots-Tag")).toBe("noindex");
const body = await res.text();
// Valid OPML 2.0 structure
expect(body).toContain('<?xml version="1.0" encoding="UTF-8"?>');
expect(body).toContain('<opml version="2.0">');
expect(body).toContain("<head>");
expect(body).toContain("<title>kill-the-news feeds</title>");
expect(body).toContain("<body>");
// One outline per feed with correct xmlUrl
expect(body).toContain('type="rss"');
expect(body).toContain('text="My Newsletter"');
expect(body).toContain('title="My Newsletter"');
expect(body).toContain('xmlUrl="https://test.getmynews.app/rss/feed-abc"');
expect(body).toContain('description="Daily news"');
expect(body).toContain('text="Tech Digest"');
expect(body).toContain('xmlUrl="https://test.getmynews.app/rss/feed-xyz"');
// feed-xyz has no description — attribute must not appear
const feedXyzLine =
body.split("\n").find((l) => l.includes("feed-xyz")) ?? "";
expect(feedXyzLine).not.toContain("description=");
});
it("should XML-escape special characters in title and description", async () => {
await mockEnv.EMAIL_STORAGE.put(
"feeds:list",
JSON.stringify({
feeds: [
{
id: "feed-special",
title: "News & <Updates>",
description: 'Say "hello" & goodbye',
},
],
}),
);
const authCookie = await loginAndGetCookie();
const res = await request("/admin/opml", {
headers: {
Cookie: authCookie,
Origin: "https://test.getmynews.app",
},
});
expect(res.status).toBe(200);
const body = await res.text();
// Raw special chars must not appear unescaped in attribute values
const outlineLine =
body.split("\n").find((l) => l.includes("feed-special")) ?? "";
expect(outlineLine).toContain("News &amp; &lt;Updates&gt;");
expect(outlineLine).toContain("Say &quot;hello&quot; &amp; goodbye");
expect(outlineLine).not.toContain('title="News & <');
});
it("should return empty body element when there are no feeds", async () => {
const authCookie = await loginAndGetCookie();
const res = await request("/admin/opml", {
headers: {
Cookie: authCookie,
Origin: "https://test.getmynews.app",
},
});
expect(res.status).toBe(200);
const body = await res.text();
expect(body).toContain("<body>");
expect(body).not.toContain("<outline");
});
});
+56
View File
@@ -0,0 +1,56 @@
import { Context } from "hono";
import { Env } from "../types";
import { FeedRepository } from "../infrastructure/feed-repository";
import { feedRssUrl } from "../infrastructure/urls";
/**
* Escape a string for use in an XML attribute value.
* Replaces &, <, >, and " with their XML entity equivalents.
*/
function escapeXmlAttr(value: string): string {
return value
.replace(/&/g, "&amp;")
.replace(/</g, "&lt;")
.replace(/>/g, "&gt;")
.replace(/"/g, "&quot;");
}
/**
* Handler for GET /admin/opml
* Exports all feeds as an OPML 2.0 document.
* Protected by the admin auth middleware (inherits from admin Hono app).
*/
export async function handleOpml(c: Context<{ Bindings: Env }>) {
const env = c.env;
const feeds = await FeedRepository.from(env).listFeeds();
const outlines = feeds
.map((feed) => {
const title = escapeXmlAttr(feed.title);
const xmlUrl = escapeXmlAttr(feedRssUrl(feed.id, env));
const descAttr = feed.description
? ` description="${escapeXmlAttr(feed.description)}"`
: "";
return ` <outline type="rss" text="${title}" title="${title}" xmlUrl="${xmlUrl}"${descAttr}/>`;
})
.join("\n");
const opml = `<?xml version="1.0" encoding="UTF-8"?>
<opml version="2.0">
<head>
<title>kill-the-news feeds</title>
</head>
<body>
${outlines}
</body>
</opml>`;
return new Response(opml, {
status: 200,
headers: {
"Content-Type": "text/x-opml; charset=utf-8",
"Content-Disposition": 'attachment; filename="feeds.opml"',
"X-Robots-Tag": "noindex",
},
});
}
+129
View File
@@ -53,4 +53,133 @@ describe("RSS Feed Route", () => {
expect(link).toContain(`rel="self"`);
});
});
describe("conditional GET (ETag + Last-Modified)", () => {
const FEED_ID = "test-feed-rss-cget";
const EMAIL_RECEIVED_AT = 1700000001000;
beforeEach(async () => {
const emailKey = `feed:${FEED_ID}:${EMAIL_RECEIVED_AT}`;
await mockEnv.EMAIL_STORAGE.put(
emailKey,
JSON.stringify({
subject: "RSS Subject",
from: "Sender <sender@example.com>",
content: "<p>Body</p>",
receivedAt: EMAIL_RECEIVED_AT,
headers: {},
}),
);
await mockEnv.EMAIL_STORAGE.put(
`feed:${FEED_ID}:metadata`,
JSON.stringify({
emails: [
{
key: emailKey,
subject: "RSS Subject",
receivedAt: EMAIL_RECEIVED_AT,
},
],
}),
);
await mockEnv.EMAIL_STORAGE.put(
`feed:${FEED_ID}:config`,
JSON.stringify({
title: "RSS Cget Feed",
language: "en",
created_at: 1700000000000,
}),
);
});
it("first GET returns 200 with ETag and Last-Modified headers", async () => {
const res = await testApp.request(`/${FEED_ID}`, {}, mockEnv);
expect(res.status).toBe(200);
expect(res.headers.get("ETag")).toBeTruthy();
expect(res.headers.get("Last-Modified")).toBeTruthy();
});
it("GET with matching If-None-Match returns 304 with empty body", async () => {
const first = await testApp.request(`/${FEED_ID}`, {}, mockEnv);
const etag = first.headers.get("ETag")!;
const res = await testApp.request(
`/${FEED_ID}`,
{ headers: { "If-None-Match": etag } },
mockEnv,
);
expect(res.status).toBe(304);
expect(await res.text()).toBe("");
});
it("GET with If-Modified-Since in the future returns 304", async () => {
const future = new Date(EMAIL_RECEIVED_AT + 1000).toUTCString();
const res = await testApp.request(
`/${FEED_ID}`,
{ headers: { "If-Modified-Since": future } },
mockEnv,
);
expect(res.status).toBe(304);
});
it("stale If-None-Match after new email results in 200", async () => {
// Get ETag before new email
const first = await testApp.request(`/${FEED_ID}`, {}, mockEnv);
const oldEtag = first.headers.get("ETag")!;
// Add a newer email
const newReceivedAt = EMAIL_RECEIVED_AT + 5000;
const newEmailKey = `feed:${FEED_ID}:${newReceivedAt}`;
await mockEnv.EMAIL_STORAGE.put(
newEmailKey,
JSON.stringify({
subject: "Newer Email",
from: "Sender <sender@example.com>",
content: "<p>New body</p>",
receivedAt: newReceivedAt,
headers: {},
}),
);
await mockEnv.EMAIL_STORAGE.put(
`feed:${FEED_ID}:metadata`,
JSON.stringify({
emails: [
{
key: newEmailKey,
subject: "Newer Email",
receivedAt: newReceivedAt,
},
{
key: `feed:${FEED_ID}:${EMAIL_RECEIVED_AT}`,
subject: "RSS Subject",
receivedAt: EMAIL_RECEIVED_AT,
},
],
}),
);
const res = await testApp.request(
`/${FEED_ID}`,
{ headers: { "If-None-Match": oldEtag } },
mockEnv,
);
expect(res.status).toBe(200);
const newEtag = res.headers.get("ETag");
expect(newEtag).not.toBe(oldEtag);
});
it("RSS and Atom ETags for the same feed differ", async () => {
const rssRes = await testApp.request(`/${FEED_ID}`, {}, mockEnv);
const rssEtag = rssRes.headers.get("ETag")!;
// Use a separate atom app to get the atom ETag
const { handle: atomHandle } = await import("./atom");
const atomApp = new Hono();
atomApp.get("/:feedId", atomHandle);
const atomRes = await atomApp.request(`/${FEED_ID}`, {}, mockEnv);
const atomEtag = atomRes.headers.get("ETag")!;
expect(rssEtag).not.toBe(atomEtag);
});
});
});
+18
View File
@@ -5,6 +5,11 @@ import { fetchFeedData } from "../application/feed-fetcher";
import { baseUrl, feedRssUrl } from "../infrastructure/urls";
import { isExpired } from "../domain/feed";
import { FeedId } from "../domain/value-objects/feed-id";
import {
computeFeedValidators,
isNotModified,
notModifiedResponse,
} from "../infrastructure/http-cache";
export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
try {
@@ -21,6 +26,17 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
return new Response("Feed has expired", { status: 410 });
}
const validators = computeFeedValidators(
"rss",
feedId,
feedData.feedConfig,
feedData.emails,
);
if (isNotModified(c.req.raw, validators)) {
return notModifiedResponse(validators);
}
const base = baseUrl(c.env);
const selfUrl = new URL(c.req.url).origin + `/rss/${feedId}`;
const rssXml = generateRssFeed(
@@ -42,6 +58,8 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
"Cache-Control": "max-age=1800",
"X-Robots-Tag": "noindex",
Link: linkHeader,
ETag: validators.etag,
"Last-Modified": validators.lastModified,
},
});
} catch (error) {
+3
View File
@@ -68,6 +68,8 @@ export interface EmailMetadata {
size?: number;
attachmentIds?: string[]; // Downloadable attachments (shown to the user)
inlineAttachmentIds?: string[]; // Inline images: hidden from lists, still cleaned up
messageId?: string; // RFC 2822 Message-ID header (dedup primary key)
dedupHash?: string; // SHA-256 hex of normalized subject+content (dedup fallback)
}
// Feed list interface
@@ -92,6 +94,7 @@ export interface Counters {
// Subset of emails_rejected: non-feed mail forwarded to FALLBACK_FORWARD_ADDRESS
// instead of dropped. Dropped count = emails_rejected emails_forwarded.
emails_forwarded: number;
emails_deduplicated: number; // Duplicate deliveries silently skipped (not stored)
unsubscribes_sent: number;
last_email_at?: string; // ISO 8601
last_feed_created_at?: string; // ISO 8601