feat: reader-compat batch — JSON Feed, OPML export, conditional GET, dedup

Batch of four reader-facing improvements (TODO "Compat lecteurs + dedup"):

- JSON Feed at /json/:feedId (feed lib .json1()); all formats cross-link
- OPML export at /admin/opml (admin-protected; the registry lists every
  feed URL, so it must not be public)
- Conditional GET on /rss + /atom: strong ETag + Last-Modified, 304 on
  If-None-Match/If-Modified-Since, validators shared via http-cache.ts
- Duplicate-send dedup in ingestion: match by Message-ID, fall back to a
  SHA-256 of normalized subject+content; a duplicate is a no-op and bumps
  the new emails_deduplicated counter (status page + /api/v1/stats)

429 tests green, tsc clean, build dry-run OK. Docs (README/CLAUDE/TODO +
landing cards) updated.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Julien Herr
2026-05-24 20:47:54 +02:00
parent 334713fbd9
commit 0abd5f306c
23 changed files with 1015 additions and 11 deletions
+4
View File
@@ -18,6 +18,7 @@ import {
} from "../infrastructure/urls";
import { feedsRouter } from "./admin/feeds";
import { emailsRouter } from "./admin/emails";
import { handleOpml } from "./opml";
import { dashboardScript } from "../scripts/generated/dashboard";
type AppEnv = { Bindings: Env };
@@ -975,6 +976,9 @@ app.get("/", async (c) => {
);
});
// OPML export (admin-protected)
app.get("/opml", handleOpml);
// Mount sub-routers
app.route("/feeds", feedsRouter);
app.route("/", emailsRouter);
+113
View File
@@ -144,4 +144,117 @@ describe("Atom Feed Route", () => {
expect(body).toContain('xmlns="http://www.w3.org/2005/Atom"');
});
});
describe("conditional GET (ETag + Last-Modified)", () => {
const FEED_ID = "test-feed-atom-cget";
const EMAIL_RECEIVED_AT = 1700000001000;
beforeEach(async () => {
const emailKey = `feed:${FEED_ID}:${EMAIL_RECEIVED_AT}`;
await mockEnv.EMAIL_STORAGE.put(
emailKey,
JSON.stringify({
subject: "Atom Subject",
from: "Sender <sender@example.com>",
content: "<p>Body</p>",
receivedAt: EMAIL_RECEIVED_AT,
headers: {},
}),
);
await mockEnv.EMAIL_STORAGE.put(
`feed:${FEED_ID}:metadata`,
JSON.stringify({
emails: [
{
key: emailKey,
subject: "Atom Subject",
receivedAt: EMAIL_RECEIVED_AT,
},
],
}),
);
await mockEnv.EMAIL_STORAGE.put(
`feed:${FEED_ID}:config`,
JSON.stringify({
title: "Atom Cget Feed",
language: "en",
created_at: 1700000000000,
}),
);
});
it("first GET returns 200 with ETag and Last-Modified headers", async () => {
const res = await testApp.request(`/${FEED_ID}`, {}, mockEnv);
expect(res.status).toBe(200);
expect(res.headers.get("ETag")).toBeTruthy();
expect(res.headers.get("Last-Modified")).toBeTruthy();
});
it("GET with matching If-None-Match returns 304 with empty body", async () => {
const first = await testApp.request(`/${FEED_ID}`, {}, mockEnv);
const etag = first.headers.get("ETag")!;
const res = await testApp.request(
`/${FEED_ID}`,
{ headers: { "If-None-Match": etag } },
mockEnv,
);
expect(res.status).toBe(304);
expect(await res.text()).toBe("");
});
it("GET with If-Modified-Since in the future returns 304", async () => {
const future = new Date(EMAIL_RECEIVED_AT + 1000).toUTCString();
const res = await testApp.request(
`/${FEED_ID}`,
{ headers: { "If-Modified-Since": future } },
mockEnv,
);
expect(res.status).toBe(304);
});
it("stale If-None-Match after new email results in 200", async () => {
const first = await testApp.request(`/${FEED_ID}`, {}, mockEnv);
const oldEtag = first.headers.get("ETag")!;
const newReceivedAt = EMAIL_RECEIVED_AT + 5000;
const newEmailKey = `feed:${FEED_ID}:${newReceivedAt}`;
await mockEnv.EMAIL_STORAGE.put(
newEmailKey,
JSON.stringify({
subject: "Newer Atom Email",
from: "Sender <sender@example.com>",
content: "<p>New body</p>",
receivedAt: newReceivedAt,
headers: {},
}),
);
await mockEnv.EMAIL_STORAGE.put(
`feed:${FEED_ID}:metadata`,
JSON.stringify({
emails: [
{
key: newEmailKey,
subject: "Newer Atom Email",
receivedAt: newReceivedAt,
},
{
key: `feed:${FEED_ID}:${EMAIL_RECEIVED_AT}`,
subject: "Atom Subject",
receivedAt: EMAIL_RECEIVED_AT,
},
],
}),
);
const res = await testApp.request(
`/${FEED_ID}`,
{ headers: { "If-None-Match": oldEtag } },
mockEnv,
);
expect(res.status).toBe(200);
const newEtag = res.headers.get("ETag");
expect(newEtag).not.toBe(oldEtag);
});
});
});
+18
View File
@@ -5,6 +5,11 @@ import { fetchFeedData } from "../application/feed-fetcher";
import { baseUrl, feedAtomUrl } from "../infrastructure/urls";
import { isExpired } from "../domain/feed";
import { FeedId } from "../domain/value-objects/feed-id";
import {
computeFeedValidators,
isNotModified,
notModifiedResponse,
} from "../infrastructure/http-cache";
export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
try {
@@ -21,6 +26,17 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
return new Response("Feed has expired", { status: 410 });
}
const validators = computeFeedValidators(
"atom",
feedId,
feedData.feedConfig,
feedData.emails,
);
if (isNotModified(c.req.raw, validators)) {
return notModifiedResponse(validators);
}
const base = baseUrl(c.env);
const selfUrl = new URL(c.req.url).origin + `/atom/${feedId}`;
const atomXml = generateAtomFeed(
@@ -42,6 +58,8 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
"Cache-Control": "max-age=1800",
"X-Robots-Tag": "noindex",
Link: linkHeader,
ETag: validators.etag,
"Last-Modified": validators.lastModified,
},
});
} catch (error) {
+1
View File
@@ -166,6 +166,7 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
label="Forwarded (catch-all)"
value={stats.emails_forwarded}
/>
<Stat label="Deduplicated" value={stats.emails_deduplicated} />
<Stat
label="Acceptance rate"
value={acceptanceRate}
+143
View File
@@ -0,0 +1,143 @@
import { describe, it, expect, beforeEach } from "vitest";
import { Hono } from "hono";
import { handle } from "./json";
import { createMockEnv } from "../test/setup";
import { Env } from "../types";
describe("JSON Feed Route", () => {
let testApp: Hono;
let mockEnv: Env;
beforeEach(() => {
mockEnv = createMockEnv() as unknown as Env;
testApp = new Hono();
testApp.get("/:feedId", handle);
});
describe("unknown feed", () => {
it("returns 404 when no metadata exists in KV", async () => {
const res = await testApp.request("/nonexistent-feed", {}, mockEnv);
expect(res.status).toBe(404);
expect(await res.text()).toBe("Feed not found");
});
});
describe("valid feed with no emails", () => {
beforeEach(async () => {
await mockEnv.EMAIL_STORAGE.put(
"feed:empty-feed:metadata",
JSON.stringify({ emails: [] }),
);
});
it("returns 200 with application/feed+json content type", async () => {
const res = await testApp.request("/empty-feed", {}, mockEnv);
expect(res.status).toBe(200);
expect(res.headers.get("Content-Type")).toContain(
"application/feed+json",
);
});
it("includes Cache-Control header", async () => {
const res = await testApp.request("/empty-feed", {}, mockEnv);
expect(res.headers.get("Cache-Control")).toBe("max-age=1800");
});
it("sets X-Robots-Tag: noindex", async () => {
const res = await testApp.request("/empty-feed", {}, mockEnv);
expect(res.headers.get("X-Robots-Tag")).toBe("noindex");
});
it("Link header advertises hub and self", async () => {
const res = await testApp.request("/empty-feed", {}, mockEnv);
const link = res.headers.get("Link") ?? "";
expect(link).toContain(`rel="hub"`);
expect(link).toContain(`rel="self"`);
});
it("body parses as JSON with jsonfeed version 1.1", async () => {
const res = await testApp.request("/empty-feed", {}, mockEnv);
const body = (await res.json()) as { version: string; items: unknown[] };
expect(body.version).toBe("https://jsonfeed.org/version/1");
expect(Array.isArray(body.items)).toBe(true);
expect(body.items).toHaveLength(0);
});
});
describe("valid feed with emails", () => {
const FEED_ID = "test-feed-json";
const EMAIL_RECEIVED_AT = 1700000001000;
beforeEach(async () => {
const emailKey = `feed:${FEED_ID}:${EMAIL_RECEIVED_AT}`;
await mockEnv.EMAIL_STORAGE.put(
emailKey,
JSON.stringify({
subject: "JSON Feed Subject",
from: "Sender <sender@example.com>",
content: "<p>Body content</p>",
receivedAt: EMAIL_RECEIVED_AT,
headers: {},
}),
);
await mockEnv.EMAIL_STORAGE.put(
`feed:${FEED_ID}:metadata`,
JSON.stringify({
emails: [
{
key: emailKey,
subject: "JSON Feed Subject",
receivedAt: EMAIL_RECEIVED_AT,
},
],
}),
);
await mockEnv.EMAIL_STORAGE.put(
`feed:${FEED_ID}:config`,
JSON.stringify({
title: "My JSON Feed",
language: "en",
created_at: 1700000000000,
}),
);
});
it("returns 200 with items containing the seeded email", async () => {
const res = await testApp.request(`/${FEED_ID}`, {}, mockEnv);
expect(res.status).toBe(200);
const body = (await res.json()) as {
version: string;
items: Array<{ title: string }>;
};
expect(body.version).toBe("https://jsonfeed.org/version/1");
expect(Array.isArray(body.items)).toBe(true);
expect(body.items).toHaveLength(1);
expect(body.items[0].title).toBe("JSON Feed Subject");
});
});
describe("expired feed", () => {
beforeEach(async () => {
const pastTimestamp = Date.now() - 1000 * 60 * 60 * 24; // 1 day ago
await mockEnv.EMAIL_STORAGE.put(
"feed:expired-feed:metadata",
JSON.stringify({ emails: [] }),
);
await mockEnv.EMAIL_STORAGE.put(
"feed:expired-feed:config",
JSON.stringify({
title: "Expired Feed",
language: "en",
created_at: pastTimestamp,
expires_at: pastTimestamp,
}),
);
});
it("returns 410 for expired feed", async () => {
const res = await testApp.request("/expired-feed", {}, mockEnv);
expect(res.status).toBe(410);
expect(await res.text()).toBe("Feed has expired");
});
});
});
+51
View File
@@ -0,0 +1,51 @@
import { Context } from "hono";
import { Env } from "../types";
import { generateJsonFeed } from "../infrastructure/feed-generator";
import { fetchFeedData } from "../application/feed-fetcher";
import { baseUrl } from "../infrastructure/urls";
import { isExpired } from "../domain/feed";
import { FeedId } from "../domain/value-objects/feed-id";
export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
try {
const feedId = c.req.param("feedId");
if (!feedId) {
return new Response("Feed ID is required", { status: 400 });
}
const feedData = await fetchFeedData(FeedId.unchecked(feedId), c.env);
if (!feedData) {
return new Response("Feed not found", { status: 404 });
}
if (isExpired(feedData.feedConfig)) {
return new Response("Feed has expired", { status: 410 });
}
const base = baseUrl(c.env);
const selfUrl = new URL(c.req.url).origin + `/json/${feedId}`;
const jsonFeed = generateJsonFeed(
feedData.feedConfig,
feedData.emails,
base,
feedId,
selfUrl,
);
const linkHeader = [
`<${base}/hub>; rel="hub"`,
`<${selfUrl}>; rel="self"`,
].join(", ");
return new Response(jsonFeed, {
status: 200,
headers: {
"Content-Type": "application/feed+json",
"Cache-Control": "max-age=1800",
"X-Robots-Tag": "noindex",
Link: linkHeader,
},
});
} catch (error) {
console.error("Error generating JSON feed:", error);
return new Response("Internal Server Error", { status: 500 });
}
}
+139
View File
@@ -0,0 +1,139 @@
import { describe, it, expect, beforeEach } from "vitest";
import { Hono } from "hono";
import app from "./admin";
import { createMockEnv } from "../test/setup";
import { Env } from "../types";
describe("OPML export — GET /admin/opml", () => {
let testApp: Hono;
let mockEnv: Env;
let request: (path: string, init?: RequestInit) => Promise<Response>;
let loginAndGetCookie: () => Promise<string>;
beforeEach(() => {
mockEnv = createMockEnv() as unknown as Env;
testApp = new Hono();
testApp.route("/admin", app);
request = (path, init = {}) =>
Promise.resolve(testApp.request(path, init, mockEnv));
loginAndGetCookie = async () => {
const formData = new FormData();
formData.append("password", "test-password");
const response = await request("/admin/login", {
method: "POST",
body: formData,
});
expect(response.status).toBe(302);
const setCookie = response.headers.get("Set-Cookie");
expect(setCookie).toBeTruthy();
return (setCookie as string).split(";")[0];
};
});
it("should return 302 redirect to login when not authenticated", async () => {
const res = await request("/admin/opml");
expect(res.status).toBe(302);
expect(res.headers.get("Location")).toBe("/admin/login");
});
it("should return 200 with OPML content when authenticated", async () => {
// Seed two feeds in the registry
await mockEnv.EMAIL_STORAGE.put(
"feeds:list",
JSON.stringify({
feeds: [
{ id: "feed-abc", title: "My Newsletter", description: "Daily news" },
{ id: "feed-xyz", title: "Tech Digest" },
],
}),
);
const authCookie = await loginAndGetCookie();
const res = await request("/admin/opml", {
headers: {
Cookie: authCookie,
Origin: "https://test.getmynews.app",
},
});
expect(res.status).toBe(200);
const contentType = res.headers.get("Content-Type") ?? "";
expect(contentType).toContain("text/x-opml");
expect(res.headers.get("Content-Disposition")).toBe(
'attachment; filename="feeds.opml"',
);
expect(res.headers.get("X-Robots-Tag")).toBe("noindex");
const body = await res.text();
// Valid OPML 2.0 structure
expect(body).toContain('<?xml version="1.0" encoding="UTF-8"?>');
expect(body).toContain('<opml version="2.0">');
expect(body).toContain("<head>");
expect(body).toContain("<title>kill-the-news feeds</title>");
expect(body).toContain("<body>");
// One outline per feed with correct xmlUrl
expect(body).toContain('type="rss"');
expect(body).toContain('text="My Newsletter"');
expect(body).toContain('title="My Newsletter"');
expect(body).toContain('xmlUrl="https://test.getmynews.app/rss/feed-abc"');
expect(body).toContain('description="Daily news"');
expect(body).toContain('text="Tech Digest"');
expect(body).toContain('xmlUrl="https://test.getmynews.app/rss/feed-xyz"');
// feed-xyz has no description — attribute must not appear
const feedXyzLine =
body.split("\n").find((l) => l.includes("feed-xyz")) ?? "";
expect(feedXyzLine).not.toContain("description=");
});
it("should XML-escape special characters in title and description", async () => {
await mockEnv.EMAIL_STORAGE.put(
"feeds:list",
JSON.stringify({
feeds: [
{
id: "feed-special",
title: "News & <Updates>",
description: 'Say "hello" & goodbye',
},
],
}),
);
const authCookie = await loginAndGetCookie();
const res = await request("/admin/opml", {
headers: {
Cookie: authCookie,
Origin: "https://test.getmynews.app",
},
});
expect(res.status).toBe(200);
const body = await res.text();
// Raw special chars must not appear unescaped in attribute values
const outlineLine =
body.split("\n").find((l) => l.includes("feed-special")) ?? "";
expect(outlineLine).toContain("News &amp; &lt;Updates&gt;");
expect(outlineLine).toContain("Say &quot;hello&quot; &amp; goodbye");
expect(outlineLine).not.toContain('title="News & <');
});
it("should return empty body element when there are no feeds", async () => {
const authCookie = await loginAndGetCookie();
const res = await request("/admin/opml", {
headers: {
Cookie: authCookie,
Origin: "https://test.getmynews.app",
},
});
expect(res.status).toBe(200);
const body = await res.text();
expect(body).toContain("<body>");
expect(body).not.toContain("<outline");
});
});
+56
View File
@@ -0,0 +1,56 @@
import { Context } from "hono";
import { Env } from "../types";
import { FeedRepository } from "../infrastructure/feed-repository";
import { feedRssUrl } from "../infrastructure/urls";
/**
* Escape a string for use in an XML attribute value.
* Replaces &, <, >, and " with their XML entity equivalents.
*/
function escapeXmlAttr(value: string): string {
return value
.replace(/&/g, "&amp;")
.replace(/</g, "&lt;")
.replace(/>/g, "&gt;")
.replace(/"/g, "&quot;");
}
/**
* Handler for GET /admin/opml
* Exports all feeds as an OPML 2.0 document.
* Protected by the admin auth middleware (inherits from admin Hono app).
*/
export async function handleOpml(c: Context<{ Bindings: Env }>) {
const env = c.env;
const feeds = await FeedRepository.from(env).listFeeds();
const outlines = feeds
.map((feed) => {
const title = escapeXmlAttr(feed.title);
const xmlUrl = escapeXmlAttr(feedRssUrl(feed.id, env));
const descAttr = feed.description
? ` description="${escapeXmlAttr(feed.description)}"`
: "";
return ` <outline type="rss" text="${title}" title="${title}" xmlUrl="${xmlUrl}"${descAttr}/>`;
})
.join("\n");
const opml = `<?xml version="1.0" encoding="UTF-8"?>
<opml version="2.0">
<head>
<title>kill-the-news feeds</title>
</head>
<body>
${outlines}
</body>
</opml>`;
return new Response(opml, {
status: 200,
headers: {
"Content-Type": "text/x-opml; charset=utf-8",
"Content-Disposition": 'attachment; filename="feeds.opml"',
"X-Robots-Tag": "noindex",
},
});
}
+129
View File
@@ -53,4 +53,133 @@ describe("RSS Feed Route", () => {
expect(link).toContain(`rel="self"`);
});
});
describe("conditional GET (ETag + Last-Modified)", () => {
const FEED_ID = "test-feed-rss-cget";
const EMAIL_RECEIVED_AT = 1700000001000;
beforeEach(async () => {
const emailKey = `feed:${FEED_ID}:${EMAIL_RECEIVED_AT}`;
await mockEnv.EMAIL_STORAGE.put(
emailKey,
JSON.stringify({
subject: "RSS Subject",
from: "Sender <sender@example.com>",
content: "<p>Body</p>",
receivedAt: EMAIL_RECEIVED_AT,
headers: {},
}),
);
await mockEnv.EMAIL_STORAGE.put(
`feed:${FEED_ID}:metadata`,
JSON.stringify({
emails: [
{
key: emailKey,
subject: "RSS Subject",
receivedAt: EMAIL_RECEIVED_AT,
},
],
}),
);
await mockEnv.EMAIL_STORAGE.put(
`feed:${FEED_ID}:config`,
JSON.stringify({
title: "RSS Cget Feed",
language: "en",
created_at: 1700000000000,
}),
);
});
it("first GET returns 200 with ETag and Last-Modified headers", async () => {
const res = await testApp.request(`/${FEED_ID}`, {}, mockEnv);
expect(res.status).toBe(200);
expect(res.headers.get("ETag")).toBeTruthy();
expect(res.headers.get("Last-Modified")).toBeTruthy();
});
it("GET with matching If-None-Match returns 304 with empty body", async () => {
const first = await testApp.request(`/${FEED_ID}`, {}, mockEnv);
const etag = first.headers.get("ETag")!;
const res = await testApp.request(
`/${FEED_ID}`,
{ headers: { "If-None-Match": etag } },
mockEnv,
);
expect(res.status).toBe(304);
expect(await res.text()).toBe("");
});
it("GET with If-Modified-Since in the future returns 304", async () => {
const future = new Date(EMAIL_RECEIVED_AT + 1000).toUTCString();
const res = await testApp.request(
`/${FEED_ID}`,
{ headers: { "If-Modified-Since": future } },
mockEnv,
);
expect(res.status).toBe(304);
});
it("stale If-None-Match after new email results in 200", async () => {
// Get ETag before new email
const first = await testApp.request(`/${FEED_ID}`, {}, mockEnv);
const oldEtag = first.headers.get("ETag")!;
// Add a newer email
const newReceivedAt = EMAIL_RECEIVED_AT + 5000;
const newEmailKey = `feed:${FEED_ID}:${newReceivedAt}`;
await mockEnv.EMAIL_STORAGE.put(
newEmailKey,
JSON.stringify({
subject: "Newer Email",
from: "Sender <sender@example.com>",
content: "<p>New body</p>",
receivedAt: newReceivedAt,
headers: {},
}),
);
await mockEnv.EMAIL_STORAGE.put(
`feed:${FEED_ID}:metadata`,
JSON.stringify({
emails: [
{
key: newEmailKey,
subject: "Newer Email",
receivedAt: newReceivedAt,
},
{
key: `feed:${FEED_ID}:${EMAIL_RECEIVED_AT}`,
subject: "RSS Subject",
receivedAt: EMAIL_RECEIVED_AT,
},
],
}),
);
const res = await testApp.request(
`/${FEED_ID}`,
{ headers: { "If-None-Match": oldEtag } },
mockEnv,
);
expect(res.status).toBe(200);
const newEtag = res.headers.get("ETag");
expect(newEtag).not.toBe(oldEtag);
});
it("RSS and Atom ETags for the same feed differ", async () => {
const rssRes = await testApp.request(`/${FEED_ID}`, {}, mockEnv);
const rssEtag = rssRes.headers.get("ETag")!;
// Use a separate atom app to get the atom ETag
const { handle: atomHandle } = await import("./atom");
const atomApp = new Hono();
atomApp.get("/:feedId", atomHandle);
const atomRes = await atomApp.request(`/${FEED_ID}`, {}, mockEnv);
const atomEtag = atomRes.headers.get("ETag")!;
expect(rssEtag).not.toBe(atomEtag);
});
});
});
+18
View File
@@ -5,6 +5,11 @@ import { fetchFeedData } from "../application/feed-fetcher";
import { baseUrl, feedRssUrl } from "../infrastructure/urls";
import { isExpired } from "../domain/feed";
import { FeedId } from "../domain/value-objects/feed-id";
import {
computeFeedValidators,
isNotModified,
notModifiedResponse,
} from "../infrastructure/http-cache";
export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
try {
@@ -21,6 +26,17 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
return new Response("Feed has expired", { status: 410 });
}
const validators = computeFeedValidators(
"rss",
feedId,
feedData.feedConfig,
feedData.emails,
);
if (isNotModified(c.req.raw, validators)) {
return notModifiedResponse(validators);
}
const base = baseUrl(c.env);
const selfUrl = new URL(c.req.url).origin + `/rss/${feedId}`;
const rssXml = generateRssFeed(
@@ -42,6 +58,8 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
"Cache-Control": "max-age=1800",
"X-Robots-Tag": "noindex",
Link: linkHeader,
ETag: validators.etag,
"Last-Modified": validators.lastModified,
},
});
} catch (error) {