mirror of
https://github.com/juherr/kill-the-news.git
synced 2026-06-20 22:03:48 +00:00
feat: reader-compat batch — JSON Feed, OPML export, conditional GET, dedup
Batch of four reader-facing improvements (TODO "Compat lecteurs + dedup"): - JSON Feed at /json/:feedId (feed lib .json1()); all formats cross-link - OPML export at /admin/opml (admin-protected; the registry lists every feed URL, so it must not be public) - Conditional GET on /rss + /atom: strong ETag + Last-Modified, 304 on If-None-Match/If-Modified-Since, validators shared via http-cache.ts - Duplicate-send dedup in ingestion: match by Message-ID, fall back to a SHA-256 of normalized subject+content; a duplicate is a no-op and bumps the new emails_deduplicated counter (status page + /api/v1/stats) 429 tests green, tsc clean, build dry-run OK. Docs (README/CLAUDE/TODO + landing cards) updated. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -595,6 +595,134 @@ describe("processEmail — attachments", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("processEmail — deduplication", () => {
|
||||
let env: ReturnType<typeof createMockEnv>;
|
||||
|
||||
beforeEach(async () => {
|
||||
env = createMockEnv();
|
||||
await env.EMAIL_STORAGE.put(
|
||||
`feed:${VALID_FEED_ID}:config`,
|
||||
JSON.stringify({}),
|
||||
);
|
||||
});
|
||||
|
||||
it("stores only one email when the same Message-ID is delivered twice", async () => {
|
||||
const headers = { "Message-ID": "<abc123@example.com>" };
|
||||
await processEmail(makeInput({ headers }), env as any);
|
||||
await processEmail(makeInput({ headers }), env as any);
|
||||
|
||||
const metadata = await env.EMAIL_STORAGE.get(
|
||||
`feed:${VALID_FEED_ID}:metadata`,
|
||||
"json",
|
||||
);
|
||||
expect(metadata.emails).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("increments emails_deduplicated counter on the second delivery", async () => {
|
||||
const headers = { "Message-ID": "<dup42@example.com>" };
|
||||
await processEmail(makeInput({ headers }), env as any);
|
||||
await processEmail(makeInput({ headers }), env as any);
|
||||
|
||||
const counters = await getCounters(env.EMAIL_STORAGE as any);
|
||||
expect(counters.emails_deduplicated).toBe(1);
|
||||
});
|
||||
|
||||
it("deduplicates by hash when no Message-ID header is present", async () => {
|
||||
const input = makeInput({
|
||||
subject: "Weekly Digest",
|
||||
content: "<p>Same content</p>",
|
||||
});
|
||||
await processEmail(input, env as any);
|
||||
await processEmail(input, env as any);
|
||||
|
||||
const metadata = await env.EMAIL_STORAGE.get(
|
||||
`feed:${VALID_FEED_ID}:metadata`,
|
||||
"json",
|
||||
);
|
||||
expect(metadata.emails).toHaveLength(1);
|
||||
|
||||
const counters = await getCounters(env.EMAIL_STORAGE as any);
|
||||
expect(counters.emails_deduplicated).toBe(1);
|
||||
});
|
||||
|
||||
it("does not deduplicate emails with different subjects (no Message-ID)", async () => {
|
||||
await processEmail(
|
||||
makeInput({ subject: "First", content: "<p>body</p>" }),
|
||||
env as any,
|
||||
);
|
||||
await processEmail(
|
||||
makeInput({ subject: "Second", content: "<p>body</p>" }),
|
||||
env as any,
|
||||
);
|
||||
|
||||
const metadata = await env.EMAIL_STORAGE.get(
|
||||
`feed:${VALID_FEED_ID}:metadata`,
|
||||
"json",
|
||||
);
|
||||
expect(metadata.emails).toHaveLength(2);
|
||||
|
||||
const counters = await getCounters(env.EMAIL_STORAGE as any);
|
||||
expect(counters.emails_deduplicated).toBe(0);
|
||||
});
|
||||
|
||||
it("does not false-positive against pre-feature entries lacking messageId/dedupHash", async () => {
|
||||
// Seed a legacy metadata entry with no messageId or dedupHash
|
||||
await env.EMAIL_STORAGE.put(
|
||||
`feed:${VALID_FEED_ID}:metadata`,
|
||||
JSON.stringify({
|
||||
emails: [
|
||||
{
|
||||
key: `feed:${VALID_FEED_ID}:999`,
|
||||
subject: "Old Subject",
|
||||
receivedAt: 999,
|
||||
size: 50,
|
||||
// intentionally no messageId, no dedupHash
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
// A new, distinct email should be stored without triggering false dedup
|
||||
const res = await processEmail(
|
||||
makeInput({ subject: "New Distinct Email", content: "<p>fresh</p>" }),
|
||||
env as any,
|
||||
);
|
||||
expect(res.ok).toBe(true);
|
||||
|
||||
const metadata = await env.EMAIL_STORAGE.get(
|
||||
`feed:${VALID_FEED_ID}:metadata`,
|
||||
"json",
|
||||
);
|
||||
expect(metadata.emails).toHaveLength(2);
|
||||
|
||||
const counters = await getCounters(env.EMAIL_STORAGE as any);
|
||||
expect(counters.emails_deduplicated).toBe(0);
|
||||
});
|
||||
|
||||
it("returns { ok: true } for a genuine duplicate (not a rejection)", async () => {
|
||||
const headers = { "Message-ID": "<nodrop@example.com>" };
|
||||
await processEmail(makeInput({ headers }), env as any);
|
||||
const res = await processEmail(makeInput({ headers }), env as any);
|
||||
expect(res).toMatchObject({ ok: true });
|
||||
});
|
||||
|
||||
it("stores messageId and dedupHash in the email metadata entry", async () => {
|
||||
const headers = { "Message-ID": "<stored@example.com>" };
|
||||
await processEmail(
|
||||
makeInput({ subject: "Sub", content: "<p>c</p>", headers }),
|
||||
env as any,
|
||||
);
|
||||
|
||||
const metadata = await env.EMAIL_STORAGE.get(
|
||||
`feed:${VALID_FEED_ID}:metadata`,
|
||||
"json",
|
||||
);
|
||||
expect(metadata.emails[0].messageId).toBe("<stored@example.com>");
|
||||
expect(typeof metadata.emails[0].dedupHash).toBe("string");
|
||||
expect(metadata.emails[0].dedupHash).toHaveLength(64); // SHA-256 hex
|
||||
});
|
||||
});
|
||||
|
||||
describe("processEmail — monitoring counters", () => {
|
||||
it("increments emails_received and sets last_email_at on success", async () => {
|
||||
const env = createMockEnv();
|
||||
@@ -709,6 +837,7 @@ describe("processEmail — unsubscribe capture", () => {
|
||||
it("keeps one entry per sender and overwrites with the latest URL", async () => {
|
||||
await processEmail(
|
||||
makeInput({
|
||||
subject: "Issue 1 from A",
|
||||
senders: ["a@one.com"],
|
||||
headers: {
|
||||
"list-unsubscribe": "<https://one.com/u/1>",
|
||||
@@ -719,6 +848,7 @@ describe("processEmail — unsubscribe capture", () => {
|
||||
);
|
||||
await processEmail(
|
||||
makeInput({
|
||||
subject: "Issue 1 from B",
|
||||
senders: ["b@two.com"],
|
||||
headers: {
|
||||
"list-unsubscribe": "<https://two.com/u/1>",
|
||||
@@ -729,6 +859,7 @@ describe("processEmail — unsubscribe capture", () => {
|
||||
);
|
||||
await processEmail(
|
||||
makeInput({
|
||||
subject: "Issue 2 from A",
|
||||
senders: ["a@one.com"],
|
||||
headers: {
|
||||
"list-unsubscribe": "<https://one.com/u/2>",
|
||||
|
||||
@@ -109,12 +109,62 @@ async function loadAcceptingFeed(
|
||||
return { ok: true, feed };
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute a SHA-256 hex digest of a normalised string combining subject and
|
||||
* content. Used as a dedup fallback when no Message-ID header is present.
|
||||
* "Normalised" means lower-cased and all whitespace runs collapsed to a single
|
||||
* space — so minor whitespace differences in re-sent mails still match.
|
||||
*/
|
||||
async function computeDedupHash(
|
||||
subject: string,
|
||||
content: string,
|
||||
): Promise<string> {
|
||||
const normalize = (s: string) => s.toLowerCase().replace(/\s+/g, " ").trim();
|
||||
const raw = `${normalize(subject)}\n${normalize(content)}`;
|
||||
const buf = await crypto.subtle.digest(
|
||||
"SHA-256",
|
||||
new TextEncoder().encode(raw),
|
||||
);
|
||||
return Array.from(new Uint8Array(buf))
|
||||
.map((b) => b.toString(16).padStart(2, "0"))
|
||||
.join("");
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the Message-ID from request headers (case-insensitive key lookup).
|
||||
* Returns undefined when absent or empty.
|
||||
*/
|
||||
function extractMessageId(
|
||||
headers: Record<string, string> | undefined,
|
||||
): string | undefined {
|
||||
if (!headers) return undefined;
|
||||
const value = Object.entries(headers).find(
|
||||
([k]) => k.toLowerCase() === "message-id",
|
||||
)?.[1];
|
||||
const trimmed = value?.trim();
|
||||
return trimmed || undefined;
|
||||
}
|
||||
|
||||
async function storeEmail(
|
||||
feed: Feed,
|
||||
input: ProcessEmailInput,
|
||||
env: Env,
|
||||
ctx?: ExecutionContext,
|
||||
): Promise<void> {
|
||||
): Promise<boolean> {
|
||||
// ── Dedup check ──────────────────────────────────────────────────────────
|
||||
// Compute both dedup signals up-front (hash is async) so we only do it once.
|
||||
const messageId = extractMessageId(input.headers);
|
||||
const dedupHash = await computeDedupHash(input.subject, input.content);
|
||||
|
||||
if (feed.hasDuplicate(messageId, dedupHash)) {
|
||||
logger.info("Duplicate email skipped", {
|
||||
feedId: feed.id.value,
|
||||
...(messageId ? { messageId } : { dedupHash }),
|
||||
});
|
||||
await bumpCounters(env.EMAIL_STORAGE, { emails_deduplicated: 1 });
|
||||
return false; // signal: skipped (not stored)
|
||||
}
|
||||
|
||||
const attachmentBucket = getAttachmentBucket(env);
|
||||
const inlineCids = extractInlineCids(input.content);
|
||||
const storedAttachments: AttachmentData[] =
|
||||
@@ -149,6 +199,8 @@ async function storeEmail(
|
||||
size: serialisedSize,
|
||||
...(downloadableIds.length > 0 ? { attachmentIds: downloadableIds } : {}),
|
||||
...(inlineIds.length > 0 ? { inlineAttachmentIds: inlineIds } : {}),
|
||||
...(messageId ? { messageId } : {}),
|
||||
dedupHash,
|
||||
};
|
||||
|
||||
// Track the latest sender's domain (feed icon) and capture the RFC 8058
|
||||
@@ -198,6 +250,7 @@ async function storeEmail(
|
||||
? (p) => ctx.waitUntil(p)
|
||||
: () => {};
|
||||
await dispatchFeedEvents(feed, env, schedule);
|
||||
return true; // signal: stored
|
||||
}
|
||||
|
||||
export async function processEmail(
|
||||
|
||||
@@ -12,6 +12,7 @@ const EMPTY_COUNTERS: Counters = {
|
||||
emails_received: 0,
|
||||
emails_rejected: 0,
|
||||
emails_forwarded: 0,
|
||||
emails_deduplicated: 0,
|
||||
unsubscribes_sent: 0,
|
||||
};
|
||||
|
||||
@@ -43,6 +44,7 @@ export async function bumpCounters(
|
||||
current.emails_received += changes.emails_received ?? 0;
|
||||
current.emails_rejected += changes.emails_rejected ?? 0;
|
||||
current.emails_forwarded += changes.emails_forwarded ?? 0;
|
||||
current.emails_deduplicated += changes.emails_deduplicated ?? 0;
|
||||
current.unsubscribes_sent += changes.unsubscribes_sent ?? 0;
|
||||
if (changes.last_email_at) current.last_email_at = changes.last_email_at;
|
||||
if (changes.last_feed_created_at)
|
||||
|
||||
@@ -203,6 +203,30 @@ export class Feed {
|
||||
).decide(senders);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether the email index already contains a duplicate of the incoming
|
||||
* email. Dedup uses `messageId` as the primary key (when both sides have one)
|
||||
* and falls back to `dedupHash` (SHA-256 of normalised subject+content).
|
||||
* Old entries that predate the feature and carry neither field are never
|
||||
* matched — they cannot cause false positives.
|
||||
*/
|
||||
hasDuplicate(messageId?: string, dedupHash?: string): boolean {
|
||||
for (const entry of this._metadata.emails) {
|
||||
if (messageId && entry.messageId && entry.messageId === messageId) {
|
||||
return true;
|
||||
}
|
||||
if (
|
||||
!messageId &&
|
||||
dedupHash &&
|
||||
entry.dedupHash &&
|
||||
entry.dedupHash === dedupHash
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add an email to the front of the index, refresh the icon domain and the
|
||||
* per-sender unsubscribe link, then trim the oldest entries back under the
|
||||
|
||||
@@ -3,6 +3,7 @@ import { cors } from "hono/cors";
|
||||
import { handle as handleInbound } from "./routes/inbound";
|
||||
import { handle as handleRSS } from "./routes/rss";
|
||||
import { handle as handleAtom } from "./routes/atom";
|
||||
import { handle as handleJSON } from "./routes/json";
|
||||
import { handle as handleAdmin } from "./routes/admin";
|
||||
import { handle as handleEntry } from "./routes/entries";
|
||||
import { handle as handleFiles } from "./routes/files";
|
||||
@@ -116,6 +117,7 @@ app.use(
|
||||
const api = new Hono<AppEnv>();
|
||||
const rss = new Hono<AppEnv>();
|
||||
const atom = new Hono<AppEnv>();
|
||||
const json = new Hono<AppEnv>();
|
||||
const entries = new Hono<AppEnv>();
|
||||
const files = new Hono<AppEnv>();
|
||||
const admin = new Hono<AppEnv>();
|
||||
@@ -151,6 +153,9 @@ rss.get("/:feedId", handleRSS);
|
||||
// Atom feed routes (public)
|
||||
atom.get("/:feedId", handleAtom);
|
||||
|
||||
// JSON Feed routes (public)
|
||||
json.get("/:feedId", handleJSON);
|
||||
|
||||
// Email entry HTML view (public)
|
||||
entries.get("/:feedId/:entryId", handleEntry);
|
||||
|
||||
@@ -166,6 +171,7 @@ app.route("/api", api);
|
||||
app.route("/api", apiApp);
|
||||
app.route("/rss", rss);
|
||||
app.route("/atom", atom);
|
||||
app.route("/json", json);
|
||||
app.route("/entries", entries);
|
||||
app.route("/files", files);
|
||||
app.route("/admin", admin);
|
||||
|
||||
@@ -15,6 +15,7 @@ describe("CountersRepository", () => {
|
||||
emails_received: 2,
|
||||
emails_rejected: 0,
|
||||
emails_forwarded: 0,
|
||||
emails_deduplicated: 0,
|
||||
unsubscribes_sent: 0,
|
||||
});
|
||||
expect(await repo.getRaw()).toMatchObject({ emails_received: 2 });
|
||||
|
||||
@@ -30,7 +30,7 @@ function buildFeed(
|
||||
emails: EmailData[],
|
||||
baseUrl: string,
|
||||
feedId: string,
|
||||
selfUrl?: { rss?: string; atom?: string },
|
||||
selfUrl?: { rss?: string; atom?: string; json?: string },
|
||||
): Feed {
|
||||
const iconUrl = `${baseUrl}/favicon/${feedId}`;
|
||||
const feed = new Feed({
|
||||
@@ -52,6 +52,7 @@ function buildFeed(
|
||||
feedLinks: {
|
||||
rss: selfUrl?.rss ?? `${baseUrl}/rss/${feedId}`,
|
||||
atom: selfUrl?.atom ?? `${baseUrl}/atom/${feedId}`,
|
||||
json: selfUrl?.json ?? `${baseUrl}/json/${feedId}`,
|
||||
},
|
||||
author: feedConfig.author
|
||||
? {
|
||||
@@ -127,3 +128,19 @@ export function generateAtomFeed(
|
||||
).atom1(),
|
||||
);
|
||||
}
|
||||
|
||||
export function generateJsonFeed(
|
||||
feedConfig: FeedConfig,
|
||||
emails: EmailData[],
|
||||
baseUrl: string,
|
||||
feedId: string,
|
||||
selfUrl?: string,
|
||||
): string {
|
||||
return buildFeed(
|
||||
feedConfig,
|
||||
emails,
|
||||
baseUrl,
|
||||
feedId,
|
||||
selfUrl ? { json: selfUrl } : undefined,
|
||||
).json1();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
import { FeedConfig, EmailData } from "../types";
|
||||
|
||||
export interface FeedValidators {
|
||||
etag: string;
|
||||
lastModified: string;
|
||||
maxReceivedAt: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute HTTP cache validators (ETag + Last-Modified) for a feed.
|
||||
* The ETag is derived from the feed format prefix, feedId, email count, and max
|
||||
* receivedAt, making it a strong deterministic validator that changes whenever
|
||||
* the feed content changes.
|
||||
*/
|
||||
export function computeFeedValidators(
|
||||
format: "rss" | "atom",
|
||||
feedId: string,
|
||||
feedConfig: FeedConfig,
|
||||
emails: EmailData[],
|
||||
): FeedValidators {
|
||||
const maxReceivedAt =
|
||||
emails.length > 0
|
||||
? Math.max(...emails.map((e) => e.receivedAt))
|
||||
: (feedConfig.created_at ?? 0);
|
||||
|
||||
const hash = `${format}-${feedId}-${emails.length}-${maxReceivedAt}`;
|
||||
const etag = `"${hash}"`;
|
||||
const lastModified = new Date(maxReceivedAt).toUTCString();
|
||||
|
||||
return { etag, lastModified, maxReceivedAt };
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if the request carries a matching conditional GET header,
|
||||
* meaning a 304 Not Modified response is appropriate.
|
||||
*/
|
||||
export function isNotModified(
|
||||
req: Request,
|
||||
validators: FeedValidators,
|
||||
): boolean {
|
||||
const ifNoneMatch = req.headers.get("If-None-Match");
|
||||
if (ifNoneMatch !== null) {
|
||||
return ifNoneMatch === validators.etag;
|
||||
}
|
||||
|
||||
const ifModifiedSince = req.headers.get("If-Modified-Since");
|
||||
if (ifModifiedSince !== null) {
|
||||
const clientTime = new Date(ifModifiedSince).getTime();
|
||||
return !isNaN(clientTime) && clientTime >= validators.maxReceivedAt;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a 304 Not Modified response with the standard cache validator headers.
|
||||
*/
|
||||
export function notModifiedResponse(validators: FeedValidators): Response {
|
||||
return new Response(null, {
|
||||
status: 304,
|
||||
headers: {
|
||||
ETag: validators.etag,
|
||||
"Last-Modified": validators.lastModified,
|
||||
"Cache-Control": "max-age=1800",
|
||||
"X-Robots-Tag": "noindex",
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
} from "../infrastructure/urls";
|
||||
import { feedsRouter } from "./admin/feeds";
|
||||
import { emailsRouter } from "./admin/emails";
|
||||
import { handleOpml } from "./opml";
|
||||
import { dashboardScript } from "../scripts/generated/dashboard";
|
||||
|
||||
type AppEnv = { Bindings: Env };
|
||||
@@ -975,6 +976,9 @@ app.get("/", async (c) => {
|
||||
);
|
||||
});
|
||||
|
||||
// OPML export (admin-protected)
|
||||
app.get("/opml", handleOpml);
|
||||
|
||||
// Mount sub-routers
|
||||
app.route("/feeds", feedsRouter);
|
||||
app.route("/", emailsRouter);
|
||||
|
||||
@@ -144,4 +144,117 @@ describe("Atom Feed Route", () => {
|
||||
expect(body).toContain('xmlns="http://www.w3.org/2005/Atom"');
|
||||
});
|
||||
});
|
||||
|
||||
describe("conditional GET (ETag + Last-Modified)", () => {
|
||||
const FEED_ID = "test-feed-atom-cget";
|
||||
const EMAIL_RECEIVED_AT = 1700000001000;
|
||||
|
||||
beforeEach(async () => {
|
||||
const emailKey = `feed:${FEED_ID}:${EMAIL_RECEIVED_AT}`;
|
||||
await mockEnv.EMAIL_STORAGE.put(
|
||||
emailKey,
|
||||
JSON.stringify({
|
||||
subject: "Atom Subject",
|
||||
from: "Sender <sender@example.com>",
|
||||
content: "<p>Body</p>",
|
||||
receivedAt: EMAIL_RECEIVED_AT,
|
||||
headers: {},
|
||||
}),
|
||||
);
|
||||
await mockEnv.EMAIL_STORAGE.put(
|
||||
`feed:${FEED_ID}:metadata`,
|
||||
JSON.stringify({
|
||||
emails: [
|
||||
{
|
||||
key: emailKey,
|
||||
subject: "Atom Subject",
|
||||
receivedAt: EMAIL_RECEIVED_AT,
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
await mockEnv.EMAIL_STORAGE.put(
|
||||
`feed:${FEED_ID}:config`,
|
||||
JSON.stringify({
|
||||
title: "Atom Cget Feed",
|
||||
language: "en",
|
||||
created_at: 1700000000000,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("first GET returns 200 with ETag and Last-Modified headers", async () => {
|
||||
const res = await testApp.request(`/${FEED_ID}`, {}, mockEnv);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers.get("ETag")).toBeTruthy();
|
||||
expect(res.headers.get("Last-Modified")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("GET with matching If-None-Match returns 304 with empty body", async () => {
|
||||
const first = await testApp.request(`/${FEED_ID}`, {}, mockEnv);
|
||||
const etag = first.headers.get("ETag")!;
|
||||
|
||||
const res = await testApp.request(
|
||||
`/${FEED_ID}`,
|
||||
{ headers: { "If-None-Match": etag } },
|
||||
mockEnv,
|
||||
);
|
||||
expect(res.status).toBe(304);
|
||||
expect(await res.text()).toBe("");
|
||||
});
|
||||
|
||||
it("GET with If-Modified-Since in the future returns 304", async () => {
|
||||
const future = new Date(EMAIL_RECEIVED_AT + 1000).toUTCString();
|
||||
const res = await testApp.request(
|
||||
`/${FEED_ID}`,
|
||||
{ headers: { "If-Modified-Since": future } },
|
||||
mockEnv,
|
||||
);
|
||||
expect(res.status).toBe(304);
|
||||
});
|
||||
|
||||
it("stale If-None-Match after new email results in 200", async () => {
|
||||
const first = await testApp.request(`/${FEED_ID}`, {}, mockEnv);
|
||||
const oldEtag = first.headers.get("ETag")!;
|
||||
|
||||
const newReceivedAt = EMAIL_RECEIVED_AT + 5000;
|
||||
const newEmailKey = `feed:${FEED_ID}:${newReceivedAt}`;
|
||||
await mockEnv.EMAIL_STORAGE.put(
|
||||
newEmailKey,
|
||||
JSON.stringify({
|
||||
subject: "Newer Atom Email",
|
||||
from: "Sender <sender@example.com>",
|
||||
content: "<p>New body</p>",
|
||||
receivedAt: newReceivedAt,
|
||||
headers: {},
|
||||
}),
|
||||
);
|
||||
await mockEnv.EMAIL_STORAGE.put(
|
||||
`feed:${FEED_ID}:metadata`,
|
||||
JSON.stringify({
|
||||
emails: [
|
||||
{
|
||||
key: newEmailKey,
|
||||
subject: "Newer Atom Email",
|
||||
receivedAt: newReceivedAt,
|
||||
},
|
||||
{
|
||||
key: `feed:${FEED_ID}:${EMAIL_RECEIVED_AT}`,
|
||||
subject: "Atom Subject",
|
||||
receivedAt: EMAIL_RECEIVED_AT,
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
const res = await testApp.request(
|
||||
`/${FEED_ID}`,
|
||||
{ headers: { "If-None-Match": oldEtag } },
|
||||
mockEnv,
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
const newEtag = res.headers.get("ETag");
|
||||
expect(newEtag).not.toBe(oldEtag);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -5,6 +5,11 @@ import { fetchFeedData } from "../application/feed-fetcher";
|
||||
import { baseUrl, feedAtomUrl } from "../infrastructure/urls";
|
||||
import { isExpired } from "../domain/feed";
|
||||
import { FeedId } from "../domain/value-objects/feed-id";
|
||||
import {
|
||||
computeFeedValidators,
|
||||
isNotModified,
|
||||
notModifiedResponse,
|
||||
} from "../infrastructure/http-cache";
|
||||
|
||||
export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
|
||||
try {
|
||||
@@ -21,6 +26,17 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
|
||||
return new Response("Feed has expired", { status: 410 });
|
||||
}
|
||||
|
||||
const validators = computeFeedValidators(
|
||||
"atom",
|
||||
feedId,
|
||||
feedData.feedConfig,
|
||||
feedData.emails,
|
||||
);
|
||||
|
||||
if (isNotModified(c.req.raw, validators)) {
|
||||
return notModifiedResponse(validators);
|
||||
}
|
||||
|
||||
const base = baseUrl(c.env);
|
||||
const selfUrl = new URL(c.req.url).origin + `/atom/${feedId}`;
|
||||
const atomXml = generateAtomFeed(
|
||||
@@ -42,6 +58,8 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
|
||||
"Cache-Control": "max-age=1800",
|
||||
"X-Robots-Tag": "noindex",
|
||||
Link: linkHeader,
|
||||
ETag: validators.etag,
|
||||
"Last-Modified": validators.lastModified,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
|
||||
@@ -166,6 +166,7 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
|
||||
label="Forwarded (catch-all)"
|
||||
value={stats.emails_forwarded}
|
||||
/>
|
||||
<Stat label="Deduplicated" value={stats.emails_deduplicated} />
|
||||
<Stat
|
||||
label="Acceptance rate"
|
||||
value={acceptanceRate}
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
import { describe, it, expect, beforeEach } from "vitest";
|
||||
import { Hono } from "hono";
|
||||
import { handle } from "./json";
|
||||
import { createMockEnv } from "../test/setup";
|
||||
import { Env } from "../types";
|
||||
|
||||
describe("JSON Feed Route", () => {
|
||||
let testApp: Hono;
|
||||
let mockEnv: Env;
|
||||
|
||||
beforeEach(() => {
|
||||
mockEnv = createMockEnv() as unknown as Env;
|
||||
testApp = new Hono();
|
||||
testApp.get("/:feedId", handle);
|
||||
});
|
||||
|
||||
describe("unknown feed", () => {
|
||||
it("returns 404 when no metadata exists in KV", async () => {
|
||||
const res = await testApp.request("/nonexistent-feed", {}, mockEnv);
|
||||
expect(res.status).toBe(404);
|
||||
expect(await res.text()).toBe("Feed not found");
|
||||
});
|
||||
});
|
||||
|
||||
describe("valid feed with no emails", () => {
|
||||
beforeEach(async () => {
|
||||
await mockEnv.EMAIL_STORAGE.put(
|
||||
"feed:empty-feed:metadata",
|
||||
JSON.stringify({ emails: [] }),
|
||||
);
|
||||
});
|
||||
|
||||
it("returns 200 with application/feed+json content type", async () => {
|
||||
const res = await testApp.request("/empty-feed", {}, mockEnv);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers.get("Content-Type")).toContain(
|
||||
"application/feed+json",
|
||||
);
|
||||
});
|
||||
|
||||
it("includes Cache-Control header", async () => {
|
||||
const res = await testApp.request("/empty-feed", {}, mockEnv);
|
||||
expect(res.headers.get("Cache-Control")).toBe("max-age=1800");
|
||||
});
|
||||
|
||||
it("sets X-Robots-Tag: noindex", async () => {
|
||||
const res = await testApp.request("/empty-feed", {}, mockEnv);
|
||||
expect(res.headers.get("X-Robots-Tag")).toBe("noindex");
|
||||
});
|
||||
|
||||
it("Link header advertises hub and self", async () => {
|
||||
const res = await testApp.request("/empty-feed", {}, mockEnv);
|
||||
const link = res.headers.get("Link") ?? "";
|
||||
expect(link).toContain(`rel="hub"`);
|
||||
expect(link).toContain(`rel="self"`);
|
||||
});
|
||||
|
||||
it("body parses as JSON with jsonfeed version 1.1", async () => {
|
||||
const res = await testApp.request("/empty-feed", {}, mockEnv);
|
||||
const body = (await res.json()) as { version: string; items: unknown[] };
|
||||
expect(body.version).toBe("https://jsonfeed.org/version/1");
|
||||
expect(Array.isArray(body.items)).toBe(true);
|
||||
expect(body.items).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("valid feed with emails", () => {
|
||||
const FEED_ID = "test-feed-json";
|
||||
const EMAIL_RECEIVED_AT = 1700000001000;
|
||||
|
||||
beforeEach(async () => {
|
||||
const emailKey = `feed:${FEED_ID}:${EMAIL_RECEIVED_AT}`;
|
||||
await mockEnv.EMAIL_STORAGE.put(
|
||||
emailKey,
|
||||
JSON.stringify({
|
||||
subject: "JSON Feed Subject",
|
||||
from: "Sender <sender@example.com>",
|
||||
content: "<p>Body content</p>",
|
||||
receivedAt: EMAIL_RECEIVED_AT,
|
||||
headers: {},
|
||||
}),
|
||||
);
|
||||
await mockEnv.EMAIL_STORAGE.put(
|
||||
`feed:${FEED_ID}:metadata`,
|
||||
JSON.stringify({
|
||||
emails: [
|
||||
{
|
||||
key: emailKey,
|
||||
subject: "JSON Feed Subject",
|
||||
receivedAt: EMAIL_RECEIVED_AT,
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
await mockEnv.EMAIL_STORAGE.put(
|
||||
`feed:${FEED_ID}:config`,
|
||||
JSON.stringify({
|
||||
title: "My JSON Feed",
|
||||
language: "en",
|
||||
created_at: 1700000000000,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("returns 200 with items containing the seeded email", async () => {
|
||||
const res = await testApp.request(`/${FEED_ID}`, {}, mockEnv);
|
||||
expect(res.status).toBe(200);
|
||||
const body = (await res.json()) as {
|
||||
version: string;
|
||||
items: Array<{ title: string }>;
|
||||
};
|
||||
expect(body.version).toBe("https://jsonfeed.org/version/1");
|
||||
expect(Array.isArray(body.items)).toBe(true);
|
||||
expect(body.items).toHaveLength(1);
|
||||
expect(body.items[0].title).toBe("JSON Feed Subject");
|
||||
});
|
||||
});
|
||||
|
||||
describe("expired feed", () => {
|
||||
beforeEach(async () => {
|
||||
const pastTimestamp = Date.now() - 1000 * 60 * 60 * 24; // 1 day ago
|
||||
await mockEnv.EMAIL_STORAGE.put(
|
||||
"feed:expired-feed:metadata",
|
||||
JSON.stringify({ emails: [] }),
|
||||
);
|
||||
await mockEnv.EMAIL_STORAGE.put(
|
||||
"feed:expired-feed:config",
|
||||
JSON.stringify({
|
||||
title: "Expired Feed",
|
||||
language: "en",
|
||||
created_at: pastTimestamp,
|
||||
expires_at: pastTimestamp,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("returns 410 for expired feed", async () => {
|
||||
const res = await testApp.request("/expired-feed", {}, mockEnv);
|
||||
expect(res.status).toBe(410);
|
||||
expect(await res.text()).toBe("Feed has expired");
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,51 @@
|
||||
import { Context } from "hono";
|
||||
import { Env } from "../types";
|
||||
import { generateJsonFeed } from "../infrastructure/feed-generator";
|
||||
import { fetchFeedData } from "../application/feed-fetcher";
|
||||
import { baseUrl } from "../infrastructure/urls";
|
||||
import { isExpired } from "../domain/feed";
|
||||
import { FeedId } from "../domain/value-objects/feed-id";
|
||||
|
||||
export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
|
||||
try {
|
||||
const feedId = c.req.param("feedId");
|
||||
if (!feedId) {
|
||||
return new Response("Feed ID is required", { status: 400 });
|
||||
}
|
||||
|
||||
const feedData = await fetchFeedData(FeedId.unchecked(feedId), c.env);
|
||||
if (!feedData) {
|
||||
return new Response("Feed not found", { status: 404 });
|
||||
}
|
||||
if (isExpired(feedData.feedConfig)) {
|
||||
return new Response("Feed has expired", { status: 410 });
|
||||
}
|
||||
|
||||
const base = baseUrl(c.env);
|
||||
const selfUrl = new URL(c.req.url).origin + `/json/${feedId}`;
|
||||
const jsonFeed = generateJsonFeed(
|
||||
feedData.feedConfig,
|
||||
feedData.emails,
|
||||
base,
|
||||
feedId,
|
||||
selfUrl,
|
||||
);
|
||||
const linkHeader = [
|
||||
`<${base}/hub>; rel="hub"`,
|
||||
`<${selfUrl}>; rel="self"`,
|
||||
].join(", ");
|
||||
|
||||
return new Response(jsonFeed, {
|
||||
status: 200,
|
||||
headers: {
|
||||
"Content-Type": "application/feed+json",
|
||||
"Cache-Control": "max-age=1800",
|
||||
"X-Robots-Tag": "noindex",
|
||||
Link: linkHeader,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("Error generating JSON feed:", error);
|
||||
return new Response("Internal Server Error", { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
import { describe, it, expect, beforeEach } from "vitest";
|
||||
import { Hono } from "hono";
|
||||
import app from "./admin";
|
||||
import { createMockEnv } from "../test/setup";
|
||||
import { Env } from "../types";
|
||||
|
||||
describe("OPML export — GET /admin/opml", () => {
|
||||
let testApp: Hono;
|
||||
let mockEnv: Env;
|
||||
let request: (path: string, init?: RequestInit) => Promise<Response>;
|
||||
let loginAndGetCookie: () => Promise<string>;
|
||||
|
||||
beforeEach(() => {
|
||||
mockEnv = createMockEnv() as unknown as Env;
|
||||
testApp = new Hono();
|
||||
testApp.route("/admin", app);
|
||||
request = (path, init = {}) =>
|
||||
Promise.resolve(testApp.request(path, init, mockEnv));
|
||||
loginAndGetCookie = async () => {
|
||||
const formData = new FormData();
|
||||
formData.append("password", "test-password");
|
||||
const response = await request("/admin/login", {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
expect(response.status).toBe(302);
|
||||
const setCookie = response.headers.get("Set-Cookie");
|
||||
expect(setCookie).toBeTruthy();
|
||||
return (setCookie as string).split(";")[0];
|
||||
};
|
||||
});
|
||||
|
||||
it("should return 302 redirect to login when not authenticated", async () => {
|
||||
const res = await request("/admin/opml");
|
||||
expect(res.status).toBe(302);
|
||||
expect(res.headers.get("Location")).toBe("/admin/login");
|
||||
});
|
||||
|
||||
it("should return 200 with OPML content when authenticated", async () => {
|
||||
// Seed two feeds in the registry
|
||||
await mockEnv.EMAIL_STORAGE.put(
|
||||
"feeds:list",
|
||||
JSON.stringify({
|
||||
feeds: [
|
||||
{ id: "feed-abc", title: "My Newsletter", description: "Daily news" },
|
||||
{ id: "feed-xyz", title: "Tech Digest" },
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
const authCookie = await loginAndGetCookie();
|
||||
const res = await request("/admin/opml", {
|
||||
headers: {
|
||||
Cookie: authCookie,
|
||||
Origin: "https://test.getmynews.app",
|
||||
},
|
||||
});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
const contentType = res.headers.get("Content-Type") ?? "";
|
||||
expect(contentType).toContain("text/x-opml");
|
||||
expect(res.headers.get("Content-Disposition")).toBe(
|
||||
'attachment; filename="feeds.opml"',
|
||||
);
|
||||
expect(res.headers.get("X-Robots-Tag")).toBe("noindex");
|
||||
|
||||
const body = await res.text();
|
||||
|
||||
// Valid OPML 2.0 structure
|
||||
expect(body).toContain('<?xml version="1.0" encoding="UTF-8"?>');
|
||||
expect(body).toContain('<opml version="2.0">');
|
||||
expect(body).toContain("<head>");
|
||||
expect(body).toContain("<title>kill-the-news feeds</title>");
|
||||
expect(body).toContain("<body>");
|
||||
|
||||
// One outline per feed with correct xmlUrl
|
||||
expect(body).toContain('type="rss"');
|
||||
expect(body).toContain('text="My Newsletter"');
|
||||
expect(body).toContain('title="My Newsletter"');
|
||||
expect(body).toContain('xmlUrl="https://test.getmynews.app/rss/feed-abc"');
|
||||
expect(body).toContain('description="Daily news"');
|
||||
|
||||
expect(body).toContain('text="Tech Digest"');
|
||||
expect(body).toContain('xmlUrl="https://test.getmynews.app/rss/feed-xyz"');
|
||||
|
||||
// feed-xyz has no description — attribute must not appear
|
||||
const feedXyzLine =
|
||||
body.split("\n").find((l) => l.includes("feed-xyz")) ?? "";
|
||||
expect(feedXyzLine).not.toContain("description=");
|
||||
});
|
||||
|
||||
it("should XML-escape special characters in title and description", async () => {
|
||||
await mockEnv.EMAIL_STORAGE.put(
|
||||
"feeds:list",
|
||||
JSON.stringify({
|
||||
feeds: [
|
||||
{
|
||||
id: "feed-special",
|
||||
title: "News & <Updates>",
|
||||
description: 'Say "hello" & goodbye',
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
const authCookie = await loginAndGetCookie();
|
||||
const res = await request("/admin/opml", {
|
||||
headers: {
|
||||
Cookie: authCookie,
|
||||
Origin: "https://test.getmynews.app",
|
||||
},
|
||||
});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
const body = await res.text();
|
||||
|
||||
// Raw special chars must not appear unescaped in attribute values
|
||||
const outlineLine =
|
||||
body.split("\n").find((l) => l.includes("feed-special")) ?? "";
|
||||
expect(outlineLine).toContain("News & <Updates>");
|
||||
expect(outlineLine).toContain("Say "hello" & goodbye");
|
||||
expect(outlineLine).not.toContain('title="News & <');
|
||||
});
|
||||
|
||||
it("should return empty body element when there are no feeds", async () => {
|
||||
const authCookie = await loginAndGetCookie();
|
||||
const res = await request("/admin/opml", {
|
||||
headers: {
|
||||
Cookie: authCookie,
|
||||
Origin: "https://test.getmynews.app",
|
||||
},
|
||||
});
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
const body = await res.text();
|
||||
expect(body).toContain("<body>");
|
||||
expect(body).not.toContain("<outline");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,56 @@
|
||||
import { Context } from "hono";
|
||||
import { Env } from "../types";
|
||||
import { FeedRepository } from "../infrastructure/feed-repository";
|
||||
import { feedRssUrl } from "../infrastructure/urls";
|
||||
|
||||
/**
|
||||
* Escape a string for use in an XML attribute value.
|
||||
* Replaces &, <, >, and " with their XML entity equivalents.
|
||||
*/
|
||||
function escapeXmlAttr(value: string): string {
|
||||
return value
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, """);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handler for GET /admin/opml
|
||||
* Exports all feeds as an OPML 2.0 document.
|
||||
* Protected by the admin auth middleware (inherits from admin Hono app).
|
||||
*/
|
||||
export async function handleOpml(c: Context<{ Bindings: Env }>) {
|
||||
const env = c.env;
|
||||
const feeds = await FeedRepository.from(env).listFeeds();
|
||||
|
||||
const outlines = feeds
|
||||
.map((feed) => {
|
||||
const title = escapeXmlAttr(feed.title);
|
||||
const xmlUrl = escapeXmlAttr(feedRssUrl(feed.id, env));
|
||||
const descAttr = feed.description
|
||||
? ` description="${escapeXmlAttr(feed.description)}"`
|
||||
: "";
|
||||
return ` <outline type="rss" text="${title}" title="${title}" xmlUrl="${xmlUrl}"${descAttr}/>`;
|
||||
})
|
||||
.join("\n");
|
||||
|
||||
const opml = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<opml version="2.0">
|
||||
<head>
|
||||
<title>kill-the-news feeds</title>
|
||||
</head>
|
||||
<body>
|
||||
${outlines}
|
||||
</body>
|
||||
</opml>`;
|
||||
|
||||
return new Response(opml, {
|
||||
status: 200,
|
||||
headers: {
|
||||
"Content-Type": "text/x-opml; charset=utf-8",
|
||||
"Content-Disposition": 'attachment; filename="feeds.opml"',
|
||||
"X-Robots-Tag": "noindex",
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -53,4 +53,133 @@ describe("RSS Feed Route", () => {
|
||||
expect(link).toContain(`rel="self"`);
|
||||
});
|
||||
});
|
||||
|
||||
describe("conditional GET (ETag + Last-Modified)", () => {
|
||||
const FEED_ID = "test-feed-rss-cget";
|
||||
const EMAIL_RECEIVED_AT = 1700000001000;
|
||||
|
||||
beforeEach(async () => {
|
||||
const emailKey = `feed:${FEED_ID}:${EMAIL_RECEIVED_AT}`;
|
||||
await mockEnv.EMAIL_STORAGE.put(
|
||||
emailKey,
|
||||
JSON.stringify({
|
||||
subject: "RSS Subject",
|
||||
from: "Sender <sender@example.com>",
|
||||
content: "<p>Body</p>",
|
||||
receivedAt: EMAIL_RECEIVED_AT,
|
||||
headers: {},
|
||||
}),
|
||||
);
|
||||
await mockEnv.EMAIL_STORAGE.put(
|
||||
`feed:${FEED_ID}:metadata`,
|
||||
JSON.stringify({
|
||||
emails: [
|
||||
{
|
||||
key: emailKey,
|
||||
subject: "RSS Subject",
|
||||
receivedAt: EMAIL_RECEIVED_AT,
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
await mockEnv.EMAIL_STORAGE.put(
|
||||
`feed:${FEED_ID}:config`,
|
||||
JSON.stringify({
|
||||
title: "RSS Cget Feed",
|
||||
language: "en",
|
||||
created_at: 1700000000000,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("first GET returns 200 with ETag and Last-Modified headers", async () => {
|
||||
const res = await testApp.request(`/${FEED_ID}`, {}, mockEnv);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers.get("ETag")).toBeTruthy();
|
||||
expect(res.headers.get("Last-Modified")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("GET with matching If-None-Match returns 304 with empty body", async () => {
|
||||
const first = await testApp.request(`/${FEED_ID}`, {}, mockEnv);
|
||||
const etag = first.headers.get("ETag")!;
|
||||
|
||||
const res = await testApp.request(
|
||||
`/${FEED_ID}`,
|
||||
{ headers: { "If-None-Match": etag } },
|
||||
mockEnv,
|
||||
);
|
||||
expect(res.status).toBe(304);
|
||||
expect(await res.text()).toBe("");
|
||||
});
|
||||
|
||||
it("GET with If-Modified-Since in the future returns 304", async () => {
|
||||
const future = new Date(EMAIL_RECEIVED_AT + 1000).toUTCString();
|
||||
const res = await testApp.request(
|
||||
`/${FEED_ID}`,
|
||||
{ headers: { "If-Modified-Since": future } },
|
||||
mockEnv,
|
||||
);
|
||||
expect(res.status).toBe(304);
|
||||
});
|
||||
|
||||
it("stale If-None-Match after new email results in 200", async () => {
|
||||
// Get ETag before new email
|
||||
const first = await testApp.request(`/${FEED_ID}`, {}, mockEnv);
|
||||
const oldEtag = first.headers.get("ETag")!;
|
||||
|
||||
// Add a newer email
|
||||
const newReceivedAt = EMAIL_RECEIVED_AT + 5000;
|
||||
const newEmailKey = `feed:${FEED_ID}:${newReceivedAt}`;
|
||||
await mockEnv.EMAIL_STORAGE.put(
|
||||
newEmailKey,
|
||||
JSON.stringify({
|
||||
subject: "Newer Email",
|
||||
from: "Sender <sender@example.com>",
|
||||
content: "<p>New body</p>",
|
||||
receivedAt: newReceivedAt,
|
||||
headers: {},
|
||||
}),
|
||||
);
|
||||
await mockEnv.EMAIL_STORAGE.put(
|
||||
`feed:${FEED_ID}:metadata`,
|
||||
JSON.stringify({
|
||||
emails: [
|
||||
{
|
||||
key: newEmailKey,
|
||||
subject: "Newer Email",
|
||||
receivedAt: newReceivedAt,
|
||||
},
|
||||
{
|
||||
key: `feed:${FEED_ID}:${EMAIL_RECEIVED_AT}`,
|
||||
subject: "RSS Subject",
|
||||
receivedAt: EMAIL_RECEIVED_AT,
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
const res = await testApp.request(
|
||||
`/${FEED_ID}`,
|
||||
{ headers: { "If-None-Match": oldEtag } },
|
||||
mockEnv,
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
const newEtag = res.headers.get("ETag");
|
||||
expect(newEtag).not.toBe(oldEtag);
|
||||
});
|
||||
|
||||
it("RSS and Atom ETags for the same feed differ", async () => {
|
||||
const rssRes = await testApp.request(`/${FEED_ID}`, {}, mockEnv);
|
||||
const rssEtag = rssRes.headers.get("ETag")!;
|
||||
|
||||
// Use a separate atom app to get the atom ETag
|
||||
const { handle: atomHandle } = await import("./atom");
|
||||
const atomApp = new Hono();
|
||||
atomApp.get("/:feedId", atomHandle);
|
||||
const atomRes = await atomApp.request(`/${FEED_ID}`, {}, mockEnv);
|
||||
const atomEtag = atomRes.headers.get("ETag")!;
|
||||
|
||||
expect(rssEtag).not.toBe(atomEtag);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -5,6 +5,11 @@ import { fetchFeedData } from "../application/feed-fetcher";
|
||||
import { baseUrl, feedRssUrl } from "../infrastructure/urls";
|
||||
import { isExpired } from "../domain/feed";
|
||||
import { FeedId } from "../domain/value-objects/feed-id";
|
||||
import {
|
||||
computeFeedValidators,
|
||||
isNotModified,
|
||||
notModifiedResponse,
|
||||
} from "../infrastructure/http-cache";
|
||||
|
||||
export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
|
||||
try {
|
||||
@@ -21,6 +26,17 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
|
||||
return new Response("Feed has expired", { status: 410 });
|
||||
}
|
||||
|
||||
const validators = computeFeedValidators(
|
||||
"rss",
|
||||
feedId,
|
||||
feedData.feedConfig,
|
||||
feedData.emails,
|
||||
);
|
||||
|
||||
if (isNotModified(c.req.raw, validators)) {
|
||||
return notModifiedResponse(validators);
|
||||
}
|
||||
|
||||
const base = baseUrl(c.env);
|
||||
const selfUrl = new URL(c.req.url).origin + `/rss/${feedId}`;
|
||||
const rssXml = generateRssFeed(
|
||||
@@ -42,6 +58,8 @@ export async function handle(c: Context<{ Bindings: Env }>): Promise<Response> {
|
||||
"Cache-Control": "max-age=1800",
|
||||
"X-Robots-Tag": "noindex",
|
||||
Link: linkHeader,
|
||||
ETag: validators.etag,
|
||||
"Last-Modified": validators.lastModified,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
|
||||
@@ -68,6 +68,8 @@ export interface EmailMetadata {
|
||||
size?: number;
|
||||
attachmentIds?: string[]; // Downloadable attachments (shown to the user)
|
||||
inlineAttachmentIds?: string[]; // Inline images: hidden from lists, still cleaned up
|
||||
messageId?: string; // RFC 2822 Message-ID header (dedup primary key)
|
||||
dedupHash?: string; // SHA-256 hex of normalized subject+content (dedup fallback)
|
||||
}
|
||||
|
||||
// Feed list interface
|
||||
@@ -92,6 +94,7 @@ export interface Counters {
|
||||
// Subset of emails_rejected: non-feed mail forwarded to FALLBACK_FORWARD_ADDRESS
|
||||
// instead of dropped. Dropped count = emails_rejected − emails_forwarded.
|
||||
emails_forwarded: number;
|
||||
emails_deduplicated: number; // Duplicate deliveries silently skipped (not stored)
|
||||
unsubscribes_sent: number;
|
||||
last_email_at?: string; // ISO 8601
|
||||
last_feed_created_at?: string; // ISO 8601
|
||||
|
||||
Reference in New Issue
Block a user