mirror of
https://github.com/juherr/kill-the-news.git
synced 2026-06-20 22:03:48 +00:00
feat: reader-compat batch — JSON Feed, OPML export, conditional GET, dedup
Batch of four reader-facing improvements (TODO "Compat lecteurs + dedup"): - JSON Feed at /json/:feedId (feed lib .json1()); all formats cross-link - OPML export at /admin/opml (admin-protected; the registry lists every feed URL, so it must not be public) - Conditional GET on /rss + /atom: strong ETag + Last-Modified, 304 on If-None-Match/If-Modified-Since, validators shared via http-cache.ts - Duplicate-send dedup in ingestion: match by Message-ID, fall back to a SHA-256 of normalized subject+content; a duplicate is a no-op and bumps the new emails_deduplicated counter (status page + /api/v1/stats) 429 tests green, tsc clean, build dry-run OK. Docs (README/CLAUDE/TODO + landing cards) updated. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -15,6 +15,7 @@ describe("CountersRepository", () => {
|
||||
emails_received: 2,
|
||||
emails_rejected: 0,
|
||||
emails_forwarded: 0,
|
||||
emails_deduplicated: 0,
|
||||
unsubscribes_sent: 0,
|
||||
});
|
||||
expect(await repo.getRaw()).toMatchObject({ emails_received: 2 });
|
||||
|
||||
@@ -30,7 +30,7 @@ function buildFeed(
|
||||
emails: EmailData[],
|
||||
baseUrl: string,
|
||||
feedId: string,
|
||||
selfUrl?: { rss?: string; atom?: string },
|
||||
selfUrl?: { rss?: string; atom?: string; json?: string },
|
||||
): Feed {
|
||||
const iconUrl = `${baseUrl}/favicon/${feedId}`;
|
||||
const feed = new Feed({
|
||||
@@ -52,6 +52,7 @@ function buildFeed(
|
||||
feedLinks: {
|
||||
rss: selfUrl?.rss ?? `${baseUrl}/rss/${feedId}`,
|
||||
atom: selfUrl?.atom ?? `${baseUrl}/atom/${feedId}`,
|
||||
json: selfUrl?.json ?? `${baseUrl}/json/${feedId}`,
|
||||
},
|
||||
author: feedConfig.author
|
||||
? {
|
||||
@@ -127,3 +128,19 @@ export function generateAtomFeed(
|
||||
).atom1(),
|
||||
);
|
||||
}
|
||||
|
||||
export function generateJsonFeed(
|
||||
feedConfig: FeedConfig,
|
||||
emails: EmailData[],
|
||||
baseUrl: string,
|
||||
feedId: string,
|
||||
selfUrl?: string,
|
||||
): string {
|
||||
return buildFeed(
|
||||
feedConfig,
|
||||
emails,
|
||||
baseUrl,
|
||||
feedId,
|
||||
selfUrl ? { json: selfUrl } : undefined,
|
||||
).json1();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
import { FeedConfig, EmailData } from "../types";
|
||||
|
||||
export interface FeedValidators {
|
||||
etag: string;
|
||||
lastModified: string;
|
||||
maxReceivedAt: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute HTTP cache validators (ETag + Last-Modified) for a feed.
|
||||
* The ETag is derived from the feed format prefix, feedId, email count, and max
|
||||
* receivedAt, making it a strong deterministic validator that changes whenever
|
||||
* the feed content changes.
|
||||
*/
|
||||
export function computeFeedValidators(
|
||||
format: "rss" | "atom",
|
||||
feedId: string,
|
||||
feedConfig: FeedConfig,
|
||||
emails: EmailData[],
|
||||
): FeedValidators {
|
||||
const maxReceivedAt =
|
||||
emails.length > 0
|
||||
? Math.max(...emails.map((e) => e.receivedAt))
|
||||
: (feedConfig.created_at ?? 0);
|
||||
|
||||
const hash = `${format}-${feedId}-${emails.length}-${maxReceivedAt}`;
|
||||
const etag = `"${hash}"`;
|
||||
const lastModified = new Date(maxReceivedAt).toUTCString();
|
||||
|
||||
return { etag, lastModified, maxReceivedAt };
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if the request carries a matching conditional GET header,
|
||||
* meaning a 304 Not Modified response is appropriate.
|
||||
*/
|
||||
export function isNotModified(
|
||||
req: Request,
|
||||
validators: FeedValidators,
|
||||
): boolean {
|
||||
const ifNoneMatch = req.headers.get("If-None-Match");
|
||||
if (ifNoneMatch !== null) {
|
||||
return ifNoneMatch === validators.etag;
|
||||
}
|
||||
|
||||
const ifModifiedSince = req.headers.get("If-Modified-Since");
|
||||
if (ifModifiedSince !== null) {
|
||||
const clientTime = new Date(ifModifiedSince).getTime();
|
||||
return !isNaN(clientTime) && clientTime >= validators.maxReceivedAt;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a 304 Not Modified response with the standard cache validator headers.
|
||||
*/
|
||||
export function notModifiedResponse(validators: FeedValidators): Response {
|
||||
return new Response(null, {
|
||||
status: 304,
|
||||
headers: {
|
||||
ETag: validators.etag,
|
||||
"Last-Modified": validators.lastModified,
|
||||
"Cache-Control": "max-age=1800",
|
||||
"X-Robots-Tag": "noindex",
|
||||
},
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user