diff --git a/CLAUDE.md b/CLAUDE.md index 321a246..4f1cff4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,6 +42,7 @@ Single Cloudflare Worker built with Hono. Routes: | `GET /admin` | Password-protected admin UI | | `/hub` | WebSub hub (subscribe/publish) | | `GET /favicon.svg`, `/favicon.ico` | Project favicon (envelope logo); fallback for per-feed favicons | +| `GET /favicon/:feedId` | Per-feed favicon from the last sender's domain (falls back to project) | | `GET /health` | Health check | | `email` | Cloudflare Email routing handler (alternative to ForwardEmail webhook) | diff --git a/README.md b/README.md index 8e6437a..15f189a 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ kill-the-news keeps the same workflow while avoiding shared domains and shared d - Optional per-feed sender allowlist (`email@domain.com` or `domain.com`) - RSS generation on demand (`/rss/:feedId`) - Atom feed at `/atom/:feedId` +- Per-feed favicon derived from the last sender's domain (`/favicon/:feedId`), cached and shown in feeds + admin - Email attachments stored in Cloudflare R2 and exposed as RSS enclosures (optional) - Cloudflare KV storage for feed config + email metadata/content - Password-protected admin UI diff --git a/TODO.md b/TODO.md index c7737aa..7c422a9 100644 --- a/TODO.md +++ b/TODO.md @@ -20,7 +20,7 @@ Feature gaps identified by comparing with [kill-the-newsletter](https://github.c - [x] **Authelia / external auth provider support** — allow delegating admin authentication to an external identity provider (e.g. Authelia, Authentik) via a trusted header (`Remote-User`, `X-Forwarded-User`) set by a reverse proxy. The Worker would accept the header as proof of authentication instead of checking the cookie, with a configurable secret or IP allowlist to trust only the proxy. -- [ ] **Per-feed favicon from the last sender's domain** — give each feed an icon by fetching the favicon of the last sender's domain, so feeds are visually distinguishable in readers and the admin UI. Resolve the domain from the most recent email's `from`, fetch its favicon (e.g. `https:///favicon.ico` or a parsed ``, with a fallback service), and cache the result aggressively (KV/R2 + Cache API with a long TTL) so it isn't re-fetched on every request. Expose it via the RSS `` / Atom `` and the admin feed list. +- [x] **Per-feed favicon from the last sender's domain** — give each feed an icon by fetching the favicon of the last sender's domain, so feeds are visually distinguishable in readers and the admin UI. Resolve the domain from the most recent email's `from`, fetch its favicon (e.g. `https:///favicon.ico` or a parsed ``, with a fallback service), and cache the result aggressively (KV/R2 + Cache API with a long TTL) so it isn't re-fetched on every request. Expose it via the RSS `` / Atom `` and the admin feed list. - [ ] **RFC 8058 one-click unsubscribe on feed deletion** — when a feed is deleted, automatically unsubscribe from the newsletters that fed it so messages stop arriving at the now-dead address. Parse and store the `List-Unsubscribe` / `List-Unsubscribe-Post` headers ([RFC 8058](https://www.rfc-editor.org/rfc/rfc8058.txt)) from incoming emails, then on deletion POST `List-Unsubscribe=One-Click` to each stored unsubscribe URL. Requires capturing the headers during ingestion (`src/lib/email-processor.ts`) and firing the outbound requests from the feed-delete paths (`src/routes/admin/feeds.tsx`), ideally via `ctx.waitUntil`. @@ -38,14 +38,14 @@ Feature gaps identified by comparing with [kill-the-newsletter](https://github.c Breakdown of the _"Per-feed favicon from the last sender's domain"_ item above. Goal: each feed shows an icon derived from its newsletter source, fetched once and cached so it never re-fetches on a normal request. -- [ ] **Resolve the sender domain** — on ingestion, extract the domain from the latest email's `from` address (already parsed in `src/lib/email-processor.ts`) and persist it on the feed (e.g. `icon_domain` in `FeedConfig` / feed metadata) so the icon tracks the most recent sender. +- [x] **Resolve the sender domain** — on ingestion, extract the domain from the latest email's `from` address (`extractEmailDomain` in `src/utils/favicon-fetcher.ts`) and persist it as `iconDomain` on the feed metadata so the icon tracks the most recent sender. -- [ ] **Fetch the favicon** — resolve an icon URL for the domain: try `https:///favicon.ico`, optionally parse the homepage for ``, and fall back to a third-party service (e.g. `https://icons.duckduckgo.com/ip3/.ico`). Run lazily/async (`ctx.waitUntil`) so it never blocks email processing. +- [x] **Fetch the favicon** — resolve an icon URL for the domain: try `https:///favicon.ico`, then fall back to `https://icons.duckduckgo.com/ip3/.ico`. Runs async via `ctx.waitUntil` so it never blocks email processing. -- [ ] **Cache aggressively** — store the fetched bytes keyed by domain (R2 for the image bytes, or KV for small icons) with a long TTL (e.g. refresh ~weekly), and serve them through the Cloudflare Cache API. The domain is the cache key so feeds from the same sender share one fetch. +- [x] **Cache aggressively** — store the fetched bytes (base64) keyed by domain in KV with a 1-week TTL (`ICON_TTL_SECONDS`). The domain is the cache key so feeds from the same sender share one fetch; the fetch only fires when the cache entry is absent/expired. -- [ ] **Serve endpoint** — add a route like `GET /icons/:domain` (or `GET /favicon/:feedId`) returning the cached bytes with the correct `Content-Type` and a long `Cache-Control`, falling back to the project favicon (see _Project favicon_ in Quick wins) when no domain icon is found. +- [x] **Serve endpoint** — `GET /favicon/:feedId` returns the cached bytes with the correct `Content-Type` and a long `Cache-Control`, falling back to the project favicon when no domain icon is found. -- [ ] **Expose in outputs** — reference the icon from the RSS `` and Atom ``/`` in `src/utils/feed-generator.ts`, and render it next to each feed in the admin list (`src/routes/admin/feeds.tsx`). +- [x] **Expose in outputs** — the icon is referenced from the RSS `` and Atom ``/`` in `src/utils/feed-generator.ts`, and rendered next to each feed in the admin list/table (`src/routes/admin.tsx`). -- [ ] **Failure handling** — missing/blocked favicons must degrade gracefully to the project favicon fallback; never let an icon fetch error surface to ingestion or feed rendering. +- [x] **Failure handling** — missing/blocked favicons degrade gracefully to the project favicon fallback (negative cache entry); icon fetch errors never surface to ingestion or feed rendering. diff --git a/docs/index.html b/docs/index.html index 97fe0e9..19dca5d 100644 --- a/docs/index.html +++ b/docs/index.html @@ -808,6 +808,14 @@

Email attachments are stored in Cloudflare R2 and exposed as RSS enclosures — no extra hosting needed.

+
+
+ +
+

Per-Feed Icons

+

Each feed picks up the favicon of its newsletter's sender domain, so feeds are easy to tell apart in your reader and the admin UI.

+
+
diff --git a/src/config/constants.ts b/src/config/constants.ts index af04fd4..6d9906e 100644 --- a/src/config/constants.ts +++ b/src/config/constants.ts @@ -24,3 +24,12 @@ export const FEEDS_LIST_KEY = "feeds:list"; /** KV key for the monitoring counters singleton. */ export const STATS_KEY = "stats:counters"; + +/** Default TTL for a cached per-domain favicon (seconds). */ +export const ICON_TTL_SECONDS = 7 * 24 * 60 * 60; // 1 week + +/** Maximum accepted favicon size (bytes); larger responses are rejected. */ +export const MAX_ICON_BYTES = 100 * 1024; // 100 KB + +/** Timeout for an outbound favicon fetch (milliseconds). */ +export const ICON_FETCH_TIMEOUT_MS = 5000; diff --git a/src/index.ts b/src/index.ts index 265e31f..6fb245c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,7 +8,7 @@ import { handle as handleEntry } from "./routes/entries"; import { handle as handleFiles } from "./routes/files"; import { handle as handleStats } from "./routes/stats"; import { handle as handleHome } from "./routes/home"; -import { handle as handleFavicon } from "./routes/favicon"; +import { handle as handleFavicon, handleFeedFavicon } from "./routes/favicon"; import { hubRouter } from "./routes/hub"; import { handleCloudflareEmail } from "./lib/cloudflare-email"; import { Env } from "./types"; @@ -169,10 +169,13 @@ app.route("/files", files); app.route("/admin", admin); app.route("/hub", hubRouter); -// Project favicon (also the fallback for the future per-feed favicon) +// Project favicon (also the fallback for the per-feed favicon) app.get("/favicon.svg", handleFavicon); app.get("/favicon.ico", handleFavicon); // readers/browsers that hardcode .ico +// Per-feed favicon derived from the last sender's domain +app.get("/favicon/:feedId", handleFeedFavicon); + // Health check endpoint for monitoring app.get("/health", (c) => c.json({ status: "ok", timestamp: Date.now() })); diff --git a/src/lib/email-processor.test.ts b/src/lib/email-processor.test.ts index a11c1fd..8c7f2d9 100644 --- a/src/lib/email-processor.test.ts +++ b/src/lib/email-processor.test.ts @@ -1,12 +1,13 @@ import { describe, it, expect, beforeEach } from "vitest"; -import "../test/setup"; -import { createMockEnv, MockR2 } from "../test/setup"; +import { http, HttpResponse } from "msw"; +import { createMockEnv, MockR2, server } from "../test/setup"; import { processEmail, ProcessEmailInput, RawAttachment, } from "./email-processor"; import { getCounters } from "../utils/stats"; +import { iconKey } from "../utils/storage"; const VALID_FEED_ID = "apple.mountain.42"; const VALID_TO = `${VALID_FEED_ID}@test.getmynews.app`; @@ -496,3 +497,54 @@ describe("processEmail — monitoring counters", () => { expect(counters.emails_received).toBe(0); }); }); + +describe("processEmail — feed icon", () => { + let env: ReturnType; + + beforeEach(async () => { + env = createMockEnv(); + await env.EMAIL_STORAGE.put( + `feed:${VALID_FEED_ID}:config`, + JSON.stringify({}), + ); + }); + + it("persists the latest sender domain on the feed metadata", async () => { + await processEmail( + makeInput({ from: "News " }), + env as any, + ); + + const metadata = (await env.EMAIL_STORAGE.get( + `feed:${VALID_FEED_ID}:metadata`, + "json", + )) as { iconDomain?: string }; + expect(metadata.iconDomain).toBe("github.com"); + }); + + it("triggers a background favicon fetch via ctx.waitUntil", async () => { + let fetched = false; + server.use( + http.get("https://github.com/favicon.ico", () => { + fetched = true; + return new HttpResponse(new Uint8Array([1, 2, 3]), { + headers: { "Content-Type": "image/png" }, + }); + }), + ); + + const pending: Promise[] = []; + const ctx = { + waitUntil: (p: Promise) => pending.push(p), + passThroughOnException: () => {}, + } as unknown as ExecutionContext; + + await processEmail(makeInput({ from: "news@github.com" }), env as any, ctx); + await Promise.all(pending); + + expect(fetched).toBe(true); + expect( + await env.EMAIL_STORAGE.get(iconKey("github.com"), "json"), + ).toMatchObject({ contentType: "image/png" }); + }); +}); diff --git a/src/lib/email-processor.ts b/src/lib/email-processor.ts index e804734..9b69899 100644 --- a/src/lib/email-processor.ts +++ b/src/lib/email-processor.ts @@ -8,6 +8,10 @@ import { } from "../types"; import { notifySubscribers } from "../utils/websub"; import { bumpCounters } from "../utils/stats"; +import { + cacheFaviconForDomain, + extractEmailDomain, +} from "../utils/favicon-fetcher"; import { logger } from "./logger"; import { FEED_MAX_BYTES } from "../config/constants"; @@ -213,6 +217,12 @@ export async function storeEmail( }; feedMetadata.emails.unshift(newEntry); + // Track the latest sender's domain so the feed icon follows the source. + const iconDomain = extractEmailDomain(input.from); + if (iconDomain) { + feedMetadata.iconDomain = iconDomain; + } + let totalSize = feedMetadata.emails.reduce( (sum, e) => sum + (e.size ?? 0), 0, @@ -240,6 +250,9 @@ export async function storeEmail( logger.info("Email processed", { feedId }); if (ctx) { ctx.waitUntil(notifySubscribers(feedId, env)); + if (iconDomain) { + ctx.waitUntil(cacheFaviconForDomain(iconDomain, env)); + } } } diff --git a/src/routes/admin.tsx b/src/routes/admin.tsx index abcdd9c..2af2186 100644 --- a/src/routes/admin.tsx +++ b/src/routes/admin.tsx @@ -145,7 +145,12 @@ app.get("/login", (c) => { fill="none" xmlns="http://www.w3.org/2000/svg" > - + {

kill-the-news

- {errorMessage && ( -
{errorMessage}
- )} + {errorMessage &&
{errorMessage}
}
@@ -641,7 +644,11 @@ app.get("/", async (c) => { title="Resize" >
- +