mirror of
https://github.com/juherr/kill-the-news.git
synced 2026-06-21 06:13:48 +00:00
feat(favicon): per-feed icon from the last sender's domain
Resolve each feed's most recent sender domain and serve its favicon at GET /favicon/:feedId, falling back to the project icon. Icons are fetched in the background on ingestion (direct /favicon.ico then a DuckDuckGo fallback), cached base64 in KV keyed by domain with a 1-week TTL so the fetch only fires when absent. Exposed via RSS <image> / Atom <icon>/<logo> and rendered in the admin feed list, plus a landing-page feature card. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -24,3 +24,12 @@ export const FEEDS_LIST_KEY = "feeds:list";
|
||||
|
||||
/** KV key for the monitoring counters singleton. */
|
||||
export const STATS_KEY = "stats:counters";
|
||||
|
||||
/** Default TTL for a cached per-domain favicon (seconds). */
|
||||
export const ICON_TTL_SECONDS = 7 * 24 * 60 * 60; // 1 week
|
||||
|
||||
/** Maximum accepted favicon size (bytes); larger responses are rejected. */
|
||||
export const MAX_ICON_BYTES = 100 * 1024; // 100 KB
|
||||
|
||||
/** Timeout for an outbound favicon fetch (milliseconds). */
|
||||
export const ICON_FETCH_TIMEOUT_MS = 5000;
|
||||
|
||||
+5
-2
@@ -8,7 +8,7 @@ import { handle as handleEntry } from "./routes/entries";
|
||||
import { handle as handleFiles } from "./routes/files";
|
||||
import { handle as handleStats } from "./routes/stats";
|
||||
import { handle as handleHome } from "./routes/home";
|
||||
import { handle as handleFavicon } from "./routes/favicon";
|
||||
import { handle as handleFavicon, handleFeedFavicon } from "./routes/favicon";
|
||||
import { hubRouter } from "./routes/hub";
|
||||
import { handleCloudflareEmail } from "./lib/cloudflare-email";
|
||||
import { Env } from "./types";
|
||||
@@ -169,10 +169,13 @@ app.route("/files", files);
|
||||
app.route("/admin", admin);
|
||||
app.route("/hub", hubRouter);
|
||||
|
||||
// Project favicon (also the fallback for the future per-feed favicon)
|
||||
// Project favicon (also the fallback for the per-feed favicon)
|
||||
app.get("/favicon.svg", handleFavicon);
|
||||
app.get("/favicon.ico", handleFavicon); // readers/browsers that hardcode .ico
|
||||
|
||||
// Per-feed favicon derived from the last sender's domain
|
||||
app.get("/favicon/:feedId", handleFeedFavicon);
|
||||
|
||||
// Health check endpoint for monitoring
|
||||
app.get("/health", (c) => c.json({ status: "ok", timestamp: Date.now() }));
|
||||
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
import { describe, it, expect, beforeEach } from "vitest";
|
||||
import "../test/setup";
|
||||
import { createMockEnv, MockR2 } from "../test/setup";
|
||||
import { http, HttpResponse } from "msw";
|
||||
import { createMockEnv, MockR2, server } from "../test/setup";
|
||||
import {
|
||||
processEmail,
|
||||
ProcessEmailInput,
|
||||
RawAttachment,
|
||||
} from "./email-processor";
|
||||
import { getCounters } from "../utils/stats";
|
||||
import { iconKey } from "../utils/storage";
|
||||
|
||||
const VALID_FEED_ID = "apple.mountain.42";
|
||||
const VALID_TO = `${VALID_FEED_ID}@test.getmynews.app`;
|
||||
@@ -496,3 +497,54 @@ describe("processEmail — monitoring counters", () => {
|
||||
expect(counters.emails_received).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("processEmail — feed icon", () => {
|
||||
let env: ReturnType<typeof createMockEnv>;
|
||||
|
||||
beforeEach(async () => {
|
||||
env = createMockEnv();
|
||||
await env.EMAIL_STORAGE.put(
|
||||
`feed:${VALID_FEED_ID}:config`,
|
||||
JSON.stringify({}),
|
||||
);
|
||||
});
|
||||
|
||||
it("persists the latest sender domain on the feed metadata", async () => {
|
||||
await processEmail(
|
||||
makeInput({ from: "News <news@github.com>" }),
|
||||
env as any,
|
||||
);
|
||||
|
||||
const metadata = (await env.EMAIL_STORAGE.get(
|
||||
`feed:${VALID_FEED_ID}:metadata`,
|
||||
"json",
|
||||
)) as { iconDomain?: string };
|
||||
expect(metadata.iconDomain).toBe("github.com");
|
||||
});
|
||||
|
||||
it("triggers a background favicon fetch via ctx.waitUntil", async () => {
|
||||
let fetched = false;
|
||||
server.use(
|
||||
http.get("https://github.com/favicon.ico", () => {
|
||||
fetched = true;
|
||||
return new HttpResponse(new Uint8Array([1, 2, 3]), {
|
||||
headers: { "Content-Type": "image/png" },
|
||||
});
|
||||
}),
|
||||
);
|
||||
|
||||
const pending: Promise<unknown>[] = [];
|
||||
const ctx = {
|
||||
waitUntil: (p: Promise<unknown>) => pending.push(p),
|
||||
passThroughOnException: () => {},
|
||||
} as unknown as ExecutionContext;
|
||||
|
||||
await processEmail(makeInput({ from: "news@github.com" }), env as any, ctx);
|
||||
await Promise.all(pending);
|
||||
|
||||
expect(fetched).toBe(true);
|
||||
expect(
|
||||
await env.EMAIL_STORAGE.get(iconKey("github.com"), "json"),
|
||||
).toMatchObject({ contentType: "image/png" });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -8,6 +8,10 @@ import {
|
||||
} from "../types";
|
||||
import { notifySubscribers } from "../utils/websub";
|
||||
import { bumpCounters } from "../utils/stats";
|
||||
import {
|
||||
cacheFaviconForDomain,
|
||||
extractEmailDomain,
|
||||
} from "../utils/favicon-fetcher";
|
||||
import { logger } from "./logger";
|
||||
import { FEED_MAX_BYTES } from "../config/constants";
|
||||
|
||||
@@ -213,6 +217,12 @@ export async function storeEmail(
|
||||
};
|
||||
feedMetadata.emails.unshift(newEntry);
|
||||
|
||||
// Track the latest sender's domain so the feed icon follows the source.
|
||||
const iconDomain = extractEmailDomain(input.from);
|
||||
if (iconDomain) {
|
||||
feedMetadata.iconDomain = iconDomain;
|
||||
}
|
||||
|
||||
let totalSize = feedMetadata.emails.reduce(
|
||||
(sum, e) => sum + (e.size ?? 0),
|
||||
0,
|
||||
@@ -240,6 +250,9 @@ export async function storeEmail(
|
||||
logger.info("Email processed", { feedId });
|
||||
if (ctx) {
|
||||
ctx.waitUntil(notifySubscribers(feedId, env));
|
||||
if (iconDomain) {
|
||||
ctx.waitUntil(cacheFaviconForDomain(iconDomain, env));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+47
-17
@@ -145,7 +145,12 @@ app.get("/login", (c) => {
|
||||
fill="none"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
>
|
||||
<rect width="24" height="24" rx="12" fill="var(--color-primary)" />
|
||||
<rect
|
||||
width="24"
|
||||
height="24"
|
||||
rx="12"
|
||||
fill="var(--color-primary)"
|
||||
/>
|
||||
<path
|
||||
d="M17 9C17 7.89543 16.1046 7 15 7H9C7.89543 7 7 7.89543 7 9V15C7 16.1046 7.89543 17 9 17H15C16.1046 17 17 16.1046 17 15V9Z"
|
||||
stroke="white"
|
||||
@@ -161,9 +166,7 @@ app.get("/login", (c) => {
|
||||
</svg>
|
||||
</div>
|
||||
<h1 class="auth-title">kill-the-news</h1>
|
||||
{errorMessage && (
|
||||
<div class="auth-error">{errorMessage}</div>
|
||||
)}
|
||||
{errorMessage && <div class="auth-error">{errorMessage}</div>}
|
||||
<form class="auth-form" action="/admin/login" method="post">
|
||||
<div class="form-group">
|
||||
<label for="password">Password</label>
|
||||
@@ -641,7 +644,11 @@ app.get("/", async (c) => {
|
||||
title="Resize"
|
||||
></div>
|
||||
</th>
|
||||
<th class="th-resizable" data-sort-key="atom" aria-sort="none">
|
||||
<th
|
||||
class="th-resizable"
|
||||
data-sort-key="atom"
|
||||
aria-sort="none"
|
||||
>
|
||||
<button
|
||||
type="button"
|
||||
class="th-button"
|
||||
@@ -689,7 +696,10 @@ app.get("/", async (c) => {
|
||||
const sortEmail = emailAddress.toLowerCase();
|
||||
const sortRss = rssUrl.toLowerCase();
|
||||
const sortAtom = atomUrl.toLowerCase();
|
||||
const descDisplay = clampText(feed.description || "", 220);
|
||||
const descDisplay = clampText(
|
||||
feed.description || "",
|
||||
220,
|
||||
);
|
||||
const descHover = clampText(feed.description || "", 1000);
|
||||
const searchHaystack =
|
||||
`${clampText(feed.title, 320)} ${feed.id} ${clampText(feed.description || "", 320)}`.toLowerCase();
|
||||
@@ -718,18 +728,30 @@ app.get("/", async (c) => {
|
||||
/>
|
||||
</td>
|
||||
<td>
|
||||
<strong class="truncate" title={titleHover}>
|
||||
{titleDisplay}
|
||||
</strong>
|
||||
{feed.description && (
|
||||
<div
|
||||
class="muted truncate"
|
||||
style="font-size: var(--font-size-sm); margin-top: 4px;"
|
||||
title={descHover}
|
||||
>
|
||||
{descDisplay}
|
||||
<div class="feed-title-cell">
|
||||
<img
|
||||
class="feed-icon"
|
||||
src={`/favicon/${feed.id}`}
|
||||
alt=""
|
||||
width="20"
|
||||
height="20"
|
||||
loading="lazy"
|
||||
/>
|
||||
<div>
|
||||
<strong class="truncate" title={titleHover}>
|
||||
{titleDisplay}
|
||||
</strong>
|
||||
{feed.description && (
|
||||
<div
|
||||
class="muted truncate"
|
||||
style="font-size: var(--font-size-sm); margin-top: 4px;"
|
||||
title={descHover}
|
||||
>
|
||||
{descDisplay}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</td>
|
||||
<td>
|
||||
<code>{feed.id}</code>
|
||||
@@ -842,6 +864,14 @@ app.get("/", async (c) => {
|
||||
>
|
||||
<div class="feed-header">
|
||||
<h3 class="feed-title" title={titleHover}>
|
||||
<img
|
||||
class="feed-icon"
|
||||
src={`/favicon/${feed.id}`}
|
||||
alt=""
|
||||
width="20"
|
||||
height="20"
|
||||
loading="lazy"
|
||||
/>
|
||||
{titleDisplay}
|
||||
</h3>
|
||||
{feed.expires_at && (
|
||||
|
||||
@@ -1,12 +1,19 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import worker from "../index";
|
||||
import { createMockEnv } from "../test/setup";
|
||||
import { iconKey } from "../utils/storage";
|
||||
import type { Env } from "../types";
|
||||
|
||||
function req(path: string): Request {
|
||||
return new Request(`https://test.getmynews.app${path}`);
|
||||
}
|
||||
|
||||
const PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 9, 8, 7]);
|
||||
|
||||
function toBase64(bytes: Uint8Array): string {
|
||||
return btoa(String.fromCharCode(...bytes));
|
||||
}
|
||||
|
||||
describe("project favicon", () => {
|
||||
it("serves an SVG favicon at /favicon.svg", async () => {
|
||||
const env = createMockEnv() as unknown as Env;
|
||||
@@ -27,3 +34,58 @@ describe("project favicon", () => {
|
||||
expect(body).toContain("<svg");
|
||||
});
|
||||
});
|
||||
|
||||
describe("per-feed favicon", () => {
|
||||
it("serves the cached domain icon when available", async () => {
|
||||
const env = createMockEnv() as unknown as Env;
|
||||
await env.EMAIL_STORAGE.put(
|
||||
"feed:abc:metadata",
|
||||
JSON.stringify({ emails: [], iconDomain: "github.com" }),
|
||||
);
|
||||
await env.EMAIL_STORAGE.put(
|
||||
iconKey("github.com"),
|
||||
JSON.stringify({ data: toBase64(PNG), contentType: "image/png" }),
|
||||
);
|
||||
|
||||
const res = await worker.fetch(req("/favicon/abc"), env);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers.get("Content-Type")).toBe("image/png");
|
||||
expect(new Uint8Array(await res.arrayBuffer())).toEqual(PNG);
|
||||
});
|
||||
|
||||
it("falls back to the project SVG when the feed has no icon domain", async () => {
|
||||
const env = createMockEnv() as unknown as Env;
|
||||
await env.EMAIL_STORAGE.put(
|
||||
"feed:abc:metadata",
|
||||
JSON.stringify({ emails: [] }),
|
||||
);
|
||||
|
||||
const res = await worker.fetch(req("/favicon/abc"), env);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers.get("Content-Type")).toMatch(/^image\/svg\+xml/);
|
||||
expect(await res.text()).toContain("<svg");
|
||||
});
|
||||
|
||||
it("falls back to the project SVG for a negative cache entry", async () => {
|
||||
const env = createMockEnv() as unknown as Env;
|
||||
await env.EMAIL_STORAGE.put(
|
||||
"feed:abc:metadata",
|
||||
JSON.stringify({ emails: [], iconDomain: "nope.test" }),
|
||||
);
|
||||
await env.EMAIL_STORAGE.put(
|
||||
iconKey("nope.test"),
|
||||
JSON.stringify({ data: null, contentType: "" }),
|
||||
);
|
||||
|
||||
const res = await worker.fetch(req("/favicon/abc"), env);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers.get("Content-Type")).toMatch(/^image\/svg\+xml/);
|
||||
});
|
||||
|
||||
it("falls back to the project SVG for an unknown feed", async () => {
|
||||
const env = createMockEnv() as unknown as Env;
|
||||
const res = await worker.fetch(req("/favicon/missing"), env);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers.get("Content-Type")).toMatch(/^image\/svg\+xml/);
|
||||
});
|
||||
});
|
||||
|
||||
+43
-1
@@ -1,5 +1,7 @@
|
||||
import { Context } from "hono";
|
||||
import { Env } from "../types";
|
||||
import { getFeedMetadata } from "../utils/storage";
|
||||
import { cacheFaviconForDomain, getCachedIcon } from "../utils/favicon-fetcher";
|
||||
|
||||
export const FAVICON_PATH = "/favicon.svg";
|
||||
|
||||
@@ -13,7 +15,7 @@ export const FAVICON_SVG = `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0
|
||||
</g>
|
||||
</svg>`;
|
||||
|
||||
export function handle(_c: Context<{ Bindings: Env }>): Response {
|
||||
function projectFavicon(): Response {
|
||||
return new Response(FAVICON_SVG, {
|
||||
headers: {
|
||||
"Content-Type": "image/svg+xml; charset=utf-8",
|
||||
@@ -21,3 +23,43 @@ export function handle(_c: Context<{ Bindings: Env }>): Response {
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function handle(_c: Context<{ Bindings: Env }>): Response {
|
||||
return projectFavicon();
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-feed favicon. Resolves the feed's most recent sender domain and serves
|
||||
* its cached icon; falls back to the project icon for any unresolved case
|
||||
* (no domain, cache miss, or negative cache entry).
|
||||
*/
|
||||
export async function handleFeedFavicon(
|
||||
c: Context<{ Bindings: Env }>,
|
||||
): Promise<Response> {
|
||||
const env = c.env;
|
||||
const feedId = c.req.param("feedId");
|
||||
if (!feedId) return projectFavicon();
|
||||
|
||||
const metadata = await getFeedMetadata(env.EMAIL_STORAGE, feedId);
|
||||
const domain = metadata?.iconDomain;
|
||||
if (!domain) return projectFavicon();
|
||||
|
||||
const icon = await getCachedIcon(domain, env);
|
||||
if (icon) {
|
||||
return new Response(icon.bytes, {
|
||||
headers: {
|
||||
"Content-Type": icon.contentType,
|
||||
"Cache-Control": "public, max-age=86400",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Known domain but nothing cached yet: warm the cache in the background and
|
||||
// serve the fallback for now.
|
||||
try {
|
||||
c.executionCtx.waitUntil(cacheFaviconForDomain(domain, env));
|
||||
} catch {
|
||||
// No ExecutionContext (e.g. tests) — fallback is served regardless.
|
||||
}
|
||||
return projectFavicon();
|
||||
}
|
||||
|
||||
@@ -24,6 +24,26 @@
|
||||
color: var(--color-text-primary);
|
||||
}
|
||||
|
||||
/* Per-feed favicon (list + table views) */
|
||||
.feed-icon {
|
||||
width: 20px;
|
||||
height: 20px;
|
||||
border-radius: 4px;
|
||||
object-fit: contain;
|
||||
vertical-align: middle;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.feed-title .feed-icon {
|
||||
margin-right: var(--spacing-xs);
|
||||
}
|
||||
|
||||
.feed-title-cell {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
gap: var(--spacing-sm);
|
||||
}
|
||||
|
||||
.feed-description {
|
||||
font-size: var(--font-size-md);
|
||||
color: var(--color-text-secondary);
|
||||
|
||||
+6
-1
@@ -29,7 +29,12 @@ class MockKV {
|
||||
return type === "json" ? JSON.parse(value) : value;
|
||||
}
|
||||
|
||||
async put(key: string, value: any) {
|
||||
async put(
|
||||
key: string,
|
||||
value: any,
|
||||
_options?: { expirationTtl?: number; expiration?: number },
|
||||
) {
|
||||
// TTL options are accepted for API parity but not simulated in tests.
|
||||
this.store.set(
|
||||
key,
|
||||
typeof value === "string" ? value : JSON.stringify(value),
|
||||
|
||||
@@ -45,6 +45,7 @@ export interface FeedConfig {
|
||||
// Feed metadata interface
|
||||
export interface FeedMetadata {
|
||||
emails: EmailMetadata[];
|
||||
iconDomain?: string; // Most recent sender's domain, used to resolve the feed icon
|
||||
}
|
||||
|
||||
// Email metadata interface (summary info for listing)
|
||||
@@ -111,6 +112,7 @@ declare global {
|
||||
put(
|
||||
key: string,
|
||||
value: string | ArrayBuffer | ReadableStream | FormData,
|
||||
options?: { expirationTtl?: number; expiration?: number },
|
||||
): Promise<void>;
|
||||
delete(key: string): Promise<void>;
|
||||
list(options?: {
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { http, HttpResponse } from "msw";
|
||||
import { server, createMockEnv } from "../test/setup";
|
||||
import {
|
||||
cacheFaviconForDomain,
|
||||
extractEmailDomain,
|
||||
getCachedIcon,
|
||||
} from "./favicon-fetcher";
|
||||
import { iconKey } from "./storage";
|
||||
import { MAX_ICON_BYTES } from "../config/constants";
|
||||
import type { Env } from "../types";
|
||||
|
||||
const PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 1, 2, 3, 4]);
|
||||
|
||||
function imageResponse(bytes: Uint8Array, contentType = "image/png") {
|
||||
return new HttpResponse(bytes, { headers: { "Content-Type": contentType } });
|
||||
}
|
||||
|
||||
describe("extractEmailDomain", () => {
|
||||
it("parses a bare address", () => {
|
||||
expect(extractEmailDomain("news@github.com")).toBe("github.com");
|
||||
});
|
||||
|
||||
it("parses a display-form address", () => {
|
||||
expect(extractEmailDomain("GitHub <news@GitHub.com>")).toBe("github.com");
|
||||
});
|
||||
|
||||
it("strips a trailing dot and lowercases", () => {
|
||||
expect(extractEmailDomain("a@Example.COM.")).toBe("example.com");
|
||||
});
|
||||
|
||||
it("returns null when there is no address", () => {
|
||||
expect(extractEmailDomain("not an email")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("cacheFaviconForDomain", () => {
|
||||
it("caches the direct /favicon.ico when available", async () => {
|
||||
const env = createMockEnv() as unknown as Env;
|
||||
server.use(
|
||||
http.get("https://github.com/favicon.ico", () => imageResponse(PNG)),
|
||||
);
|
||||
|
||||
await cacheFaviconForDomain("github.com", env);
|
||||
|
||||
const record = await env.EMAIL_STORAGE.get(iconKey("github.com"), "json");
|
||||
expect(record).toMatchObject({ contentType: "image/png" });
|
||||
expect((record as { data: string }).data).toBeTruthy();
|
||||
expect(record).not.toHaveProperty("fetchedAt");
|
||||
|
||||
const icon = await getCachedIcon("github.com", env);
|
||||
expect(icon?.contentType).toBe("image/png");
|
||||
expect(new Uint8Array(icon!.bytes)).toEqual(PNG);
|
||||
});
|
||||
|
||||
it("falls back to DuckDuckGo when the direct icon 404s", async () => {
|
||||
const env = createMockEnv() as unknown as Env;
|
||||
server.use(
|
||||
http.get("https://acme.test/favicon.ico", () =>
|
||||
HttpResponse.text("nope", { status: 404 }),
|
||||
),
|
||||
http.get("https://icons.duckduckgo.com/ip3/acme.test.ico", () =>
|
||||
imageResponse(PNG, "image/x-icon"),
|
||||
),
|
||||
);
|
||||
|
||||
await cacheFaviconForDomain("acme.test", env);
|
||||
|
||||
const icon = await getCachedIcon("acme.test", env);
|
||||
expect(icon?.contentType).toBe("image/x-icon");
|
||||
});
|
||||
|
||||
it("writes a negative entry when no icon is found", async () => {
|
||||
const env = createMockEnv() as unknown as Env;
|
||||
server.use(
|
||||
http.get("https://nope.test/favicon.ico", () =>
|
||||
HttpResponse.text("", { status: 404 }),
|
||||
),
|
||||
http.get("https://icons.duckduckgo.com/ip3/nope.test.ico", () =>
|
||||
HttpResponse.text("", { status: 404 }),
|
||||
),
|
||||
);
|
||||
|
||||
await cacheFaviconForDomain("nope.test", env);
|
||||
|
||||
const record = await env.EMAIL_STORAGE.get(iconKey("nope.test"), "json");
|
||||
expect(record).toEqual({ data: null, contentType: "" });
|
||||
expect(await getCachedIcon("nope.test", env)).toBeNull();
|
||||
});
|
||||
|
||||
it("rejects oversized responses as negative", async () => {
|
||||
const env = createMockEnv() as unknown as Env;
|
||||
const big = new Uint8Array(MAX_ICON_BYTES + 1);
|
||||
server.use(
|
||||
http.get("https://big.test/favicon.ico", () => imageResponse(big)),
|
||||
http.get("https://icons.duckduckgo.com/ip3/big.test.ico", () =>
|
||||
HttpResponse.text("", { status: 404 }),
|
||||
),
|
||||
);
|
||||
|
||||
await cacheFaviconForDomain("big.test", env);
|
||||
expect(await getCachedIcon("big.test", env)).toBeNull();
|
||||
});
|
||||
|
||||
it("rejects non-image content types as negative", async () => {
|
||||
const env = createMockEnv() as unknown as Env;
|
||||
server.use(
|
||||
http.get("https://html.test/favicon.ico", () =>
|
||||
HttpResponse.text("<html>", {
|
||||
headers: { "Content-Type": "text/html" },
|
||||
}),
|
||||
),
|
||||
http.get("https://icons.duckduckgo.com/ip3/html.test.ico", () =>
|
||||
HttpResponse.text("", { status: 404 }),
|
||||
),
|
||||
);
|
||||
|
||||
await cacheFaviconForDomain("html.test", env);
|
||||
expect(await getCachedIcon("html.test", env)).toBeNull();
|
||||
});
|
||||
|
||||
it("short-circuits when an entry already exists (no outbound fetch)", async () => {
|
||||
const env = createMockEnv() as unknown as Env;
|
||||
// Pre-seed a record; with MSW onUnhandledRequest:"error", any fetch fails.
|
||||
await env.EMAIL_STORAGE.put(
|
||||
iconKey("cached.test"),
|
||||
JSON.stringify({ data: null, contentType: "" }),
|
||||
);
|
||||
|
||||
await expect(
|
||||
cacheFaviconForDomain("cached.test", env),
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("never throws on network errors", async () => {
|
||||
const env = createMockEnv() as unknown as Env;
|
||||
server.use(
|
||||
http.get("https://err.test/favicon.ico", () => HttpResponse.error()),
|
||||
http.get("https://icons.duckduckgo.com/ip3/err.test.ico", () =>
|
||||
HttpResponse.error(),
|
||||
),
|
||||
);
|
||||
|
||||
await expect(
|
||||
cacheFaviconForDomain("err.test", env),
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,131 @@
|
||||
import { Env } from "../types";
|
||||
import {
|
||||
ICON_FETCH_TIMEOUT_MS,
|
||||
ICON_TTL_SECONDS,
|
||||
MAX_ICON_BYTES,
|
||||
} from "../config/constants";
|
||||
import { iconKey } from "./storage";
|
||||
import { logger } from "../lib/logger";
|
||||
|
||||
interface IconRecord {
|
||||
data: string | null; // base64 icon bytes, or null for a negative cache entry
|
||||
contentType: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the lowercased domain from a `from` value, accepting either a bare
|
||||
* address (`a@b.com`) or a display form (`Name <a@b.com>`). Returns null when
|
||||
* no plausible address can be parsed.
|
||||
*/
|
||||
export function extractEmailDomain(from: string): string | null {
|
||||
const match = from.match(/[^\s<>@]+@([^\s<>@]+\.[^\s<>@]+)/);
|
||||
if (!match) return null;
|
||||
const domain = match[1].trim().toLowerCase().replace(/\.+$/, "");
|
||||
return domain || null;
|
||||
}
|
||||
|
||||
function arrayBufferToBase64(buffer: ArrayBuffer): string {
|
||||
const bytes = new Uint8Array(buffer);
|
||||
let binary = "";
|
||||
const chunkSize = 0x8000;
|
||||
for (let i = 0; i < bytes.length; i += chunkSize) {
|
||||
binary += String.fromCharCode(...bytes.subarray(i, i + chunkSize));
|
||||
}
|
||||
return btoa(binary);
|
||||
}
|
||||
|
||||
function base64ToArrayBuffer(base64: string): ArrayBuffer {
|
||||
const binary = atob(base64);
|
||||
const bytes = new Uint8Array(binary.length);
|
||||
for (let i = 0; i < binary.length; i++) {
|
||||
bytes[i] = binary.charCodeAt(i);
|
||||
}
|
||||
return bytes.buffer;
|
||||
}
|
||||
|
||||
async function fetchIconFrom(
|
||||
url: string,
|
||||
): Promise<{ buffer: ArrayBuffer; contentType: string } | null> {
|
||||
const res = await fetch(url, {
|
||||
redirect: "follow",
|
||||
signal: AbortSignal.timeout(ICON_FETCH_TIMEOUT_MS),
|
||||
headers: { "User-Agent": "kill-the-news/1.0" },
|
||||
});
|
||||
if (!res.ok) return null;
|
||||
|
||||
const contentType = res.headers.get("content-type") ?? "";
|
||||
if (!contentType.startsWith("image/")) return null;
|
||||
|
||||
const buffer = await res.arrayBuffer();
|
||||
if (buffer.byteLength === 0 || buffer.byteLength > MAX_ICON_BYTES)
|
||||
return null;
|
||||
|
||||
return { buffer, contentType: contentType.split(";")[0].trim() };
|
||||
}
|
||||
|
||||
async function resolveIcon(
|
||||
domain: string,
|
||||
): Promise<{ buffer: ArrayBuffer; contentType: string } | null> {
|
||||
const candidates = [
|
||||
`https://${domain}/favicon.ico`,
|
||||
`https://icons.duckduckgo.com/ip3/${domain}.ico`,
|
||||
];
|
||||
for (const url of candidates) {
|
||||
try {
|
||||
const icon = await fetchIconFrom(url);
|
||||
if (icon) return icon;
|
||||
} catch {
|
||||
// Try the next candidate; network/timeout errors must never propagate.
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve and cache the favicon for a sender domain. Idempotent and never
|
||||
* throws: if a (success or negative) cache entry already exists it returns
|
||||
* immediately, so callers can fire this on every email without refetching.
|
||||
* The KV TTL is the sole expiry mechanism.
|
||||
*/
|
||||
export async function cacheFaviconForDomain(
|
||||
domain: string,
|
||||
env: Env,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const key = iconKey(domain);
|
||||
const existing = await env.EMAIL_STORAGE.get(key, "text");
|
||||
if (existing !== null) return; // present (incl. negative) → nothing to do
|
||||
|
||||
const icon = await resolveIcon(domain);
|
||||
const record: IconRecord = icon
|
||||
? {
|
||||
data: arrayBufferToBase64(icon.buffer),
|
||||
contentType: icon.contentType,
|
||||
}
|
||||
: { data: null, contentType: "" };
|
||||
|
||||
await env.EMAIL_STORAGE.put(key, JSON.stringify(record), {
|
||||
expirationTtl: ICON_TTL_SECONDS,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.warn("Favicon cache failed", { domain, error: String(error) });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a cached icon for a domain. Returns null on a miss or a negative entry.
|
||||
*/
|
||||
export async function getCachedIcon(
|
||||
domain: string,
|
||||
env: Env,
|
||||
): Promise<{ bytes: ArrayBuffer; contentType: string } | null> {
|
||||
const record = (await env.EMAIL_STORAGE.get(
|
||||
iconKey(domain),
|
||||
"json",
|
||||
)) as IconRecord | null;
|
||||
if (!record || record.data === null) return null;
|
||||
return {
|
||||
bytes: base64ToArrayBuffer(record.data),
|
||||
contentType: record.contentType,
|
||||
};
|
||||
}
|
||||
@@ -72,6 +72,17 @@ describe("generateRssFeed", () => {
|
||||
expect(result).toContain("<title>Test Newsletter</title>");
|
||||
});
|
||||
|
||||
it("includes the per-feed icon as the channel <image>", () => {
|
||||
const result = generateRssFeed(
|
||||
mockFeedConfig,
|
||||
mockEmails,
|
||||
BASE_URL,
|
||||
FEED_ID,
|
||||
);
|
||||
expect(result).toContain("<image>");
|
||||
expect(result).toContain(`${BASE_URL}/favicon/${FEED_ID}`);
|
||||
});
|
||||
|
||||
it("includes <enclosure> element for email with attachment", () => {
|
||||
const result = generateRssFeed(
|
||||
mockFeedConfig,
|
||||
@@ -172,6 +183,18 @@ describe("generateAtomFeed", () => {
|
||||
expect(result).toContain('xmlns="http://www.w3.org/2005/Atom"');
|
||||
});
|
||||
|
||||
it("includes the per-feed icon as <icon> and <logo>", () => {
|
||||
const result = generateAtomFeed(
|
||||
mockFeedConfig,
|
||||
mockEmails,
|
||||
BASE_URL,
|
||||
FEED_ID,
|
||||
);
|
||||
const iconUrl = `${BASE_URL}/favicon/${FEED_ID}`;
|
||||
expect(result).toContain(`<icon>${iconUrl}</icon>`);
|
||||
expect(result).toContain(`<logo>${iconUrl}</logo>`);
|
||||
});
|
||||
|
||||
it("contains <feed> root element", () => {
|
||||
const result = generateAtomFeed(
|
||||
mockFeedConfig,
|
||||
|
||||
@@ -23,9 +23,14 @@ function buildFeed(
|
||||
feedId: string,
|
||||
selfUrl?: { rss?: string; atom?: string },
|
||||
): Feed {
|
||||
const iconUrl = `${baseUrl}/favicon/${feedId}`;
|
||||
const feed = new Feed({
|
||||
title: feedConfig.title,
|
||||
description: feedConfig.description || "",
|
||||
// Per-feed icon derived from the last sender's domain (self-falls-back to
|
||||
// the project icon). image → RSS <image>/Atom <logo>; favicon → Atom <icon>.
|
||||
image: iconUrl,
|
||||
favicon: iconUrl,
|
||||
// Computed dynamically so the id is always canonical regardless of what
|
||||
// was stored in KV at feed-creation time (which may have used a stale domain).
|
||||
id: `${baseUrl}/rss/${feedId}`,
|
||||
|
||||
@@ -7,6 +7,13 @@ import {
|
||||
} from "../types";
|
||||
import { MAX_METADATA_EMAILS } from "../config/constants";
|
||||
|
||||
/**
|
||||
* KV key for a domain's cached favicon (shared across feeds from the same sender).
|
||||
*/
|
||||
export function iconKey(domain: string): string {
|
||||
return `icon:${domain}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store email data in KV
|
||||
*/
|
||||
|
||||
Reference in New Issue
Block a user