mirror of
https://github.com/juherr/kill-the-news.git
synced 2026-06-20 22:03:48 +00:00
refactor: parallelize KV writes, cap metadata, remove redundant normalization
- email-processor: run email put + metadata get in parallel (saves one KV round-trip per email) - email-processor: add missing 50-email metadata cap (was unbounded unlike storage.ts) - email-processor: remove redundant normalizeEmail call on already-normalized allowedSender - email-parser: strip WHAT-comments and boilerplate JSDoc throughout Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
+18
-15
@@ -17,21 +17,20 @@ function normalizeEmail(value: string): string {
|
|||||||
|
|
||||||
function senderMatchesAllowlist(
|
function senderMatchesAllowlist(
|
||||||
sender: string,
|
sender: string,
|
||||||
allowedSender: string,
|
allowedSender: string, // already normalized by caller
|
||||||
): boolean {
|
): boolean {
|
||||||
|
if (!allowedSender) return false;
|
||||||
|
|
||||||
const normalizedSender = normalizeEmail(sender);
|
const normalizedSender = normalizeEmail(sender);
|
||||||
const normalizedAllowed = normalizeEmail(allowedSender);
|
|
||||||
|
|
||||||
if (!normalizedAllowed) return false;
|
if (allowedSender.includes("@")) {
|
||||||
|
return normalizedSender === allowedSender;
|
||||||
if (normalizedAllowed.includes("@")) {
|
|
||||||
return normalizedSender === normalizedAllowed;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const senderDomain = normalizedSender.split("@")[1] || "";
|
const senderDomain = normalizedSender.split("@")[1] || "";
|
||||||
const normalizedDomain = normalizedAllowed.startsWith("@")
|
const normalizedDomain = allowedSender.startsWith("@")
|
||||||
? normalizedAllowed.slice(1)
|
? allowedSender.slice(1)
|
||||||
: normalizedAllowed;
|
: allowedSender;
|
||||||
return senderDomain === normalizedDomain;
|
return senderDomain === normalizedDomain;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -84,13 +83,14 @@ export async function processEmail(
|
|||||||
};
|
};
|
||||||
|
|
||||||
const emailKey = `feed:${feedId}:${Date.now()}`;
|
const emailKey = `feed:${feedId}:${Date.now()}`;
|
||||||
await env.EMAIL_STORAGE.put(emailKey, JSON.stringify(emailData));
|
|
||||||
|
|
||||||
const feedMetadataKey = `feed:${feedId}:metadata`;
|
const feedMetadataKey = `feed:${feedId}:metadata`;
|
||||||
const feedMetadata = ((await env.EMAIL_STORAGE.get(
|
|
||||||
feedMetadataKey,
|
const [, rawMetadata] = await Promise.all([
|
||||||
"json",
|
env.EMAIL_STORAGE.put(emailKey, JSON.stringify(emailData)),
|
||||||
)) || {
|
env.EMAIL_STORAGE.get(feedMetadataKey, "json"),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const feedMetadata = ((rawMetadata as FeedMetadata | null) || {
|
||||||
emails: [],
|
emails: [],
|
||||||
}) as FeedMetadata;
|
}) as FeedMetadata;
|
||||||
feedMetadata.emails.unshift({
|
feedMetadata.emails.unshift({
|
||||||
@@ -98,6 +98,9 @@ export async function processEmail(
|
|||||||
subject: emailData.subject,
|
subject: emailData.subject,
|
||||||
receivedAt: emailData.receivedAt,
|
receivedAt: emailData.receivedAt,
|
||||||
});
|
});
|
||||||
|
if (feedMetadata.emails.length > 50) {
|
||||||
|
feedMetadata.emails = feedMetadata.emails.slice(0, 50);
|
||||||
|
}
|
||||||
await env.EMAIL_STORAGE.put(feedMetadataKey, JSON.stringify(feedMetadata));
|
await env.EMAIL_STORAGE.put(feedMetadataKey, JSON.stringify(feedMetadata));
|
||||||
|
|
||||||
console.log(`Successfully processed email for feed ${feedId}`);
|
console.log(`Successfully processed email for feed ${feedId}`);
|
||||||
|
|||||||
@@ -1,48 +1,26 @@
|
|||||||
import { EmailData } from "../types";
|
import { EmailData } from "../types";
|
||||||
|
|
||||||
/**
|
|
||||||
* Simple email parser specialized for ForwardEmail.net's webhook format
|
|
||||||
*/
|
|
||||||
export class EmailParser {
|
export class EmailParser {
|
||||||
/**
|
// Matches noun1.noun2.XY (the feed ID format) before the @ symbol
|
||||||
* Extract the feed ID from an email address
|
|
||||||
* @param emailAddress The email address (e.g., apple.mountain.42@domain.com)
|
|
||||||
* @returns The feed ID or null if not found
|
|
||||||
*/
|
|
||||||
static extractFeedId(emailAddress: string): string | null {
|
static extractFeedId(emailAddress: string): string | null {
|
||||||
// Match pattern for noun1.noun2.XY before the @ symbol
|
|
||||||
const match = emailAddress.match(/^([a-z]+\.[a-z]+\.\d{2})@/i);
|
const match = emailAddress.match(/^([a-z]+\.[a-z]+\.\d{2})@/i);
|
||||||
return match ? match[1] : null;
|
return match ? match[1] : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Parse email data from ForwardEmail.net's webhook payload
|
|
||||||
* @param payload ForwardEmail.net webhook payload
|
|
||||||
*/
|
|
||||||
static parseForwardEmailPayload(payload: any): EmailData {
|
static parseForwardEmailPayload(payload: any): EmailData {
|
||||||
if (!payload) {
|
if (!payload) {
|
||||||
throw new Error("Missing or invalid webhook payload");
|
throw new Error("Missing or invalid webhook payload");
|
||||||
}
|
}
|
||||||
|
|
||||||
// Extract the "to" address
|
|
||||||
const toAddress = payload.recipients?.[0] || "";
|
|
||||||
|
|
||||||
// Extract the sender information using ForwardEmail's structure
|
|
||||||
const fromAddress =
|
const fromAddress =
|
||||||
payload.from?.text ||
|
payload.from?.text ||
|
||||||
(payload.from?.value?.[0]?.address
|
(payload.from?.value?.[0]?.address
|
||||||
? `${payload.from.value[0].name || ""} <${payload.from.value[0].address}>`
|
? `${payload.from.value[0].name || ""} <${payload.from.value[0].address}>`
|
||||||
: "Unknown Sender");
|
: "Unknown Sender");
|
||||||
|
|
||||||
// Extract subject
|
const subject = this.decodeEncodedWords(payload.subject || "No Subject");
|
||||||
let subject = payload.subject || "No Subject";
|
|
||||||
// Decode any encoded words in the subject
|
|
||||||
subject = this.decodeEncodedWords(subject);
|
|
||||||
|
|
||||||
// Get content, preferring HTML over plain text
|
|
||||||
const content = payload.html || payload.text || "";
|
const content = payload.html || payload.text || "";
|
||||||
|
|
||||||
// Create simple email data object
|
|
||||||
return {
|
return {
|
||||||
subject,
|
subject,
|
||||||
from: fromAddress,
|
from: fromAddress,
|
||||||
@@ -52,13 +30,9 @@ export class EmailParser {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Extract headers from ForwardEmail payload
|
|
||||||
*/
|
|
||||||
private static extractHeaders(payload: any): Record<string, string> {
|
private static extractHeaders(payload: any): Record<string, string> {
|
||||||
const headers: Record<string, string> = {};
|
const headers: Record<string, string> = {};
|
||||||
|
|
||||||
// Extract headers from headerLines if available
|
|
||||||
if (payload.headerLines && Array.isArray(payload.headerLines)) {
|
if (payload.headerLines && Array.isArray(payload.headerLines)) {
|
||||||
payload.headerLines.forEach((h: { key: string; line: string }) => {
|
payload.headerLines.forEach((h: { key: string; line: string }) => {
|
||||||
const key = h.key.toLowerCase();
|
const key = h.key.toLowerCase();
|
||||||
@@ -67,9 +41,7 @@ export class EmailParser {
|
|||||||
.trim();
|
.trim();
|
||||||
headers[key] = value;
|
headers[key] = value;
|
||||||
});
|
});
|
||||||
}
|
} else if (typeof payload.headers === "string") {
|
||||||
// Or from headers string if provided
|
|
||||||
else if (typeof payload.headers === "string") {
|
|
||||||
payload.headers.split(/\r?\n/).forEach((line: string) => {
|
payload.headers.split(/\r?\n/).forEach((line: string) => {
|
||||||
const match = line.match(/^([^:]+):\s*(.*)$/);
|
const match = line.match(/^([^:]+):\s*(.*)$/);
|
||||||
if (match) {
|
if (match) {
|
||||||
@@ -81,27 +53,19 @@ export class EmailParser {
|
|||||||
return headers;
|
return headers;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Decode RFC 2047 encoded words in headers
|
|
||||||
* @param text Text that may contain encoded words like =?UTF-8?Q?Hello_World?=
|
|
||||||
*/
|
|
||||||
static decodeEncodedWords(text: string): string {
|
static decodeEncodedWords(text: string): string {
|
||||||
if (!text) return "";
|
if (!text) return "";
|
||||||
|
|
||||||
// Simple RFC 2047 encoded-word decoder
|
|
||||||
return text.replace(
|
return text.replace(
|
||||||
/=\?([^?]+)\?([BQ])\?([^?]+)\?=/gi,
|
/=\?([^?]+)\?([BQ])\?([^?]+)\?=/gi,
|
||||||
(_, charset, encoding, text) => {
|
(_, charset, encoding, text) => {
|
||||||
if (encoding.toUpperCase() === "B") {
|
if (encoding.toUpperCase() === "B") {
|
||||||
// Base64 encoding
|
|
||||||
try {
|
try {
|
||||||
const decoded = atob(text);
|
return atob(text);
|
||||||
return decoded;
|
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
return text;
|
return text;
|
||||||
}
|
}
|
||||||
} else if (encoding.toUpperCase() === "Q") {
|
} else if (encoding.toUpperCase() === "Q") {
|
||||||
// Quoted-printable encoding
|
|
||||||
return this.decodeQuotedPrintable(text.replace(/_/g, " "));
|
return this.decodeQuotedPrintable(text.replace(/_/g, " "));
|
||||||
}
|
}
|
||||||
return text;
|
return text;
|
||||||
@@ -109,10 +73,6 @@ export class EmailParser {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Decode quoted-printable encoded text
|
|
||||||
* @param text Quoted-printable encoded text
|
|
||||||
*/
|
|
||||||
private static decodeQuotedPrintable(text: string): string {
|
private static decodeQuotedPrintable(text: string): string {
|
||||||
return text.replace(/=([0-9A-F]{2})/gi, (_, hex) => {
|
return text.replace(/=([0-9A-F]{2})/gi, (_, hex) => {
|
||||||
return String.fromCharCode(parseInt(hex, 16));
|
return String.fromCharCode(parseInt(hex, 16));
|
||||||
|
|||||||
Reference in New Issue
Block a user