From 8839aac24bbfed4a959463267744d1e62b5e384c Mon Sep 17 00:00:00 2001 From: Young Lee <8462583+yl8976@users.noreply.github.com> Date: Thu, 27 Feb 2025 14:51:38 -0800 Subject: [PATCH] Set up initial project and files --- .cursor/rules/project-implementation-plan.mdc | 57 ++ .gitignore | 28 + README.md | 81 ++ package.json | 27 + setup.sh | 29 + src/index.ts | 75 ++ src/routes/admin.ts | 811 ++++++++++++++++++ src/routes/inbound.ts | 81 ++ src/routes/rss.ts | 70 ++ src/types/hono.d.ts | 8 + src/types/index.ts | 68 ++ src/types/mailparser.d.ts | 17 + src/types/rss.d.ts | 51 ++ src/utils/email-parser.ts | 114 +++ src/utils/feed-generator.ts | 53 ++ src/utils/storage.ts | 136 +++ tsconfig.json | 18 + wrangler.toml | 30 + 18 files changed, 1754 insertions(+) create mode 100644 .cursor/rules/project-implementation-plan.mdc create mode 100644 .gitignore create mode 100644 README.md create mode 100644 package.json create mode 100755 setup.sh create mode 100644 src/index.ts create mode 100644 src/routes/admin.ts create mode 100644 src/routes/inbound.ts create mode 100644 src/routes/rss.ts create mode 100644 src/types/hono.d.ts create mode 100644 src/types/index.ts create mode 100644 src/types/mailparser.d.ts create mode 100644 src/types/rss.d.ts create mode 100644 src/utils/email-parser.ts create mode 100644 src/utils/feed-generator.ts create mode 100644 src/utils/storage.ts create mode 100644 tsconfig.json create mode 100644 wrangler.toml diff --git a/.cursor/rules/project-implementation-plan.mdc b/.cursor/rules/project-implementation-plan.mdc new file mode 100644 index 0000000..f7a14b4 --- /dev/null +++ b/.cursor/rules/project-implementation-plan.mdc @@ -0,0 +1,57 @@ +--- +description: Project Implementation Plan +globs: * +alwaysApply: false +--- +# Detailed Project Implementation Plan + +This section is a top-level "blueprint" describing the solution architecture and how each component fits together. + +## Overview + +**Goal:** Build a service that turns email newsletters into RSS feeds, so you can subscribe in an RSS reader like Reeder. The service should provide unique email addresses per feed, a front-end admin panel, indefinite (or long-term) storage of newsletters, and minimal cost—preferably using Cloudflare services plus ForwardEmail.net. + +## Key Components + +1. **ForwardEmail.net** + - Accept incoming newsletters on your custom domain’s email addresses. + - Forward them (via webhook) to your API endpoint for processing. + - Free inbound plan includes JSON + raw MIME data. +2. **Cloudflare Workers** + - **Inbound Worker:** Receives the webhook from ForwardEmail.net, parses/stores newsletter data in KV (or R2). + - **RSS Worker:** Serves RSS feeds by reading from KV and outputting XML. + - **Admin Worker (potential):** Could serve a small UI or JSON API for feed management. +3. **Cloudflare KV** + - Key-value store for storing newsletter items (subject, date, HTML, etc.). + - Minimal cost for text data. + - Indefinite retention if you keep usage under limits. +4. **Cloudflare Pages (Optional)** + - Could host a separate front-end for admin tasks. + - Alternatively, build a simple admin UI directly within the Worker. +5. **Admin Dashboard** + - Basic login and feed creation (generate random email addresses). + - List newsletters and optionally delete them or rename feed titles. + - For a simple approach, implement a minimal password-protected area or JSON endpoints. +6. **Domain / DNS Setup** + - Use your custom domain (e.g. `mynewsletters.dev`). + - Add DNS records so ForwardEmail.net is the MX handler. + - Configure Cloudflare for general DNS (with “Orange Cloud” or not, depending on your proxying preferences). + - Verify your domain following ForwardEmail.net’s instructions. + +## Data Flow + +1. A newsletter arrives at `newsletterXYZ@mynewsletters.dev`. +2. ForwardEmail.net triggers a webhook to `https://your-worker.example.com/api/inbound?feed=XYZ` with JSON + raw MIME. +3. The Worker parses the email, extracts relevant information (date, subject, HTML body), and stores it in KV under a key like `feed:XYZ:timestamp`. +4. When your RSS reader (e.g. Reeder) requests `GET https://your-worker.example.com/rss/XYZ`, the Worker fetches all items from KV for that feed, builds an RSS XML response, and returns it. +5. *(Optional)* The Admin Dashboard (via a password-protected route or a separate Cloudflare Pages front-end) can create new feed IDs, display items, etc. + +## Summary of Implementation Steps + +1. Set up the Domain and ForwardEmail.net for inbound mail. +2. Create a Cloudflare Worker to handle the inbound webhook. +3. Parse the email (using ForwardEmail.net’s parsed data or parsing the raw MIME if necessary). +4. Store the data in KV. +5. Create an RSS Worker endpoint to retrieve the data and output XML. +6. *(Optional)* Develop an Admin UI to create new feeds, list items, and manage them. +7. Deploy and test the solution. Subscribe to the feed with Reeder. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..062db28 --- /dev/null +++ b/.gitignore @@ -0,0 +1,28 @@ +# Dependencies +node_modules/ +package-lock.json +yarn.lock + +# Build +dist/ +.wrangler/ +.dev.vars + +# Environment +.env + +# IDE +.vscode/ +.idea/ +*.iml + +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# System +.DS_Store +Thumbs.db \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..f5c01a1 --- /dev/null +++ b/README.md @@ -0,0 +1,81 @@ +# Email-to-RSS + +A modern service that turns email newsletters into RSS feeds, built with Cloudflare Workers. This service provides unique email addresses per feed, a front-end admin panel, and long-term storage of newsletters. + +## Features + +- **Minimal Dependencies**: Built using modern, web-friendly libraries without Node.js-specific dependencies +- **Lightweight**: Entire worker bundle is only ~360KB (gzipped: ~65KB) +- **Email Processing**: Handles emails from ForwardEmail.net webhook +- **RSS Generation**: Serves standards-compliant RSS feeds +- **Admin Interface**: Simple management UI for feeds and emails +- **Storage**: Uses Cloudflare KV for efficient, low-cost storage + +## Architecture + +### Email Flow +1. A newsletter arrives at `newsletter-XYZ@yourdomain.com` +2. ForwardEmail.net forwards it to your Cloudflare Worker +3. The Worker parses the email, extracts content, and stores it in KV +4. The RSS feed is updated with the new content + +### Key Components + +- **Email Parser**: Lightweight custom parser that works in edge environments +- **Feed Generator**: Modern RSS feed generator with minimal dependencies +- **Admin UI**: Simple interface to manage feeds and view emails + +## Development + +This project uses a modern build process with Wrangler's built-in bundling (powered by esbuild): + +```bash +# Install dependencies +npm install + +# Run development server +npm run dev + +# Build for production +npm run build + +# Deploy to Cloudflare +npm run deploy +``` + +## Environment Variables + +- `ADMIN_PASSWORD`: Password for the admin interface +- `DOMAIN`: Your custom domain for receiving emails +- `FORWARDEMAIL_TOKEN`: Token for ForwardEmail.net webhook authentication + +## Technology Stack + +- **Cloudflare Workers**: Edge computing platform +- **Cloudflare KV**: Key-value storage +- **Hono**: Lightweight web framework +- **TypeScript**: Type-safe JavaScript +- **Feed**: Modern RSS feed generator +- **Zod**: Schema validation + +## Setup + +1. Clone this repository +2. Install dependencies with `npm install` +3. Copy `wrangler.toml.example` to `wrangler.toml` and set your values +4. Run `npm run dev` to start the development server +5. Deploy with `npm run deploy` + +## Minimalist Approach + +This project follows a minimalist approach: + +- No unnecessary dependencies +- Web-standard APIs where possible +- No Node.js-specific modules or polyfills +- Modern TypeScript features +- Clean, maintainable code structure + +## License + +MIT diff --git a/package.json b/package.json new file mode 100644 index 0000000..a79b739 --- /dev/null +++ b/package.json @@ -0,0 +1,27 @@ +{ + "name": "email-to-rss", + "version": "0.1.0", + "description": "A service that converts email newsletters to RSS feeds using Cloudflare Workers", + "main": "dist/worker.js", + "scripts": { + "build": "wrangler deploy --dry-run --outdir=dist", + "format": "prettier --write '**/*.{js,ts,css,json,md}'", + "dev": "wrangler dev", + "deploy": "wrangler deploy" + }, + "author": "", + "license": "MIT", + "devDependencies": { + "@cloudflare/workers-types": "^4.20250224.0", + "@types/mailparser": "^3.4.5", + "@types/rss": "^0.0.32", + "prettier": "^3.5.2", + "typescript": "^5.7.3", + "wrangler": "^3.111.0" + }, + "dependencies": { + "feed": "^4.2.2", + "hono": "^3.12.8", + "zod": "^3.22.4" + } +} diff --git a/setup.sh b/setup.sh new file mode 100755 index 0000000..ff1622d --- /dev/null +++ b/setup.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Email to RSS setup script + +echo "🚀 Setting up Email to RSS service..." + +# Install dependencies +echo "📦 Installing dependencies..." +npm install + +# Create KV namespaces +echo "🗄️ Creating KV namespaces..." +echo "You'll need to update wrangler.toml with these IDs." +npx wrangler kv:namespace create EMAIL_STORAGE +npx wrangler kv:namespace create EMAIL_STORAGE --preview + +# Set up admin password +echo "🔐 Setting up admin password..." +read -p "Enter admin password: " admin_password +echo "$admin_password" | npx wrangler secret put ADMIN_PASSWORD + +# Prompt for domain +read -p "Enter your domain (e.g., yourdomain.com): " domain +echo "📝 Please update your domain in wrangler.toml" + +echo "✅ Setup complete! Next steps:" +echo "1. Update wrangler.toml with your KV namespace IDs and domain" +echo "2. Set up MX records for your domain with ForwardEmail.net" +echo "3. Deploy with 'npm run deploy'" \ No newline at end of file diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..67d188d --- /dev/null +++ b/src/index.ts @@ -0,0 +1,75 @@ +import { Hono } from 'hono'; +import { handle as handleInbound } from './routes/inbound'; +import { handle as handleRSS } from './routes/rss'; +import { handle as handleAdmin } from './routes/admin'; +import { Context, Next } from 'hono'; +import { Env } from './types'; + +// Define allowed origins for CORS +const ALLOWED_ORIGINS = ['https://getmynews.app', 'https://api.getmynews.app']; + +// Create the main Hono app +const app = new Hono(); + +// CORS middleware +app.use('*', async (c, next) => { + const origin = c.req.header('Origin'); + if (origin && ALLOWED_ORIGINS.includes(origin)) { + c.header('Access-Control-Allow-Origin', origin); + c.header('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, OPTIONS'); + c.header('Access-Control-Allow-Headers', 'Content-Type, Authorization'); + c.header('Access-Control-Max-Age', '86400'); + } + + // Handle preflight requests + if (c.req.method === 'OPTIONS') { + return c.text('', 204); + } + + await next(); +}); + +// Create auth middleware function +const authMiddleware = async (c: Context, next: Next) => { + const authHeader = c.req.header('Authorization'); + + if (!authHeader || !authHeader.startsWith('Basic ')) { + c.header('WWW-Authenticate', 'Basic realm="Admin Area"'); + return c.text('Unauthorized', 401); + } + + const base64Credentials = authHeader.split(' ')[1]; + const credentials = atob(base64Credentials); + const [username, password] = credentials.split(':'); + + // Check against environment variable + const env = c.env as unknown as Env; + const adminPassword = env.ADMIN_PASSWORD; + + if (username !== 'admin' || password !== adminPassword) { + c.header('WWW-Authenticate', 'Basic realm="Admin Area"'); + return c.text('Unauthorized', 401); + } + + await next(); +}; + +// Apply auth middleware to admin routes +app.use('/admin/*', authMiddleware); + +// Also apply auth middleware to root path +app.use('/', authMiddleware); + +// Route handlers +app.post('/api/inbound', handleInbound); +app.get('/rss/:feedId', handleRSS); +app.route('/admin', handleAdmin); + +// Root path uses admin handler +app.route('/', handleAdmin); + +// Catch-all for 404s +app.all('*', (c) => c.text('Not Found', 404)); + +// Export the worker handler +export default app; \ No newline at end of file diff --git a/src/routes/admin.ts b/src/routes/admin.ts new file mode 100644 index 0000000..4af0f9a --- /dev/null +++ b/src/routes/admin.ts @@ -0,0 +1,811 @@ +import { Hono } from 'hono'; +import { html } from 'hono/html'; +import { z } from 'zod'; +import { Env, FeedConfig, FeedList, FeedMetadata, EmailMetadata, EmailData } from '../types'; + +// Create a Hono app for admin routes +const app = new Hono(); + +// Schema for feed creation +const createFeedSchema = z.object({ + title: z.string().min(1, 'Title is required'), + description: z.string().optional(), + language: z.string().optional().default('en') +}); + +// Schema for feed updates +const updateFeedSchema = z.object({ + title: z.string().min(1, 'Title is required'), + description: z.string().optional(), + language: z.string().optional().default('en') +}); + +// Admin dashboard route +app.get('/', async (c) => { + // Type assertion for environment variables + const env = c.env as unknown as Env; + const emailStorage = env.EMAIL_STORAGE; + + // List all feeds + const feedList = await listAllFeeds(emailStorage); + + return c.html( + html` + + + Email to RSS Admin + + + + + + +
+

Email to RSS Admin

+

Manage your email newsletter feeds

+
+ +

Create New Feed

+
+
+ + +
+ +
+ + +
+ +
+ + +
+ + +
+ +

Your Feeds

+ ${feedList.length > 0 ? + html`` : + html`

You don't have any feeds yet. Create one above.

` + } + + ` + ); +}); + +// Create a new feed +app.post('/feeds', async (c) => { + // Type assertion for environment variables + const env = c.env as unknown as Env; + const emailStorage = env.EMAIL_STORAGE; + + try { + const formData = await c.req.formData(); + const title = formData.get('title')?.toString() || ''; + const description = formData.get('description')?.toString(); + const language = formData.get('language')?.toString() || 'en'; + + // Validate inputs + const parsedData = createFeedSchema.parse({ + title, + description, + language + }); + + // Generate a unique feed ID + const feedId = generateRandomId(); + + // Store feed configuration + const feedConfigKey = `feed:${feedId}:config`; + await emailStorage.put(feedConfigKey, JSON.stringify({ + title: parsedData.title, + description: parsedData.description, + language: parsedData.language, + site_url: `https://api.${env.DOMAIN}/rss/${feedId}`, + feed_url: `https://api.${env.DOMAIN}/rss/${feedId}`, + created_at: Date.now() + })); + + // Create empty metadata for the feed + const feedMetadataKey = `feed:${feedId}:metadata`; + await emailStorage.put(feedMetadataKey, JSON.stringify({ + emails: [] + })); + + // Add feed to the list of all feeds + await addFeedToList(emailStorage, feedId, parsedData.title); + + // Redirect back to admin page + return c.redirect('/admin'); + } catch (error) { + console.error('Error creating feed:', error); + return c.text('Error creating feed. Please try again.', 400); + } +}); + +// Edit feed form +app.get('/feeds/:feedId/edit', async (c) => { + // Type assertion for environment variables + const env = c.env as unknown as Env; + const emailStorage = env.EMAIL_STORAGE; + const feedId = c.req.param('feedId'); + + // Get feed configuration + const feedConfigKey = `feed:${feedId}:config`; + const feedConfig = await emailStorage.get(feedConfigKey, 'json') as FeedConfig | null; + + if (!feedConfig) { + return c.text('Feed not found', 404); + } + + return c.html( + html` + + + Edit Feed - Email to RSS Admin + + + + + +
+

Edit Feed

+

Back to Dashboard

+
+ +
+
+ + +
+ +
+ + +
+ +
+ + +
+ + +
+ + ` + ); +}); + +// Update feed +app.post('/feeds/:feedId/edit', async (c) => { + // Type assertion for environment variables + const env = c.env as unknown as Env; + const emailStorage = env.EMAIL_STORAGE; + const feedId = c.req.param('feedId'); + + try { + const formData = await c.req.formData(); + const title = formData.get('title')?.toString() || ''; + const description = formData.get('description')?.toString(); + const language = formData.get('language')?.toString() || 'en'; + + // Validate inputs + const parsedData = updateFeedSchema.parse({ + title, + description, + language + }); + + // Get existing feed config + const feedConfigKey = `feed:${feedId}:config`; + const existingConfig = await emailStorage.get(feedConfigKey, 'json') as FeedConfig | null; + + if (!existingConfig) { + return c.text('Feed not found', 404); + } + + // Update feed configuration + await emailStorage.put(feedConfigKey, JSON.stringify({ + ...existingConfig, + title: parsedData.title, + description: parsedData.description, + language: parsedData.language, + updated_at: Date.now() + })); + + // Update feed in the list of all feeds + await updateFeedInList(emailStorage, feedId, parsedData.title); + + // Redirect back to admin page + return c.redirect('/admin'); + } catch (error) { + console.error('Error updating feed:', error); + return c.text('Error updating feed. Please try again.', 400); + } +}); + +// Delete feed +app.post('/feeds/:feedId/delete', async (c) => { + // Type assertion for environment variables + const env = c.env as unknown as Env; + const emailStorage = env.EMAIL_STORAGE; + const feedId = c.req.param('feedId'); + + try { + // Get feed metadata to find all email keys + const feedMetadataKey = `feed:${feedId}:metadata`; + const feedMetadata = await emailStorage.get(feedMetadataKey, 'json') as FeedMetadata | null; + + if (!feedMetadata) { + return c.text('Feed not found', 404); + } + + // Delete all emails for this feed + for (const email of feedMetadata.emails) { + await emailStorage.delete(email.key); + } + + // Delete feed configuration and metadata + await emailStorage.delete(`feed:${feedId}:config`); + await emailStorage.delete(feedMetadataKey); + + // Remove feed from the list of all feeds + await removeFeedFromList(emailStorage, feedId); + + // Redirect back to admin page + return c.redirect('/admin'); + } catch (error) { + console.error('Error deleting feed:', error); + return c.text('Error deleting feed. Please try again.', 400); + } +}); + +// View emails for a feed +app.get('/feeds/:feedId/emails', async (c) => { + // Type assertion for environment variables + const env = c.env as unknown as Env; + const emailStorage = env.EMAIL_STORAGE; + const feedId = c.req.param('feedId'); + + // Get feed configuration + const feedConfigKey = `feed:${feedId}:config`; + const feedConfig = await emailStorage.get(feedConfigKey, 'json') as FeedConfig | null; + + if (!feedConfig) { + return c.text('Feed not found', 404); + } + + // Get feed metadata (list of emails) + const feedMetadataKey = `feed:${feedId}:metadata`; + const feedMetadata = (await emailStorage.get(feedMetadataKey, 'json') || { emails: [] }) as FeedMetadata; + + return c.html( + html` + + + ${feedConfig.title} - Emails + + + + + +
+

${feedConfig.title} - Emails

+

Back to Dashboard

+
+ +
+

Email Address: newsletter-${feedId}@${env.DOMAIN}

+

RSS Feed: https://api.${env.DOMAIN}/rss/${feedId}

+
+ +

Emails (${feedMetadata.emails.length})

+ ${feedMetadata.emails.length > 0 ? + html`` : + html`

No emails received yet. Subscribe to newsletters using the email address above.

` + } + + + + ` + ); +}); + +// View email content +app.get('/emails/:emailKey', async (c) => { + // Type assertion for environment variables + const env = c.env as unknown as Env; + const emailStorage = env.EMAIL_STORAGE; + const emailKey = c.req.param('emailKey'); + + // Get email content + const emailData = await emailStorage.get(emailKey, 'json') as EmailData | null; + + if (!emailData) { + return c.text('Email not found', 404); + } + + // Extract feed ID from the key + const feedId = emailKey.split(':')[1]; + + // Create a sanitized HTML content with CSS for the iframe + const htmlContent = ` + + + + + + + + + ${emailData.content} + + + `; + + // Properly encode the HTML content to handle Unicode characters + const encodedHtmlContent = (() => { + // Convert the string to UTF-8 + const encoder = new TextEncoder(); + const bytes = encoder.encode(htmlContent); + // Convert bytes to base64 + return btoa(String.fromCharCode(...new Uint8Array(bytes))); + })(); + + return c.html( + html` + + + ${emailData.subject} + + + + + + +
+

${emailData.subject}

+

Back to Emails

+
+ +
+

From: ${emailData.from}

+

Received: ${new Date(emailData.receivedAt).toLocaleString()}

+
+ +
+ + +
+ +
+
+ +
+ + +
+ + ` + ); +}); + +// Delete an email +app.post('/emails/:emailKey/delete', async (c) => { + // Type assertion for environment variables + const env = c.env as unknown as Env; + const emailStorage = env.EMAIL_STORAGE; + const emailKey = c.req.param('emailKey'); + + try { + // Get the feed ID from form data + const formData = await c.req.formData(); + const feedId = formData.get('feedId')?.toString() || ''; + + if (!feedId) { + return c.text('Missing feed ID', 400); + } + + // Delete the email from KV storage + await emailStorage.delete(emailKey); + + // Remove the email from the feed metadata + const feedMetadataKey = `feed:${feedId}:metadata`; + const feedMetadata = (await emailStorage.get(feedMetadataKey, 'json') || { emails: [] }) as FeedMetadata; + + // Filter out the deleted email + feedMetadata.emails = feedMetadata.emails.filter(email => email.key !== emailKey); + + // Update the feed metadata + await emailStorage.put(feedMetadataKey, JSON.stringify(feedMetadata)); + + // Redirect back to the emails list + return c.redirect(`/admin/feeds/${feedId}/emails`); + } catch (error) { + console.error('Error deleting email:', error); + return c.text('Error deleting email. Please try again.', 400); + } +}); + +// Helper function to generate a random feed ID +function generateRandomId(length = 8): string { + const chars = 'abcdefghijklmnopqrstuvwxyz0123456789'; + let result = ''; + for (let i = 0; i < length; i++) { + result += chars.charAt(Math.floor(Math.random() * chars.length)); + } + return result; +} + +// Helper function to list all feeds +async function listAllFeeds(emailStorage: KVNamespace): Promise { + try { + const feedListKey = 'feeds:list'; + const feedList = await emailStorage.get(feedListKey, 'json') as FeedList | null || { feeds: [] }; + + // Fetch detailed information for each feed + const feeds = []; + for (const feed of feedList.feeds) { + const feedConfigKey = `feed:${feed.id}:config`; + const feedConfig = await emailStorage.get(feedConfigKey, 'json') as FeedConfig | null; + + if (feedConfig) { + feeds.push({ + id: feed.id, + title: feedConfig.title, + description: feedConfig.description + }); + } + } + + return feeds; + } catch (error) { + console.error('Error listing feeds:', error); + return []; + } +} + +// Helper function to add a feed to the list of all feeds +async function addFeedToList(emailStorage: KVNamespace, feedId: string, title: string): Promise { + const feedListKey = 'feeds:list'; + const feedList = await emailStorage.get(feedListKey, 'json') as FeedList | null || { feeds: [] }; + + feedList.feeds.push({ + id: feedId, + title + }); + + await emailStorage.put(feedListKey, JSON.stringify(feedList)); +} + +// Helper function to update a feed in the list of all feeds +async function updateFeedInList(emailStorage: KVNamespace, feedId: string, title: string): Promise { + const feedListKey = 'feeds:list'; + const feedList = await emailStorage.get(feedListKey, 'json') as FeedList | null || { feeds: [] }; + + const feedIndex = feedList.feeds.findIndex((feed) => feed.id === feedId); + + if (feedIndex >= 0) { + feedList.feeds[feedIndex].title = title; + await emailStorage.put(feedListKey, JSON.stringify(feedList)); + } +} + +// Helper function to remove a feed from the list of all feeds +async function removeFeedFromList(emailStorage: KVNamespace, feedId: string): Promise { + const feedListKey = 'feeds:list'; + const feedList = await emailStorage.get(feedListKey, 'json') as FeedList | null || { feeds: [] }; + + feedList.feeds = feedList.feeds.filter((feed) => feed.id !== feedId); + + await emailStorage.put(feedListKey, JSON.stringify(feedList)); +} + +// Export the Hono app +export const handle = app; \ No newline at end of file diff --git a/src/routes/inbound.ts b/src/routes/inbound.ts new file mode 100644 index 0000000..6a3948a --- /dev/null +++ b/src/routes/inbound.ts @@ -0,0 +1,81 @@ +import { Context } from 'hono'; +import { EmailParser } from '../utils/email-parser'; +import { Env, FeedMetadata } from '../types'; + +// Interface for ForwardEmail.net webhook payload +interface ForwardEmailPayload { + recipients?: string[]; + from?: { + value?: Array<{address?: string; name?: string}>; + text?: string; + html?: string; + }; + subject?: string; + text?: string; + html?: string; + date?: string; + messageId?: string; + headerLines?: Array<{key: string; line: string}>; + headers?: string; + raw?: string; + attachments?: Array; +} + +/** + * Handle incoming emails from ForwardEmail.net webhook + */ +export async function handle(c: Context): Promise { + try { + // Type assertion for environment variables + const env = c.env as unknown as Env; + + // Parse the incoming JSON payload + const payload: ForwardEmailPayload = await c.req.json(); + + // Log basic information about the incoming email + console.log("Received email:", { + to: payload.recipients?.[0], + from: payload.from?.text || 'Unknown', + subject: payload.subject, + contentType: payload.html ? 'HTML' : 'Text' + }); + + // Extract feed ID from email address (e.g., newsletter-xyz@domain.com -> xyz) + const toAddress = payload.recipients?.[0] || ''; + const feedId = EmailParser.extractFeedId(toAddress); + + if (!feedId) { + console.error(`Invalid email address format: ${toAddress}`); + return new Response('Invalid email address format', { status: 400 }); + } + + // Parse the email using our simplified parser + const emailData = EmailParser.parseForwardEmailPayload(payload); + + // Generate a unique key for this email in KV storage + const emailKey = `feed:${feedId}:${Date.now()}`; + + // Store the email data in KV + await env.EMAIL_STORAGE.put(emailKey, JSON.stringify(emailData)); + + // Get existing feed metadata + const feedMetadataKey = `feed:${feedId}:metadata`; + const feedMetadata = (await env.EMAIL_STORAGE.get(feedMetadataKey, 'json') || { emails: [] }) as FeedMetadata; + + // Add this email to the feed metadata + feedMetadata.emails.unshift({ + key: emailKey, + subject: emailData.subject, + receivedAt: emailData.receivedAt + }); + + // Store updated feed metadata + await env.EMAIL_STORAGE.put(feedMetadataKey, JSON.stringify(feedMetadata)); + + console.log(`Successfully processed email for feed ${feedId}`); + return new Response('Email processed successfully', { status: 200 }); + } catch (error) { + console.error('Error processing email:', error); + return new Response('Error processing email', { status: 500 }); + } +} \ No newline at end of file diff --git a/src/routes/rss.ts b/src/routes/rss.ts new file mode 100644 index 0000000..33f7d01 --- /dev/null +++ b/src/routes/rss.ts @@ -0,0 +1,70 @@ +import { Context } from 'hono'; +import { Env, FeedConfig, FeedMetadata, EmailData } from '../types'; +import { generateRssFeed } from '../utils/feed-generator'; + +/** + * Generates an RSS feed for a specific feed ID + */ +export async function handle(c: Context): Promise { + try { + // Type assertion for environment variables + const env = c.env as unknown as Env; + + // Extract the feed ID from the route params + const feedId = c.req.param('feedId'); + + if (!feedId) { + return new Response('Feed ID is required', { status: 400 }); + } + + // Get the KV namespace + const emailStorage = env.EMAIL_STORAGE; + + // Check if the feed exists + const feedMetadataKey = `feed:${feedId}:metadata`; + const feedMetadata = await emailStorage.get(feedMetadataKey, 'json') as FeedMetadata | null; + + if (!feedMetadata) { + return new Response('Feed not found', { status: 404 }); + } + + // Get feed configuration (title, description, etc.) + const feedConfigKey = `feed:${feedId}:config`; + const feedConfig = await emailStorage.get(feedConfigKey, 'json') as FeedConfig | null || { + title: `Newsletter Feed ${feedId}`, + description: 'Converted email newsletter', + site_url: `https://api.${env.DOMAIN}/rss/${feedId}`, + feed_url: `https://api.${env.DOMAIN}/rss/${feedId}`, + language: 'en', + created_at: Date.now() + }; + + // Get the emails for this feed (up to the last 20) + const emails = feedMetadata.emails.slice(0, 20); + const emailsData: EmailData[] = []; + + // Fetch all email content + for (const email of emails) { + const emailData = await emailStorage.get(email.key, 'json') as EmailData | null; + if (emailData) { + emailsData.push(emailData); + } + } + + // Generate the RSS feed XML + const baseUrl = `https://api.${env.DOMAIN}`; + const rssXml = generateRssFeed(feedConfig, emailsData, baseUrl); + + // Return the RSS feed with appropriate content type + return new Response(rssXml, { + status: 200, + headers: { + 'Content-Type': 'application/rss+xml', + 'Cache-Control': 'max-age=1800' // 30 minutes cache + } + }); + } catch (error) { + console.error('Error generating RSS feed:', error); + return new Response('Internal Server Error', { status: 500 }); + } +} \ No newline at end of file diff --git a/src/types/hono.d.ts b/src/types/hono.d.ts new file mode 100644 index 0000000..745c070 --- /dev/null +++ b/src/types/hono.d.ts @@ -0,0 +1,8 @@ +import { Env } from './index'; + +// Extend Hono's types to include our custom environment +declare module 'hono' { + interface ContextVariableMap { + env: Env; + } +} \ No newline at end of file diff --git a/src/types/index.ts b/src/types/index.ts new file mode 100644 index 0000000..2437fce --- /dev/null +++ b/src/types/index.ts @@ -0,0 +1,68 @@ +// Global environment interface for Cloudflare Workers +export interface Env { + EMAIL_STORAGE: KVNamespace; + ADMIN_PASSWORD: string; + DOMAIN: string; +} + +// Email interface for stored emails +export interface EmailData { + subject: string; + from: string; + content: string; + receivedAt: number; + headers: Record; +} + +// Feed configuration interface +export interface FeedConfig { + title: string; + description?: string; + language: string; + site_url: string; + feed_url: string; + author?: string; + created_at: number; + updated_at?: number; +} + +// Feed metadata interface +export interface FeedMetadata { + emails: EmailMetadata[]; +} + +// Email metadata interface (summary info for listing) +export interface EmailMetadata { + key: string; + subject: string; + receivedAt: number; +} + +// Feed list interface +export interface FeedList { + feeds: FeedListItem[]; +} + +// Feed summary interface (for the global feed list) +export interface FeedListItem { + id: string; + title: string; +} + +// Declare KVNamespace for TypeScript +declare global { + // This is not an ideal solution but works for our example + interface KVNamespace { + get(key: string, options?: { type: 'text' }): Promise; + get(key: string, options: { type: 'json' }): Promise; + get(key: string, options: { type: 'arrayBuffer' }): Promise; + get(key: string, options: { type: 'stream' }): Promise; + put(key: string, value: string | ArrayBuffer | ReadableStream | FormData): Promise; + delete(key: string): Promise; + list(options?: { prefix?: string; limit?: number; cursor?: string }): Promise<{ + keys: { name: string; expiration?: number }[]; + list_complete: boolean; + cursor?: string; + }>; + } +} \ No newline at end of file diff --git a/src/types/mailparser.d.ts b/src/types/mailparser.d.ts new file mode 100644 index 0000000..474719b --- /dev/null +++ b/src/types/mailparser.d.ts @@ -0,0 +1,17 @@ +// Extend mailparser types for Buffer in worker environment +declare module 'buffer-polyfill' { + global { + var Buffer: { + from(data: string, encoding?: string): { + toString(encoding?: string): string; + }; + }; + } +} + +// Add missing atob declaration +declare module 'atob-polyfill' { + global { + function atob(data: string): string; + } +} \ No newline at end of file diff --git a/src/types/rss.d.ts b/src/types/rss.d.ts new file mode 100644 index 0000000..a59f6a4 --- /dev/null +++ b/src/types/rss.d.ts @@ -0,0 +1,51 @@ +declare module 'rss' { + interface RSSOptions { + title: string; + description: string; + feed_url: string; + site_url: string; + language?: string; + image_url?: string; + docs?: string; + managingEditor?: string; + webMaster?: string; + copyright?: string; + pubDate?: Date; + ttl?: number; + generator?: string; + categories?: string[]; + custom_namespaces?: Record; + custom_elements?: any[]; + } + + interface RSSItemOptions { + title: string; + description: string; + url: string; + guid?: string; + categories?: string[]; + author?: string; + date?: Date; + lat?: number; + long?: number; + enclosure?: { + url: string; + file?: string; + size?: number; + type?: string; + }; + custom_elements?: any[]; + } + + interface RSSXMLOptions { + indent?: boolean; + } + + class RSS { + constructor(options: RSSOptions); + item(options: RSSItemOptions): void; + xml(options?: RSSXMLOptions): string; + } + + export = RSS; +} \ No newline at end of file diff --git a/src/utils/email-parser.ts b/src/utils/email-parser.ts new file mode 100644 index 0000000..8ee1222 --- /dev/null +++ b/src/utils/email-parser.ts @@ -0,0 +1,114 @@ +import { EmailData } from '../types'; + +/** + * Simple email parser specialized for ForwardEmail.net's webhook format + */ +export class EmailParser { + /** + * Extract the feed ID from an email address + * @param emailAddress The email address (e.g., newsletter-xyz@domain.com) + * @returns The feed ID or null if not found + */ + static extractFeedId(emailAddress: string): string | null { + const match = emailAddress.match(/^newsletter-([a-zA-Z0-9]+)@/); + return match ? match[1] : null; + } + + /** + * Parse email data from ForwardEmail.net's webhook payload + * @param payload ForwardEmail.net webhook payload + */ + static parseForwardEmailPayload(payload: any): EmailData { + if (!payload) { + throw new Error('Missing or invalid webhook payload'); + } + + // Extract the "to" address + const toAddress = payload.recipients?.[0] || ''; + + // Extract the sender information using ForwardEmail's structure + const fromAddress = payload.from?.text || + (payload.from?.value?.[0]?.address ? + `${payload.from.value[0].name || ''} <${payload.from.value[0].address}>` : + 'Unknown Sender'); + + // Extract subject + let subject = payload.subject || 'No Subject'; + // Decode any encoded words in the subject + subject = this.decodeEncodedWords(subject); + + // Get content, preferring HTML over plain text + const content = payload.html || payload.text || ''; + + // Create simple email data object + return { + subject, + from: fromAddress, + content, + receivedAt: payload.date ? new Date(payload.date).getTime() : Date.now(), + headers: this.extractHeaders(payload) + }; + } + + /** + * Extract headers from ForwardEmail payload + */ + private static extractHeaders(payload: any): Record { + const headers: Record = {}; + + // Extract headers from headerLines if available + if (payload.headerLines && Array.isArray(payload.headerLines)) { + payload.headerLines.forEach((h: {key: string; line: string}) => { + const key = h.key.toLowerCase(); + const value = h.line.replace(new RegExp(`^${h.key}:\\s*`, 'i'), '').trim(); + headers[key] = value; + }); + } + // Or from headers string if provided + else if (typeof payload.headers === 'string') { + payload.headers.split(/\r?\n/).forEach((line: string) => { + const match = line.match(/^([^:]+):\s*(.*)$/); + if (match) { + headers[match[1].toLowerCase()] = match[2]; + } + }); + } + + return headers; + } + + /** + * Decode RFC 2047 encoded words in headers + * @param text Text that may contain encoded words like =?UTF-8?Q?Hello_World?= + */ + static decodeEncodedWords(text: string): string { + if (!text) return ''; + + // Simple RFC 2047 encoded-word decoder + return text.replace(/=\?([^?]+)\?([BQ])\?([^?]+)\?=/gi, (_, charset, encoding, text) => { + if (encoding.toUpperCase() === 'B') { + // Base64 encoding + try { + const decoded = atob(text); + return decoded; + } catch (e) { + return text; + } + } else if (encoding.toUpperCase() === 'Q') { + // Quoted-printable encoding + return this.decodeQuotedPrintable(text.replace(/_/g, ' ')); + } + return text; + }); + } + + /** + * Decode quoted-printable encoded text + * @param text Quoted-printable encoded text + */ + private static decodeQuotedPrintable(text: string): string { + return text.replace(/=([0-9A-F]{2})/gi, (_, hex) => { + return String.fromCharCode(parseInt(hex, 16)); + }); + } +} \ No newline at end of file diff --git a/src/utils/feed-generator.ts b/src/utils/feed-generator.ts new file mode 100644 index 0000000..df5b8ba --- /dev/null +++ b/src/utils/feed-generator.ts @@ -0,0 +1,53 @@ +import { Feed } from 'feed'; +import { FeedConfig, EmailData } from '../types'; + +/** + * Generate an RSS feed from a list of emails + */ +export function generateRssFeed( + feedConfig: FeedConfig, + emails: EmailData[], + baseUrl: string +): string { + // Create a new feed + const feed = new Feed({ + title: feedConfig.title, + description: feedConfig.description || '', + id: feedConfig.feed_url, + link: feedConfig.site_url, + language: feedConfig.language, + updated: new Date(), + generator: 'Email-to-RSS', + copyright: `Copyright © ${new Date().getFullYear()} ${feedConfig.title}`, + feedLinks: { + rss: feedConfig.feed_url + }, + author: feedConfig.author ? { + name: feedConfig.author, + email: `noreply@${new URL(feedConfig.site_url).hostname}` + } : undefined + }); + + // Add each email as a feed item + for (const email of emails) { + const date = new Date(email.receivedAt); + const uniqueId = `${email.receivedAt}-${Buffer.from(email.subject).toString('base64').substring(0, 10)}`; + + feed.addItem({ + title: email.subject, + id: uniqueId, + link: `${baseUrl}/emails/${uniqueId}`, + description: email.content, + content: email.content, + author: [ + { + name: email.from, + }, + ], + date: date, + }); + } + + // Return the RSS feed as XML + return feed.rss2(); +} \ No newline at end of file diff --git a/src/utils/storage.ts b/src/utils/storage.ts new file mode 100644 index 0000000..ad80b75 --- /dev/null +++ b/src/utils/storage.ts @@ -0,0 +1,136 @@ +import { EmailData, FeedConfig, FeedMetadata, FeedList, EmailMetadata } from '../types'; + +/** + * Store email data in KV + */ +export async function storeEmail( + kv: KVNamespace, + feedId: string, + emailData: EmailData +): Promise { + // Generate a unique key for this email + const timestamp = Date.now(); + const key = `feed:${feedId}:email:${timestamp}`; + + // Store the email content + await kv.put(key, JSON.stringify(emailData)); + + // Update the feed's metadata (list of emails) + await updateFeedMetadata(kv, feedId, { + key, + subject: emailData.subject, + receivedAt: timestamp + }); + + return key; +} + +/** + * Update feed metadata with a new email + */ +async function updateFeedMetadata( + kv: KVNamespace, + feedId: string, + emailMetadata: EmailMetadata +): Promise { + const feedMetadataKey = `feed:${feedId}:metadata`; + const existingMetadata = await kv.get(feedMetadataKey, { type: 'json' }) as FeedMetadata | null; + + const metadata: FeedMetadata = existingMetadata || { emails: [] }; + + // Add new email to the beginning of the list + metadata.emails.unshift(emailMetadata); + + // Keep only the last 50 emails in the metadata + if (metadata.emails.length > 50) { + metadata.emails = metadata.emails.slice(0, 50); + } + + // Store updated metadata + await kv.put(feedMetadataKey, JSON.stringify(metadata)); +} + +/** + * Get feed metadata + */ +export async function getFeedMetadata( + kv: KVNamespace, + feedId: string +): Promise { + const feedMetadataKey = `feed:${feedId}:metadata`; + return await kv.get(feedMetadataKey, { type: 'json' }) as FeedMetadata | null; +} + +/** + * Get feed configuration + */ +export async function getFeedConfig( + kv: KVNamespace, + feedId: string +): Promise { + const feedConfigKey = `feed:${feedId}:config`; + return await kv.get(feedConfigKey, { type: 'json' }) as FeedConfig | null; +} + +/** + * Get email data + */ +export async function getEmailData( + kv: KVNamespace, + key: string +): Promise { + return await kv.get(key, { type: 'json' }) as EmailData | null; +} + +/** + * Create a new feed + */ +export async function createFeed( + kv: KVNamespace, + feedId: string, + feedConfig: FeedConfig +): Promise { + // Store feed configuration + const feedConfigKey = `feed:${feedId}:config`; + await kv.put(feedConfigKey, JSON.stringify(feedConfig)); + + // Create empty metadata for the feed + const feedMetadataKey = `feed:${feedId}:metadata`; + await kv.put(feedMetadataKey, JSON.stringify({ + emails: [] + })); + + // Add feed to the list of all feeds + await addFeedToList(kv, feedId, feedConfig.title); +} + +/** + * Add a feed to the global list + */ +export async function addFeedToList( + kv: KVNamespace, + feedId: string, + title: string +): Promise { + const feedListKey = 'feeds:list'; + const existingList = await kv.get(feedListKey, { type: 'json' }) as FeedList | null; + + const feedList: FeedList = existingList || { feeds: [] }; + + feedList.feeds.push({ + id: feedId, + title + }); + + await kv.put(feedListKey, JSON.stringify(feedList)); +} + +/** + * Get all feeds + */ +export async function getAllFeeds(kv: KVNamespace): Promise { + const feedListKey = 'feeds:list'; + const feedList = await kv.get(feedListKey, { type: 'json' }) as FeedList | null; + + return feedList || { feeds: [] }; +} \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..60334e4 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "ESNext", + "moduleResolution": "node", + "esModuleInterop": true, + "strict": true, + "lib": ["ES2020"], + "types": ["@cloudflare/workers-types"], + "outDir": "dist", + "noEmit": true, + "isolatedModules": true, + "allowJs": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} \ No newline at end of file diff --git a/wrangler.toml b/wrangler.toml new file mode 100644 index 0000000..e909dd8 --- /dev/null +++ b/wrangler.toml @@ -0,0 +1,30 @@ +name = "email-to-rss" +main = "src/index.ts" +compatibility_date = "2024-09-23" +compatibility_flags = ["nodejs_compat"] + +# KV Namespace bindings +kv_namespaces = [ + { binding = "EMAIL_STORAGE", id = "721e2789af9a41eba56a77e7891fd85a", preview_id = "b741d3713dd0416ca80b34ae6539736e" } +] + +# Environment variables +[vars] +ADMIN_PASSWORD = "" # Set this using wrangler secret +DOMAIN = "getmynews.app" # Your custom domain for emails + +# Development environment +[env.dev] +# Add any development-specific configuration here +workers_dev = true + +# Production environment +[env.production] +# Add any production-specific configuration here +workers_dev = false +routes = [ + "https://api.getmynews.app/api/*", + "https://api.getmynews.app/rss/*", + "https://getmynews.app/*", + "https://www.getmynews.app/*" +] \ No newline at end of file