From 2da8b065d08438aa2adea397d8ab2a560e996af6 Mon Sep 17 00:00:00 2001 From: Dries Augustyns Date: Tue, 21 Apr 2026 21:28:31 +0200 Subject: [PATCH] feat: implement phishing detection using OpenRouter API with configurable sampling rate --- CLAUDE.md | 4 + apps/api/src/app/constants.ts | 7 + apps/api/src/jobs/email-processor.ts | 31 ++++ apps/api/src/services/SecurityService.ts | 214 ++++++++++++++++++++++- turbo.json | 15 +- 5 files changed, 267 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 5aa15fb..9bdaf8a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -154,6 +154,10 @@ Required for builds and deployment (see turbo.json and .env.example): plus-addressing, domain existence, and MX records - Security (optional): `AUTO_PROJECT_DISABLE` (default: true) - Controls whether projects are automatically disabled when bounce/complaint rate thresholds are exceeded +- Phishing Detection (optional): + - `OPENROUTER_API_KEY` - API key for OpenRouter (enables phishing detection) + - `OPENROUTER_MODEL` (default: anthropic/claude-3-haiku) - LLM model to use for content analysis + - `PHISHING_DETECTION_SAMPLE_RATE` (default: 0.1) - Percentage of emails to check (0.0-1.0, e.g., 0.1 = 10%) **Important Notes:** diff --git a/apps/api/src/app/constants.ts b/apps/api/src/app/constants.ts index 06c5b2f..f260bb3 100644 --- a/apps/api/src/app/constants.ts +++ b/apps/api/src/app/constants.ts @@ -115,3 +115,10 @@ export const TOKEN_EXPIRY_SECONDS = 3600; // 1 hour export const EMAIL_VERIFICATION_RATE_LIMIT = 3; // Max 3 emails per hour export const PASSWORD_RESET_RATE_LIMIT = 3; // Max 3 emails per hour export const EMAIL_VERIFICATION_RATE_WINDOW = 3600; // 1 hour in seconds + +// Phishing Detection (optional) +// OpenRouter API integration for content safety checks +export const OPENROUTER_API_KEY = validateEnv('OPENROUTER_API_KEY', ''); +export const OPENROUTER_MODEL = validateEnv('OPENROUTER_MODEL', 'anthropic/claude-3-haiku'); +export const PHISHING_DETECTION_SAMPLE_RATE = Number(validateEnv('PHISHING_DETECTION_SAMPLE_RATE', '0.1')); // Default 10% of emails +export const PHISHING_DETECTION_ENABLED = OPENROUTER_API_KEY !== ''; diff --git a/apps/api/src/jobs/email-processor.ts b/apps/api/src/jobs/email-processor.ts index bf78a6a..e245c66 100644 --- a/apps/api/src/jobs/email-processor.ts +++ b/apps/api/src/jobs/email-processor.ts @@ -14,6 +14,7 @@ import {EmailService} from '../services/EmailService.js'; import {EventService} from '../services/EventService.js'; import {MeterService} from '../services/MeterService.js'; import {emailQueue} from '../services/QueueService.js'; +import {SecurityService} from '../services/SecurityService.js'; import {getSendingQuota, sendRawEmail} from '../services/SESService.js'; /** @@ -146,6 +147,36 @@ export async function createEmailWorker() { // Determine tracking based on project settings and email type const shouldTrack = EmailService.shouldTrackEmail(email.project.tracking, email.sourceType); + // Check for phishing/dangerous content before sending + const phishingCheck = await SecurityService.checkPhishingContent( + email.projectId, + formattedEmail.subject, + compiledHtml, + ); + + if (phishingCheck.shouldDisable) { + // Disable project immediately + await SecurityService.disableProjectForPhishing( + email.projectId, + formattedEmail.subject, + phishingCheck.confidence, + 'Phishing content detected', + ); + + // Mark email as failed + await prisma.email.update({ + where: {id: emailId}, + data: { + status: EmailStatus.FAILED, + error: `Phishing content detected with ${phishingCheck.confidence}% confidence - project disabled`, + }, + }); + + throw new Error( + `Phishing content detected with ${phishingCheck.confidence}% confidence - project ${email.projectId} disabled`, + ); + } + // Send via AWS SES const result = await sendRawEmail({ from: { diff --git a/apps/api/src/services/SecurityService.ts b/apps/api/src/services/SecurityService.ts index 5281da2..9fbd889 100644 --- a/apps/api/src/services/SecurityService.ts +++ b/apps/api/src/services/SecurityService.ts @@ -10,7 +10,15 @@ import {Keys} from './keys.js'; import {MembershipService} from './MembershipService.js'; import {NtfyService} from './NtfyService.js'; import {QueueService} from './QueueService.js'; -import {AUTO_PROJECT_DISABLE, DASHBOARD_URI, LANDING_URI} from '../app/constants.js'; +import { + AUTO_PROJECT_DISABLE, + DASHBOARD_URI, + LANDING_URI, + OPENROUTER_API_KEY, + OPENROUTER_MODEL, + PHISHING_DETECTION_ENABLED, + PHISHING_DETECTION_SAMPLE_RATE, +} from '../app/constants.js'; /** * Security thresholds for bounce and complaint rates @@ -699,4 +707,208 @@ export class SecurityService { signale.error(`[SECURITY] Failed to disable project ${projectId}:`, error); } } + + /** + * Check email content for phishing or dangerous content using LLM + * Uses sampling to reduce costs - only checks a percentage of emails + * @returns { isPhishing: boolean, confidence: number, shouldDisable: boolean } + */ + public static async checkPhishingContent( + projectId: string, + subject: string, + body: string, + ): Promise<{isPhishing: boolean; confidence: number; shouldDisable: boolean}> { + // Default safe response + const safeResponse = {isPhishing: false, confidence: 0, shouldDisable: false}; + + try { + // Check if phishing detection is enabled + if (!PHISHING_DETECTION_ENABLED) { + return safeResponse; + } + + // Sample-based checking - only check a percentage of emails + if (Math.random() > PHISHING_DETECTION_SAMPLE_RATE) { + return safeResponse; + } + + signale.info(`[PHISHING] Checking email for project ${projectId} (sampled)`); + + // Strip HTML tags for content analysis (basic HTML removal) + const strippedBody = body + .replace(/<[^>]*>/g, ' ') + .replace(/\s+/g, ' ') + .trim(); + + // Call OpenRouter API + const response = await fetch('https://openrouter.ai/api/v1/chat/completions', { + method: 'POST', + headers: { + 'Authorization': `Bearer ${OPENROUTER_API_KEY}`, + 'Content-Type': 'application/json', + 'HTTP-Referer': 'https://useplunk.com', + 'X-Title': 'Plunk Email Platform', + }, + body: JSON.stringify({ + model: OPENROUTER_MODEL, + messages: [ + { + role: 'system', + content: `You are a security expert analyzing emails for phishing, scams, and dangerous content. Analyze the email subject and body and respond ONLY with a JSON object in this exact format: +{ + "is_phishing": true/false, + "confidence": 0-100, + "reason": "brief explanation" +} + +Criteria for phishing/dangerous content: +- Credential theft attempts (fake login pages, password requests) +- Financial scams (fake invoices, lottery scams, advance-fee fraud) +- Malicious links or attachments references +- Impersonation of banks, government, or well-known companies +- Urgent threats or fear-based manipulation +- Suspicious cryptocurrency or investment schemes +- Requests for sensitive personal information + +Be strict but fair. Marketing emails and legitimate transactional emails are NOT phishing. +Only flag content that is CLEARLY attempting to deceive or harm recipients. +Set confidence to 100 only if you are absolutely certain it's phishing.`, + }, + { + role: 'user', + content: `Subject: ${subject}\n\nBody:\n${strippedBody.substring(0, 2000)}`, + }, + ], + temperature: 0.1, + max_tokens: 200, + }), + }); + + if (!response.ok) { + signale.error(`[PHISHING] OpenRouter API error: ${response.status} ${response.statusText}`); + return safeResponse; + } + + const data = (await response.json()) as { + choices?: Array<{message?: {content?: string}}>; + }; + + const content = data.choices?.[0]?.message?.content; + if (!content) { + signale.error('[PHISHING] No content in OpenRouter response'); + return safeResponse; + } + + // Parse the JSON response + const result = JSON.parse(content) as { + is_phishing: boolean; + confidence: number; + reason: string; + }; + + const isPhishing = result.is_phishing === true; + const confidence = Math.min(100, Math.max(0, result.confidence || 0)); + + if (isPhishing) { + signale.warn( + `[PHISHING] Detected phishing content for project ${projectId} - Confidence: ${confidence}% - Reason: ${result.reason}`, + ); + } + + // Auto-disable project if 100% confidence + const shouldDisable = isPhishing && confidence === 100; + + if (shouldDisable) { + signale.error( + `[PHISHING] High confidence phishing detected (${confidence}%) - will disable project ${projectId}`, + ); + } + + return { + isPhishing, + confidence, + shouldDisable, + }; + } catch (error) { + // Log error but don't throw - we don't want phishing checks to break email sending + // Better to let a phishing email through than to break legitimate emails + signale.error(`[PHISHING] Failed to check content for project ${projectId}:`, error); + return safeResponse; + } + } + + /** + * Disable a project due to phishing detection + */ + public static async disableProjectForPhishing( + projectId: string, + subject: string, + confidence: number, + reason?: string, + ): Promise { + try { + // Check if already disabled to avoid duplicate logs + const project = await prisma.project.findUnique({ + where: {id: projectId}, + select: {id: true, disabled: true, name: true}, + }); + + if (!project) { + signale.error(`[PHISHING] Project ${projectId} not found`); + return; + } + + if (project.disabled) { + signale.warn(`[PHISHING] Project ${projectId} (${project.name}) already disabled`); + return; + } + + // Disable the project + await prisma.project.update({ + where: {id: projectId}, + data: {disabled: true}, + }); + + const violation = `Phishing content detected with ${confidence}% confidence in email: "${subject}"${reason ? ` - ${reason}` : ''}`; + + // Log critical security event + signale.error( + `[PHISHING] Project ${projectId} (${project.name}) has been automatically disabled due to phishing detection:`, + violation, + ); + + // Cancel all pending jobs for this project + try { + await QueueService.cancelAllProjectJobs(projectId); + signale.info(`[PHISHING] Cancelled all pending jobs for project ${projectId}`); + } catch (error) { + signale.error(`[PHISHING] Failed to cancel pending jobs for project ${projectId}:`, error); + } + + // Send urgent notification about project suspension + await NtfyService.notifyProjectDisabledForSecurity(project.name, projectId, [violation]); + + // Send email notification to project members + try { + const members = await MembershipService.getMembers(projectId); + const emails = members.map(m => m.email); + if (emails.length > 0) { + const template = React.createElement(ProjectDisabledEmail, { + projectName: project.name, + projectId, + violations: [violation], + dashboardUrl: DASHBOARD_URI, + landingUrl: LANDING_URI, + }); + await Promise.all( + emails.map(email => sendPlatformEmail(email, 'Project Disabled - Phishing Detected', template)), + ); + } + } catch (emailError) { + signale.error(`[PHISHING] Failed to send project disabled email:`, emailError); + } + } catch (error) { + signale.error(`[PHISHING] Failed to disable project ${projectId}:`, error); + } + } } diff --git a/turbo.json b/turbo.json index e8e1fe9..f232387 100644 --- a/turbo.json +++ b/turbo.json @@ -77,7 +77,10 @@ "PORT_SUBMISSION", "AUTO_PROJECT_DISABLE", "DISABLE_SIGNUPS", - "VERIFY_EMAIL_ON_SIGNUP" + "VERIFY_EMAIL_ON_SIGNUP", + "OPENROUTER_API_KEY", + "OPENROUTER_MODEL", + "PHISHING_DETECTION_SAMPLE_RATE" ], "inputs": ["$TURBO_DEFAULT$", ".env*"] }, @@ -137,7 +140,10 @@ "PORT_SUBMISSION", "AUTO_PROJECT_DISABLE", "DISABLE_SIGNUPS", - "VERIFY_EMAIL_ON_SIGNUP" + "VERIFY_EMAIL_ON_SIGNUP", + "OPENROUTER_API_KEY", + "OPENROUTER_MODEL", + "PHISHING_DETECTION_SAMPLE_RATE" ] }, "dev:worker": { @@ -188,7 +194,10 @@ "PORT_SUBMISSION", "AUTO_PROJECT_DISABLE", "DISABLE_SIGNUPS", - "VERIFY_EMAIL_ON_SIGNUP" + "VERIFY_EMAIL_ON_SIGNUP", + "OPENROUTER_API_KEY", + "OPENROUTER_MODEL", + "PHISHING_DETECTION_SAMPLE_RATE" ] }, "clean": {