Compare commits
21
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2700e62935 | ||
|
|
517b93cc95 | ||
|
|
8e3fb9595d | ||
|
|
58be4abc31 | ||
|
|
480f0c1034 | ||
|
|
80beb2bb99 | ||
|
|
6ab4d77ca9 | ||
|
|
edfc399061 | ||
|
|
4a145f3488 | ||
|
|
868bdee615 | ||
|
|
6ebbb50f68 | ||
|
|
87eb56ff36 | ||
|
|
a348d37c21 | ||
|
|
5c166797b5 | ||
|
|
971b98a4cc | ||
|
|
81315eac8e | ||
|
|
1c1c95d332 | ||
|
|
844be42151 | ||
|
|
d59f4a10cd | ||
|
|
51ac88b9f5 | ||
|
|
523bcad602 |
@@ -163,6 +163,19 @@ SMTP_DOMAIN=smtp.example.com
|
||||
# Default: unset (auto-detect, falls back to 14)
|
||||
# EMAIL_RATE_LIMIT_PER_SECOND=1
|
||||
|
||||
# Number of emails the worker processes in parallel. When unset, concurrency is
|
||||
# derived from the effective rate limit (~ rate * 0.5, min 5, capped by
|
||||
# EMAIL_WORKER_MAX_CONCURRENCY) so a higher SES quota translates into higher
|
||||
# throughput automatically. Pin this only when the Prisma pool or memory is the
|
||||
# binding constraint.
|
||||
# Default: unset (auto-derived)
|
||||
# EMAIL_WORKER_CONCURRENCY=10
|
||||
|
||||
# Upper bound applied to the auto-derived concurrency. Raise this when your SES
|
||||
# quota is high AND the Prisma connection pool has been sized for it.
|
||||
# Default: 50
|
||||
# EMAIL_WORKER_MAX_CONCURRENCY=50
|
||||
|
||||
# ========================================
|
||||
# ADVANCED (rarely needed)
|
||||
# ========================================
|
||||
|
||||
@@ -161,7 +161,7 @@ Required for builds and deployment (see turbo.json and .env.example):
|
||||
- `OPENROUTER_API_KEY` - API key for OpenRouter (enables phishing detection)
|
||||
- `OPENROUTER_MODEL` (default: anthropic/claude-3-haiku) - LLM model to use for content analysis
|
||||
- `PHISHING_DETECTION_SAMPLE_RATE` (default: 0.1) - Percentage of emails to check (0.0-1.0, e.g., 0.1 = 10%)
|
||||
- `PHISHING_CONFIDENCE_THRESHOLD` (default: 85) - Minimum confidence percentage (0-100) to auto-disable project for single detection
|
||||
- `PHISHING_CONFIDENCE_THRESHOLD` (default: 95) - Minimum confidence percentage (0-100) to auto-disable project for single detection
|
||||
- `PHISHING_CUMULATIVE_THRESHOLD` (default: 3) - Number of phishing detections within time window to trigger auto-disable
|
||||
- `PHISHING_CUMULATIVE_WINDOW_MS` (default: 3600000) - Time window in milliseconds for cumulative tracking (default 1 hour)
|
||||
|
||||
@@ -174,6 +174,21 @@ Required for builds and deployment (see turbo.json and .env.example):
|
||||
- **Frontend Variables**: Next.js apps use `NEXT_PUBLIC_*` prefixed variables that are embedded at build time for
|
||||
client-side access
|
||||
|
||||
## Environment Variable Changes
|
||||
|
||||
When you add, rename, remove, or change the default/behaviour of any environment variable, you MUST update all THREE of the following in the same change:
|
||||
|
||||
1. `apps/api/.env.example` — local development defaults
|
||||
2. `.env.self-host.example` — self-hosting / production template
|
||||
3. `apps/wiki/content/docs/self-hosting/environment-variables.mdx` — user-facing reference
|
||||
|
||||
Rules:
|
||||
|
||||
- If the variable already exists in any file, **modify** its line/row/description — do not duplicate or leave a stale entry.
|
||||
- Keep section/category names consistent across all three files (e.g. "AWS SES", "Phishing Detection").
|
||||
- For dev-only or self-host-only variables, still mention them in the wiki and note the scope; only skip the example file where the variable is genuinely never applicable.
|
||||
- When in doubt about whether a variable belongs in `apps/api/.env.example` (development), include it commented out with a short note.
|
||||
|
||||
## Plugins
|
||||
|
||||
There are two plugins installed for you to use.
|
||||
|
||||
@@ -8,6 +8,8 @@
|
||||
# ==============================================================================
|
||||
NODE_ENV=development
|
||||
JWT_SECRET=hBx9Xh8J6KOMAGAsSjvcZJBT5TWyIkFX
|
||||
# Port the API server listens on (default: 8080)
|
||||
# PORT=8080
|
||||
|
||||
# ==============================================================================
|
||||
# Application URLs
|
||||
@@ -25,6 +27,8 @@ LANDING_URI=http://localhost:4000
|
||||
# ==============================================================================
|
||||
# API key for authenticating with the Plunk API (obtained from dashboard)
|
||||
PLUNK_API_KEY=
|
||||
# From address used for platform notification emails (project disabled, billing limits, etc.)
|
||||
# PLUNK_FROM_ADDRESS=
|
||||
|
||||
# ==============================================================================
|
||||
# Database & Redis
|
||||
@@ -61,6 +65,17 @@ SES_CONFIGURATION_SET_NO_TRACKING=plunk-configuration-set-no-tracking # Optiona
|
||||
# Default: Fetched from AWS (typically 14 for sandbox, higher for production accounts)
|
||||
# EMAIL_RATE_LIMIT_PER_SECOND=14
|
||||
|
||||
# Email worker concurrency (number of emails processed in parallel)
|
||||
# If not set, derived from the effective rate limit (~ rate * 0.5, min 5, capped
|
||||
# by EMAIL_WORKER_MAX_CONCURRENCY). Set this to pin a fixed value when the
|
||||
# Prisma connection pool or memory is the binding constraint.
|
||||
# EMAIL_WORKER_CONCURRENCY=10
|
||||
|
||||
# Upper bound for auto-derived worker concurrency
|
||||
# Raise this when your SES quota is high AND the Prisma pool has been sized for it.
|
||||
# Default: 50
|
||||
# EMAIL_WORKER_MAX_CONCURRENCY=50
|
||||
|
||||
# ==============================================================================
|
||||
# OAuth (Optional - for social login)
|
||||
# ==============================================================================
|
||||
@@ -85,3 +100,46 @@ STRIPE_METER_EVENT_NAME=emails # Meter event name (API key from your Stripe mete
|
||||
# Set to 'false' to disable automatic project suspension (useful for self-hosters who manage manually)
|
||||
# Default: true (automatic project disabling enabled)
|
||||
# AUTO_PROJECT_DISABLE=true
|
||||
|
||||
# ==============================================================================
|
||||
# Notifications (Optional - system notifications via ntfy)
|
||||
# ==============================================================================
|
||||
# ntfy topic URL for internal system notifications (e.g. project disabled, billing limits).
|
||||
# When unset, ntfy notifications are disabled.
|
||||
# Examples:
|
||||
# - Public ntfy.sh: https://ntfy.sh/your-unique-topic-name
|
||||
# - Self-hosted: https://your-ntfy-server.com/your-topic
|
||||
# NTFY_URL=
|
||||
|
||||
# ==============================================================================
|
||||
# Attachments (Optional)
|
||||
# ==============================================================================
|
||||
# Limits applied to attachments on transactional emails.
|
||||
# AWS SES caps total message size at 40 MB; the defaults below leave headroom.
|
||||
# MAX_ATTACHMENT_SIZE_MB=10
|
||||
# MAX_ATTACHMENTS_COUNT=10
|
||||
|
||||
# ==============================================================================
|
||||
# User Management (Optional)
|
||||
# ==============================================================================
|
||||
# When 'true', the signup endpoint rejects new user registrations.
|
||||
# Default: false
|
||||
# DISABLE_SIGNUPS=false
|
||||
# When 'true', validates emails on signup (disposable domains, plus-addressing,
|
||||
# domain existence, MX records).
|
||||
# Default: false
|
||||
# VERIFY_EMAIL_ON_SIGNUP=false
|
||||
|
||||
# ==============================================================================
|
||||
# Phishing Detection (Optional - AI-powered phishing scan via OpenRouter)
|
||||
# ==============================================================================
|
||||
# When OPENROUTER_API_KEY is set, a random sample of outgoing emails is
|
||||
# analyzed by an LLM. Projects can be auto-disabled if a single email exceeds
|
||||
# PHISHING_CONFIDENCE_THRESHOLD, or if PHISHING_CUMULATIVE_THRESHOLD emails are
|
||||
# flagged within PHISHING_CUMULATIVE_WINDOW_MS.
|
||||
# OPENROUTER_API_KEY=
|
||||
# OPENROUTER_MODEL=anthropic/claude-3-haiku
|
||||
# PHISHING_DETECTION_SAMPLE_RATE=0.1
|
||||
# PHISHING_CONFIDENCE_THRESHOLD=95
|
||||
# PHISHING_CUMULATIVE_THRESHOLD=3
|
||||
# PHISHING_CUMULATIVE_WINDOW_MS=3600000
|
||||
|
||||
@@ -58,6 +58,20 @@ export const EMAIL_RATE_LIMIT_PER_SECOND = process.env.EMAIL_RATE_LIMIT_PER_SECO
|
||||
? Number(process.env.EMAIL_RATE_LIMIT_PER_SECOND)
|
||||
: undefined;
|
||||
|
||||
// Email Worker Concurrency (optional override)
|
||||
// If not set, concurrency is derived from the effective rate limit so a higher
|
||||
// SES quota actually translates into higher throughput. Set this to pin a fixed
|
||||
// value (useful when Prisma pool size or memory is the binding constraint).
|
||||
export const EMAIL_WORKER_CONCURRENCY = process.env.EMAIL_WORKER_CONCURRENCY
|
||||
? Number(process.env.EMAIL_WORKER_CONCURRENCY)
|
||||
: undefined;
|
||||
|
||||
// Upper bound for auto-derived concurrency. Raise this if you have a large SES
|
||||
// quota AND have sized the Prisma connection pool accordingly.
|
||||
export const EMAIL_WORKER_MAX_CONCURRENCY = process.env.EMAIL_WORKER_MAX_CONCURRENCY
|
||||
? Number(process.env.EMAIL_WORKER_MAX_CONCURRENCY)
|
||||
: 50;
|
||||
|
||||
// Storage
|
||||
export const REDIS_URL = validateEnv('REDIS_URL');
|
||||
export const DATABASE_URL = validateEnv('DATABASE_URL');
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import {beforeEach, describe, expect, it} from 'vitest';
|
||||
import {factories, getPrismaClient} from '../../../../../test/helpers';
|
||||
import {ContactService} from '../../services/ContactService.js';
|
||||
import {coerceCustomValue} from '../import-processor.js';
|
||||
|
||||
/**
|
||||
* Tests for Contact Import Processor - Subscription Status Preservation
|
||||
@@ -279,3 +280,51 @@ describe('Contact Import - Subscription Status Preservation', () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('coerceCustomValue', () => {
|
||||
describe('boolean coercion', () => {
|
||||
it.each(['true', 'TRUE', 'True', ' true ', 'yes', 'YES', 'Yes'])('coerces %j to true', value => {
|
||||
expect(coerceCustomValue(value)).toBe(true);
|
||||
});
|
||||
|
||||
it.each(['false', 'FALSE', 'False', ' false ', 'no', 'NO', 'No'])('coerces %j to false', value => {
|
||||
expect(coerceCustomValue(value)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('number coercion', () => {
|
||||
it.each([
|
||||
['42', 42],
|
||||
['-7', -7],
|
||||
['3.14', 3.14],
|
||||
[' 42 ', 42],
|
||||
['0.5', 0.5],
|
||||
['-0.25', -0.25],
|
||||
['0', 0],
|
||||
['1', 1],
|
||||
])('coerces %j to %j', (value, expected) => {
|
||||
expect(coerceCustomValue(value)).toBe(expected);
|
||||
});
|
||||
|
||||
it.each(['01234', '+42', '.5', '42.', '1e10', 'NaN', 'Infinity', '1.2.3', '4-2'])(
|
||||
'leaves %j as a string (preserves IDs / rejects loose formats)',
|
||||
value => {
|
||||
expect(coerceCustomValue(value)).toBe(value);
|
||||
},
|
||||
);
|
||||
|
||||
it('"1.0" is a number (does not match the boolean truthy set)', () => {
|
||||
expect(coerceCustomValue('1.0')).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('passthrough', () => {
|
||||
it.each(['Alice', 'true!', 'yesno', 'maybe'])('leaves %j as a string', value => {
|
||||
expect(coerceCustomValue(value)).toBe(value);
|
||||
});
|
||||
|
||||
it('leaves empty string as empty string', () => {
|
||||
expect(coerceCustomValue('')).toBe('');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -8,7 +8,12 @@ import type {SendEmailJobData} from '@plunk/types';
|
||||
import {type Job, Worker} from 'bullmq';
|
||||
import signale from 'signale';
|
||||
|
||||
import {DASHBOARD_URI, EMAIL_RATE_LIMIT_PER_SECOND} from '../app/constants.js';
|
||||
import {
|
||||
DASHBOARD_URI,
|
||||
EMAIL_RATE_LIMIT_PER_SECOND,
|
||||
EMAIL_WORKER_CONCURRENCY,
|
||||
EMAIL_WORKER_MAX_CONCURRENCY,
|
||||
} from '../app/constants.js';
|
||||
import {prisma} from '../database/prisma.js';
|
||||
import {CampaignService} from '../services/CampaignService.js';
|
||||
import {EmailService} from '../services/EmailService.js';
|
||||
@@ -47,9 +52,31 @@ async function getEmailRateLimit(): Promise<number> {
|
||||
return DEFAULT_RATE_LIMIT;
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive worker concurrency from the rate limit so a higher SES quota actually
|
||||
* translates into higher throughput. The mean job duration is ~0.5s (Prisma
|
||||
* reads + HTML compile + SES call + writes), so `rate * 0.5` gives ~2× headroom
|
||||
* over the per-second cap. Clamped to keep sandbox accounts useful and to
|
||||
* protect the Prisma pool on very large quotas.
|
||||
*/
|
||||
function deriveWorkerConcurrency(rateLimit: number): number {
|
||||
if (EMAIL_WORKER_CONCURRENCY !== undefined) {
|
||||
return EMAIL_WORKER_CONCURRENCY;
|
||||
}
|
||||
|
||||
const TARGET_JOB_SECONDS = 0.5;
|
||||
const MIN_CONCURRENCY = 5;
|
||||
const derived = Math.ceil(rateLimit * TARGET_JOB_SECONDS);
|
||||
return Math.max(MIN_CONCURRENCY, Math.min(derived, EMAIL_WORKER_MAX_CONCURRENCY));
|
||||
}
|
||||
|
||||
export async function createEmailWorker() {
|
||||
// Fetch the rate limit (from env, AWS, or default)
|
||||
const rateLimit = await getEmailRateLimit();
|
||||
const concurrency = deriveWorkerConcurrency(rateLimit);
|
||||
signale.info(
|
||||
`[EMAIL-PROCESSOR] Worker concurrency: ${concurrency} (rate limit: ${rateLimit}/s)`,
|
||||
);
|
||||
const worker = new Worker<SendEmailJobData>(
|
||||
emailQueue.name,
|
||||
async (job: Job<SendEmailJobData>) => {
|
||||
@@ -253,7 +280,7 @@ export async function createEmailWorker() {
|
||||
},
|
||||
{
|
||||
connection: emailQueue.opts.connection,
|
||||
concurrency: 10, // Process up to 10 emails concurrently
|
||||
concurrency,
|
||||
limiter: {
|
||||
max: rateLimit, // Max emails per second (from env, AWS SES quota, or default)
|
||||
duration: 1000,
|
||||
|
||||
@@ -123,7 +123,11 @@ export function createImportWorker() {
|
||||
|
||||
// Extract custom data (all fields except email and subscribed)
|
||||
const {email: _, subscribed: __, ...customData} = record;
|
||||
const data = Object.keys(customData).length > 0 ? customData : undefined;
|
||||
const customEntries = Object.entries(customData);
|
||||
const data =
|
||||
customEntries.length > 0
|
||||
? Object.fromEntries(customEntries.map(([k, v]) => [k, coerceCustomValue(v)]))
|
||||
: undefined;
|
||||
|
||||
// Check if contact exists before upserting
|
||||
const existingContact = await ContactService.findByEmail(projectId, email);
|
||||
@@ -216,3 +220,30 @@ function isValidEmail(email: string): boolean {
|
||||
const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
|
||||
return emailRegex.test(email);
|
||||
}
|
||||
|
||||
// Values considered as boolean during import.
|
||||
// Numbers (0, 1) are intentionally absent.
|
||||
const BOOLEAN_TRUE = new Set(['true', 'yes']);
|
||||
const BOOLEAN_FALSE = new Set(['false', 'no']);
|
||||
|
||||
// Strict integer-or-decimal number detection pattern.
|
||||
// Valid: 0, 42, -42, 3.14
|
||||
// Rejected: 007, +42, 1.2.3, 1e5
|
||||
const NUMERIC_RE = /^-?(0|[1-9]\d*)(\.\d+)?$/;
|
||||
|
||||
/**
|
||||
* Coerces a raw string into its most natural primitive type: `boolean`,
|
||||
* `number`, or `string`. Values that match neither are
|
||||
* returned unchanged.
|
||||
*
|
||||
* @param value The raw string to coerce.
|
||||
* @returns The coerced value as `boolean`, `number`, or `string`.
|
||||
*/
|
||||
export function coerceCustomValue(value: string): string | boolean | number {
|
||||
const trimmed = value.trim();
|
||||
const lower = trimmed.toLowerCase();
|
||||
if (BOOLEAN_TRUE.has(lower)) return true;
|
||||
if (BOOLEAN_FALSE.has(lower)) return false;
|
||||
if (NUMERIC_RE.test(trimmed)) return Number(trimmed);
|
||||
return value;
|
||||
}
|
||||
|
||||
@@ -3,6 +3,20 @@ import type {NextFunction, Request, Response} from 'express';
|
||||
import {databaseRequestLogger} from '../requestLogger.js';
|
||||
import {factories, getPrismaClient} from '../../../../../test/helpers';
|
||||
|
||||
async function waitForLog(prisma: ReturnType<typeof getPrismaClient>, id: string, timeoutMs = 2000) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
const record = await prisma.apiRequest.findUnique({where: {id}});
|
||||
if (record) return record;
|
||||
await new Promise(resolve => setTimeout(resolve, 20));
|
||||
}
|
||||
return prisma.apiRequest.findUnique({where: {id}});
|
||||
}
|
||||
|
||||
async function waitForNoLog(ms = 200) {
|
||||
await new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
describe('Request Logger Middleware', () => {
|
||||
const prisma = getPrismaClient();
|
||||
let req: Partial<Request>;
|
||||
@@ -77,13 +91,7 @@ describe('Request Logger Middleware', () => {
|
||||
const responseBody = {success: true, data: {id: '123'}};
|
||||
await res.json!(responseBody);
|
||||
|
||||
// Wait for async logging to complete
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
// Verify database record was created
|
||||
const loggedRequest = await prisma.apiRequest.findUnique({
|
||||
where: {id: 'test-request-id-123'},
|
||||
});
|
||||
const loggedRequest = await waitForLog(prisma, 'test-request-id-123');
|
||||
|
||||
expect(loggedRequest).toBeDefined();
|
||||
expect(loggedRequest?.method).toBe('POST');
|
||||
@@ -109,11 +117,7 @@ describe('Request Logger Middleware', () => {
|
||||
const responseBody = {success: true};
|
||||
await res.json!(responseBody);
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
const loggedRequest = await prisma.apiRequest.findUnique({
|
||||
where: {id: 'public-request-id'},
|
||||
});
|
||||
const loggedRequest = await waitForLog(prisma, 'public-request-id');
|
||||
|
||||
expect(loggedRequest).toBeDefined();
|
||||
expect(loggedRequest?.projectId).toBeNull();
|
||||
@@ -131,11 +135,7 @@ describe('Request Logger Middleware', () => {
|
||||
|
||||
await res.json!({success: true});
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
const loggedRequest = await prisma.apiRequest.findUnique({
|
||||
where: {id: 'test-request-id-123'},
|
||||
});
|
||||
const loggedRequest = await waitForLog(prisma, 'test-request-id-123');
|
||||
|
||||
// Allow for timer imprecision (especially in CI environments)
|
||||
expect(loggedRequest?.duration).toBeGreaterThanOrEqual(45);
|
||||
@@ -162,11 +162,7 @@ describe('Request Logger Middleware', () => {
|
||||
|
||||
await res.json!(errorResponse);
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
const loggedRequest = await prisma.apiRequest.findUnique({
|
||||
where: {id: 'test-request-id-123'},
|
||||
});
|
||||
const loggedRequest = await waitForLog(prisma, 'test-request-id-123');
|
||||
|
||||
expect(loggedRequest).toBeDefined();
|
||||
expect(loggedRequest?.statusCode).toBe(400);
|
||||
@@ -189,11 +185,7 @@ describe('Request Logger Middleware', () => {
|
||||
|
||||
await res.json!(errorResponse);
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
const loggedRequest = await prisma.apiRequest.findUnique({
|
||||
where: {id: 'test-request-id-123'},
|
||||
});
|
||||
const loggedRequest = await waitForLog(prisma, 'test-request-id-123');
|
||||
|
||||
expect(loggedRequest?.statusCode).toBe(500);
|
||||
expect(loggedRequest?.errorCode).toBe('INTERNAL_SERVER_ERROR');
|
||||
@@ -209,11 +201,7 @@ describe('Request Logger Middleware', () => {
|
||||
error: {code: 'RESOURCE_NOT_FOUND', message: 'Template not found'},
|
||||
});
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
const loggedRequest = await prisma.apiRequest.findUnique({
|
||||
where: {id: 'test-request-id-123'},
|
||||
});
|
||||
const loggedRequest = await waitForLog(prisma, 'test-request-id-123');
|
||||
|
||||
expect(loggedRequest?.statusCode).toBe(404);
|
||||
expect(loggedRequest?.errorCode).toBe('RESOURCE_NOT_FOUND');
|
||||
@@ -306,11 +294,8 @@ describe('Request Logger Middleware', () => {
|
||||
|
||||
databaseRequestLogger(req as Request, res as Response, next);
|
||||
await res.json!({success: true});
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
const loggedRequest = await prisma.apiRequest.findUnique({
|
||||
where: {id: `log-${path.replace(/\//g, '-')}`},
|
||||
});
|
||||
const loggedRequest = await waitForLog(prisma, `log-${path.replace(/\//g, '-')}`);
|
||||
|
||||
expect(loggedRequest).toBeDefined();
|
||||
expect(loggedRequest?.path).toBe(path);
|
||||
@@ -352,17 +337,18 @@ describe('Request Logger Middleware', () => {
|
||||
|
||||
await res.json!({success: true});
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
// Should create a record with generated UUID
|
||||
const allRequests = await prisma.apiRequest.findMany({
|
||||
where: {
|
||||
path: '/v1/send',
|
||||
method: 'POST',
|
||||
},
|
||||
orderBy: {createdAt: 'desc'},
|
||||
take: 1,
|
||||
});
|
||||
// Should create a record with generated UUID — poll for it
|
||||
const deadline = Date.now() + 2000;
|
||||
let allRequests: Awaited<ReturnType<typeof prisma.apiRequest.findMany>> = [];
|
||||
while (Date.now() < deadline) {
|
||||
allRequests = await prisma.apiRequest.findMany({
|
||||
where: {path: '/v1/send', method: 'POST'},
|
||||
orderBy: {createdAt: 'desc'},
|
||||
take: 1,
|
||||
});
|
||||
if (allRequests.length > 0) break;
|
||||
await new Promise(resolve => setTimeout(resolve, 20));
|
||||
}
|
||||
|
||||
expect(allRequests.length).toBeGreaterThan(0);
|
||||
expect(allRequests[0].id).toBeDefined();
|
||||
@@ -418,11 +404,8 @@ describe('Request Logger Middleware', () => {
|
||||
databaseRequestLogger(reqWithSize as Request, resWithId as Response, next);
|
||||
|
||||
await resWithId.json!({success: true});
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
const loggedRequest = await prisma.apiRequest.findUnique({
|
||||
where: {id: 'test-size-5000'},
|
||||
});
|
||||
const loggedRequest = await waitForLog(prisma, 'test-size-5000');
|
||||
|
||||
expect(loggedRequest?.requestSize).toBe(5000);
|
||||
});
|
||||
@@ -445,11 +428,8 @@ describe('Request Logger Middleware', () => {
|
||||
databaseRequestLogger(reqNoSize as Request, resWithId as Response, next);
|
||||
|
||||
await resWithId.json!({success: true});
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
const loggedRequest = await prisma.apiRequest.findUnique({
|
||||
where: {id: 'test-no-size'},
|
||||
});
|
||||
const loggedRequest = await waitForLog(prisma, 'test-no-size');
|
||||
|
||||
expect(loggedRequest?.requestSize).toBeNull();
|
||||
});
|
||||
@@ -474,11 +454,8 @@ describe('Request Logger Middleware', () => {
|
||||
};
|
||||
|
||||
await resLarge.json!(largeResponse);
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
const loggedRequest = await prisma.apiRequest.findUnique({
|
||||
where: {id: 'test-large-response'},
|
||||
});
|
||||
const loggedRequest = await waitForLog(prisma, 'test-large-response');
|
||||
|
||||
const expectedSize = JSON.stringify(largeResponse).length;
|
||||
expect(loggedRequest?.responseSize).toBe(expectedSize);
|
||||
|
||||
@@ -108,7 +108,7 @@ export class EmailService {
|
||||
await BillingLimitService.incrementUsage(params.projectId, EmailSourceType.TRANSACTIONAL);
|
||||
|
||||
// Queue email for sending
|
||||
await this.queueEmail(email.id);
|
||||
await this.queueEmail(email.id, EmailSourceType.TRANSACTIONAL);
|
||||
|
||||
return email;
|
||||
}
|
||||
@@ -172,7 +172,7 @@ export class EmailService {
|
||||
await BillingLimitService.incrementUsage(params.projectId, sourceType);
|
||||
|
||||
// Queue email for sending
|
||||
await this.queueEmail(email.id);
|
||||
await this.queueEmail(email.id, sourceType);
|
||||
|
||||
return email;
|
||||
}
|
||||
@@ -278,7 +278,7 @@ export class EmailService {
|
||||
await BillingLimitService.incrementUsage(params.projectId, sourceType);
|
||||
|
||||
// Queue email for sending
|
||||
await this.queueEmail(email.id);
|
||||
await this.queueEmail(email.id, sourceType);
|
||||
|
||||
return email;
|
||||
}
|
||||
@@ -1137,7 +1137,7 @@ export class EmailService {
|
||||
* Queue an email for sending
|
||||
* Adds email to the BullMQ queue for processing by workers
|
||||
*/
|
||||
private static async queueEmail(emailId: string, delay?: number): Promise<void> {
|
||||
await QueueService.queueEmail(emailId, delay);
|
||||
private static async queueEmail(emailId: string, sourceType: EmailSourceType, delay?: number): Promise<void> {
|
||||
await QueueService.queueEmail(emailId, sourceType, delay);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import {CampaignStatus, EmailStatus} from '@plunk/db';
|
||||
import {CampaignStatus, EmailSourceType, EmailStatus} from '@plunk/db';
|
||||
import {type Job, Queue} from 'bullmq';
|
||||
import type {RedisOptions} from 'ioredis';
|
||||
import signale from 'signale';
|
||||
@@ -174,20 +174,43 @@ export const meterQueue = new Queue<MeterEventJobData>('meter', {
|
||||
},
|
||||
});
|
||||
|
||||
function emailPriorityFor(sourceType: EmailSourceType): number {
|
||||
switch (sourceType) {
|
||||
case EmailSourceType.TRANSACTIONAL:
|
||||
return 1;
|
||||
case EmailSourceType.WORKFLOW:
|
||||
return 5;
|
||||
case EmailSourceType.CAMPAIGN:
|
||||
return 10;
|
||||
default:
|
||||
return 5;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue Service - Centralized queue management
|
||||
*/
|
||||
export class QueueService {
|
||||
/**
|
||||
* Add email to queue for sending
|
||||
* Add email to queue for sending.
|
||||
*
|
||||
* Transactional emails jump the queue ahead of workflow and campaign sends
|
||||
* via BullMQ's priority (lower number = higher precedence). This prevents
|
||||
* latency-sensitive sends (login codes, password resets) from queuing behind
|
||||
* large campaign bursts on the shared `email` queue.
|
||||
*/
|
||||
public static async queueEmail(emailId: string, delay?: number): Promise<Job<SendEmailJobData>> {
|
||||
public static async queueEmail(
|
||||
emailId: string,
|
||||
sourceType: EmailSourceType,
|
||||
delay?: number,
|
||||
): Promise<Job<SendEmailJobData>> {
|
||||
return emailQueue.add(
|
||||
'send-email',
|
||||
{emailId},
|
||||
{
|
||||
delay, // Optional delay in milliseconds
|
||||
jobId: `email-${emailId}`, // Prevent duplicate jobs
|
||||
delay,
|
||||
jobId: `email-${emailId}`,
|
||||
priority: emailPriorityFor(sourceType),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -765,7 +765,36 @@ export class SecurityService {
|
||||
const uniqueUrls = [...new Set(urlMatches.map(u => u.replace(/[.,;)]+$/, '')))].slice(0, 20);
|
||||
|
||||
// Extract sender domain for context
|
||||
const senderDomain = fromEmail.includes('@') ? fromEmail.split('@')[1] : fromEmail;
|
||||
const senderDomain = (fromEmail.split('@')[1] ?? fromEmail).toLowerCase();
|
||||
|
||||
// Check whether this domain is verified by the project. A verified
|
||||
// domain means the sender proved DNS/DKIM control — strong evidence of
|
||||
// legitimacy, especially for institutional TLDs like .gov, .edu, .mil.
|
||||
const verifiedDomain = await prisma.domain.findFirst({
|
||||
where: {projectId, domain: senderDomain, verified: true},
|
||||
select: {domain: true},
|
||||
});
|
||||
const isDomainVerified = verifiedDomain !== null;
|
||||
|
||||
// Institutional TLDs that imply a vetted, real-world entity behind the
|
||||
// domain (government, military, accredited education). When combined
|
||||
// with DKIM verification these effectively cannot be phishing senders.
|
||||
const institutionalTldPattern =
|
||||
/\.(gov|mil|edu)(\.[a-z]{2,})?$|\.gc\.ca$|\.gouv\.fr$|\.gov\.uk$|\.ac\.[a-z]{2,}$/i;
|
||||
const isInstitutionalDomain = institutionalTldPattern.test(senderDomain);
|
||||
|
||||
// Skip the LLM check entirely when the sender is a verified institutional
|
||||
// domain (e.g. a .gov customer). These TLDs are gated by registries that
|
||||
// verify the real-world entity, and DKIM verification proves the project
|
||||
// controls the domain — together they make phishing effectively
|
||||
// impossible from this sender. Avoids paying for an LLM call that
|
||||
// sometimes false-positives on official government communications.
|
||||
if (isDomainVerified && isInstitutionalDomain) {
|
||||
signale.info(
|
||||
`[PHISHING] Skipping check for project ${projectId} — verified institutional domain (${senderDomain})`,
|
||||
);
|
||||
return safeResponse;
|
||||
}
|
||||
|
||||
// Call OpenRouter API
|
||||
const response = await fetch('https://openrouter.ai/api/v1/chat/completions', {
|
||||
@@ -798,7 +827,11 @@ Criteria for phishing/dangerous content:
|
||||
- Requests for sensitive personal information
|
||||
|
||||
IMPORTANT - Use sender and project context when evaluating:
|
||||
- The sender project name and domain are provided. Links to the sender's own domain(s) are expected and NOT suspicious.
|
||||
- The sender project name and domain are provided, along with whether the domain has been verified (DKIM/DNS) by this project.
|
||||
- A VERIFIED sender domain means the sender proved ownership of the domain via DNS records. This is strong evidence of legitimacy.
|
||||
- If the verified sender domain is an institutional domain (e.g. .gov, .gov.uk, .gouv.fr, .gc.ca, .mil, .edu, .ac.*), treat the email as legitimate institutional communication. Government, military, and accredited education domains cannot be obtained by phishers — do NOT flag these as impersonation of government/banks/etc. just because the content mentions official topics, taxes, benefits, court notices, etc.
|
||||
- Impersonation rules only apply when the sender domain does NOT match the brand being referenced. A verified bank domain sending a banking email is not impersonating itself.
|
||||
- Links to the sender's own domain(s) are expected and NOT suspicious.
|
||||
- URLs that match or are clearly related to the project name or sender domain add credibility.
|
||||
- Only flag a URL as suspicious if it is unrelated to or impersonates a different known brand.
|
||||
- Lack of recognizable brand does NOT make an email phishing — many legitimate businesses are not famous.
|
||||
@@ -811,6 +844,8 @@ Set confidence to 100 only if you are absolutely certain it's phishing.`,
|
||||
role: 'user',
|
||||
content: `Sender project name: ${projectName}
|
||||
Sender domain: ${senderDomain}
|
||||
Sender domain verified (DKIM/DNS confirmed by project): ${isDomainVerified ? 'yes' : 'no'}
|
||||
Sender domain is an institutional TLD (gov/mil/edu/ac/etc.): ${isInstitutionalDomain ? 'yes' : 'no'}
|
||||
${uniqueUrls.length > 0 ? `URLs found in email: ${uniqueUrls.join(', ')}` : ''}
|
||||
|
||||
Subject: ${subject}
|
||||
|
||||
@@ -758,8 +758,20 @@ describe('WorkflowService', () => {
|
||||
});
|
||||
const contact = await factories.createContact({projectId});
|
||||
|
||||
// Start first execution (still running)
|
||||
await WorkflowService.startExecution(projectId, workflow.id, contact.id);
|
||||
// Insert a RUNNING execution directly to avoid racing with the background
|
||||
// step processor that startExecution kicks off (a trigger-only workflow can
|
||||
// transition to COMPLETED before the second call observes it as RUNNING).
|
||||
const triggerStep = await prisma.workflowStep.findFirst({
|
||||
where: {workflowId: workflow.id, type: WorkflowStepType.TRIGGER},
|
||||
});
|
||||
await prisma.workflowExecution.create({
|
||||
data: {
|
||||
workflowId: workflow.id,
|
||||
contactId: contact.id,
|
||||
status: WorkflowExecutionStatus.RUNNING,
|
||||
currentStepId: triggerStep?.id,
|
||||
},
|
||||
});
|
||||
|
||||
// Second execution should fail (first still running)
|
||||
await expect(WorkflowService.startExecution(projectId, workflow.id, contact.id)).rejects.toThrow(
|
||||
|
||||
@@ -37,6 +37,8 @@ In this example, every imported contact ends up with `data.firstName`, `data.pla
|
||||
- **Email column**: must be present and valid. Rows with missing or invalid emails are reported back as errors.
|
||||
- **Reserved column names**: `id`, `subscribed`, `createdAt`, `updatedAt`, and the auto-generated URL variables (`unsubscribeUrl`, etc.) are silently filtered out. Don't include them as columns.
|
||||
- **Date columns**: use ISO 8601 (`2026-05-06T12:00:00Z`) so they're typed as dates and become usable with `within` / `olderThan` segment operators.
|
||||
- **Boolean columns**: `true`, `false`, `yes`, `no` (case-insensitive) are stored as booleans and get the boolean toggle in segment filters.
|
||||
- **Numeric columns**: plain integers and decimals (`42`, `3.14`) are stored as numbers and become usable with `gt` / `lt` segment operators. Leading-zero values (`01234`), `+`-prefixed numbers, and scientific notation stay strings so IDs, zip codes, and phone numbers aren't corrupted.
|
||||
- **Existing contacts**: if a row's email matches an existing contact, the import **updates** the contact (merging the CSV's columns into `data`). It doesn't create a duplicate or overwrite the whole record.
|
||||
|
||||
## Importing your CSV
|
||||
|
||||
@@ -36,6 +36,7 @@ Set your subdomains here. The application automatically derives all internal and
|
||||
| `AWS_SES_SECRET_ACCESS_KEY` | Yes | AWS secret access key for SES. | `wJalr...` |
|
||||
| `SES_CONFIGURATION_SET` | No | SES configuration set name used for open/click tracking. | `plunk-configuration-set` (default) |
|
||||
| `SES_CONFIGURATION_SET_NO_TRACKING` | No | A second SES configuration set without tracking. When set, projects can toggle email tracking on/off. If omitted, the tracking toggle is hidden. | `plunk-no-tracking-configuration-set` (default) |
|
||||
| `MAIL_FROM_SUBDOMAIN` | No | Subdomain prefix used when constructing the MAIL FROM hostname for a verified domain (e.g. with default `plunk` and domain `yourdomain.com`, the MAIL FROM is `plunk.yourdomain.com`). Override when the default subdomain is already in use (e.g. by an R2/CDN custom domain), since the MAIL FROM hostname needs MX + TXT records that can't coexist with a CNAME. | `plunk` |
|
||||
|
||||
## Storage (Minio)
|
||||
|
||||
@@ -129,6 +130,16 @@ Plunk bundles a self-hosted [ntfy](https://ntfy.sh) server for internal system n
|
||||
| ----------------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- |
|
||||
| `AUTO_PROJECT_DISABLE` | No | When `true`, projects are automatically suspended when bounce or complaint rate thresholds are exceeded. Set to `false` to manage project status manually. | `true` |
|
||||
| `EMAIL_RATE_LIMIT_PER_SECOND` | No | Override the email sending rate limit. If not set, Plunk automatically fetches the quota from your AWS SES account. | — |
|
||||
| `EMAIL_WORKER_CONCURRENCY` | No | Number of emails the worker processes in parallel. When unset, derived from the effective rate limit so a higher SES quota scales throughput automatically. | — |
|
||||
| `EMAIL_WORKER_MAX_CONCURRENCY`| No | Upper bound applied to the auto-derived worker concurrency. Raise this only after sizing the Prisma connection pool accordingly. | `50` |
|
||||
|
||||
## Advanced
|
||||
|
||||
Variables for unusual deployments. The defaults work for the standard Docker Compose setup — only change these if you know you need to.
|
||||
|
||||
| Variable | Required | Description | Default |
|
||||
| ------------ | -------- | ------------------------------------------------------------------------------------------------------------------------ | ------- |
|
||||
| `NGINX_PORT` | No | Host port the bundled Nginx reverse proxy binds to. Override when port 80/443 is already in use on the host (e.g. running behind another reverse proxy that forwards to a different port). | `80` |
|
||||
|
||||
## Phishing Detection
|
||||
|
||||
|
||||
+17
-58
@@ -27,39 +27,6 @@ services:
|
||||
# Infrastructure Services
|
||||
# ============================================
|
||||
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
container_name: plunk-postgres
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_DB: plunk
|
||||
POSTGRES_USER: plunk
|
||||
POSTGRES_PASSWORD: ${DB_PASSWORD:-changeme123}
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: [ "CMD-SHELL", "pg_isready -U plunk" ]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
networks:
|
||||
- plunk
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
container_name: plunk-redis
|
||||
restart: unless-stopped
|
||||
command: redis-server --appendonly yes
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
test: [ "CMD", "redis-cli", "ping" ]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
networks:
|
||||
- plunk
|
||||
|
||||
minio:
|
||||
image: minio/minio:latest
|
||||
container_name: plunk-minio
|
||||
@@ -70,10 +37,10 @@ services:
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-plunkminiopass}
|
||||
volumes:
|
||||
- minio_data:/data
|
||||
ports:
|
||||
# Expose Minio API (for S3 operations) and Console (for web UI)
|
||||
- "${MINIO_API_PORT:-9000}:9000"
|
||||
- "${MINIO_CONSOLE_PORT:-9001}:9001"
|
||||
# ports:
|
||||
# # Expose Minio API (for S3 operations) and Console (for web UI)
|
||||
# - "${MINIO_API_PORT:-9000}:9000"
|
||||
# - "${MINIO_CONSOLE_PORT:-9001}:9001"
|
||||
healthcheck:
|
||||
test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
|
||||
interval: 30s
|
||||
@@ -88,13 +55,13 @@ services:
|
||||
restart: unless-stopped
|
||||
command: serve
|
||||
environment:
|
||||
- TZ=UTC
|
||||
- TZ=CST
|
||||
volumes:
|
||||
- ntfy_cache:/var/cache/ntfy
|
||||
- ntfy_etc:/etc/ntfy
|
||||
ports:
|
||||
# Expose ntfy web UI and API
|
||||
- "${NTFY_PORT:-8080}:80"
|
||||
# ports:
|
||||
# # Expose ntfy web UI and API
|
||||
# - "${NTFY_PORT:-8080}:80"
|
||||
healthcheck:
|
||||
test: [ "CMD-SHELL", "wget -q --tries=1 http://localhost:80/v1/health -O - | grep -Eo '\"healthy\"\\s*:\\s*true' || exit 1" ]
|
||||
interval: 30s
|
||||
@@ -118,11 +85,11 @@ services:
|
||||
NODE_ENV: production
|
||||
|
||||
# Database
|
||||
DATABASE_URL: postgresql://plunk:${DB_PASSWORD:-changeme123}@postgres:5432/plunk
|
||||
DIRECT_DATABASE_URL: postgresql://plunk:${DB_PASSWORD:-changeme123}@postgres:5432/plunk
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
DIRECT_DATABASE_URL: ${DIRECT_DATABASE_URL}
|
||||
|
||||
# Redis
|
||||
REDIS_URL: redis://redis:6379
|
||||
REDIS_URL: ${REDIS_URL}
|
||||
|
||||
# Security
|
||||
JWT_SECRET: ${JWT_SECRET}
|
||||
@@ -214,19 +181,15 @@ services:
|
||||
|
||||
ports:
|
||||
# SMTP ports (for email relay)
|
||||
- "${PORT_SECURE:-465}:465" # SMTPS (implicit TLS)
|
||||
- "${PORT_SUBMISSION:-587}:587" # SMTP Submission (STARTTLS)
|
||||
# - "${PORT_SECURE:-465}:465" # SMTPS (implicit TLS)
|
||||
# - "${PORT_SUBMISSION:-587}:587" # SMTP Submission (STARTTLS)
|
||||
# Optional: Expose individual service ports for debugging
|
||||
# - "8080:8080" # API
|
||||
# - "3000:3000" # Web
|
||||
# - "4000:4000" # Landing
|
||||
# - "1000:1000" # Wiki
|
||||
- "6000:8080" # API
|
||||
- "6001:3000" # Web
|
||||
# - "6002:4000" # Landing
|
||||
# - "6003:1000" # Wiki
|
||||
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
minio:
|
||||
condition: service_healthy
|
||||
ntfy:
|
||||
@@ -244,10 +207,6 @@ services:
|
||||
- plunk
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
driver: local
|
||||
redis_data:
|
||||
driver: local
|
||||
minio_data:
|
||||
driver: local
|
||||
plunk_data:
|
||||
|
||||
Reference in New Issue
Block a user