Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2700e62935 | ||
|
|
517b93cc95 | ||
|
|
8e3fb9595d | ||
|
|
58be4abc31 | ||
|
|
480f0c1034 | ||
|
|
80beb2bb99 | ||
|
|
6ab4d77ca9 | ||
|
|
edfc399061 | ||
|
|
4a145f3488 | ||
|
|
868bdee615 | ||
|
|
6ebbb50f68 | ||
|
|
87eb56ff36 | ||
|
|
a348d37c21 | ||
|
|
5c166797b5 | ||
|
|
971b98a4cc | ||
|
|
81315eac8e | ||
|
|
1c1c95d332 | ||
|
|
844be42151 | ||
|
|
d59f4a10cd | ||
|
|
51ac88b9f5 | ||
|
|
523bcad602 | ||
|
|
32dd7bba46 | ||
|
|
71e2277643 | ||
|
|
079e1879b3 | ||
|
|
4de40f40fa | ||
|
|
94ceadbbe4 | ||
|
|
9797aed47f | ||
|
|
8b3657d056 | ||
|
|
01ec34a8cb | ||
|
|
ba3813e242 | ||
|
|
283f40239d | ||
|
|
53b631e6c6 | ||
|
|
6759bffd2a | ||
|
|
d2496bc51d | ||
|
|
bfecf04fa3 | ||
|
|
6d98d51222 | ||
|
|
c484da88ab | ||
|
|
6aee5db588 | ||
|
|
3f30a48c40 |
@@ -163,6 +163,19 @@ SMTP_DOMAIN=smtp.example.com
|
||||
# Default: unset (auto-detect, falls back to 14)
|
||||
# EMAIL_RATE_LIMIT_PER_SECOND=1
|
||||
|
||||
# Number of emails the worker processes in parallel. When unset, concurrency is
|
||||
# derived from the effective rate limit (~ rate * 0.5, min 5, capped by
|
||||
# EMAIL_WORKER_MAX_CONCURRENCY) so a higher SES quota translates into higher
|
||||
# throughput automatically. Pin this only when the Prisma pool or memory is the
|
||||
# binding constraint.
|
||||
# Default: unset (auto-derived)
|
||||
# EMAIL_WORKER_CONCURRENCY=10
|
||||
|
||||
# Upper bound applied to the auto-derived concurrency. Raise this when your SES
|
||||
# quota is high AND the Prisma connection pool has been sized for it.
|
||||
# Default: 50
|
||||
# EMAIL_WORKER_MAX_CONCURRENCY=50
|
||||
|
||||
# ========================================
|
||||
# ADVANCED (rarely needed)
|
||||
# ========================================
|
||||
|
||||
@@ -65,6 +65,18 @@ jobs:
|
||||
- name: Install dependencies
|
||||
run: yarn install --frozen-lockfile
|
||||
|
||||
- name: Tune Postgres for ephemeral CI workload
|
||||
env:
|
||||
PGPASSWORD: postgres
|
||||
run: |
|
||||
# synchronous_commit=off is the biggest single I/O win and is safe to lose
|
||||
# data on crash for a throwaway CI database.
|
||||
# synchronous_commit is dynamic — applies on reload. max_connections would
|
||||
# require a restart, so we leave it at the default of 100 and cap workers
|
||||
# at 4 × connection_limit=20 = 80 to stay under that budget.
|
||||
psql -h localhost -U postgres -d plunk_test -c "ALTER SYSTEM SET synchronous_commit = 'off';"
|
||||
psql -h localhost -U postgres -d plunk_test -c "SELECT pg_reload_conf();"
|
||||
|
||||
- name: Setup environment variables
|
||||
run: |
|
||||
cat > .env << EOF
|
||||
|
||||
@@ -161,7 +161,7 @@ Required for builds and deployment (see turbo.json and .env.example):
|
||||
- `OPENROUTER_API_KEY` - API key for OpenRouter (enables phishing detection)
|
||||
- `OPENROUTER_MODEL` (default: anthropic/claude-3-haiku) - LLM model to use for content analysis
|
||||
- `PHISHING_DETECTION_SAMPLE_RATE` (default: 0.1) - Percentage of emails to check (0.0-1.0, e.g., 0.1 = 10%)
|
||||
- `PHISHING_CONFIDENCE_THRESHOLD` (default: 85) - Minimum confidence percentage (0-100) to auto-disable project for single detection
|
||||
- `PHISHING_CONFIDENCE_THRESHOLD` (default: 95) - Minimum confidence percentage (0-100) to auto-disable project for single detection
|
||||
- `PHISHING_CUMULATIVE_THRESHOLD` (default: 3) - Number of phishing detections within time window to trigger auto-disable
|
||||
- `PHISHING_CUMULATIVE_WINDOW_MS` (default: 3600000) - Time window in milliseconds for cumulative tracking (default 1 hour)
|
||||
|
||||
@@ -174,6 +174,21 @@ Required for builds and deployment (see turbo.json and .env.example):
|
||||
- **Frontend Variables**: Next.js apps use `NEXT_PUBLIC_*` prefixed variables that are embedded at build time for
|
||||
client-side access
|
||||
|
||||
## Environment Variable Changes
|
||||
|
||||
When you add, rename, remove, or change the default/behaviour of any environment variable, you MUST update all THREE of the following in the same change:
|
||||
|
||||
1. `apps/api/.env.example` — local development defaults
|
||||
2. `.env.self-host.example` — self-hosting / production template
|
||||
3. `apps/wiki/content/docs/self-hosting/environment-variables.mdx` — user-facing reference
|
||||
|
||||
Rules:
|
||||
|
||||
- If the variable already exists in any file, **modify** its line/row/description — do not duplicate or leave a stale entry.
|
||||
- Keep section/category names consistent across all three files (e.g. "AWS SES", "Phishing Detection").
|
||||
- For dev-only or self-host-only variables, still mention them in the wiki and note the scope; only skip the example file where the variable is genuinely never applicable.
|
||||
- When in doubt about whether a variable belongs in `apps/api/.env.example` (development), include it commented out with a short note.
|
||||
|
||||
## Plugins
|
||||
|
||||
There are two plugins installed for you to use.
|
||||
|
||||
@@ -8,6 +8,8 @@
|
||||
# ==============================================================================
|
||||
NODE_ENV=development
|
||||
JWT_SECRET=hBx9Xh8J6KOMAGAsSjvcZJBT5TWyIkFX
|
||||
# Port the API server listens on (default: 8080)
|
||||
# PORT=8080
|
||||
|
||||
# ==============================================================================
|
||||
# Application URLs
|
||||
@@ -25,6 +27,8 @@ LANDING_URI=http://localhost:4000
|
||||
# ==============================================================================
|
||||
# API key for authenticating with the Plunk API (obtained from dashboard)
|
||||
PLUNK_API_KEY=
|
||||
# From address used for platform notification emails (project disabled, billing limits, etc.)
|
||||
# PLUNK_FROM_ADDRESS=
|
||||
|
||||
# ==============================================================================
|
||||
# Database & Redis
|
||||
@@ -61,6 +65,17 @@ SES_CONFIGURATION_SET_NO_TRACKING=plunk-configuration-set-no-tracking # Optiona
|
||||
# Default: Fetched from AWS (typically 14 for sandbox, higher for production accounts)
|
||||
# EMAIL_RATE_LIMIT_PER_SECOND=14
|
||||
|
||||
# Email worker concurrency (number of emails processed in parallel)
|
||||
# If not set, derived from the effective rate limit (~ rate * 0.5, min 5, capped
|
||||
# by EMAIL_WORKER_MAX_CONCURRENCY). Set this to pin a fixed value when the
|
||||
# Prisma connection pool or memory is the binding constraint.
|
||||
# EMAIL_WORKER_CONCURRENCY=10
|
||||
|
||||
# Upper bound for auto-derived worker concurrency
|
||||
# Raise this when your SES quota is high AND the Prisma pool has been sized for it.
|
||||
# Default: 50
|
||||
# EMAIL_WORKER_MAX_CONCURRENCY=50
|
||||
|
||||
# ==============================================================================
|
||||
# OAuth (Optional - for social login)
|
||||
# ==============================================================================
|
||||
@@ -85,3 +100,46 @@ STRIPE_METER_EVENT_NAME=emails # Meter event name (API key from your Stripe mete
|
||||
# Set to 'false' to disable automatic project suspension (useful for self-hosters who manage manually)
|
||||
# Default: true (automatic project disabling enabled)
|
||||
# AUTO_PROJECT_DISABLE=true
|
||||
|
||||
# ==============================================================================
|
||||
# Notifications (Optional - system notifications via ntfy)
|
||||
# ==============================================================================
|
||||
# ntfy topic URL for internal system notifications (e.g. project disabled, billing limits).
|
||||
# When unset, ntfy notifications are disabled.
|
||||
# Examples:
|
||||
# - Public ntfy.sh: https://ntfy.sh/your-unique-topic-name
|
||||
# - Self-hosted: https://your-ntfy-server.com/your-topic
|
||||
# NTFY_URL=
|
||||
|
||||
# ==============================================================================
|
||||
# Attachments (Optional)
|
||||
# ==============================================================================
|
||||
# Limits applied to attachments on transactional emails.
|
||||
# AWS SES caps total message size at 40 MB; the defaults below leave headroom.
|
||||
# MAX_ATTACHMENT_SIZE_MB=10
|
||||
# MAX_ATTACHMENTS_COUNT=10
|
||||
|
||||
# ==============================================================================
|
||||
# User Management (Optional)
|
||||
# ==============================================================================
|
||||
# When 'true', the signup endpoint rejects new user registrations.
|
||||
# Default: false
|
||||
# DISABLE_SIGNUPS=false
|
||||
# When 'true', validates emails on signup (disposable domains, plus-addressing,
|
||||
# domain existence, MX records).
|
||||
# Default: false
|
||||
# VERIFY_EMAIL_ON_SIGNUP=false
|
||||
|
||||
# ==============================================================================
|
||||
# Phishing Detection (Optional - AI-powered phishing scan via OpenRouter)
|
||||
# ==============================================================================
|
||||
# When OPENROUTER_API_KEY is set, a random sample of outgoing emails is
|
||||
# analyzed by an LLM. Projects can be auto-disabled if a single email exceeds
|
||||
# PHISHING_CONFIDENCE_THRESHOLD, or if PHISHING_CUMULATIVE_THRESHOLD emails are
|
||||
# flagged within PHISHING_CUMULATIVE_WINDOW_MS.
|
||||
# OPENROUTER_API_KEY=
|
||||
# OPENROUTER_MODEL=anthropic/claude-3-haiku
|
||||
# PHISHING_DETECTION_SAMPLE_RATE=0.1
|
||||
# PHISHING_CONFIDENCE_THRESHOLD=95
|
||||
# PHISHING_CUMULATIVE_THRESHOLD=3
|
||||
# PHISHING_CUMULATIVE_WINDOW_MS=3600000
|
||||
|
||||
@@ -58,6 +58,20 @@ export const EMAIL_RATE_LIMIT_PER_SECOND = process.env.EMAIL_RATE_LIMIT_PER_SECO
|
||||
? Number(process.env.EMAIL_RATE_LIMIT_PER_SECOND)
|
||||
: undefined;
|
||||
|
||||
// Email Worker Concurrency (optional override)
|
||||
// If not set, concurrency is derived from the effective rate limit so a higher
|
||||
// SES quota actually translates into higher throughput. Set this to pin a fixed
|
||||
// value (useful when Prisma pool size or memory is the binding constraint).
|
||||
export const EMAIL_WORKER_CONCURRENCY = process.env.EMAIL_WORKER_CONCURRENCY
|
||||
? Number(process.env.EMAIL_WORKER_CONCURRENCY)
|
||||
: undefined;
|
||||
|
||||
// Upper bound for auto-derived concurrency. Raise this if you have a large SES
|
||||
// quota AND have sized the Prisma connection pool accordingly.
|
||||
export const EMAIL_WORKER_MAX_CONCURRENCY = process.env.EMAIL_WORKER_MAX_CONCURRENCY
|
||||
? Number(process.env.EMAIL_WORKER_MAX_CONCURRENCY)
|
||||
: 50;
|
||||
|
||||
// Storage
|
||||
export const REDIS_URL = validateEnv('REDIS_URL');
|
||||
export const DATABASE_URL = validateEnv('DATABASE_URL');
|
||||
|
||||
@@ -4,7 +4,7 @@ import type {NextFunction, Request, Response} from 'express';
|
||||
|
||||
import {redis} from '../database/redis.js';
|
||||
import {NotAllowed, NotFound} from '../exceptions/index.js';
|
||||
import {isAuthenticated, requireEmailVerified} from '../middleware/auth.js';
|
||||
import {requireAuth, requireEmailVerified} from '../middleware/auth.js';
|
||||
import {DomainService} from '../services/DomainService.js';
|
||||
import {Keys} from '../services/keys.js';
|
||||
import {MembershipService} from '../services/MembershipService.js';
|
||||
@@ -17,16 +17,12 @@ export class Domains {
|
||||
* Get all domains for a project
|
||||
*/
|
||||
@Get('project/:projectId')
|
||||
@Middleware([isAuthenticated, requireEmailVerified])
|
||||
@Middleware([requireAuth, requireEmailVerified])
|
||||
@CatchAsync
|
||||
public async getProjectDomains(req: Request, res: Response, _next: NextFunction) {
|
||||
public async getProjectDomains(_req: Request, res: Response, _next: NextFunction) {
|
||||
const auth = res.locals.auth;
|
||||
const {projectId} = DomainSchemas.projectId.parse(req.params);
|
||||
|
||||
// Verify user has access to this project
|
||||
await MembershipService.requireAccess(auth.userId!, projectId);
|
||||
|
||||
const domains = await DomainService.getProjectDomains(projectId);
|
||||
const domains = await DomainService.getProjectDomains(auth.projectId!);
|
||||
|
||||
return res.status(200).json(domains);
|
||||
}
|
||||
@@ -35,19 +31,18 @@ export class Domains {
|
||||
* Add a new domain to a project
|
||||
*/
|
||||
@Post('')
|
||||
@Middleware([isAuthenticated, requireEmailVerified])
|
||||
@Middleware([requireAuth, requireEmailVerified])
|
||||
@CatchAsync
|
||||
public async addDomain(req: Request, res: Response, _next: NextFunction) {
|
||||
const auth = res.locals.auth;
|
||||
const {projectId, domain} = DomainSchemas.create.parse(req.body);
|
||||
const {domain} = DomainSchemas.create.parse(req.body);
|
||||
const projectId = auth.projectId!;
|
||||
|
||||
if (!auth.userId) {
|
||||
throw new NotFound('User authentication required');
|
||||
// Require admin role for JWT users (API keys bypass — project-scoped by design)
|
||||
if (auth.type === 'jwt') {
|
||||
await MembershipService.requireAdminAccess(auth.userId!, projectId);
|
||||
}
|
||||
|
||||
// Verify user has admin access to this project
|
||||
await MembershipService.requireAdminAccess(auth.userId!, projectId);
|
||||
|
||||
// Block domain changes on disabled projects
|
||||
const isDisabled = await SecurityService.isProjectDisabled(projectId);
|
||||
if (isDisabled) {
|
||||
@@ -68,6 +63,12 @@ export class Domains {
|
||||
const ownershipCheck = await DomainService.checkDomainOwnership(domain, auth.userId);
|
||||
|
||||
if (ownershipCheck.exists) {
|
||||
if (ownershipCheck.projectId === projectId) {
|
||||
return res.status(400).json({
|
||||
error: 'This domain is already linked to this project.',
|
||||
});
|
||||
}
|
||||
|
||||
// If domain exists and user is a member of that project, allow it
|
||||
if (ownershipCheck.isMember) {
|
||||
return res.status(400).json({
|
||||
@@ -99,7 +100,7 @@ export class Domains {
|
||||
* Check verification status for a domain
|
||||
*/
|
||||
@Get(':id/verify')
|
||||
@Middleware([isAuthenticated, requireEmailVerified])
|
||||
@Middleware([requireAuth, requireEmailVerified])
|
||||
@CatchAsync
|
||||
public async checkVerification(req: Request, res: Response, _next: NextFunction) {
|
||||
const auth = res.locals.auth;
|
||||
@@ -107,13 +108,10 @@ export class Domains {
|
||||
|
||||
const domain = await DomainService.id(id);
|
||||
|
||||
if (!domain) {
|
||||
if (!domain || domain.projectId !== auth.projectId) {
|
||||
throw new NotFound('Domain not found');
|
||||
}
|
||||
|
||||
// Verify user has access to the project this domain belongs to
|
||||
await MembershipService.requireAccess(auth.userId!, domain.projectId);
|
||||
|
||||
const verificationStatus = await DomainService.checkVerification(id);
|
||||
|
||||
// Invalidate cache if status changed
|
||||
@@ -127,7 +125,7 @@ export class Domains {
|
||||
* Remove a domain from a project
|
||||
*/
|
||||
@Delete(':id')
|
||||
@Middleware([isAuthenticated, requireEmailVerified])
|
||||
@Middleware([requireAuth, requireEmailVerified])
|
||||
@CatchAsync
|
||||
public async removeDomain(req: Request, res: Response, _next: NextFunction) {
|
||||
const auth = res.locals.auth;
|
||||
@@ -135,12 +133,14 @@ export class Domains {
|
||||
|
||||
const domain = await DomainService.id(id);
|
||||
|
||||
if (!domain) {
|
||||
if (!domain || domain.projectId !== auth.projectId) {
|
||||
throw new NotFound('Domain not found');
|
||||
}
|
||||
|
||||
// Verify user has admin access to the project this domain belongs to
|
||||
await MembershipService.requireAdminAccess(auth.userId!, domain.projectId);
|
||||
// Require admin role for JWT users (API keys bypass — project-scoped by design)
|
||||
if (auth.type === 'jwt') {
|
||||
await MembershipService.requireAdminAccess(auth.userId!, domain.projectId);
|
||||
}
|
||||
|
||||
// Block domain changes on disabled projects
|
||||
const isDisabled = await SecurityService.isProjectDisabled(domain.projectId);
|
||||
|
||||
@@ -573,6 +573,16 @@ export class Webhooks {
|
||||
|
||||
signale.success(`[WEBHOOK] Invoice paid for project ${project.name} (${project.id})`);
|
||||
|
||||
// Re-enable the project only if it was previously disabled for a failed payment.
|
||||
// Projects disabled for other reasons (reputation, phishing, manual) must stay disabled.
|
||||
if (project.disabled && project.disabledReason === 'PAYMENT_FAILED') {
|
||||
await prisma.project.update({
|
||||
where: {id: project.id},
|
||||
data: {disabled: false, disabledReason: null},
|
||||
});
|
||||
signale.success(`[WEBHOOK] Project ${project.name} (${project.id}) re-enabled after payment`);
|
||||
}
|
||||
|
||||
// Send notification about invoice payment
|
||||
await NtfyService.notifyInvoicePaid(project.name, project.id);
|
||||
break;
|
||||
@@ -606,7 +616,7 @@ export class Webhooks {
|
||||
|
||||
await prisma.project.update({
|
||||
where: {id: project.id},
|
||||
data: {disabled: true},
|
||||
data: {disabled: true, disabledReason: 'PAYMENT_FAILED'},
|
||||
});
|
||||
|
||||
await NtfyService.notifyProjectDisabledForPayment(project.name, project.id);
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import {beforeEach, describe, expect, it} from 'vitest';
|
||||
import {factories, getPrismaClient} from '../../../../../test/helpers';
|
||||
import {ContactService} from '../../services/ContactService.js';
|
||||
import {coerceCustomValue} from '../import-processor.js';
|
||||
|
||||
/**
|
||||
* Tests for Contact Import Processor - Subscription Status Preservation
|
||||
@@ -279,3 +280,51 @@ describe('Contact Import - Subscription Status Preservation', () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('coerceCustomValue', () => {
|
||||
describe('boolean coercion', () => {
|
||||
it.each(['true', 'TRUE', 'True', ' true ', 'yes', 'YES', 'Yes'])('coerces %j to true', value => {
|
||||
expect(coerceCustomValue(value)).toBe(true);
|
||||
});
|
||||
|
||||
it.each(['false', 'FALSE', 'False', ' false ', 'no', 'NO', 'No'])('coerces %j to false', value => {
|
||||
expect(coerceCustomValue(value)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('number coercion', () => {
|
||||
it.each([
|
||||
['42', 42],
|
||||
['-7', -7],
|
||||
['3.14', 3.14],
|
||||
[' 42 ', 42],
|
||||
['0.5', 0.5],
|
||||
['-0.25', -0.25],
|
||||
['0', 0],
|
||||
['1', 1],
|
||||
])('coerces %j to %j', (value, expected) => {
|
||||
expect(coerceCustomValue(value)).toBe(expected);
|
||||
});
|
||||
|
||||
it.each(['01234', '+42', '.5', '42.', '1e10', 'NaN', 'Infinity', '1.2.3', '4-2'])(
|
||||
'leaves %j as a string (preserves IDs / rejects loose formats)',
|
||||
value => {
|
||||
expect(coerceCustomValue(value)).toBe(value);
|
||||
},
|
||||
);
|
||||
|
||||
it('"1.0" is a number (does not match the boolean truthy set)', () => {
|
||||
expect(coerceCustomValue('1.0')).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('passthrough', () => {
|
||||
it.each(['Alice', 'true!', 'yesno', 'maybe'])('leaves %j as a string', value => {
|
||||
expect(coerceCustomValue(value)).toBe(value);
|
||||
});
|
||||
|
||||
it('leaves empty string as empty string', () => {
|
||||
expect(coerceCustomValue('')).toBe('');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -8,7 +8,12 @@ import type {SendEmailJobData} from '@plunk/types';
|
||||
import {type Job, Worker} from 'bullmq';
|
||||
import signale from 'signale';
|
||||
|
||||
import {DASHBOARD_URI, EMAIL_RATE_LIMIT_PER_SECOND} from '../app/constants.js';
|
||||
import {
|
||||
DASHBOARD_URI,
|
||||
EMAIL_RATE_LIMIT_PER_SECOND,
|
||||
EMAIL_WORKER_CONCURRENCY,
|
||||
EMAIL_WORKER_MAX_CONCURRENCY,
|
||||
} from '../app/constants.js';
|
||||
import {prisma} from '../database/prisma.js';
|
||||
import {CampaignService} from '../services/CampaignService.js';
|
||||
import {EmailService} from '../services/EmailService.js';
|
||||
@@ -47,9 +52,31 @@ async function getEmailRateLimit(): Promise<number> {
|
||||
return DEFAULT_RATE_LIMIT;
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive worker concurrency from the rate limit so a higher SES quota actually
|
||||
* translates into higher throughput. The mean job duration is ~0.5s (Prisma
|
||||
* reads + HTML compile + SES call + writes), so `rate * 0.5` gives ~2× headroom
|
||||
* over the per-second cap. Clamped to keep sandbox accounts useful and to
|
||||
* protect the Prisma pool on very large quotas.
|
||||
*/
|
||||
function deriveWorkerConcurrency(rateLimit: number): number {
|
||||
if (EMAIL_WORKER_CONCURRENCY !== undefined) {
|
||||
return EMAIL_WORKER_CONCURRENCY;
|
||||
}
|
||||
|
||||
const TARGET_JOB_SECONDS = 0.5;
|
||||
const MIN_CONCURRENCY = 5;
|
||||
const derived = Math.ceil(rateLimit * TARGET_JOB_SECONDS);
|
||||
return Math.max(MIN_CONCURRENCY, Math.min(derived, EMAIL_WORKER_MAX_CONCURRENCY));
|
||||
}
|
||||
|
||||
export async function createEmailWorker() {
|
||||
// Fetch the rate limit (from env, AWS, or default)
|
||||
const rateLimit = await getEmailRateLimit();
|
||||
const concurrency = deriveWorkerConcurrency(rateLimit);
|
||||
signale.info(
|
||||
`[EMAIL-PROCESSOR] Worker concurrency: ${concurrency} (rate limit: ${rateLimit}/s)`,
|
||||
);
|
||||
const worker = new Worker<SendEmailJobData>(
|
||||
emailQueue.name,
|
||||
async (job: Job<SendEmailJobData>) => {
|
||||
@@ -253,7 +280,7 @@ export async function createEmailWorker() {
|
||||
},
|
||||
{
|
||||
connection: emailQueue.opts.connection,
|
||||
concurrency: 10, // Process up to 10 emails concurrently
|
||||
concurrency,
|
||||
limiter: {
|
||||
max: rateLimit, // Max emails per second (from env, AWS SES quota, or default)
|
||||
duration: 1000,
|
||||
|
||||
@@ -123,7 +123,11 @@ export function createImportWorker() {
|
||||
|
||||
// Extract custom data (all fields except email and subscribed)
|
||||
const {email: _, subscribed: __, ...customData} = record;
|
||||
const data = Object.keys(customData).length > 0 ? customData : undefined;
|
||||
const customEntries = Object.entries(customData);
|
||||
const data =
|
||||
customEntries.length > 0
|
||||
? Object.fromEntries(customEntries.map(([k, v]) => [k, coerceCustomValue(v)]))
|
||||
: undefined;
|
||||
|
||||
// Check if contact exists before upserting
|
||||
const existingContact = await ContactService.findByEmail(projectId, email);
|
||||
@@ -216,3 +220,30 @@ function isValidEmail(email: string): boolean {
|
||||
const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
|
||||
return emailRegex.test(email);
|
||||
}
|
||||
|
||||
// Values considered as boolean during import.
|
||||
// Numbers (0, 1) are intentionally absent.
|
||||
const BOOLEAN_TRUE = new Set(['true', 'yes']);
|
||||
const BOOLEAN_FALSE = new Set(['false', 'no']);
|
||||
|
||||
// Strict integer-or-decimal number detection pattern.
|
||||
// Valid: 0, 42, -42, 3.14
|
||||
// Rejected: 007, +42, 1.2.3, 1e5
|
||||
const NUMERIC_RE = /^-?(0|[1-9]\d*)(\.\d+)?$/;
|
||||
|
||||
/**
|
||||
* Coerces a raw string into its most natural primitive type: `boolean`,
|
||||
* `number`, or `string`. Values that match neither are
|
||||
* returned unchanged.
|
||||
*
|
||||
* @param value The raw string to coerce.
|
||||
* @returns The coerced value as `boolean`, `number`, or `string`.
|
||||
*/
|
||||
export function coerceCustomValue(value: string): string | boolean | number {
|
||||
const trimmed = value.trim();
|
||||
const lower = trimmed.toLowerCase();
|
||||
if (BOOLEAN_TRUE.has(lower)) return true;
|
||||
if (BOOLEAN_FALSE.has(lower)) return false;
|
||||
if (NUMERIC_RE.test(trimmed)) return Number(trimmed);
|
||||
return value;
|
||||
}
|
||||
|
||||
@@ -3,6 +3,20 @@ import type {NextFunction, Request, Response} from 'express';
|
||||
import {databaseRequestLogger} from '../requestLogger.js';
|
||||
import {factories, getPrismaClient} from '../../../../../test/helpers';
|
||||
|
||||
async function waitForLog(prisma: ReturnType<typeof getPrismaClient>, id: string, timeoutMs = 2000) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
const record = await prisma.apiRequest.findUnique({where: {id}});
|
||||
if (record) return record;
|
||||
await new Promise(resolve => setTimeout(resolve, 20));
|
||||
}
|
||||
return prisma.apiRequest.findUnique({where: {id}});
|
||||
}
|
||||
|
||||
async function waitForNoLog(ms = 200) {
|
||||
await new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
describe('Request Logger Middleware', () => {
|
||||
const prisma = getPrismaClient();
|
||||
let req: Partial<Request>;
|
||||
@@ -77,13 +91,7 @@ describe('Request Logger Middleware', () => {
|
||||
const responseBody = {success: true, data: {id: '123'}};
|
||||
await res.json!(responseBody);
|
||||
|
||||
// Wait for async logging to complete
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
// Verify database record was created
|
||||
const loggedRequest = await prisma.apiRequest.findUnique({
|
||||
where: {id: 'test-request-id-123'},
|
||||
});
|
||||
const loggedRequest = await waitForLog(prisma, 'test-request-id-123');
|
||||
|
||||
expect(loggedRequest).toBeDefined();
|
||||
expect(loggedRequest?.method).toBe('POST');
|
||||
@@ -109,11 +117,7 @@ describe('Request Logger Middleware', () => {
|
||||
const responseBody = {success: true};
|
||||
await res.json!(responseBody);
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
const loggedRequest = await prisma.apiRequest.findUnique({
|
||||
where: {id: 'public-request-id'},
|
||||
});
|
||||
const loggedRequest = await waitForLog(prisma, 'public-request-id');
|
||||
|
||||
expect(loggedRequest).toBeDefined();
|
||||
expect(loggedRequest?.projectId).toBeNull();
|
||||
@@ -131,11 +135,7 @@ describe('Request Logger Middleware', () => {
|
||||
|
||||
await res.json!({success: true});
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
const loggedRequest = await prisma.apiRequest.findUnique({
|
||||
where: {id: 'test-request-id-123'},
|
||||
});
|
||||
const loggedRequest = await waitForLog(prisma, 'test-request-id-123');
|
||||
|
||||
// Allow for timer imprecision (especially in CI environments)
|
||||
expect(loggedRequest?.duration).toBeGreaterThanOrEqual(45);
|
||||
@@ -162,11 +162,7 @@ describe('Request Logger Middleware', () => {
|
||||
|
||||
await res.json!(errorResponse);
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
const loggedRequest = await prisma.apiRequest.findUnique({
|
||||
where: {id: 'test-request-id-123'},
|
||||
});
|
||||
const loggedRequest = await waitForLog(prisma, 'test-request-id-123');
|
||||
|
||||
expect(loggedRequest).toBeDefined();
|
||||
expect(loggedRequest?.statusCode).toBe(400);
|
||||
@@ -189,11 +185,7 @@ describe('Request Logger Middleware', () => {
|
||||
|
||||
await res.json!(errorResponse);
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
const loggedRequest = await prisma.apiRequest.findUnique({
|
||||
where: {id: 'test-request-id-123'},
|
||||
});
|
||||
const loggedRequest = await waitForLog(prisma, 'test-request-id-123');
|
||||
|
||||
expect(loggedRequest?.statusCode).toBe(500);
|
||||
expect(loggedRequest?.errorCode).toBe('INTERNAL_SERVER_ERROR');
|
||||
@@ -209,11 +201,7 @@ describe('Request Logger Middleware', () => {
|
||||
error: {code: 'RESOURCE_NOT_FOUND', message: 'Template not found'},
|
||||
});
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
const loggedRequest = await prisma.apiRequest.findUnique({
|
||||
where: {id: 'test-request-id-123'},
|
||||
});
|
||||
const loggedRequest = await waitForLog(prisma, 'test-request-id-123');
|
||||
|
||||
expect(loggedRequest?.statusCode).toBe(404);
|
||||
expect(loggedRequest?.errorCode).toBe('RESOURCE_NOT_FOUND');
|
||||
@@ -306,11 +294,8 @@ describe('Request Logger Middleware', () => {
|
||||
|
||||
databaseRequestLogger(req as Request, res as Response, next);
|
||||
await res.json!({success: true});
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
const loggedRequest = await prisma.apiRequest.findUnique({
|
||||
where: {id: `log-${path.replace(/\//g, '-')}`},
|
||||
});
|
||||
const loggedRequest = await waitForLog(prisma, `log-${path.replace(/\//g, '-')}`);
|
||||
|
||||
expect(loggedRequest).toBeDefined();
|
||||
expect(loggedRequest?.path).toBe(path);
|
||||
@@ -352,17 +337,18 @@ describe('Request Logger Middleware', () => {
|
||||
|
||||
await res.json!({success: true});
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
// Should create a record with generated UUID
|
||||
const allRequests = await prisma.apiRequest.findMany({
|
||||
where: {
|
||||
path: '/v1/send',
|
||||
method: 'POST',
|
||||
},
|
||||
orderBy: {createdAt: 'desc'},
|
||||
take: 1,
|
||||
});
|
||||
// Should create a record with generated UUID — poll for it
|
||||
const deadline = Date.now() + 2000;
|
||||
let allRequests: Awaited<ReturnType<typeof prisma.apiRequest.findMany>> = [];
|
||||
while (Date.now() < deadline) {
|
||||
allRequests = await prisma.apiRequest.findMany({
|
||||
where: {path: '/v1/send', method: 'POST'},
|
||||
orderBy: {createdAt: 'desc'},
|
||||
take: 1,
|
||||
});
|
||||
if (allRequests.length > 0) break;
|
||||
await new Promise(resolve => setTimeout(resolve, 20));
|
||||
}
|
||||
|
||||
expect(allRequests.length).toBeGreaterThan(0);
|
||||
expect(allRequests[0].id).toBeDefined();
|
||||
@@ -418,11 +404,8 @@ describe('Request Logger Middleware', () => {
|
||||
databaseRequestLogger(reqWithSize as Request, resWithId as Response, next);
|
||||
|
||||
await resWithId.json!({success: true});
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
const loggedRequest = await prisma.apiRequest.findUnique({
|
||||
where: {id: 'test-size-5000'},
|
||||
});
|
||||
const loggedRequest = await waitForLog(prisma, 'test-size-5000');
|
||||
|
||||
expect(loggedRequest?.requestSize).toBe(5000);
|
||||
});
|
||||
@@ -445,11 +428,8 @@ describe('Request Logger Middleware', () => {
|
||||
databaseRequestLogger(reqNoSize as Request, resWithId as Response, next);
|
||||
|
||||
await resWithId.json!({success: true});
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
const loggedRequest = await prisma.apiRequest.findUnique({
|
||||
where: {id: 'test-no-size'},
|
||||
});
|
||||
const loggedRequest = await waitForLog(prisma, 'test-no-size');
|
||||
|
||||
expect(loggedRequest?.requestSize).toBeNull();
|
||||
});
|
||||
@@ -474,11 +454,8 @@ describe('Request Logger Middleware', () => {
|
||||
};
|
||||
|
||||
await resLarge.json!(largeResponse);
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
const loggedRequest = await prisma.apiRequest.findUnique({
|
||||
where: {id: 'test-large-response'},
|
||||
});
|
||||
const loggedRequest = await waitForLog(prisma, 'test-large-response');
|
||||
|
||||
const expectedSize = JSON.stringify(largeResponse).length;
|
||||
expect(loggedRequest?.responseSize).toBe(expectedSize);
|
||||
|
||||
@@ -438,15 +438,14 @@ export class DomainService {
|
||||
* @param userId User ID to check membership
|
||||
* @returns Object with exists flag and membership info
|
||||
*/
|
||||
public static async checkDomainOwnership(domain: string, userId: string) {
|
||||
public static async checkDomainOwnership(domain: string, userId?: string) {
|
||||
const existingDomain = await prisma.domain.findFirst({
|
||||
where: {domain},
|
||||
include: {
|
||||
project: {
|
||||
include: {
|
||||
members: {
|
||||
where: {userId},
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -456,8 +455,20 @@ export class DomainService {
|
||||
return {exists: false};
|
||||
}
|
||||
|
||||
// Check if user is a member of the project that owns this domain
|
||||
const isMember = existingDomain.project.members.length > 0;
|
||||
let isMember = false;
|
||||
|
||||
if (userId) {
|
||||
const membership = await prisma.membership.findUnique({
|
||||
where: {
|
||||
userId_projectId: {
|
||||
userId,
|
||||
projectId: existingDomain.project.id,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
isMember = membership !== null;
|
||||
}
|
||||
|
||||
return {
|
||||
exists: true,
|
||||
|
||||
@@ -108,7 +108,7 @@ export class EmailService {
|
||||
await BillingLimitService.incrementUsage(params.projectId, EmailSourceType.TRANSACTIONAL);
|
||||
|
||||
// Queue email for sending
|
||||
await this.queueEmail(email.id);
|
||||
await this.queueEmail(email.id, EmailSourceType.TRANSACTIONAL);
|
||||
|
||||
return email;
|
||||
}
|
||||
@@ -172,7 +172,7 @@ export class EmailService {
|
||||
await BillingLimitService.incrementUsage(params.projectId, sourceType);
|
||||
|
||||
// Queue email for sending
|
||||
await this.queueEmail(email.id);
|
||||
await this.queueEmail(email.id, sourceType);
|
||||
|
||||
return email;
|
||||
}
|
||||
@@ -278,7 +278,7 @@ export class EmailService {
|
||||
await BillingLimitService.incrementUsage(params.projectId, sourceType);
|
||||
|
||||
// Queue email for sending
|
||||
await this.queueEmail(email.id);
|
||||
await this.queueEmail(email.id, sourceType);
|
||||
|
||||
return email;
|
||||
}
|
||||
@@ -659,12 +659,16 @@ export class EmailService {
|
||||
/**
|
||||
* Detects if HTML contains custom patterns that indicate it was written in the HTML editor
|
||||
* rather than the visual editor. Mirrors the same logic in apps/web/src/lib/emailStyles.ts.
|
||||
*
|
||||
* The TipTap editor loads StarterKit + TextAlign + Color + TextStyle + Link +
|
||||
* ResizableImage + VariableMention. TextStyle/Color/Link round-trip <span style="..."> and
|
||||
* <a style="..."> markup. This detection therefore PERMITS span + inline styles and only
|
||||
* REJECTS markup TipTap cannot represent (tables, divs, forms, embeds, custom attrs,
|
||||
* <style> blocks, etc).
|
||||
*/
|
||||
private static detectCustomHtmlPatterns(html: string): boolean {
|
||||
if (!html || html.trim() === '') return false;
|
||||
|
||||
const hasInlineStyles = /<[^>]+style\s*=\s*["'][^"']*["']/i.test(html);
|
||||
|
||||
const classMatches = html.matchAll(/class\s*=\s*["']([^"']*)["']/gi);
|
||||
let hasCustomClasses = false;
|
||||
for (const match of classMatches) {
|
||||
@@ -679,21 +683,21 @@ export class EmailService {
|
||||
}
|
||||
}
|
||||
|
||||
const hasCustomAttributes = /<[^>]+(?:data-|aria-|role=|id=)/i.test(html);
|
||||
const hasComplexTables = /<table[^>]*>[\s\S]*?<table/i.test(html);
|
||||
const hasCustomElements = /<(?:div|span|section|article|header|footer|nav|aside)[^>]*>/i.test(html);
|
||||
// Element-attribute-scoped regex; the leading [\s"'] guard prevents `id=` inside
|
||||
// href URLs (e.g. `?id=...`) from false-matching as an HTML id attribute.
|
||||
const hasCustomAttributes = /<[a-z][^>]*?[\s"'](?:data-|aria-|role=|id=)/i.test(html);
|
||||
|
||||
// Elements TipTap cannot round-trip with the currently-loaded extension set.
|
||||
// <span> is intentionally excluded -- TipTap's TextStyle extension handles it.
|
||||
const hasCustomElements =
|
||||
/<(?:div|section|article|header|footer|nav|aside|main|table|tr|td|th|tbody|thead|tfoot|colgroup|col|form|input|button|select|textarea|iframe|video|audio|svg|object|embed|details|summary|dialog)\b/i.test(
|
||||
html,
|
||||
);
|
||||
|
||||
const hasMediaQueries = /@media/i.test(html);
|
||||
const hasStyleTags = /<style[^>]*>/i.test(html);
|
||||
|
||||
return (
|
||||
hasInlineStyles ||
|
||||
hasCustomClasses ||
|
||||
hasCustomAttributes ||
|
||||
hasComplexTables ||
|
||||
hasCustomElements ||
|
||||
hasMediaQueries ||
|
||||
hasStyleTags
|
||||
);
|
||||
return hasCustomClasses || hasCustomAttributes || hasCustomElements || hasMediaQueries || hasStyleTags;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1133,7 +1137,7 @@ export class EmailService {
|
||||
* Queue an email for sending
|
||||
* Adds email to the BullMQ queue for processing by workers
|
||||
*/
|
||||
private static async queueEmail(emailId: string, delay?: number): Promise<void> {
|
||||
await QueueService.queueEmail(emailId, delay);
|
||||
private static async queueEmail(emailId: string, sourceType: EmailSourceType, delay?: number): Promise<void> {
|
||||
await QueueService.queueEmail(emailId, sourceType, delay);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import {CampaignStatus, EmailStatus} from '@plunk/db';
|
||||
import {CampaignStatus, EmailSourceType, EmailStatus} from '@plunk/db';
|
||||
import {type Job, Queue} from 'bullmq';
|
||||
import type {RedisOptions} from 'ioredis';
|
||||
import signale from 'signale';
|
||||
@@ -174,20 +174,43 @@ export const meterQueue = new Queue<MeterEventJobData>('meter', {
|
||||
},
|
||||
});
|
||||
|
||||
function emailPriorityFor(sourceType: EmailSourceType): number {
|
||||
switch (sourceType) {
|
||||
case EmailSourceType.TRANSACTIONAL:
|
||||
return 1;
|
||||
case EmailSourceType.WORKFLOW:
|
||||
return 5;
|
||||
case EmailSourceType.CAMPAIGN:
|
||||
return 10;
|
||||
default:
|
||||
return 5;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue Service - Centralized queue management
|
||||
*/
|
||||
export class QueueService {
|
||||
/**
|
||||
* Add email to queue for sending
|
||||
* Add email to queue for sending.
|
||||
*
|
||||
* Transactional emails jump the queue ahead of workflow and campaign sends
|
||||
* via BullMQ's priority (lower number = higher precedence). This prevents
|
||||
* latency-sensitive sends (login codes, password resets) from queuing behind
|
||||
* large campaign bursts on the shared `email` queue.
|
||||
*/
|
||||
public static async queueEmail(emailId: string, delay?: number): Promise<Job<SendEmailJobData>> {
|
||||
public static async queueEmail(
|
||||
emailId: string,
|
||||
sourceType: EmailSourceType,
|
||||
delay?: number,
|
||||
): Promise<Job<SendEmailJobData>> {
|
||||
return emailQueue.add(
|
||||
'send-email',
|
||||
{emailId},
|
||||
{
|
||||
delay, // Optional delay in milliseconds
|
||||
jobId: `email-${emailId}`, // Prevent duplicate jobs
|
||||
delay,
|
||||
jobId: `email-${emailId}`,
|
||||
priority: emailPriorityFor(sourceType),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -50,22 +50,13 @@ const SECURITY_THRESHOLDS = {
|
||||
MIN_COMPLAINTS_FOR_CRITICAL: 5,
|
||||
MIN_COMPLAINTS_FOR_WARNING: 3,
|
||||
|
||||
// === Absolute count ceilings ===
|
||||
// These trigger regardless of rate — catches high-volume spammers who dilute their bounce rate
|
||||
// 24-hour absolute ceilings
|
||||
BOUNCE_24H_CEILING_WARNING: 50,
|
||||
BOUNCE_24H_CEILING_CRITICAL: 100,
|
||||
COMPLAINT_24H_CEILING_WARNING: 10,
|
||||
COMPLAINT_24H_CEILING_CRITICAL: 25,
|
||||
|
||||
// 7-day absolute ceilings
|
||||
BOUNCE_7DAY_CEILING_WARNING: 200,
|
||||
BOUNCE_7DAY_CEILING_CRITICAL: 500,
|
||||
COMPLAINT_7DAY_CEILING_WARNING: 30,
|
||||
COMPLAINT_7DAY_CEILING_CRITICAL: 75,
|
||||
|
||||
// === New project thresholds (projects < 30 days old) ===
|
||||
// Legitimate senders ramp up gradually; spammers blast immediately
|
||||
// === Absolute count ceilings (new projects only) ===
|
||||
// These trigger regardless of rate — catches new accounts blasting emails
|
||||
// before their bounce rate has caught up. Established projects rely on
|
||||
// rate-based checks only, since high absolute counts at high volume
|
||||
// (e.g. 100 bounces out of 10K) don't indicate abuse.
|
||||
//
|
||||
// Legitimate senders ramp up gradually; spammers blast immediately.
|
||||
NEW_PROJECT_AGE_DAYS: 30,
|
||||
NEW_PROJECT_BOUNCE_24H_CEILING_WARNING: 10,
|
||||
NEW_PROJECT_BOUNCE_24H_CEILING_CRITICAL: 25,
|
||||
@@ -516,82 +507,54 @@ export class SecurityService {
|
||||
const violations: string[] = [];
|
||||
const warnings: string[] = [];
|
||||
|
||||
// Pick absolute count ceilings based on project age
|
||||
const bounceCeilings = isNewProject
|
||||
? {
|
||||
ceiling24hWarning: SECURITY_THRESHOLDS.NEW_PROJECT_BOUNCE_24H_CEILING_WARNING,
|
||||
ceiling24hCritical: SECURITY_THRESHOLDS.NEW_PROJECT_BOUNCE_24H_CEILING_CRITICAL,
|
||||
ceiling7dWarning: SECURITY_THRESHOLDS.NEW_PROJECT_BOUNCE_7DAY_CEILING_WARNING,
|
||||
ceiling7dCritical: SECURITY_THRESHOLDS.NEW_PROJECT_BOUNCE_7DAY_CEILING_CRITICAL,
|
||||
}
|
||||
: {
|
||||
ceiling24hWarning: SECURITY_THRESHOLDS.BOUNCE_24H_CEILING_WARNING,
|
||||
ceiling24hCritical: SECURITY_THRESHOLDS.BOUNCE_24H_CEILING_CRITICAL,
|
||||
ceiling7dWarning: SECURITY_THRESHOLDS.BOUNCE_7DAY_CEILING_WARNING,
|
||||
ceiling7dCritical: SECURITY_THRESHOLDS.BOUNCE_7DAY_CEILING_CRITICAL,
|
||||
};
|
||||
// === Absolute count ceiling checks (new projects only, rate-independent) ===
|
||||
// Catches new accounts blasting emails before their bounce rate catches up.
|
||||
// Established projects skip these — high absolute counts at high volume
|
||||
// (e.g. 100 bounces out of 10K) don't indicate abuse; rate checks handle them.
|
||||
if (isNewProject) {
|
||||
// 24-hour bounce ceilings
|
||||
if (twentyFourHour.bounces >= SECURITY_THRESHOLDS.NEW_PROJECT_BOUNCE_24H_CEILING_CRITICAL) {
|
||||
violations.push(
|
||||
`24-hour bounce count (new project) (${twentyFourHour.bounces} bounces) exceeds critical ceiling (${SECURITY_THRESHOLDS.NEW_PROJECT_BOUNCE_24H_CEILING_CRITICAL})`,
|
||||
);
|
||||
} else if (twentyFourHour.bounces >= SECURITY_THRESHOLDS.NEW_PROJECT_BOUNCE_24H_CEILING_WARNING) {
|
||||
warnings.push(
|
||||
`24-hour bounce count (new project) (${twentyFourHour.bounces} bounces) exceeds warning ceiling (${SECURITY_THRESHOLDS.NEW_PROJECT_BOUNCE_24H_CEILING_WARNING})`,
|
||||
);
|
||||
}
|
||||
|
||||
const complaintCeilings = isNewProject
|
||||
? {
|
||||
ceiling24hWarning: SECURITY_THRESHOLDS.NEW_PROJECT_COMPLAINT_24H_CEILING_WARNING,
|
||||
ceiling24hCritical: SECURITY_THRESHOLDS.NEW_PROJECT_COMPLAINT_24H_CEILING_CRITICAL,
|
||||
ceiling7dWarning: SECURITY_THRESHOLDS.NEW_PROJECT_COMPLAINT_7DAY_CEILING_WARNING,
|
||||
ceiling7dCritical: SECURITY_THRESHOLDS.NEW_PROJECT_COMPLAINT_7DAY_CEILING_CRITICAL,
|
||||
}
|
||||
: {
|
||||
ceiling24hWarning: SECURITY_THRESHOLDS.COMPLAINT_24H_CEILING_WARNING,
|
||||
ceiling24hCritical: SECURITY_THRESHOLDS.COMPLAINT_24H_CEILING_CRITICAL,
|
||||
ceiling7dWarning: SECURITY_THRESHOLDS.COMPLAINT_7DAY_CEILING_WARNING,
|
||||
ceiling7dCritical: SECURITY_THRESHOLDS.COMPLAINT_7DAY_CEILING_CRITICAL,
|
||||
};
|
||||
// 7-day bounce ceilings
|
||||
if (sevenDay.bounces >= SECURITY_THRESHOLDS.NEW_PROJECT_BOUNCE_7DAY_CEILING_CRITICAL) {
|
||||
violations.push(
|
||||
`7-day bounce count (new project) (${sevenDay.bounces} bounces) exceeds critical ceiling (${SECURITY_THRESHOLDS.NEW_PROJECT_BOUNCE_7DAY_CEILING_CRITICAL})`,
|
||||
);
|
||||
} else if (sevenDay.bounces >= SECURITY_THRESHOLDS.NEW_PROJECT_BOUNCE_7DAY_CEILING_WARNING) {
|
||||
warnings.push(
|
||||
`7-day bounce count (new project) (${sevenDay.bounces} bounces) exceeds warning ceiling (${SECURITY_THRESHOLDS.NEW_PROJECT_BOUNCE_7DAY_CEILING_WARNING})`,
|
||||
);
|
||||
}
|
||||
|
||||
const projectLabel = isNewProject ? ' (new project)' : '';
|
||||
// 24-hour complaint ceilings
|
||||
if (twentyFourHour.complaints >= SECURITY_THRESHOLDS.NEW_PROJECT_COMPLAINT_24H_CEILING_CRITICAL) {
|
||||
violations.push(
|
||||
`24-hour complaint count (new project) (${twentyFourHour.complaints} complaints) exceeds critical ceiling (${SECURITY_THRESHOLDS.NEW_PROJECT_COMPLAINT_24H_CEILING_CRITICAL})`,
|
||||
);
|
||||
} else if (twentyFourHour.complaints >= SECURITY_THRESHOLDS.NEW_PROJECT_COMPLAINT_24H_CEILING_WARNING) {
|
||||
warnings.push(
|
||||
`24-hour complaint count (new project) (${twentyFourHour.complaints} complaints) exceeds warning ceiling (${SECURITY_THRESHOLDS.NEW_PROJECT_COMPLAINT_24H_CEILING_WARNING})`,
|
||||
);
|
||||
}
|
||||
|
||||
// === Absolute count ceiling checks (rate-independent) ===
|
||||
// These catch high-volume spammers who dilute their bounce rate by blasting emails
|
||||
|
||||
// 24-hour bounce ceilings
|
||||
if (twentyFourHour.bounces >= bounceCeilings.ceiling24hCritical) {
|
||||
violations.push(
|
||||
`24-hour bounce count${projectLabel} (${twentyFourHour.bounces} bounces) exceeds critical ceiling (${bounceCeilings.ceiling24hCritical})`,
|
||||
);
|
||||
} else if (twentyFourHour.bounces >= bounceCeilings.ceiling24hWarning) {
|
||||
warnings.push(
|
||||
`24-hour bounce count${projectLabel} (${twentyFourHour.bounces} bounces) exceeds warning ceiling (${bounceCeilings.ceiling24hWarning})`,
|
||||
);
|
||||
}
|
||||
|
||||
// 7-day bounce ceilings
|
||||
if (sevenDay.bounces >= bounceCeilings.ceiling7dCritical) {
|
||||
violations.push(
|
||||
`7-day bounce count${projectLabel} (${sevenDay.bounces} bounces) exceeds critical ceiling (${bounceCeilings.ceiling7dCritical})`,
|
||||
);
|
||||
} else if (sevenDay.bounces >= bounceCeilings.ceiling7dWarning) {
|
||||
warnings.push(
|
||||
`7-day bounce count${projectLabel} (${sevenDay.bounces} bounces) exceeds warning ceiling (${bounceCeilings.ceiling7dWarning})`,
|
||||
);
|
||||
}
|
||||
|
||||
// 24-hour complaint ceilings
|
||||
if (twentyFourHour.complaints >= complaintCeilings.ceiling24hCritical) {
|
||||
violations.push(
|
||||
`24-hour complaint count${projectLabel} (${twentyFourHour.complaints} complaints) exceeds critical ceiling (${complaintCeilings.ceiling24hCritical})`,
|
||||
);
|
||||
} else if (twentyFourHour.complaints >= complaintCeilings.ceiling24hWarning) {
|
||||
warnings.push(
|
||||
`24-hour complaint count${projectLabel} (${twentyFourHour.complaints} complaints) exceeds warning ceiling (${complaintCeilings.ceiling24hWarning})`,
|
||||
);
|
||||
}
|
||||
|
||||
// 7-day complaint ceilings
|
||||
if (sevenDay.complaints >= complaintCeilings.ceiling7dCritical) {
|
||||
violations.push(
|
||||
`7-day complaint count${projectLabel} (${sevenDay.complaints} complaints) exceeds critical ceiling (${complaintCeilings.ceiling7dCritical})`,
|
||||
);
|
||||
} else if (sevenDay.complaints >= complaintCeilings.ceiling7dWarning) {
|
||||
warnings.push(
|
||||
`7-day complaint count${projectLabel} (${sevenDay.complaints} complaints) exceeds warning ceiling (${complaintCeilings.ceiling7dWarning})`,
|
||||
);
|
||||
// 7-day complaint ceilings
|
||||
if (sevenDay.complaints >= SECURITY_THRESHOLDS.NEW_PROJECT_COMPLAINT_7DAY_CEILING_CRITICAL) {
|
||||
violations.push(
|
||||
`7-day complaint count (new project) (${sevenDay.complaints} complaints) exceeds critical ceiling (${SECURITY_THRESHOLDS.NEW_PROJECT_COMPLAINT_7DAY_CEILING_CRITICAL})`,
|
||||
);
|
||||
} else if (sevenDay.complaints >= SECURITY_THRESHOLDS.NEW_PROJECT_COMPLAINT_7DAY_CEILING_WARNING) {
|
||||
warnings.push(
|
||||
`7-day complaint count (new project) (${sevenDay.complaints} complaints) exceeds warning ceiling (${SECURITY_THRESHOLDS.NEW_PROJECT_COMPLAINT_7DAY_CEILING_WARNING})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// === Rate-based checks (existing logic) ===
|
||||
@@ -713,7 +676,7 @@ export class SecurityService {
|
||||
// Disable the project
|
||||
await prisma.project.update({
|
||||
where: {id: projectId},
|
||||
data: {disabled: true},
|
||||
data: {disabled: true, disabledReason: 'EMAIL_REPUTATION'},
|
||||
});
|
||||
|
||||
// Log critical security event
|
||||
@@ -802,7 +765,36 @@ export class SecurityService {
|
||||
const uniqueUrls = [...new Set(urlMatches.map(u => u.replace(/[.,;)]+$/, '')))].slice(0, 20);
|
||||
|
||||
// Extract sender domain for context
|
||||
const senderDomain = fromEmail.includes('@') ? fromEmail.split('@')[1] : fromEmail;
|
||||
const senderDomain = (fromEmail.split('@')[1] ?? fromEmail).toLowerCase();
|
||||
|
||||
// Check whether this domain is verified by the project. A verified
|
||||
// domain means the sender proved DNS/DKIM control — strong evidence of
|
||||
// legitimacy, especially for institutional TLDs like .gov, .edu, .mil.
|
||||
const verifiedDomain = await prisma.domain.findFirst({
|
||||
where: {projectId, domain: senderDomain, verified: true},
|
||||
select: {domain: true},
|
||||
});
|
||||
const isDomainVerified = verifiedDomain !== null;
|
||||
|
||||
// Institutional TLDs that imply a vetted, real-world entity behind the
|
||||
// domain (government, military, accredited education). When combined
|
||||
// with DKIM verification these effectively cannot be phishing senders.
|
||||
const institutionalTldPattern =
|
||||
/\.(gov|mil|edu)(\.[a-z]{2,})?$|\.gc\.ca$|\.gouv\.fr$|\.gov\.uk$|\.ac\.[a-z]{2,}$/i;
|
||||
const isInstitutionalDomain = institutionalTldPattern.test(senderDomain);
|
||||
|
||||
// Skip the LLM check entirely when the sender is a verified institutional
|
||||
// domain (e.g. a .gov customer). These TLDs are gated by registries that
|
||||
// verify the real-world entity, and DKIM verification proves the project
|
||||
// controls the domain — together they make phishing effectively
|
||||
// impossible from this sender. Avoids paying for an LLM call that
|
||||
// sometimes false-positives on official government communications.
|
||||
if (isDomainVerified && isInstitutionalDomain) {
|
||||
signale.info(
|
||||
`[PHISHING] Skipping check for project ${projectId} — verified institutional domain (${senderDomain})`,
|
||||
);
|
||||
return safeResponse;
|
||||
}
|
||||
|
||||
// Call OpenRouter API
|
||||
const response = await fetch('https://openrouter.ai/api/v1/chat/completions', {
|
||||
@@ -835,7 +827,11 @@ Criteria for phishing/dangerous content:
|
||||
- Requests for sensitive personal information
|
||||
|
||||
IMPORTANT - Use sender and project context when evaluating:
|
||||
- The sender project name and domain are provided. Links to the sender's own domain(s) are expected and NOT suspicious.
|
||||
- The sender project name and domain are provided, along with whether the domain has been verified (DKIM/DNS) by this project.
|
||||
- A VERIFIED sender domain means the sender proved ownership of the domain via DNS records. This is strong evidence of legitimacy.
|
||||
- If the verified sender domain is an institutional domain (e.g. .gov, .gov.uk, .gouv.fr, .gc.ca, .mil, .edu, .ac.*), treat the email as legitimate institutional communication. Government, military, and accredited education domains cannot be obtained by phishers — do NOT flag these as impersonation of government/banks/etc. just because the content mentions official topics, taxes, benefits, court notices, etc.
|
||||
- Impersonation rules only apply when the sender domain does NOT match the brand being referenced. A verified bank domain sending a banking email is not impersonating itself.
|
||||
- Links to the sender's own domain(s) are expected and NOT suspicious.
|
||||
- URLs that match or are clearly related to the project name or sender domain add credibility.
|
||||
- Only flag a URL as suspicious if it is unrelated to or impersonates a different known brand.
|
||||
- Lack of recognizable brand does NOT make an email phishing — many legitimate businesses are not famous.
|
||||
@@ -848,6 +844,8 @@ Set confidence to 100 only if you are absolutely certain it's phishing.`,
|
||||
role: 'user',
|
||||
content: `Sender project name: ${projectName}
|
||||
Sender domain: ${senderDomain}
|
||||
Sender domain verified (DKIM/DNS confirmed by project): ${isDomainVerified ? 'yes' : 'no'}
|
||||
Sender domain is an institutional TLD (gov/mil/edu/ac/etc.): ${isInstitutionalDomain ? 'yes' : 'no'}
|
||||
${uniqueUrls.length > 0 ? `URLs found in email: ${uniqueUrls.join(', ')}` : ''}
|
||||
|
||||
Subject: ${subject}
|
||||
@@ -971,7 +969,7 @@ ${strippedBody.substring(0, 2000)}`,
|
||||
// Disable the project
|
||||
await prisma.project.update({
|
||||
where: {id: projectId},
|
||||
data: {disabled: true},
|
||||
data: {disabled: true, disabledReason: 'PHISHING_DETECTED'},
|
||||
});
|
||||
|
||||
const violation = `A policy violation was detected. Please contact support for more details.`;
|
||||
|
||||
@@ -908,7 +908,14 @@ export class WorkflowExecutionService {
|
||||
}
|
||||
|
||||
/**
|
||||
* WEBHOOK step - Call an external webhook
|
||||
* WEBHOOK step - Call an external webhook.
|
||||
*
|
||||
* Renders `{{vars}}` in `url`, header values, and `body`. The variable
|
||||
* scope is a superset of the SEND_EMAIL scope: id, email, contact data,
|
||||
* execution context, and subscribe/unsubscribe/manage URLs — plus a
|
||||
* webhook-only `event` namespace exposing the trigger event payload.
|
||||
* `method` is intentionally NOT rendered — it must remain a literal
|
||||
* HTTP verb.
|
||||
*/
|
||||
private static async executeWebhook(
|
||||
_step: WorkflowStep,
|
||||
@@ -924,9 +931,37 @@ export class WorkflowExecutionService {
|
||||
contact.data && typeof contact.data === 'object' && !Array.isArray(contact.data)
|
||||
? (contact.data as Record<string, unknown>)
|
||||
: {};
|
||||
const executionContext =
|
||||
execution.context && typeof execution.context === 'object' && !Array.isArray(execution.context)
|
||||
? (execution.context as Record<string, unknown>)
|
||||
: {};
|
||||
const context = execution.context || {};
|
||||
|
||||
const payload = body || {
|
||||
// Render scope: SEND_EMAIL's scope (id, email, contact data, execution
|
||||
// context, subscribe/unsubscribe/manage URLs) plus a webhook-only
|
||||
// `event` namespace carrying the trigger event payload. `method` is
|
||||
// intentionally NOT rendered — it must remain a literal HTTP verb.
|
||||
const variables = {
|
||||
id: contact.id,
|
||||
email: contact.email,
|
||||
...contactData,
|
||||
...executionContext,
|
||||
data: contactData,
|
||||
event: context,
|
||||
unsubscribeUrl: `${DASHBOARD_URI}/unsubscribe/${contact.id}`,
|
||||
subscribeUrl: `${DASHBOARD_URI}/subscribe/${contact.id}`,
|
||||
manageUrl: `${DASHBOARD_URI}/manage/${contact.id}`,
|
||||
};
|
||||
|
||||
const renderedUrl = this.renderTemplate(url, variables);
|
||||
const renderedHeaders = headers
|
||||
? Object.fromEntries(
|
||||
Object.entries(headers).map(([key, value]) => [key, this.renderTemplate(value, variables)]),
|
||||
)
|
||||
: undefined;
|
||||
const renderedBody = body ? this.renderJsonTemplate(body, variables) : undefined;
|
||||
|
||||
const payload = renderedBody || {
|
||||
contact: {
|
||||
email: contact.email,
|
||||
subscribed: contact.subscribed,
|
||||
@@ -944,11 +979,11 @@ export class WorkflowExecutionService {
|
||||
};
|
||||
|
||||
// Make HTTP request
|
||||
const response = await WorkflowExecutionService.safeFetch(url, {
|
||||
const response = await WorkflowExecutionService.safeFetch(renderedUrl, {
|
||||
method,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
...headers,
|
||||
...renderedHeaders,
|
||||
},
|
||||
body: method !== 'GET' ? JSON.stringify(payload) : undefined,
|
||||
});
|
||||
@@ -962,7 +997,7 @@ export class WorkflowExecutionService {
|
||||
}
|
||||
|
||||
return {
|
||||
url,
|
||||
url: renderedUrl,
|
||||
method,
|
||||
statusCode: response.status,
|
||||
success: response.ok,
|
||||
@@ -970,6 +1005,28 @@ export class WorkflowExecutionService {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper: Recursively render template variables in any JSON-shaped value.
|
||||
* Strings are rendered, arrays/objects are walked, and non-string scalars
|
||||
* (numbers, booleans, null) are returned untouched.
|
||||
*/
|
||||
private static renderJsonTemplate(value: unknown, variables: Record<string, unknown>): unknown {
|
||||
if (typeof value === 'string') {
|
||||
return this.renderTemplate(value, variables);
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value.map(item => this.renderJsonTemplate(item, variables));
|
||||
}
|
||||
if (value !== null && typeof value === 'object') {
|
||||
const result: Record<string, unknown> = {};
|
||||
for (const [key, child] of Object.entries(value as Record<string, unknown>)) {
|
||||
result[key] = this.renderJsonTemplate(child, variables);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* UPDATE_CONTACT step - Update contact data
|
||||
*/
|
||||
|
||||
@@ -50,27 +50,20 @@ describe('SecurityService', () => {
|
||||
const complainedCount = opts?.complainedCount ?? 0;
|
||||
const createdAt = opts?.createdAt ?? new Date();
|
||||
|
||||
const emails = [];
|
||||
for (let i = 0; i < count; i++) {
|
||||
emails.push(
|
||||
prisma.email.create({
|
||||
data: {
|
||||
projectId,
|
||||
contactId,
|
||||
subject: `Test ${i}`,
|
||||
body: '<p>test</p>',
|
||||
from: 'test@example.com',
|
||||
status: EmailStatus.SENT,
|
||||
sourceType: EmailSourceType.TRANSACTIONAL,
|
||||
sentAt: createdAt,
|
||||
createdAt,
|
||||
bouncedAt: i < bouncedCount ? createdAt : null,
|
||||
complainedAt: i >= bouncedCount && i < bouncedCount + complainedCount ? createdAt : null,
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
await Promise.all(emails);
|
||||
const data = Array.from({length: count}, (_, i) => ({
|
||||
projectId,
|
||||
contactId,
|
||||
subject: `Test ${i}`,
|
||||
body: '<p>test</p>',
|
||||
from: 'test@example.com',
|
||||
status: EmailStatus.SENT,
|
||||
sourceType: EmailSourceType.TRANSACTIONAL,
|
||||
sentAt: createdAt,
|
||||
createdAt,
|
||||
bouncedAt: i < bouncedCount ? createdAt : null,
|
||||
complainedAt: i >= bouncedCount && i < bouncedCount + complainedCount ? createdAt : null,
|
||||
}));
|
||||
await prisma.email.createMany({data});
|
||||
}
|
||||
|
||||
describe('Rate-based checks (existing behavior)', () => {
|
||||
@@ -109,14 +102,12 @@ describe('SecurityService', () => {
|
||||
await createEmails(50, {bouncedCount: 10});
|
||||
|
||||
const status = await SecurityService.getSecurityStatus(projectId);
|
||||
// Rate-based check doesn't trigger, but absolute count ceiling might
|
||||
// With 10 bounces in 24h, this is below the 50-bounce ceiling for established projects
|
||||
expect(status.violations).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Absolute count ceilings (established projects)', () => {
|
||||
// Age the project past the new-project window so standard ceilings apply
|
||||
describe('Established projects skip absolute ceilings', () => {
|
||||
// Age the project past the new-project window
|
||||
beforeEach(async () => {
|
||||
const oldDate = new Date(Date.now() - 31 * 24 * 60 * 60 * 1000);
|
||||
await prisma.project.update({
|
||||
@@ -125,45 +116,27 @@ describe('SecurityService', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('should trigger critical when 24-hour bounce count exceeds ceiling', async () => {
|
||||
// 20,000 emails, 101 bounces = 0.5% rate (well below rate threshold)
|
||||
// But 101 bounces > 100 (24h critical ceiling for established projects)
|
||||
await createEmails(20000, {bouncedCount: 101});
|
||||
|
||||
const status = await SecurityService.getSecurityStatus(projectId);
|
||||
expect(status.shouldDisable).toBe(true);
|
||||
expect(status.violations.some(v => v.includes('24-hour bounce count'))).toBe(true);
|
||||
});
|
||||
|
||||
it('should trigger warning when 24-hour bounce count exceeds warning ceiling', async () => {
|
||||
// 10,000 emails, 51 bounces = 0.51% (below rate threshold)
|
||||
// But 51 > 50 (24h warning ceiling), below 100 critical
|
||||
await createEmails(10000, {bouncedCount: 51});
|
||||
|
||||
const status = await SecurityService.getSecurityStatus(projectId);
|
||||
expect(status.isHealthy).toBe(true); // warnings don't make it unhealthy
|
||||
expect(status.warnings.some(w => w.includes('24-hour bounce count'))).toBe(true);
|
||||
});
|
||||
|
||||
it('should trigger critical when 24-hour complaint count exceeds ceiling', async () => {
|
||||
// 20,000 emails, 26 complaints = 0.13% (below complaint rate critical of 0.15%)
|
||||
// But 26 > 25 (24h complaint critical ceiling)
|
||||
await createEmails(20000, {complainedCount: 26});
|
||||
|
||||
const status = await SecurityService.getSecurityStatus(projectId);
|
||||
expect(status.shouldDisable).toBe(true);
|
||||
expect(status.violations.some(v => v.includes('24-hour complaint count'))).toBe(true);
|
||||
});
|
||||
|
||||
it('should NOT trigger ceiling when bounce count is below ceiling', async () => {
|
||||
// 20,000 emails, 40 bounces = below 50 warning ceiling for established projects
|
||||
await createEmails(20000, {bouncedCount: 40});
|
||||
it('should NOT trigger on high absolute bounce count when rate is healthy', async () => {
|
||||
// 20,000 emails, 200 bounces = 1% rate (well below rate threshold)
|
||||
// Established projects rely solely on rates — high absolute counts at
|
||||
// high volume don't indicate abuse.
|
||||
await createEmails(20000, {bouncedCount: 200});
|
||||
|
||||
const status = await SecurityService.getSecurityStatus(projectId);
|
||||
expect(status.isHealthy).toBe(true);
|
||||
expect(status.shouldDisable).toBe(false);
|
||||
expect(status.violations).toHaveLength(0);
|
||||
expect(status.warnings).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should NOT trigger on high absolute complaint count when rate is healthy', async () => {
|
||||
// 100,000 emails, 30 complaints = 0.03% (at warning floor, below critical 0.15%)
|
||||
// Old absolute ceiling (25 complaints in 7d critical) would have tripped.
|
||||
await createEmails(100000, {complainedCount: 30});
|
||||
|
||||
const status = await SecurityService.getSecurityStatus(projectId);
|
||||
expect(status.shouldDisable).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('New project stricter thresholds', () => {
|
||||
@@ -178,7 +151,7 @@ describe('SecurityService', () => {
|
||||
expect(status.violations.some(v => v.includes('new project'))).toBe(true);
|
||||
});
|
||||
|
||||
it('should apply standard ceilings for projects over 30 days old', async () => {
|
||||
it('should NOT apply absolute ceilings for projects over 30 days old', async () => {
|
||||
// Age the project to 31 days
|
||||
const oldDate = new Date(Date.now() - 31 * 24 * 60 * 60 * 1000);
|
||||
await prisma.project.update({
|
||||
@@ -186,15 +159,14 @@ describe('SecurityService', () => {
|
||||
data: {createdAt: oldDate},
|
||||
});
|
||||
|
||||
// 10,000 emails, 26 bounces (above 25 new project ceiling, below 50 standard warning ceiling)
|
||||
// 10,000 emails, 26 bounces — would trip new-project ceiling, but
|
||||
// established projects skip ceilings entirely (rate is 0.26%, healthy).
|
||||
await createEmails(10000, {bouncedCount: 26});
|
||||
|
||||
const status = await SecurityService.getSecurityStatus(projectId);
|
||||
expect(status.isNewProject).toBe(false);
|
||||
// 26 is below the 50-bounce 24h warning ceiling for established projects
|
||||
expect(status.warnings.some(w => w.includes('24-hour bounce count'))).toBe(false);
|
||||
// And below the 100-bounce 24h critical ceiling
|
||||
expect(status.violations.some(v => v.includes('24-hour bounce count'))).toBe(false);
|
||||
expect(status.warnings.some(w => w.includes('bounce count'))).toBe(false);
|
||||
expect(status.violations.some(v => v.includes('bounce count'))).toBe(false);
|
||||
});
|
||||
|
||||
it('should catch new project blasting emails with delayed bounces', async () => {
|
||||
@@ -212,8 +184,8 @@ describe('SecurityService', () => {
|
||||
|
||||
describe('checkAndEnforceSecurityLimits', () => {
|
||||
it('should disable project when critical thresholds are exceeded', async () => {
|
||||
// Create enough bounces to trigger critical
|
||||
await createEmails(20000, {bouncedCount: 101});
|
||||
// New project, 20K emails with 30 bounces — exceeds new project 24h critical ceiling
|
||||
await createEmails(20000, {bouncedCount: 30});
|
||||
|
||||
await SecurityService.checkAndEnforceSecurityLimits(projectId);
|
||||
|
||||
@@ -225,13 +197,13 @@ describe('SecurityService', () => {
|
||||
});
|
||||
|
||||
it('should NOT disable project when only warnings exist', async () => {
|
||||
// 10,000 emails, 51 bounces (above warning but below critical for established project)
|
||||
// Established project, 200 emails, 12 bounces = 6% (above 5% warning, below 10% critical)
|
||||
const oldDate = new Date(Date.now() - 31 * 24 * 60 * 60 * 1000);
|
||||
await prisma.project.update({
|
||||
where: {id: projectId},
|
||||
data: {createdAt: oldDate},
|
||||
});
|
||||
await createEmails(10000, {bouncedCount: 51});
|
||||
await createEmails(200, {bouncedCount: 12});
|
||||
|
||||
await SecurityService.checkAndEnforceSecurityLimits(projectId);
|
||||
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
|
||||
import {StepExecutionStatus, WorkflowExecutionStatus, WorkflowStepType, WorkflowTriggerType} from '@plunk/db';
|
||||
import {WorkflowExecutionService} from '../WorkflowExecutionService';
|
||||
import {factories, getPrismaClient} from '../../../../../test/helpers';
|
||||
|
||||
/**
|
||||
* Tests for WEBHOOK step config templating.
|
||||
*
|
||||
* `executeWebhook` is a private static method but is invokable at runtime
|
||||
* through a `as any` cast. We mock `safeFetch` (also private) via the
|
||||
* same mechanism so we can capture the rendered request without making a
|
||||
* real network call.
|
||||
*/
|
||||
describe('WorkflowExecutionService.executeWebhook templating', () => {
|
||||
let projectId: string;
|
||||
const prisma = getPrismaClient();
|
||||
|
||||
// Capture (url, options) passed to safeFetch
|
||||
let safeFetchSpy: ReturnType<typeof vi.spyOn>;
|
||||
let captured: {url: string; options: RequestInit} | null = null;
|
||||
|
||||
beforeEach(async () => {
|
||||
const {project} = await factories.createUserWithProject();
|
||||
projectId = project.id;
|
||||
|
||||
captured = null;
|
||||
safeFetchSpy = vi
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
.spyOn(WorkflowExecutionService as any, 'safeFetch')
|
||||
.mockImplementation(async (...args: unknown[]) => {
|
||||
const [url, options] = args as [string, RequestInit];
|
||||
captured = {url, options};
|
||||
return new Response('{"ok":true}', {
|
||||
status: 200,
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
safeFetchSpy.mockRestore();
|
||||
});
|
||||
|
||||
/**
|
||||
* Helper: build a workflow with a single WEBHOOK step using the given
|
||||
* config, plus a contact and a RUNNING execution. Returns the args
|
||||
* shape `executeWebhook` expects.
|
||||
*/
|
||||
async function setup(
|
||||
webhookConfig: Record<string, unknown>,
|
||||
contactOverrides: {data?: Record<string, unknown>} = {},
|
||||
executionContext: Record<string, unknown> = {},
|
||||
) {
|
||||
const contact = await factories.createContact({
|
||||
projectId,
|
||||
data: contactOverrides.data,
|
||||
});
|
||||
|
||||
const workflow = await factories.createWorkflow({
|
||||
projectId,
|
||||
enabled: true,
|
||||
triggerType: WorkflowTriggerType.EVENT,
|
||||
triggerConfig: {eventName: 'test.event'},
|
||||
});
|
||||
|
||||
const step = await prisma.workflowStep.create({
|
||||
data: {
|
||||
workflowId: workflow.id,
|
||||
type: WorkflowStepType.WEBHOOK,
|
||||
name: 'Webhook',
|
||||
position: {x: 0, y: 0},
|
||||
config: webhookConfig,
|
||||
},
|
||||
});
|
||||
|
||||
const execution = await prisma.workflowExecution.create({
|
||||
data: {
|
||||
workflowId: workflow.id,
|
||||
contactId: contact.id,
|
||||
status: WorkflowExecutionStatus.RUNNING,
|
||||
context: executionContext,
|
||||
},
|
||||
include: {contact: true, workflow: true},
|
||||
});
|
||||
|
||||
const stepExecution = await prisma.workflowStepExecution.create({
|
||||
data: {
|
||||
executionId: execution.id,
|
||||
stepId: step.id,
|
||||
status: StepExecutionStatus.RUNNING,
|
||||
startedAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
return {step, execution, stepExecution};
|
||||
}
|
||||
|
||||
async function invokeWebhook(
|
||||
step: unknown,
|
||||
execution: unknown,
|
||||
stepExecution: unknown,
|
||||
config: unknown,
|
||||
) {
|
||||
// Call through `as any` because executeWebhook is private at the
|
||||
// TypeScript level. JS has no actual access control.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return (WorkflowExecutionService as any).executeWebhook(step, execution, stepExecution, config);
|
||||
}
|
||||
|
||||
it('renders {{vars}} in the URL from contact.data', async () => {
|
||||
const {step, execution, stepExecution} = await setup(
|
||||
{
|
||||
url: 'https://example.com/api/users/{{userId}}',
|
||||
method: 'GET',
|
||||
},
|
||||
{data: {userId: 'abc-123'}},
|
||||
);
|
||||
|
||||
await invokeWebhook(step, execution, stepExecution, step.config);
|
||||
|
||||
expect(captured).not.toBeNull();
|
||||
expect(captured!.url).toBe('https://example.com/api/users/abc-123');
|
||||
});
|
||||
|
||||
it('renders {{vars}} in header values', async () => {
|
||||
const {step, execution, stepExecution} = await setup(
|
||||
{
|
||||
url: 'https://example.com/hook',
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: 'Bearer {{apiToken}}',
|
||||
'X-Static': 'literal',
|
||||
},
|
||||
},
|
||||
{data: {apiToken: 'secret-token-xyz'}},
|
||||
);
|
||||
|
||||
await invokeWebhook(step, execution, stepExecution, step.config);
|
||||
|
||||
expect(captured).not.toBeNull();
|
||||
const headers = captured!.options.headers as Record<string, string>;
|
||||
expect(headers.Authorization).toBe('Bearer secret-token-xyz');
|
||||
expect(headers['X-Static']).toBe('literal');
|
||||
});
|
||||
|
||||
it('renders {{vars}} in nested object body leaves and JSON-encodes', async () => {
|
||||
const {step, execution, stepExecution} = await setup(
|
||||
{
|
||||
url: 'https://example.com/hook',
|
||||
method: 'POST',
|
||||
body: {
|
||||
user: {
|
||||
email: '{{email}}',
|
||||
name: '{{firstName}}',
|
||||
},
|
||||
ref: 'literal-ref',
|
||||
tags: ['plan:{{plan}}', 'static'],
|
||||
},
|
||||
},
|
||||
{data: {firstName: 'Ada', plan: 'gold'}},
|
||||
{campaignId: 'camp-9'},
|
||||
);
|
||||
|
||||
await invokeWebhook(step, execution, stepExecution, step.config);
|
||||
|
||||
expect(captured).not.toBeNull();
|
||||
const body = JSON.parse(captured!.options.body as string);
|
||||
expect(body.user.email).toBe(execution.contact.email);
|
||||
expect(body.user.name).toBe('Ada');
|
||||
expect(body.ref).toBe('literal-ref');
|
||||
expect(body.tags).toEqual(['plan:gold', 'static']);
|
||||
});
|
||||
|
||||
it('leaves non-string body leaves untouched', async () => {
|
||||
const {step, execution, stepExecution} = await setup({
|
||||
url: 'https://example.com/hook',
|
||||
method: 'POST',
|
||||
body: {
|
||||
score: 42,
|
||||
active: true,
|
||||
deleted: null,
|
||||
meta: {
|
||||
count: 7,
|
||||
enabled: false,
|
||||
},
|
||||
tags: ['{{plan ?? free}}', 100, false],
|
||||
},
|
||||
});
|
||||
|
||||
await invokeWebhook(step, execution, stepExecution, step.config);
|
||||
|
||||
expect(captured).not.toBeNull();
|
||||
const body = JSON.parse(captured!.options.body as string);
|
||||
expect(body.score).toBe(42);
|
||||
expect(body.active).toBe(true);
|
||||
expect(body.deleted).toBe(null);
|
||||
expect(body.meta).toEqual({count: 7, enabled: false});
|
||||
// String leaf rendered (with default), non-string leaves preserved.
|
||||
expect(body.tags).toEqual(['free', 100, false]);
|
||||
});
|
||||
|
||||
it('renders {{event.*}} variables from the trigger payload', async () => {
|
||||
const {step, execution, stepExecution} = await setup(
|
||||
{
|
||||
url: 'https://example.com/hooks/{{event.referrer}}',
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'X-Email-Id': '{{event.emailId}}',
|
||||
},
|
||||
body: {
|
||||
referrer: '{{event.referrer}}',
|
||||
subject: '{{event.subject}}',
|
||||
},
|
||||
},
|
||||
{},
|
||||
{referrer: 'newsletter-may', emailId: 'eml_abc123', subject: 'Welcome'},
|
||||
);
|
||||
|
||||
await invokeWebhook(step, execution, stepExecution, step.config);
|
||||
|
||||
expect(captured).not.toBeNull();
|
||||
expect(captured!.url).toBe('https://example.com/hooks/newsletter-may');
|
||||
const headers = captured!.options.headers as Record<string, string>;
|
||||
expect(headers['X-Email-Id']).toBe('eml_abc123');
|
||||
const body = JSON.parse(captured!.options.body as string);
|
||||
expect(body.referrer).toBe('newsletter-may');
|
||||
expect(body.subject).toBe('Welcome');
|
||||
});
|
||||
});
|
||||
@@ -758,8 +758,20 @@ describe('WorkflowService', () => {
|
||||
});
|
||||
const contact = await factories.createContact({projectId});
|
||||
|
||||
// Start first execution (still running)
|
||||
await WorkflowService.startExecution(projectId, workflow.id, contact.id);
|
||||
// Insert a RUNNING execution directly to avoid racing with the background
|
||||
// step processor that startExecution kicks off (a trigger-only workflow can
|
||||
// transition to COMPLETED before the second call observes it as RUNNING).
|
||||
const triggerStep = await prisma.workflowStep.findFirst({
|
||||
where: {workflowId: workflow.id, type: WorkflowStepType.TRIGGER},
|
||||
});
|
||||
await prisma.workflowExecution.create({
|
||||
data: {
|
||||
workflowId: workflow.id,
|
||||
contactId: contact.id,
|
||||
status: WorkflowExecutionStatus.RUNNING,
|
||||
currentStepId: triggerStep?.id,
|
||||
},
|
||||
});
|
||||
|
||||
// Second execution should fail (first still running)
|
||||
await expect(WorkflowService.startExecution(projectId, workflow.id, contact.id)).rejects.toThrow(
|
||||
|
||||
@@ -28,7 +28,7 @@ function getQ(types: Array<{ type: string; q: number }>, target: string): number
|
||||
function negotiate(accept: string): Negotiated {
|
||||
if (!accept) return 'html';
|
||||
const types = parseAccept(accept);
|
||||
const mdQ = getQ(types, 'text/markdown');
|
||||
const mdQ = types.find(t => t.type === 'text/markdown')?.q ?? -1;
|
||||
const htmlQ = getQ(types, 'text/html');
|
||||
if (mdQ <= 0 && htmlQ <= 0) return 'none';
|
||||
if (mdQ > 0 && mdQ >= htmlQ) return 'markdown';
|
||||
|
||||
@@ -725,6 +725,7 @@ export default function Index() {
|
||||
viewport={{once: true}}
|
||||
transition={{duration: 0.8, delay: 0.9, ease: [0.22, 1, 0.36, 1]}}
|
||||
className={'mx-auto max-w-xl'}
|
||||
data-nosnippet
|
||||
>
|
||||
<div className={'overflow-hidden rounded-[20px] border border-neutral-200 bg-white'}>
|
||||
<div className={'flex items-center gap-5 p-6'}>
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
import {describe, expect, it} from 'vitest';
|
||||
import {detectCustomHtmlPatterns} from '../emailStyles';
|
||||
|
||||
describe('detectCustomHtmlPatterns', () => {
|
||||
describe('empty / whitespace input', () => {
|
||||
it('returns false for empty string', () => {
|
||||
expect(detectCustomHtmlPatterns('')).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false for whitespace-only string', () => {
|
||||
expect(detectCustomHtmlPatterns(' \n\t ')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('content TipTap can round-trip (should NOT be flagged as custom)', () => {
|
||||
it('returns false for a basic paragraph', () => {
|
||||
expect(detectCustomHtmlPatterns('<p>Hello</p>')).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false for headings, lists, blockquote, bold, italic', () => {
|
||||
expect(
|
||||
detectCustomHtmlPatterns(
|
||||
'<h1>Title</h1><p><strong>bold</strong> <em>italic</em></p><ul><li>one</li></ul><blockquote>quote</blockquote>',
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false for <span> with inline color style (TipTap TextStyle output)', () => {
|
||||
expect(detectCustomHtmlPatterns('<span style="color: rgb(220, 38, 38)">red</span>')).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false for <span> with background-color: initial (TipTap export artifact)', () => {
|
||||
expect(detectCustomHtmlPatterns('<span style="background-color: initial">stuff</span>')).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false for <span> with background-color: transparent (TipTap export artifact)', () => {
|
||||
expect(detectCustomHtmlPatterns('<span style="background-color: transparent">stuff</span>')).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false for <a> with inline style (TipTap Link output)', () => {
|
||||
expect(detectCustomHtmlPatterns('<a href="https://example.com" style="color: red">link</a>')).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false for paragraph with inline text-align style', () => {
|
||||
expect(detectCustomHtmlPatterns('<p style="text-align: center">centered</p>')).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false when an href URL contains "id=" or "contactId=" (must not match custom-attr regex)', () => {
|
||||
expect(
|
||||
detectCustomHtmlPatterns('<a href="https://example.com/u?contactId=abc&id=123">unsub</a>'),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false for allowed class prefixes', () => {
|
||||
expect(detectCustomHtmlPatterns('<p class="prose">x</p>')).toBe(false);
|
||||
expect(detectCustomHtmlPatterns('<span class="variable-mention">x</span>')).toBe(false);
|
||||
expect(detectCustomHtmlPatterns('<img class="email-image" src="x" />')).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false for a TipTap-style colored span wrapped in a paragraph', () => {
|
||||
expect(
|
||||
detectCustomHtmlPatterns('<p>Hello <span style="color: rgb(220, 38, 38);">world</span>!</p>'),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('content TipTap can NOT round-trip (should be flagged as custom)', () => {
|
||||
it('returns true for <div>', () => {
|
||||
expect(detectCustomHtmlPatterns('<div>stuff</div>')).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true for <table> markup (no TipTap Table extension loaded)', () => {
|
||||
expect(detectCustomHtmlPatterns('<table><tr><td>x</td></tr></table>')).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true for a single <table> tag', () => {
|
||||
expect(detectCustomHtmlPatterns('<table>x</table>')).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true for <style> tag', () => {
|
||||
expect(detectCustomHtmlPatterns('<style>p { color: red; }</style>')).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true for @media query inside a style block', () => {
|
||||
expect(detectCustomHtmlPatterns('@media (max-width: 600px) { ... }')).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true for custom data-* attribute', () => {
|
||||
expect(detectCustomHtmlPatterns('<p data-foo="bar">x</p>')).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true for aria-* attribute', () => {
|
||||
expect(detectCustomHtmlPatterns('<p aria-label="x">y</p>')).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true for role= attribute', () => {
|
||||
expect(detectCustomHtmlPatterns('<p role="presentation">x</p>')).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true for id= attribute on an element', () => {
|
||||
expect(detectCustomHtmlPatterns('<p id="main">x</p>')).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true for a disallowed CSS class', () => {
|
||||
expect(detectCustomHtmlPatterns('<p class="custom">x</p>')).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true for <section>, <article>, <header>, <footer>, <nav>, <aside>, <main>', () => {
|
||||
expect(detectCustomHtmlPatterns('<section>x</section>')).toBe(true);
|
||||
expect(detectCustomHtmlPatterns('<article>x</article>')).toBe(true);
|
||||
expect(detectCustomHtmlPatterns('<header>x</header>')).toBe(true);
|
||||
expect(detectCustomHtmlPatterns('<footer>x</footer>')).toBe(true);
|
||||
expect(detectCustomHtmlPatterns('<nav>x</nav>')).toBe(true);
|
||||
expect(detectCustomHtmlPatterns('<aside>x</aside>')).toBe(true);
|
||||
expect(detectCustomHtmlPatterns('<main>x</main>')).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true for form/input/button/iframe/svg', () => {
|
||||
expect(detectCustomHtmlPatterns('<form>x</form>')).toBe(true);
|
||||
expect(detectCustomHtmlPatterns('<input type="text" />')).toBe(true);
|
||||
expect(detectCustomHtmlPatterns('<button>x</button>')).toBe(true);
|
||||
expect(detectCustomHtmlPatterns('<iframe src="x"></iframe>')).toBe(true);
|
||||
expect(detectCustomHtmlPatterns('<svg><circle /></svg>')).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,10 +1,15 @@
|
||||
// Detects if HTML contains custom patterns that indicate it was written in the HTML editor
|
||||
// rather than the visual editor. Custom HTML should render as-is without prose wrapper.
|
||||
//
|
||||
// The TipTap editor in EmailEditor.tsx loads: StarterKit (paragraphs, headings, lists,
|
||||
// blockquote, code, hr, bold, italic, strike, etc.), TextAlign, Color, TextStyle, Link,
|
||||
// ResizableImage, and VariableMention. Of these, TextStyle + Color + Link natively
|
||||
// round-trip <span style="color: ..."> / <a style="color: ..."> markup that TipTap itself
|
||||
// generates when you change text color or style a link. We must therefore PERMIT what
|
||||
// TipTap can represent and REJECT only what it can't.
|
||||
export const detectCustomHtmlPatterns = (html: string): boolean => {
|
||||
if (!html || html.trim() === '') return false;
|
||||
|
||||
const hasInlineStyles = /<[^>]+style\s*=\s*["'][^"']*["']/i.test(html);
|
||||
|
||||
const classMatches = html.matchAll(/class\s*=\s*["']([^"']*)["']/gi);
|
||||
let hasCustomClasses = false;
|
||||
for (const match of classMatches) {
|
||||
@@ -28,21 +33,26 @@ export const detectCustomHtmlPatterns = (html: string): boolean => {
|
||||
}
|
||||
}
|
||||
|
||||
const hasCustomAttributes = /<[^>]+(?:data-|aria-|role=|id=)/i.test(html);
|
||||
const hasComplexTables = /<table[^>]*>[\s\S]*?<table/i.test(html);
|
||||
const hasCustomElements = /<(?:div|span|section|article|header|footer|nav|aside)[^>]*>/i.test(html);
|
||||
// Custom attributes that carry semantics TipTap doesn't preserve. We require an
|
||||
// attribute-boundary (whitespace, `=`, or quote) before the prefix so that query
|
||||
// strings like `?id=...` inside an `href="..."` value don't false-match.
|
||||
const hasCustomAttributes = /<[a-z][^>]*?[\s"'](?:data-|aria-|role=|id=)/i.test(html);
|
||||
|
||||
// Elements TipTap cannot round-trip with the currently-loaded extension set.
|
||||
// - No Table/TableRow/TableCell extensions are loaded -> all table markup is custom.
|
||||
// - No Div/Section/etc. block-layout extensions -> reject layout containers.
|
||||
// - Form/embed/media/interactive elements have no TipTap representation here.
|
||||
// <span> is intentionally NOT in this list: TipTap's TextStyle extension emits and
|
||||
// accepts <span style="..."> for things like text color.
|
||||
const hasCustomElements =
|
||||
/<(?:div|section|article|header|footer|nav|aside|main|table|tr|td|th|tbody|thead|tfoot|colgroup|col|form|input|button|select|textarea|iframe|video|audio|svg|object|embed|details|summary|dialog)\b/i.test(
|
||||
html,
|
||||
);
|
||||
|
||||
const hasMediaQueries = /@media/i.test(html);
|
||||
const hasStyleTags = /<style[^>]*>/i.test(html);
|
||||
|
||||
return (
|
||||
hasInlineStyles ||
|
||||
hasCustomClasses ||
|
||||
hasCustomAttributes ||
|
||||
hasComplexTables ||
|
||||
hasCustomElements ||
|
||||
hasMediaQueries ||
|
||||
hasStyleTags
|
||||
);
|
||||
return hasCustomClasses || hasCustomAttributes || hasCustomElements || hasMediaQueries || hasStyleTags;
|
||||
};
|
||||
|
||||
export const wrapEmailWithStyles = (htmlBody: string): string => {
|
||||
|
||||
@@ -252,7 +252,7 @@ export default function CampaignsPage() {
|
||||
</div>
|
||||
|
||||
{/* Search & Filters */}
|
||||
<div className="flex flex-col sm:flex-row gap-3">
|
||||
<div className="flex flex-col sm:flex-row sm:items-center gap-3">
|
||||
<div className="relative flex-1">
|
||||
<Search className="absolute left-3 top-1/2 -translate-y-1/2 h-4 w-4 text-neutral-400" />
|
||||
<Input
|
||||
@@ -260,7 +260,7 @@ export default function CampaignsPage() {
|
||||
placeholder="Search campaigns..."
|
||||
value={searchInput}
|
||||
onChange={e => setSearchInput(e.target.value)}
|
||||
className="pl-10 pr-10"
|
||||
className="pl-10 pr-10 h-8 text-xs"
|
||||
/>
|
||||
{searchInput && (
|
||||
<button
|
||||
|
||||
@@ -405,7 +405,12 @@ export default function ContactsPage() {
|
||||
) : (
|
||||
<MailX className="h-4 w-4 text-red-600" />
|
||||
)}
|
||||
<span className="text-sm font-medium text-neutral-900">{contact.email}</span>
|
||||
<Link
|
||||
href={`/contacts/${contact.id}`}
|
||||
className="text-sm font-medium text-neutral-900 hover:text-neutral-700 focus-visible:outline-none focus-visible:underline"
|
||||
>
|
||||
{contact.email}
|
||||
</Link>
|
||||
</div>
|
||||
</td>
|
||||
<td className="px-6 py-4 whitespace-nowrap">
|
||||
@@ -455,7 +460,12 @@ export default function ContactsPage() {
|
||||
) : (
|
||||
<MailX className="h-4 w-4 text-red-600 flex-shrink-0" />
|
||||
)}
|
||||
<span className="text-sm font-medium text-neutral-900 truncate">{contact.email}</span>
|
||||
<Link
|
||||
href={`/contacts/${contact.id}`}
|
||||
className="text-sm font-medium text-neutral-900 truncate hover:text-neutral-700 focus-visible:outline-none focus-visible:underline"
|
||||
>
|
||||
{contact.email}
|
||||
</Link>
|
||||
</div>
|
||||
<span
|
||||
className={`inline-flex items-center px-2 py-0.5 rounded-full text-xs font-medium flex-shrink-0 ${
|
||||
|
||||
@@ -90,7 +90,7 @@ export default function TemplatesPage() {
|
||||
</div>
|
||||
|
||||
{/* Search & Filters */}
|
||||
<div className="flex flex-col sm:flex-row gap-3">
|
||||
<div className="flex flex-col sm:flex-row sm:items-center gap-3">
|
||||
<div className="relative flex-1">
|
||||
<Search className="absolute left-3 top-1/2 -translate-y-1/2 h-4 w-4 text-neutral-400" />
|
||||
<Input
|
||||
@@ -98,7 +98,7 @@ export default function TemplatesPage() {
|
||||
placeholder="Search templates..."
|
||||
value={searchInput}
|
||||
onChange={e => setSearchInput(e.target.value)}
|
||||
className="pl-10 pr-10"
|
||||
className="pl-10 pr-10 h-8 text-xs"
|
||||
/>
|
||||
{searchInput && (
|
||||
<button
|
||||
|
||||
@@ -117,7 +117,7 @@ export default function WorkflowsPage() {
|
||||
placeholder="Search workflows..."
|
||||
value={searchInput}
|
||||
onChange={e => setSearchInput(e.target.value)}
|
||||
className="pl-10 pr-10"
|
||||
className="pl-10 pr-10 h-8 text-xs"
|
||||
/>
|
||||
{searchInput && (
|
||||
<button
|
||||
|
||||
@@ -6,3 +6,16 @@
|
||||
body {
|
||||
font-family: 'Inter', sans-serif;
|
||||
}
|
||||
|
||||
/*
|
||||
* Two-tone palette: white content, gray chrome.
|
||||
* `--color-fd-background` paints the page (content area + nav).
|
||||
* `--color-fd-card` paints the sidebar (via `bg-fd-card` on `#nd-sidebar`)
|
||||
* and the `<Cards>` component — both read well as soft gray against white.
|
||||
*/
|
||||
:root {
|
||||
--color-fd-background: hsl(0, 0%, 100%);
|
||||
--color-fd-card: hsl(0, 0%, 96.5%);
|
||||
--color-fd-secondary: hsl(0, 0%, 95%);
|
||||
--color-fd-border: hsla(0, 0%, 80%, 60%);
|
||||
}
|
||||
|
||||
@@ -37,7 +37,7 @@ A workflow always begins with a single auto-created `TRIGGER` step. You build th
|
||||
| `DELAY` | Pauses the execution for a fixed duration before continuing. | `amount`, `unit` (`minutes` / `hours` / `days`) |
|
||||
| `WAIT_FOR_EVENT` | Pauses until a specified event is tracked on the contact, with a timeout fallback. | `eventName`, `timeout` (seconds) |
|
||||
| `CONDITION` | Branches the execution based on contact data or event data. Each `CONDITION` step has two outgoing transitions tagged `yes` / `no`. | A filter expression (same shape as segment filters) |
|
||||
| `WEBHOOK` | Calls an external HTTPS endpoint with contact + execution context as the JSON body. | `url`, optional `method`, `headers` |
|
||||
| `WEBHOOK` | Calls an external HTTPS endpoint with contact + execution context as the JSON body. `url`, header values, and `body` support `{{variables}}`. | `url`, optional `method`, `headers`, `body` |
|
||||
| `UPDATE_CONTACT` | Patches contact data — useful for tagging contacts as they progress (`{ stage: "activated" }`). | `data` object |
|
||||
| `EXIT` | Terminates the execution. Optionally records an `exitReason` for analytics. | optional `reason` |
|
||||
|
||||
|
||||
@@ -37,6 +37,8 @@ In this example, every imported contact ends up with `data.firstName`, `data.pla
|
||||
- **Email column**: must be present and valid. Rows with missing or invalid emails are reported back as errors.
|
||||
- **Reserved column names**: `id`, `subscribed`, `createdAt`, `updatedAt`, and the auto-generated URL variables (`unsubscribeUrl`, etc.) are silently filtered out. Don't include them as columns.
|
||||
- **Date columns**: use ISO 8601 (`2026-05-06T12:00:00Z`) so they're typed as dates and become usable with `within` / `olderThan` segment operators.
|
||||
- **Boolean columns**: `true`, `false`, `yes`, `no` (case-insensitive) are stored as booleans and get the boolean toggle in segment filters.
|
||||
- **Numeric columns**: plain integers and decimals (`42`, `3.14`) are stored as numbers and become usable with `gt` / `lt` segment operators. Leading-zero values (`01234`), `+`-prefixed numbers, and scientific notation stay strings so IDs, zip codes, and phone numbers aren't corrupted.
|
||||
- **Existing contacts**: if a row's email matches an existing contact, the import **updates** the contact (merging the CSV's columns into `data`). It doesn't create a duplicate or overwrite the whole record.
|
||||
|
||||
## Importing your CSV
|
||||
|
||||
@@ -84,6 +84,53 @@ After the trigger, add a **Webhook** step and configure it:
|
||||
}
|
||||
```
|
||||
|
||||
- **Body** (optional): Custom request body. When omitted, Plunk sends the [default payload](#webhook-payload) shown below. When provided, the value replaces the default payload entirely and is JSON-encoded before being sent.
|
||||
|
||||
</Step>
|
||||
|
||||
<Step>
|
||||
|
||||
### Use variables in the request (optional)
|
||||
|
||||
The `url`, header values, and `body` all support `{{variable}}` interpolation. The available scope is the same as `SEND_EMAIL` templates, plus a webhook-only `event` namespace exposing the trigger event payload:
|
||||
|
||||
| Variable | Value |
|
||||
| --------------------------------------------------------- | --------------------------------------------------------------------------- |
|
||||
| `{{id}}`, `{{email}}` | The contact's ID and email. |
|
||||
| `{{<key>}}` (top-level) | Any key from the contact's `data` JSON (e.g. `{{firstName}}`, `{{plan}}`). |
|
||||
| `{{data.<key>}}` | The same contact data, addressed via the `data` namespace. |
|
||||
| `{{event.<key>}}` | Webhook-only. Fields from the trigger event payload (e.g. `{{event.subject}}`). |
|
||||
| `{{<key>}}` (from execution context) | Keys passed in as `context` when starting a `MANUAL` execution. |
|
||||
| `{{unsubscribeUrl}}`, `{{subscribeUrl}}`, `{{manageUrl}}` | Per-contact subscription management URLs. |
|
||||
|
||||
The HTTP `method` is **not** templated — it must be a literal verb (`GET`, `POST`, `PUT`, `PATCH`, `DELETE`). The `url` must include a static scheme (`http://` or `https://`); placeholders are supported inside the URL but cannot replace the scheme.
|
||||
|
||||
Example — forward a contact event to your own API, parameterised by contact data:
|
||||
|
||||
**URL**
|
||||
|
||||
```text
|
||||
https://api.example.com/users/{{id}}/events
|
||||
```
|
||||
|
||||
**Headers**
|
||||
|
||||
```json
|
||||
{
|
||||
"Authorization": "Bearer your-secret-token"
|
||||
}
|
||||
```
|
||||
|
||||
**Body**
|
||||
|
||||
```json
|
||||
{
|
||||
"email": "{{email}}",
|
||||
"plan": "{{plan}}",
|
||||
"referrer": "{{event.referrer}}"
|
||||
}
|
||||
```
|
||||
|
||||
</Step>
|
||||
|
||||
<Step>
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
"---Docs---",
|
||||
"concepts",
|
||||
"guides",
|
||||
"---Recipes---",
|
||||
"recipes",
|
||||
"---API Reference---",
|
||||
"api-reference",
|
||||
"---Self-Hosting---",
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
---
|
||||
title: Double opt-in
|
||||
description: Require a confirmation click before a new signup starts receiving marketing email
|
||||
icon: MailCheck
|
||||
---
|
||||
|
||||
Double opt-in adds a confirmation step between "user signs up" and "user starts getting marketing email." It's the standard way to avoid mailing typoed addresses, role accounts, and anyone who didn't actually consent.
|
||||
|
||||
The trick is `{{subscribeUrl}}`: a per-contact link Plunk auto-injects into every send. Clicking it flips `subscribed` to `true` and fires a `contact.subscribed` event.
|
||||
|
||||
## Setup
|
||||
|
||||
import {Step, Steps} from 'fumadocs-ui/components/steps';
|
||||
|
||||
<Steps>
|
||||
|
||||
<Step>
|
||||
|
||||
### Create two templates
|
||||
|
||||
- A **Transactional** template for the confirmation email, containing `{{subscribeUrl}}`:
|
||||
|
||||
```html
|
||||
<p>Hi {{firstName}}, please confirm your email to start receiving updates:</p>
|
||||
<p><a href="{{subscribeUrl}}">Confirm my email</a></p>
|
||||
```
|
||||
|
||||
- A **Marketing** template for the welcome email that goes out *after* they confirm.
|
||||
|
||||
<Callout title="The confirmation must be transactional" type="warn">
|
||||
A marketing template targeted at an unsubscribed contact is [silently skipped](/concepts/contacts#emails-by-subscription-state). Use a transactional template for the confirmation specifically — it bypasses the subscription check.
|
||||
</Callout>
|
||||
|
||||
</Step>
|
||||
|
||||
<Step>
|
||||
|
||||
### Trigger the signup from your backend
|
||||
|
||||
Two calls with your secret key (`sk_*`): create the contact unsubscribed, then track the event that fires the confirmation workflow.
|
||||
|
||||
```bash
|
||||
curl https://next-api.useplunk.com/contacts \
|
||||
-H "Authorization: Bearer sk_your_secret_key" \
|
||||
-d '{ "email": "ada@example.com", "subscribed": false, "data": { "firstName": "Ada" } }'
|
||||
|
||||
curl https://next-api.useplunk.com/v1/track \
|
||||
-H "Authorization: Bearer sk_your_secret_key" \
|
||||
-d '{ "event": "signup.pending", "email": "ada@example.com", "subscribed": false }'
|
||||
```
|
||||
|
||||
Both calls pass `subscribed: false`. If you skip the first call and rely on `/v1/track` alone, tracking on an unknown email creates the contact — but defaults it to subscribed, which defeats the point.
|
||||
|
||||
</Step>
|
||||
|
||||
<Step>
|
||||
|
||||
### Workflow A: send the confirmation
|
||||
|
||||
**Workflows → New workflow**:
|
||||
|
||||
- **Trigger**: `EVENT` on `signup.pending`
|
||||
- `SEND_EMAIL` step → transactional confirmation template
|
||||
|
||||
Enable it.
|
||||
|
||||
</Step>
|
||||
|
||||
<Step>
|
||||
|
||||
### Workflow B: welcome them after confirmation
|
||||
|
||||
**Workflows → New workflow**:
|
||||
|
||||
- **Trigger**: `EVENT` on `contact.subscribed`
|
||||
- `SEND_EMAIL` step → marketing welcome template
|
||||
|
||||
Enable it. `contact.subscribed` fires whenever a contact opts in — including via `{{subscribeUrl}}`, the preferences page, or the API — so this workflow handles both first-time confirmations and resubscribes.
|
||||
|
||||
</Step>
|
||||
|
||||
</Steps>
|
||||
|
||||
## Reminder if they don't confirm
|
||||
|
||||
Extend Workflow A with a `WAIT_FOR_EVENT` step after the send:
|
||||
|
||||
- **Event**: `contact.subscribed`
|
||||
- **Timeout**: `86400` (24 hours)
|
||||
|
||||
On timeout, send a single reminder (also transactional). Keep the number of reminders small — repeated confirmation prompts look like spam to mailbox providers as much as to recipients.
|
||||
|
||||
## What's next
|
||||
|
||||
<Cards>
|
||||
<Card title="Unsubscribe & preferences pages" href="/guides/unsubscribe-pages">
|
||||
Detail on `{{subscribeUrl}}` and the hosted pages.
|
||||
</Card>
|
||||
<Card title="Templates" href="/concepts/templates">
|
||||
The difference between Marketing, Transactional, and Headless templates.
|
||||
</Card>
|
||||
</Cards>
|
||||
@@ -0,0 +1,19 @@
|
||||
---
|
||||
title: Recipes
|
||||
description: End-to-end walkthroughs for common patterns built on Plunk events and workflows
|
||||
icon: ChefHat
|
||||
---
|
||||
|
||||
Recipes are concrete, step-by-step builds for patterns we see most often in Plunk projects. Each one assumes you already understand the underlying [concepts](/concepts/workflows) and walks you through the exact API calls, workflow steps, and template variables involved.
|
||||
|
||||
<Cards>
|
||||
<Card title="Waitlist with confirmation email" href="/recipes/waitlist">
|
||||
Capture signups with a single tracked event, then automatically email each person who joins.
|
||||
</Card>
|
||||
<Card title="Sync unsubscribes to your database" href="/recipes/sync-unsubscribes">
|
||||
Keep your own user table in step with Plunk's subscription state using a webhook step.
|
||||
</Card>
|
||||
<Card title="Double opt-in" href="/recipes/double-opt-in">
|
||||
Add a confirmation step before a contact starts receiving marketing email, using `{{subscribeUrl}}`.
|
||||
</Card>
|
||||
</Cards>
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"pages": ["index", "waitlist", "sync-unsubscribes", "double-opt-in"]
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
---
|
||||
title: Sync unsubscribes to your database
|
||||
description: Mirror Plunk's subscription state into your own user table using a workflow + webhook
|
||||
icon: RefreshCw
|
||||
---
|
||||
|
||||
Every flip of a contact's `subscribed` state — manual edits, the hosted unsubscribe page, bounces, complaints — fires a `contact.unsubscribed` event. Wire a workflow with a `WEBHOOK` step to forward that to your backend.
|
||||
|
||||
## Setup
|
||||
|
||||
import {Step, Steps} from 'fumadocs-ui/components/steps';
|
||||
|
||||
<Steps>
|
||||
|
||||
<Step>
|
||||
|
||||
### Build the receiving endpoint
|
||||
|
||||
A public HTTPS endpoint that verifies a shared secret and updates the user row. Webhook requests time out after 10 seconds, so do the work async if it's slow.
|
||||
|
||||
```ts
|
||||
app.post('/plunk/unsubscribes', async (req, res) => {
|
||||
if (req.header('authorization') !== `Bearer ${process.env.PLUNK_WEBHOOK_SECRET}`) {
|
||||
return res.status(401).end();
|
||||
}
|
||||
|
||||
const { contact, event } = req.body;
|
||||
await db.user.update({
|
||||
where: { email: contact.email },
|
||||
data: {
|
||||
emailSubscribed: false,
|
||||
emailUnsubscribedReason: event.reason ?? 'user_action',
|
||||
},
|
||||
});
|
||||
|
||||
res.status(204).end();
|
||||
});
|
||||
```
|
||||
|
||||
`event.reason` is `"bounce"` or `"complaint"` for automatic unsubscribes, and absent for manual / self-service ones.
|
||||
|
||||
</Step>
|
||||
|
||||
<Step>
|
||||
|
||||
### Create the workflow
|
||||
|
||||
**Workflows → New workflow**:
|
||||
|
||||
- **Trigger**: `EVENT` on `contact.unsubscribed`
|
||||
- Add a `WEBHOOK` step:
|
||||
- **URL**: `https://api.example.com/plunk/unsubscribes`
|
||||
- **Headers**: `{ "Authorization": "Bearer your-shared-secret" }`
|
||||
- Leave the body blank to get the [default payload](/guides/webhooks#webhook-payload).
|
||||
|
||||
Enable the workflow.
|
||||
|
||||
</Step>
|
||||
|
||||
</Steps>
|
||||
|
||||
## Mirroring resubscribes
|
||||
|
||||
Build a second workflow with the same shape, triggered by `contact.subscribed`. Keep it separate from the unsubscribe flow — two short workflows are easier to monitor than one branched one.
|
||||
|
||||
## The reverse direction
|
||||
|
||||
If your product is the source of truth (a user toggles their email preference in your settings UI), call `PATCH /contacts/:id` from your backend:
|
||||
|
||||
```bash
|
||||
curl -X PATCH https://next-api.useplunk.com/contacts/cnt_abc \
|
||||
-H "Authorization: Bearer sk_your_secret_key" \
|
||||
-d '{"subscribed": false}'
|
||||
```
|
||||
|
||||
That flip also fires `contact.unsubscribed`, meaning your own webhook will round-trip back into your handler. That's usually harmless because the update is idempotent — but be aware of it.
|
||||
|
||||
## What's next
|
||||
|
||||
<Cards>
|
||||
<Card title="Webhooks" href="/guides/webhooks">
|
||||
Webhook step reference, payload shape, and safety.
|
||||
</Card>
|
||||
<Card title="Unsubscribe pages" href="/guides/unsubscribe-pages">
|
||||
The hosted pages and template URL variables.
|
||||
</Card>
|
||||
</Cards>
|
||||
@@ -0,0 +1,89 @@
|
||||
---
|
||||
title: Waitlist with confirmation email
|
||||
description: Track signups as a custom event, store everyone who joins as a contact, and automatically email them
|
||||
icon: ListOrdered
|
||||
---
|
||||
|
||||
A waitlist is the simplest possible Plunk workflow: one tracked event from your app, one workflow that listens for it, one email.
|
||||
|
||||
## Setup
|
||||
|
||||
import {Step, Steps} from 'fumadocs-ui/components/steps';
|
||||
|
||||
<Steps>
|
||||
|
||||
<Step>
|
||||
|
||||
### Create the confirmation template
|
||||
|
||||
In **Templates → New template**, create a **Marketing** template. Use `{{variable}}` placeholders for anything you want to personalise from contact data:
|
||||
|
||||
```text
|
||||
Subject: You're on the list, {{firstName}}
|
||||
|
||||
Hi {{firstName}}, thanks for joining the {{product}} waitlist.
|
||||
We'll let you know as soon as your spot opens up.
|
||||
```
|
||||
|
||||
</Step>
|
||||
|
||||
<Step>
|
||||
|
||||
### Track the signup from your backend
|
||||
|
||||
Call `POST /v1/track` when a user submits the form. Use a secret key (`sk_*`) — never call this from the browser.
|
||||
|
||||
```bash
|
||||
curl https://next-api.useplunk.com/v1/track \
|
||||
-H "Authorization: Bearer sk_your_secret_key" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"event": "waitlist.joined",
|
||||
"email": "ada@example.com",
|
||||
"data": { "firstName": "Ada", "product": "Beta" }
|
||||
}'
|
||||
```
|
||||
|
||||
This call upserts the contact (subscribed by default) and records `waitlist.joined` on them. Anything you put in `data` lands on the contact and is available as `{{firstName}}`, `{{product}}`, etc. in the template.
|
||||
|
||||
<Callout title="Pick a stable event name" type="info">
|
||||
A workflow's trigger event **cannot be changed after the first execution**. Namespace it (`waitlist.joined`) rather than something generic you might want to reuse.
|
||||
</Callout>
|
||||
|
||||
</Step>
|
||||
|
||||
<Step>
|
||||
|
||||
### Create the workflow
|
||||
|
||||
**Workflows → New workflow**:
|
||||
|
||||
- **Trigger**: `EVENT` on `waitlist.joined`
|
||||
- Add a `SEND_EMAIL` step pointing at the template from step 1
|
||||
|
||||
Enable the workflow. Workflows are created disabled — until the toggle is on, nothing fires.
|
||||
|
||||
</Step>
|
||||
|
||||
</Steps>
|
||||
|
||||
## Tagging signups for later
|
||||
|
||||
If you want to segment on waitlist signups later, add an `UPDATE_CONTACT` step before the email:
|
||||
|
||||
```json
|
||||
{ "stage": "waitlist", "waitlistSource": "{{event.referrer}}" }
|
||||
```
|
||||
|
||||
You can then build a [segment](/concepts/segments) of contacts where `stage == "waitlist"` to target with follow-up campaigns. This is cleaner than filtering on "ever fired `waitlist.joined`."
|
||||
|
||||
## What's next
|
||||
|
||||
<Cards>
|
||||
<Card title="Workflows" href="/concepts/workflows">
|
||||
Step types and trigger semantics.
|
||||
</Card>
|
||||
<Card title="Track event API" href="/api-reference/public-api/trackEvent">
|
||||
Full reference for `POST /v1/track`.
|
||||
</Card>
|
||||
</Cards>
|
||||
@@ -36,6 +36,7 @@ Set your subdomains here. The application automatically derives all internal and
|
||||
| `AWS_SES_SECRET_ACCESS_KEY` | Yes | AWS secret access key for SES. | `wJalr...` |
|
||||
| `SES_CONFIGURATION_SET` | No | SES configuration set name used for open/click tracking. | `plunk-configuration-set` (default) |
|
||||
| `SES_CONFIGURATION_SET_NO_TRACKING` | No | A second SES configuration set without tracking. When set, projects can toggle email tracking on/off. If omitted, the tracking toggle is hidden. | `plunk-no-tracking-configuration-set` (default) |
|
||||
| `MAIL_FROM_SUBDOMAIN` | No | Subdomain prefix used when constructing the MAIL FROM hostname for a verified domain (e.g. with default `plunk` and domain `yourdomain.com`, the MAIL FROM is `plunk.yourdomain.com`). Override when the default subdomain is already in use (e.g. by an R2/CDN custom domain), since the MAIL FROM hostname needs MX + TXT records that can't coexist with a CNAME. | `plunk` |
|
||||
|
||||
## Storage (Minio)
|
||||
|
||||
@@ -129,6 +130,16 @@ Plunk bundles a self-hosted [ntfy](https://ntfy.sh) server for internal system n
|
||||
| ----------------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- |
|
||||
| `AUTO_PROJECT_DISABLE` | No | When `true`, projects are automatically suspended when bounce or complaint rate thresholds are exceeded. Set to `false` to manage project status manually. | `true` |
|
||||
| `EMAIL_RATE_LIMIT_PER_SECOND` | No | Override the email sending rate limit. If not set, Plunk automatically fetches the quota from your AWS SES account. | — |
|
||||
| `EMAIL_WORKER_CONCURRENCY` | No | Number of emails the worker processes in parallel. When unset, derived from the effective rate limit so a higher SES quota scales throughput automatically. | — |
|
||||
| `EMAIL_WORKER_MAX_CONCURRENCY`| No | Upper bound applied to the auto-derived worker concurrency. Raise this only after sizing the Prisma connection pool accordingly. | `50` |
|
||||
|
||||
## Advanced
|
||||
|
||||
Variables for unusual deployments. The defaults work for the standard Docker Compose setup — only change these if you know you need to.
|
||||
|
||||
| Variable | Required | Description | Default |
|
||||
| ------------ | -------- | ------------------------------------------------------------------------------------------------------------------------ | ------- |
|
||||
| `NGINX_PORT` | No | Host port the bundled Nginx reverse proxy binds to. Override when port 80/443 is already in use on the host (e.g. running behind another reverse proxy that forwards to a different port). | `80` |
|
||||
|
||||
## Phishing Detection
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ function getQ(types: Array<{ type: string; q: number }>, target: string): number
|
||||
function negotiate(accept: string): Negotiated {
|
||||
if (!accept) return 'html';
|
||||
const types = parseAccept(accept);
|
||||
const mdQ = getQ(types, 'text/markdown');
|
||||
const mdQ = types.find(t => t.type === 'text/markdown')?.q ?? -1;
|
||||
const htmlQ = getQ(types, 'text/html');
|
||||
if (mdQ <= 0 && htmlQ <= 0) return 'none';
|
||||
if (mdQ > 0 && mdQ >= htmlQ) return 'markdown';
|
||||
|
||||
+17
-58
@@ -27,39 +27,6 @@ services:
|
||||
# Infrastructure Services
|
||||
# ============================================
|
||||
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
container_name: plunk-postgres
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_DB: plunk
|
||||
POSTGRES_USER: plunk
|
||||
POSTGRES_PASSWORD: ${DB_PASSWORD:-changeme123}
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: [ "CMD-SHELL", "pg_isready -U plunk" ]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
networks:
|
||||
- plunk
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
container_name: plunk-redis
|
||||
restart: unless-stopped
|
||||
command: redis-server --appendonly yes
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
test: [ "CMD", "redis-cli", "ping" ]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
networks:
|
||||
- plunk
|
||||
|
||||
minio:
|
||||
image: minio/minio:latest
|
||||
container_name: plunk-minio
|
||||
@@ -70,10 +37,10 @@ services:
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-plunkminiopass}
|
||||
volumes:
|
||||
- minio_data:/data
|
||||
ports:
|
||||
# Expose Minio API (for S3 operations) and Console (for web UI)
|
||||
- "${MINIO_API_PORT:-9000}:9000"
|
||||
- "${MINIO_CONSOLE_PORT:-9001}:9001"
|
||||
# ports:
|
||||
# # Expose Minio API (for S3 operations) and Console (for web UI)
|
||||
# - "${MINIO_API_PORT:-9000}:9000"
|
||||
# - "${MINIO_CONSOLE_PORT:-9001}:9001"
|
||||
healthcheck:
|
||||
test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
|
||||
interval: 30s
|
||||
@@ -88,13 +55,13 @@ services:
|
||||
restart: unless-stopped
|
||||
command: serve
|
||||
environment:
|
||||
- TZ=UTC
|
||||
- TZ=CST
|
||||
volumes:
|
||||
- ntfy_cache:/var/cache/ntfy
|
||||
- ntfy_etc:/etc/ntfy
|
||||
ports:
|
||||
# Expose ntfy web UI and API
|
||||
- "${NTFY_PORT:-8080}:80"
|
||||
# ports:
|
||||
# # Expose ntfy web UI and API
|
||||
# - "${NTFY_PORT:-8080}:80"
|
||||
healthcheck:
|
||||
test: [ "CMD-SHELL", "wget -q --tries=1 http://localhost:80/v1/health -O - | grep -Eo '\"healthy\"\\s*:\\s*true' || exit 1" ]
|
||||
interval: 30s
|
||||
@@ -118,11 +85,11 @@ services:
|
||||
NODE_ENV: production
|
||||
|
||||
# Database
|
||||
DATABASE_URL: postgresql://plunk:${DB_PASSWORD:-changeme123}@postgres:5432/plunk
|
||||
DIRECT_DATABASE_URL: postgresql://plunk:${DB_PASSWORD:-changeme123}@postgres:5432/plunk
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
DIRECT_DATABASE_URL: ${DIRECT_DATABASE_URL}
|
||||
|
||||
# Redis
|
||||
REDIS_URL: redis://redis:6379
|
||||
REDIS_URL: ${REDIS_URL}
|
||||
|
||||
# Security
|
||||
JWT_SECRET: ${JWT_SECRET}
|
||||
@@ -214,19 +181,15 @@ services:
|
||||
|
||||
ports:
|
||||
# SMTP ports (for email relay)
|
||||
- "${PORT_SECURE:-465}:465" # SMTPS (implicit TLS)
|
||||
- "${PORT_SUBMISSION:-587}:587" # SMTP Submission (STARTTLS)
|
||||
# - "${PORT_SECURE:-465}:465" # SMTPS (implicit TLS)
|
||||
# - "${PORT_SUBMISSION:-587}:587" # SMTP Submission (STARTTLS)
|
||||
# Optional: Expose individual service ports for debugging
|
||||
# - "8080:8080" # API
|
||||
# - "3000:3000" # Web
|
||||
# - "4000:4000" # Landing
|
||||
# - "1000:1000" # Wiki
|
||||
- "6000:8080" # API
|
||||
- "6001:3000" # Web
|
||||
# - "6002:4000" # Landing
|
||||
# - "6003:1000" # Wiki
|
||||
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
minio:
|
||||
condition: service_healthy
|
||||
ntfy:
|
||||
@@ -244,10 +207,6 @@ services:
|
||||
- plunk
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
driver: local
|
||||
redis_data:
|
||||
driver: local
|
||||
minio_data:
|
||||
driver: local
|
||||
plunk_data:
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
-- CreateEnum
|
||||
CREATE TYPE "ProjectDisabledReason" AS ENUM ('PAYMENT_FAILED', 'EMAIL_REPUTATION', 'PHISHING_DETECTED', 'MANUAL');
|
||||
|
||||
-- AlterTable
|
||||
ALTER TABLE "projects" ADD COLUMN "disabledReason" "ProjectDisabledReason";
|
||||
@@ -43,7 +43,8 @@ model Project {
|
||||
secret String @unique
|
||||
|
||||
// Admin
|
||||
disabled Boolean @default(false)
|
||||
disabled Boolean @default(false)
|
||||
disabledReason ProjectDisabledReason?
|
||||
|
||||
// Billing
|
||||
customer String? @unique
|
||||
@@ -632,6 +633,13 @@ model Event {
|
||||
// ENUMS
|
||||
// ============================================
|
||||
|
||||
enum ProjectDisabledReason {
|
||||
PAYMENT_FAILED // Subscription renewal payment failed
|
||||
EMAIL_REPUTATION // Bounce or complaint rate thresholds exceeded
|
||||
PHISHING_DETECTED // Phishing content detected by LLM scan
|
||||
MANUAL // Disabled by support/admin (e.g. directly in DB)
|
||||
}
|
||||
|
||||
enum AuthMethod {
|
||||
PASSWORD
|
||||
GOOGLE_OAUTH
|
||||
|
||||
+86
-78
@@ -1,6 +1,50 @@
|
||||
import {PrismaClient} from '@plunk/db';
|
||||
import {execSync} from 'child_process';
|
||||
|
||||
// Snake-cased table names from prisma schema (see @@map directives).
|
||||
// Order doesn't matter — TRUNCATE with CASCADE handles FK dependencies in one statement.
|
||||
const TRUNCATE_TABLES = [
|
||||
'events',
|
||||
'workflow_step_executions',
|
||||
'emails',
|
||||
'workflow_executions',
|
||||
'workflow_transitions',
|
||||
'workflow_steps',
|
||||
'workflows',
|
||||
'campaigns',
|
||||
'templates',
|
||||
'segment_memberships',
|
||||
'segments',
|
||||
'contacts',
|
||||
'domains',
|
||||
'memberships',
|
||||
'projects',
|
||||
'users',
|
||||
];
|
||||
|
||||
/**
|
||||
* Connects to the admin `postgres` database to ensure the worker's test DB exists.
|
||||
* Postgres has no `CREATE DATABASE IF NOT EXISTS`, so we check pg_database first.
|
||||
*/
|
||||
async function ensureDatabaseExists(databaseUrl: string, workerDbName: string) {
|
||||
const adminUrl = new URL(databaseUrl);
|
||||
adminUrl.pathname = '/postgres';
|
||||
adminUrl.searchParams.delete('connection_limit');
|
||||
adminUrl.searchParams.delete('pool_timeout');
|
||||
|
||||
const admin = new PrismaClient({datasources: {db: {url: adminUrl.toString()}}});
|
||||
try {
|
||||
const rows = await admin.$queryRawUnsafe<{exists: boolean}[]>(
|
||||
`SELECT EXISTS(SELECT 1 FROM pg_database WHERE datname = '${workerDbName}') AS exists`,
|
||||
);
|
||||
if (!rows[0]?.exists) {
|
||||
await admin.$executeRawUnsafe(`CREATE DATABASE "${workerDbName}"`);
|
||||
}
|
||||
} finally {
|
||||
await admin.$disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Test database helper
|
||||
* Manages test database isolation and cleanup
|
||||
@@ -9,44 +53,56 @@ class TestDatabase {
|
||||
private prisma: PrismaClient | null = null;
|
||||
|
||||
async initialize() {
|
||||
// Use test database URL if provided, otherwise use main database
|
||||
// setup.ts has already rewritten DATABASE_URL to include the per-worker DB name
|
||||
// (e.g. plunk_test_w1, plunk_test_w2). We create that DB if missing, migrate it,
|
||||
// then open the long-lived client we use for tests.
|
||||
const databaseUrl = process.env.TEST_DATABASE_URL || process.env.DATABASE_URL;
|
||||
|
||||
if (!databaseUrl) {
|
||||
throw new Error('DATABASE_URL or TEST_DATABASE_URL must be set for testing');
|
||||
}
|
||||
|
||||
// Create Prisma client with connection pool limits
|
||||
this.prisma = new PrismaClient({
|
||||
datasources: {
|
||||
db: {
|
||||
url: databaseUrl,
|
||||
},
|
||||
},
|
||||
// Limit connection pool to prevent memory issues in tests
|
||||
// @ts-ignore - These options exist but may not be in types
|
||||
__internal: {
|
||||
engine: {
|
||||
connection_limit: 5,
|
||||
},
|
||||
},
|
||||
});
|
||||
const url = new URL(databaseUrl);
|
||||
const workerDbName = url.pathname.replace(/^\//, '');
|
||||
if (!workerDbName) {
|
||||
throw new Error('DATABASE_URL must include a database name');
|
||||
}
|
||||
|
||||
// Connect to database
|
||||
await this.prisma.$connect();
|
||||
// Bump the pool above Prisma's default (~5 on CI). Test Postgres has
|
||||
// max_connections=100; with N workers we want N*20 ≤ 100 — fine up to 4 workers.
|
||||
if (!url.searchParams.has('connection_limit')) {
|
||||
url.searchParams.set('connection_limit', '20');
|
||||
}
|
||||
if (!url.searchParams.has('pool_timeout')) {
|
||||
url.searchParams.set('pool_timeout', '20');
|
||||
}
|
||||
|
||||
// Run migrations (only once per test suite)
|
||||
await ensureDatabaseExists(databaseUrl, workerDbName);
|
||||
|
||||
// Run pending migrations against this worker's DB. `migrate deploy` is a no-op
|
||||
// when up-to-date and avoids the drift prompts that `migrate dev` does.
|
||||
try {
|
||||
execSync('yarn workspace @plunk/db migrate:dev', {
|
||||
execSync('yarn workspace @plunk/db migrate:prod', {
|
||||
env: {
|
||||
...process.env,
|
||||
DATABASE_URL: databaseUrl,
|
||||
DATABASE_URL: url.toString(),
|
||||
DIRECT_DATABASE_URL: process.env.DIRECT_DATABASE_URL || url.toString(),
|
||||
},
|
||||
stdio: 'ignore',
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
} catch (error) {
|
||||
console.warn('Migration warning (may already be up to date):', error);
|
||||
const err = error as {stdout?: string; stderr?: string; message?: string};
|
||||
console.error('Migration failed for', workerDbName);
|
||||
if (err.stdout) console.error('stdout:', err.stdout);
|
||||
if (err.stderr) console.error('stderr:', err.stderr);
|
||||
if (!err.stdout && !err.stderr) console.error(err.message);
|
||||
throw error;
|
||||
}
|
||||
|
||||
this.prisma = new PrismaClient({
|
||||
datasources: {db: {url: url.toString()}},
|
||||
});
|
||||
await this.prisma.$connect();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -60,80 +116,32 @@ class TestDatabase {
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up database after each test
|
||||
* Deletes all records in reverse order of dependencies
|
||||
* Uses batched deletes to prevent memory issues with large datasets
|
||||
* Retries on deadlock to handle race conditions with background event tracking
|
||||
* Wipe all per-test data with a single TRUNCATE ... CASCADE statement.
|
||||
* Roughly an order of magnitude faster than 14 sequential deleteMany calls
|
||||
* — TRUNCATE skips the row scan and only touches table headers.
|
||||
*/
|
||||
async cleanup() {
|
||||
if (!this.prisma) return;
|
||||
|
||||
const tables = TRUNCATE_TABLES.map(t => `"${t}"`).join(', ');
|
||||
const maxRetries = 3;
|
||||
let lastError: Error | null = null;
|
||||
|
||||
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
||||
try {
|
||||
// Use a transaction to ensure all deletes happen atomically
|
||||
// This prevents foreign key constraint violations and race conditions
|
||||
await this.prisma.$transaction([
|
||||
// Level 1: Delete deepest dependencies first
|
||||
this.prisma.event.deleteMany(),
|
||||
this.prisma.workflowStepExecution.deleteMany(),
|
||||
|
||||
// Level 2: Delete entities that depend on Level 1
|
||||
this.prisma.email.deleteMany(),
|
||||
this.prisma.workflowExecution.deleteMany(),
|
||||
|
||||
// Level 3: Delete workflow structure
|
||||
this.prisma.workflowTransition.deleteMany(),
|
||||
this.prisma.workflowStep.deleteMany(),
|
||||
this.prisma.workflow.deleteMany(),
|
||||
|
||||
// Level 4: Delete campaigns and templates
|
||||
this.prisma.campaign.deleteMany(),
|
||||
this.prisma.template.deleteMany(),
|
||||
|
||||
// Level 5: Delete segment relationships
|
||||
this.prisma.segmentMembership.deleteMany(),
|
||||
this.prisma.segment.deleteMany(),
|
||||
|
||||
// Level 6: Delete contacts
|
||||
this.prisma.contact.deleteMany(),
|
||||
|
||||
// Level 7: Delete domains
|
||||
this.prisma.domain.deleteMany(),
|
||||
|
||||
// Level 8: Delete memberships (has FK to both user and project)
|
||||
this.prisma.membership.deleteMany(),
|
||||
|
||||
// Level 9: Delete projects
|
||||
this.prisma.project.deleteMany(),
|
||||
|
||||
// Level 10: Delete users last
|
||||
this.prisma.user.deleteMany(),
|
||||
]);
|
||||
|
||||
// Success - exit retry loop
|
||||
await this.prisma.$executeRawUnsafe(`TRUNCATE TABLE ${tables} RESTART IDENTITY CASCADE`);
|
||||
return;
|
||||
} catch (error) {
|
||||
lastError = error as Error;
|
||||
|
||||
// Check if this is a deadlock error (PostgreSQL error code 40P01)
|
||||
const isDeadlock = error instanceof Error && error.message?.includes('deadlock detected');
|
||||
|
||||
if (isDeadlock && attempt < maxRetries) {
|
||||
// Wait before retrying (exponential backoff)
|
||||
const delay = Math.pow(2, attempt) * 50; // 100ms, 200ms, 400ms
|
||||
await new Promise(resolve => setTimeout(resolve, delay));
|
||||
await new Promise(resolve => setTimeout(resolve, Math.pow(2, attempt) * 50));
|
||||
continue;
|
||||
}
|
||||
|
||||
// Not a deadlock or out of retries
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// If we get here, all retries failed
|
||||
console.error(`Error cleaning up database after ${maxRetries} attempts:`, lastError);
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
@@ -108,7 +108,9 @@ export class TestFactories {
|
||||
async createUser(options: UserFactoryOptions = {}) {
|
||||
const email = options.email || `user-${uniqueId()}@test.com`;
|
||||
const password = options.password || 'password123';
|
||||
const hashedPassword = await bcrypt.hash(password, 10);
|
||||
// Cost factor 4 is the bcrypt minimum — ~100x faster than the production cost of 10.
|
||||
// Test users don't need real-world hash strength.
|
||||
const hashedPassword = await bcrypt.hash(password, 4);
|
||||
|
||||
return this.prisma.user.create({
|
||||
data: {
|
||||
|
||||
+35
-22
@@ -1,39 +1,52 @@
|
||||
import { beforeAll, afterAll, afterEach, vi } from 'vitest';
|
||||
import { testDatabase } from './helpers/database';
|
||||
// IMPORTANT: this file runs before each test file's imports execute.
|
||||
// We rewrite DATABASE_URL and REDIS_URL here so per-worker isolation is
|
||||
// applied before any service module constructs a Prisma/Redis client.
|
||||
|
||||
import dotenv from 'dotenv';
|
||||
import path from 'path';
|
||||
import {afterAll, afterEach, beforeAll, vi} from 'vitest';
|
||||
|
||||
// Load environment variables from root .env file
|
||||
dotenv.config({ path: path.resolve(__dirname, '../.env') });
|
||||
dotenv.config({path: path.resolve(__dirname, '../.env')});
|
||||
|
||||
// Vitest assigns each worker a 1-based pool id; defaults to "1" for single-worker runs.
|
||||
const workerId = process.env.VITEST_POOL_ID || '1';
|
||||
|
||||
if (process.env.DATABASE_URL) {
|
||||
const url = new URL(process.env.DATABASE_URL);
|
||||
const baseDb = url.pathname.replace(/^\//, '') || 'plunk_test';
|
||||
url.pathname = `/${baseDb}_w${workerId}`;
|
||||
process.env.DATABASE_URL = url.toString();
|
||||
// Mirror onto DIRECT_DATABASE_URL so prisma migrate uses the same worker DB.
|
||||
if (process.env.DIRECT_DATABASE_URL) {
|
||||
const direct = new URL(process.env.DIRECT_DATABASE_URL);
|
||||
direct.pathname = `/${baseDb}_w${workerId}`;
|
||||
process.env.DIRECT_DATABASE_URL = direct.toString();
|
||||
}
|
||||
}
|
||||
|
||||
if (process.env.REDIS_URL) {
|
||||
const url = new URL(process.env.REDIS_URL);
|
||||
url.pathname = `/${(parseInt(workerId, 10) - 1) % 16}`;
|
||||
process.env.REDIS_URL = url.toString();
|
||||
}
|
||||
|
||||
process.env.NODE_ENV = 'test';
|
||||
process.env.JWT_SECRET = process.env.JWT_SECRET || 'test-jwt-secret-key-for-testing';
|
||||
|
||||
// Static import is safe: database.ts only reads env in initialize(), which runs
|
||||
// in beforeAll — well after the env mutations above.
|
||||
import {testDatabase} from './helpers/database';
|
||||
|
||||
// Global test setup
|
||||
beforeAll(async () => {
|
||||
// Initialize test database
|
||||
await testDatabase.initialize();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
// Clear all mocks first
|
||||
vi.clearAllMocks();
|
||||
|
||||
// Restore real timers
|
||||
vi.useRealTimers();
|
||||
|
||||
// Clean up database after each test
|
||||
// This must be last to ensure proper cleanup order
|
||||
await testDatabase.cleanup();
|
||||
|
||||
// Force garbage collection hint (if available in test environment)
|
||||
if (global.gc) {
|
||||
global.gc();
|
||||
}
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
// Disconnect from database
|
||||
await testDatabase.disconnect();
|
||||
});
|
||||
|
||||
// Set test environment variables
|
||||
process.env.NODE_ENV = 'test';
|
||||
process.env.JWT_SECRET = process.env.JWT_SECRET || 'test-jwt-secret-key-for-testing';
|
||||
|
||||
+8
-7
@@ -22,18 +22,19 @@ export default defineConfig({
|
||||
},
|
||||
testTimeout: 30000,
|
||||
hookTimeout: 30000,
|
||||
// Memory optimization: Run tests in sequence to prevent memory issues
|
||||
// This is critical for tests that create large datasets
|
||||
// Each fork is a worker with an isolated Postgres database and Redis db-number
|
||||
// (see test/setup.ts). That isolation is what lets us run files in parallel
|
||||
// without the cross-test interference we used to hit with a shared DB.
|
||||
pool: 'forks',
|
||||
poolOptions: {
|
||||
forks: {
|
||||
singleFork: true, // Run all tests in a single fork to limit memory
|
||||
// Cap at 4 to stay within Postgres' default max_connections=100
|
||||
// when each worker uses connection_limit=20.
|
||||
maxForks: 4,
|
||||
minForks: 1,
|
||||
},
|
||||
},
|
||||
// Run tests sequentially to avoid database cleanup conflicts
|
||||
fileParallelism: false,
|
||||
// Limit concurrent test files to reduce memory pressure
|
||||
maxConcurrency: 3,
|
||||
maxConcurrency: 5,
|
||||
// Only include our test files, not dependency tests
|
||||
include: [
|
||||
'apps/**/__tests__/**/*.{test,spec}.{ts,tsx}',
|
||||
|
||||
Reference in New Issue
Block a user