From 0a67a8278f45d89b1abdf55be1cf251a470595cf Mon Sep 17 00:00:00 2001 From: Dries Augustyns Date: Mon, 12 Jan 2026 20:02:08 +0100 Subject: [PATCH] feat: Add additional checks for website, NS records and personal emails --- .../src/services/EmailVerificationService.ts | 98 ++++++++++++++++--- .../tools/EmailVerificationResult.tsx | 39 +++++++- apps/landing/src/lib/emailVerification.ts | 13 +-- apps/landing/src/lib/toolsContent.ts | 7 +- apps/landing/src/pages/api/verify-email.ts | 18 +--- apps/landing/src/pages/tools/verify-email.tsx | 3 +- apps/wiki/openapi.json | 25 ++++- packages/types/src/api/verification.ts | 6 +- 8 files changed, 163 insertions(+), 46 deletions(-) diff --git a/apps/api/src/services/EmailVerificationService.ts b/apps/api/src/services/EmailVerificationService.ts index b0812f4..b245adb 100644 --- a/apps/api/src/services/EmailVerificationService.ts +++ b/apps/api/src/services/EmailVerificationService.ts @@ -6,6 +6,9 @@ import {redis} from '../database/redis.js'; const DISPOSABLE_DOMAINS_URL = 'https://raw.githubusercontent.com/disposable-email-domains/disposable-email-domains/main/disposable_email_blocklist.conf'; const DISPOSABLE_DOMAINS_CACHE_KEY = 'email:disposable_domains'; +const PERSONAL_DOMAINS_URL = + 'https://gist.githubusercontent.com/ammarshah/f5c2624d767f91a7cbdc4e54db8dd0bf/raw/660fd949eba09c0b86574d9d3aa0f2137161fc7c/all_email_provider_domains.txt'; +const PERSONAL_DOMAINS_CACHE_KEY = 'email:personal_domains'; const CACHE_TTL_SECONDS = 24 * 60 * 60; // 24 hours (list updates daily) // Known email forwarding/alias services @@ -39,12 +42,15 @@ const FORWARDING_DOMAINS = new Set([ export class EmailVerificationService { private static disposableDomainsSet: Set | null = null; + private static personalDomainsSet: Set | null = null; /** * Verify an email address + * - Checks for NS records (proves domain exists in DNS) * - Checks for MX records (required for receiving email) - * - Checks if domain exists (DNS A/AAAA records) - informational only + * - Checks for A/AAAA records (informational - indicates if domain has a website) * - Detects disposable email addresses + * - Detects personal/free email providers (Gmail, Hotmail, etc.) * - Detects forwarding/alias email addresses * - Suggests corrections for common typos */ @@ -56,7 +62,9 @@ export class EmailVerificationService { isAlias: false, isTypo: false, isPlusAddressed: false, + isPersonalEmail: false, domainExists: false, + hasWebsite: false, hasMxRecords: false, reasons: [], }; @@ -74,6 +82,9 @@ export class EmailVerificationService { // Check if email is from a disposable domain using GitHub list result.isDisposable = await this.isDisposableDomain(domain); + // Check if email is from a personal/free email provider + result.isPersonalEmail = await this.isPersonalEmailDomain(domain); + // Check if email is from a known forwarding/alias service result.isAlias = this.isForwardingDomain(domain); @@ -88,34 +99,46 @@ export class EmailVerificationService { result.isTypo = true; } - // Check MX records first - this is what matters for email delivery - // A domain can receive email with only MX records, no A/AAAA records needed + // Step 1: Check NS records - proves the domain exists in DNS + try { + const nsRecords = await dns.resolveNs(domain); + result.domainExists = nsRecords && nsRecords.length > 0; + } catch { + result.domainExists = false; + result.valid = false; + result.reasons.push('Domain does not exist (no nameservers found)'); + // If domain doesn't exist, no point checking MX/A records + return result; + } + + // Step 2: Check MX records - required for receiving email try { const mxRecords = await dns.resolveMx(domain); result.hasMxRecords = mxRecords && mxRecords.length > 0; if (!result.hasMxRecords) { result.valid = false; - result.reasons.push('No MX records found for domain'); + result.reasons.push('Domain cannot receive email (no MX records found)'); } } catch { result.hasMxRecords = false; result.valid = false; - result.reasons.push('No MX records found for domain'); + result.reasons.push('Domain cannot receive email (no MX records found)'); } - // Check if domain exists (has A/AAAA records) - informational only - // This doesn't affect validity since email delivery only requires MX records + // Step 3: Check if domain has A/AAAA records - informational only + // This indicates if the domain has a website/web server + // Doesn't affect email validity since email only requires MX records try { await dns.resolve(domain, 'A'); - result.domainExists = true; + result.hasWebsite = true; } catch { // Try AAAA records if A records fail try { await dns.resolve(domain, 'AAAA'); - result.domainExists = true; + result.hasWebsite = true; } catch { - // Domain doesn't have A/AAAA records, but this is OK if it has MX records - result.domainExists = false; + // Domain doesn't have A/AAAA records (no website), but this is OK for email + result.hasWebsite = false; } } @@ -186,4 +209,57 @@ export class EmailVerificationService { private static isForwardingDomain(domain: string): boolean { return FORWARDING_DOMAINS.has(domain.toLowerCase()); } + + /** + * Fetch and cache the personal email domains list from GitHub + * Uses Redis for caching with 24-hour TTL + * Falls back to in-memory cache if Redis fails + */ + private static async getPersonalEmailDomains(): Promise> { + // Return in-memory cache if available + if (this.personalDomainsSet) { + return this.personalDomainsSet; + } + + try { + // Try to get from Redis cache first + const cached = await redis.get(PERSONAL_DOMAINS_CACHE_KEY); + if (cached) { + const domains = JSON.parse(cached) as string[]; + this.personalDomainsSet = new Set(domains); + return this.personalDomainsSet; + } + + // Fetch from GitHub if not in cache + const response = await fetch(PERSONAL_DOMAINS_URL); + if (!response.ok) { + throw new Error(`Failed to fetch personal email domains: ${response.statusText}`); + } + + const text = await response.text(); + const domains = text + .split('\n') + .map(line => line.trim()) + .filter(line => line && !line.startsWith('#')); // Filter empty lines and comments + + // Cache in Redis + await redis.set(PERSONAL_DOMAINS_CACHE_KEY, JSON.stringify(domains), 'EX', CACHE_TTL_SECONDS); + + // Cache in memory + this.personalDomainsSet = new Set(domains); + return this.personalDomainsSet; + } catch (error) { + console.error('Error fetching personal email domains:', error); + // Return empty set as fallback - don't block email verification + return new Set(); + } + } + + /** + * Check if a domain is a personal/free email provider + */ + private static async isPersonalEmailDomain(domain: string): Promise { + const personalDomains = await this.getPersonalEmailDomains(); + return personalDomains.has(domain.toLowerCase()); + } } diff --git a/apps/landing/src/components/tools/EmailVerificationResult.tsx b/apps/landing/src/components/tools/EmailVerificationResult.tsx index a6fe5ec..649600c 100644 --- a/apps/landing/src/components/tools/EmailVerificationResult.tsx +++ b/apps/landing/src/components/tools/EmailVerificationResult.tsx @@ -2,15 +2,16 @@ import { AlertCircle, AlertTriangle, CheckCircle, + Forward, Info, Mail, Server, Shield, Trash2, + User, XCircle, - Forward, } from 'lucide-react'; -import type {EmailVerificationResult as VerificationResult} from '../../lib/emailVerification'; +import type {EmailVerificationResult as VerificationResult} from '@plunk/types'; interface EmailVerificationResultProps { result: VerificationResult; @@ -52,7 +53,7 @@ export function EmailVerificationResult({result}: EmailVerificationResultProps)

Domain Exists

-

DNS A/AAAA records found

+

Domain has nameservers (NS records)

{result.domainExists ? ( @@ -78,6 +79,22 @@ export function EmailVerificationResult({result}: EmailVerificationResultProps) )} + {/* Website/A Records */} +
+
+ +
+

Has Website

+

DNS A/AAAA records found

+
+
+ {result.hasWebsite ? ( + + ) : ( + + )} +
+ {/* Disposable Email */}
@@ -94,6 +111,22 @@ export function EmailVerificationResult({result}: EmailVerificationResultProps) )}
+ {/* Personal Email */} +
+
+ +
+

Personal Email

+

Free email provider (Gmail, Hotmail, etc.)

+
+
+ {result.isPersonalEmail ? ( + + ) : ( + No + )} +
+ {/* Alias/Forwarding Email */}
diff --git a/apps/landing/src/lib/emailVerification.ts b/apps/landing/src/lib/emailVerification.ts index 0f80b80..c2a0556 100644 --- a/apps/landing/src/lib/emailVerification.ts +++ b/apps/landing/src/lib/emailVerification.ts @@ -1,15 +1,4 @@ -export interface EmailVerificationResult { - email: string; - valid: boolean; - isDisposable: boolean; - isAlias: boolean; - isTypo: boolean; - isPlusAddressed: boolean; - domainExists: boolean; - hasMxRecords: boolean; - suggestedEmail?: string; - reasons: string[]; -} +import type {EmailVerificationResult} from '@plunk/types'; export async function verifyEmail(email: string): Promise { const response = await fetch('/api/verify-email', { diff --git a/apps/landing/src/lib/toolsContent.ts b/apps/landing/src/lib/toolsContent.ts index 6ffc3e0..70e95d2 100644 --- a/apps/landing/src/lib/toolsContent.ts +++ b/apps/landing/src/lib/toolsContent.ts @@ -1,4 +1,4 @@ -import {CheckCircle, Mail, Search, Shield} from 'lucide-react'; +import {CheckCircle, Mail, Search, Shield, User} from 'lucide-react'; /** * Educational content for the email verification tool @@ -29,6 +29,11 @@ export const EMAIL_VERIFICATION_FEATURES = [ description: 'Identify temporary email addresses that are often used for spam or fake signups.', icon: Mail, }, + { + title: 'Personal Email Detection', + description: 'Detect personal/free email providers like Gmail, Hotmail, Yahoo for B2B validation.', + icon: User, + }, { title: 'Plus Addressing', description: 'Detect plus-addressed emails (user+tag@domain.com) which can be useful for tracking.', diff --git a/apps/landing/src/pages/api/verify-email.ts b/apps/landing/src/pages/api/verify-email.ts index a193703..0cafdee 100644 --- a/apps/landing/src/pages/api/verify-email.ts +++ b/apps/landing/src/pages/api/verify-email.ts @@ -1,24 +1,16 @@ import type {NextApiRequest, NextApiResponse} from 'next'; +import type {EmailVerificationResult} from '@plunk/types'; import {UtilitySchemas} from '@plunk/shared'; import {API_URI} from '../../lib/constants'; -interface VerifyEmailResponse { - email: string; - valid: boolean; - isDisposable: boolean; - isTypo: boolean; - isPlusAddressed: boolean; - domainExists: boolean; - hasMxRecords: boolean; - suggestedEmail?: string; - reasons: string[]; -} - interface ErrorResponse { error: string; } -export default async function handler(req: NextApiRequest, res: NextApiResponse) { +export default async function handler( + req: NextApiRequest, + res: NextApiResponse, +) { // Only allow POST requests if (req.method !== 'POST') { return res.status(405).json({error: 'Method not allowed'}); diff --git a/apps/landing/src/pages/tools/verify-email.tsx b/apps/landing/src/pages/tools/verify-email.tsx index afd5859..e3d9295 100644 --- a/apps/landing/src/pages/tools/verify-email.tsx +++ b/apps/landing/src/pages/tools/verify-email.tsx @@ -4,7 +4,8 @@ import {DASHBOARD_URI} from '../../lib/constants'; import React, {useState} from 'react'; import {NextSeo} from 'next-seo'; import {ArrowRight, CheckCircle, Loader2, Search} from 'lucide-react'; -import {type EmailVerificationResult as VerificationResult, verifyEmail} from '../../lib/emailVerification'; +import type {EmailVerificationResult as VerificationResult} from '@plunk/types'; +import {verifyEmail} from '../../lib/emailVerification'; import {EmailVerificationResult} from '../../components/tools/EmailVerificationResult'; import {Button, Input} from '@plunk/ui'; import {EMAIL_VERIFICATION_FEATURES} from '../../lib/toolsContent'; diff --git a/apps/wiki/openapi.json b/apps/wiki/openapi.json index e0baa7d..39d43c2 100644 --- a/apps/wiki/openapi.json +++ b/apps/wiki/openapi.json @@ -596,7 +596,7 @@ "post": { "tags": ["Public API"], "summary": "Verify email address", - "description": "Verify an email address for validity, check if it's from a disposable domain, verify MX records, and detect potential typos with suggestions.", + "description": "Verify an email address for validity, check if it's from a disposable domain or personal email provider, verify MX records, and detect potential typos with suggestions.", "operationId": "verifyEmail", "requestBody": { "required": true, @@ -676,9 +676,17 @@ "type": "boolean", "description": "Whether the email uses plus addressing (contains a + in the local part)" }, + "isPersonalEmail": { + "type": "boolean", + "description": "Whether the email is from a personal/free email provider (Gmail, Hotmail, Yahoo, etc.)" + }, "domainExists": { "type": "boolean", - "description": "Whether the domain exists (has DNS A or AAAA records)" + "description": "Whether the domain exists in DNS (has NS records)" + }, + "hasWebsite": { + "type": "boolean", + "description": "Whether the domain has a website (has DNS A or AAAA records) - informational only" }, "hasMxRecords": { "type": "boolean", @@ -705,7 +713,9 @@ "isAlias", "isTypo", "isPlusAddressed", + "isPersonalEmail", "domainExists", + "hasWebsite", "hasMxRecords", "reasons" ] @@ -724,7 +734,9 @@ "isAlias": false, "isTypo": false, "isPlusAddressed": false, + "isPersonalEmail": true, "domainExists": true, + "hasWebsite": true, "hasMxRecords": true, "reasons": ["Email appears to be valid"] } @@ -741,10 +753,15 @@ "isAlias": false, "isTypo": true, "isPlusAddressed": false, + "isPersonalEmail": false, "domainExists": false, + "hasWebsite": false, "hasMxRecords": false, "suggestedEmail": "user@gmail.com", - "reasons": ["Possible typo detected, did you mean gmail.com?", "Domain does not exist"] + "reasons": [ + "Possible typo detected, did you mean gmail.com?", + "Domain does not exist (no nameservers found)" + ] } } }, @@ -759,7 +776,9 @@ "isAlias": false, "isTypo": false, "isPlusAddressed": false, + "isPersonalEmail": false, "domainExists": true, + "hasWebsite": true, "hasMxRecords": true, "reasons": ["Email appears to be valid"] } diff --git a/packages/types/src/api/verification.ts b/packages/types/src/api/verification.ts index f2037e7..cdfede4 100644 --- a/packages/types/src/api/verification.ts +++ b/packages/types/src/api/verification.ts @@ -12,8 +12,10 @@ export interface EmailVerificationResult { isAlias: boolean; isTypo: boolean; isPlusAddressed: boolean; - domainExists: boolean; - hasMxRecords: boolean; + isPersonalEmail: boolean; // Email is from a personal/free provider (Gmail, Hotmail, etc.) + domainExists: boolean; // Domain exists in DNS (has NS records) + hasWebsite: boolean; // Domain has A/AAAA records (informational - not required for email) + hasMxRecords: boolean; // Domain has MX records (required for receiving email) suggestedEmail?: string; reasons: string[]; }