feat: Add additional checks for website, NS records and personal emails

2026-01-12 20:02:08 +01:00
parent 26800a8553
commit 0a67a8278f
8 changed files with 163 additions and 46 deletions
@@ -6,6 +6,9 @@ import {redis} from '../database/redis.js';
 const DISPOSABLE_DOMAINS_URL =
  'https://raw.githubusercontent.com/disposable-email-domains/disposable-email-domains/main/disposable_email_blocklist.conf';
 const DISPOSABLE_DOMAINS_CACHE_KEY = 'email:disposable_domains';
+const PERSONAL_DOMAINS_URL =
+  'https://gist.githubusercontent.com/ammarshah/f5c2624d767f91a7cbdc4e54db8dd0bf/raw/660fd949eba09c0b86574d9d3aa0f2137161fc7c/all_email_provider_domains.txt';
+const PERSONAL_DOMAINS_CACHE_KEY = 'email:personal_domains';
 const CACHE_TTL_SECONDS = 24 * 60 * 60; // 24 hours (list updates daily)

 // Known email forwarding/alias services
@@ -39,12 +42,15 @@ const FORWARDING_DOMAINS = new Set([

 export class EmailVerificationService {
  private static disposableDomainsSet: Set<string> | null = null;
+  private static personalDomainsSet: Set<string> | null = null;

  /**
   * Verify an email address
+   * - Checks for NS records (proves domain exists in DNS)
   * - Checks for MX records (required for receiving email)
-   * - Checks if domain exists (DNS A/AAAA records) - informational only
+   * - Checks for A/AAAA records (informational - indicates if domain has a website)
   * - Detects disposable email addresses
+   * - Detects personal/free email providers (Gmail, Hotmail, etc.)
   * - Detects forwarding/alias email addresses
   * - Suggests corrections for common typos
   */
@@ -56,7 +62,9 @@ export class EmailVerificationService {
      isAlias: false,
      isTypo: false,
      isPlusAddressed: false,
+      isPersonalEmail: false,
      domainExists: false,
+      hasWebsite: false,
      hasMxRecords: false,
      reasons: [],
    };
@@ -74,6 +82,9 @@ export class EmailVerificationService {
    // Check if email is from a disposable domain using GitHub list
    result.isDisposable = await this.isDisposableDomain(domain);

+    // Check if email is from a personal/free email provider
+    result.isPersonalEmail = await this.isPersonalEmailDomain(domain);
+
    // Check if email is from a known forwarding/alias service
    result.isAlias = this.isForwardingDomain(domain);

@@ -88,34 +99,46 @@ export class EmailVerificationService {
      result.isTypo = true;
    }

-    // Check MX records first - this is what matters for email delivery
-    // A domain can receive email with only MX records, no A/AAAA records needed
+    // Step 1: Check NS records - proves the domain exists in DNS
+    try {
+      const nsRecords = await dns.resolveNs(domain);
+      result.domainExists = nsRecords && nsRecords.length > 0;
+    } catch {
+      result.domainExists = false;
+      result.valid = false;
+      result.reasons.push('Domain does not exist (no nameservers found)');
+      // If domain doesn't exist, no point checking MX/A records
+      return result;
+    }
+
+    // Step 2: Check MX records - required for receiving email
    try {
      const mxRecords = await dns.resolveMx(domain);
      result.hasMxRecords = mxRecords && mxRecords.length > 0;
      if (!result.hasMxRecords) {
        result.valid = false;
-        result.reasons.push('No MX records found for domain');
+        result.reasons.push('Domain cannot receive email (no MX records found)');
      }
    } catch {
      result.hasMxRecords = false;
      result.valid = false;
-      result.reasons.push('No MX records found for domain');
+      result.reasons.push('Domain cannot receive email (no MX records found)');
    }

-    // Check if domain exists (has A/AAAA records) - informational only
-    // This doesn't affect validity since email delivery only requires MX records
+    // Step 3: Check if domain has A/AAAA records - informational only
+    // This indicates if the domain has a website/web server
+    // Doesn't affect email validity since email only requires MX records
    try {
      await dns.resolve(domain, 'A');
-      result.domainExists = true;
+      result.hasWebsite = true;
    } catch {
      // Try AAAA records if A records fail
      try {
        await dns.resolve(domain, 'AAAA');
-        result.domainExists = true;
+        result.hasWebsite = true;
      } catch {
-        // Domain doesn't have A/AAAA records, but this is OK if it has MX records
-        result.domainExists = false;
+        // Domain doesn't have A/AAAA records (no website), but this is OK for email
+        result.hasWebsite = false;
      }
    }

@@ -186,4 +209,57 @@ export class EmailVerificationService {
  private static isForwardingDomain(domain: string): boolean {
    return FORWARDING_DOMAINS.has(domain.toLowerCase());
  }
+
+  /**
+   * Fetch and cache the personal email domains list from GitHub
+   * Uses Redis for caching with 24-hour TTL
+   * Falls back to in-memory cache if Redis fails
+   */
+  private static async getPersonalEmailDomains(): Promise<Set<string>> {
+    // Return in-memory cache if available
+    if (this.personalDomainsSet) {
+      return this.personalDomainsSet;
+    }
+
+    try {
+      // Try to get from Redis cache first
+      const cached = await redis.get(PERSONAL_DOMAINS_CACHE_KEY);
+      if (cached) {
+        const domains = JSON.parse(cached) as string[];
+        this.personalDomainsSet = new Set(domains);
+        return this.personalDomainsSet;
+      }
+
+      // Fetch from GitHub if not in cache
+      const response = await fetch(PERSONAL_DOMAINS_URL);
+      if (!response.ok) {
+        throw new Error(`Failed to fetch personal email domains: ${response.statusText}`);
+      }
+
+      const text = await response.text();
+      const domains = text
+        .split('\n')
+        .map(line => line.trim())
+        .filter(line => line && !line.startsWith('#')); // Filter empty lines and comments
+
+      // Cache in Redis
+      await redis.set(PERSONAL_DOMAINS_CACHE_KEY, JSON.stringify(domains), 'EX', CACHE_TTL_SECONDS);
+
+      // Cache in memory
+      this.personalDomainsSet = new Set(domains);
+      return this.personalDomainsSet;
+    } catch (error) {
+      console.error('Error fetching personal email domains:', error);
+      // Return empty set as fallback - don't block email verification
+      return new Set<string>();
+    }
+  }
+
+  /**
+   * Check if a domain is a personal/free email provider
+   */
+  private static async isPersonalEmailDomain(domain: string): Promise<boolean> {
+    const personalDomains = await this.getPersonalEmailDomains();
+    return personalDomains.has(domain.toLowerCase());
+  }
 }
@@ -2,15 +2,16 @@ import {
  AlertCircle,
  AlertTriangle,
  CheckCircle,
+  Forward,
  Info,
  Mail,
  Server,
  Shield,
  Trash2,
+  User,
  XCircle,
-  Forward,
 } from 'lucide-react';
-import type {EmailVerificationResult as VerificationResult} from '../../lib/emailVerification';
+import type {EmailVerificationResult as VerificationResult} from '@plunk/types';

 interface EmailVerificationResultProps {
  result: VerificationResult;
@@ -52,7 +53,7 @@ export function EmailVerificationResult({result}: EmailVerificationResultProps)
              <Server className="h-5 w-5 text-neutral-600" />
              <div>
                <p className="font-medium text-neutral-900">Domain Exists</p>
-                <p className="text-sm text-neutral-600">DNS A/AAAA records found</p>
+                <p className="text-sm text-neutral-600">Domain has nameservers (NS records)</p>
              </div>
            </div>
            {result.domainExists ? (
@@ -78,6 +79,22 @@ export function EmailVerificationResult({result}: EmailVerificationResultProps)
            )}
          </div>

+          {/* Website/A Records */}
+          <div className="flex items-center justify-between px-6 py-4">
+            <div className="flex items-center gap-3">
+              <Server className="h-5 w-5 text-neutral-600" />
+              <div>
+                <p className="font-medium text-neutral-900">Has Website</p>
+                <p className="text-sm text-neutral-600">DNS A/AAAA records found</p>
+              </div>
+            </div>
+            {result.hasWebsite ? (
+              <CheckCircle className="h-5 w-5 text-green-600" />
+            ) : (
+              <Info className="h-5 w-5 text-neutral-400" />
+            )}
+          </div>
+
          {/* Disposable Email */}
          <div className="flex items-center justify-between px-6 py-4">
            <div className="flex items-center gap-3">
@@ -94,6 +111,22 @@ export function EmailVerificationResult({result}: EmailVerificationResultProps)
            )}
          </div>

+          {/* Personal Email */}
+          <div className="flex items-center justify-between px-6 py-4">
+            <div className="flex items-center gap-3">
+              <User className="h-5 w-5 text-neutral-600" />
+              <div>
+                <p className="font-medium text-neutral-900">Personal Email</p>
+                <p className="text-sm text-neutral-600">Free email provider (Gmail, Hotmail, etc.)</p>
+              </div>
+            </div>
+            {result.isPersonalEmail ? (
+              <Info className="h-5 w-5 text-blue-600" />
+            ) : (
+              <span className="text-sm text-neutral-500">No</span>
+            )}
+          </div>
+
          {/* Alias/Forwarding Email */}
          <div className="flex items-center justify-between px-6 py-4">
            <div className="flex items-center gap-3">
@@ -1,15 +1,4 @@
-export interface EmailVerificationResult {
-  email: string;
-  valid: boolean;
-  isDisposable: boolean;
-  isAlias: boolean;
-  isTypo: boolean;
-  isPlusAddressed: boolean;
-  domainExists: boolean;
-  hasMxRecords: boolean;
-  suggestedEmail?: string;
-  reasons: string[];
-}
+import type {EmailVerificationResult} from '@plunk/types';

 export async function verifyEmail(email: string): Promise<EmailVerificationResult> {
  const response = await fetch('/api/verify-email', {
@@ -1,4 +1,4 @@
-import {CheckCircle, Mail, Search, Shield} from 'lucide-react';
+import {CheckCircle, Mail, Search, Shield, User} from 'lucide-react';

 /**
 * Educational content for the email verification tool
@@ -29,6 +29,11 @@ export const EMAIL_VERIFICATION_FEATURES = [
    description: 'Identify temporary email addresses that are often used for spam or fake signups.',
    icon: Mail,
  },
+  {
+    title: 'Personal Email Detection',
+    description: 'Detect personal/free email providers like Gmail, Hotmail, Yahoo for B2B validation.',
+    icon: User,
+  },
  {
    title: 'Plus Addressing',
    description: 'Detect plus-addressed emails (user+tag@domain.com) which can be useful for tracking.',
@@ -1,24 +1,16 @@
 import type {NextApiRequest, NextApiResponse} from 'next';
+import type {EmailVerificationResult} from '@plunk/types';
 import {UtilitySchemas} from '@plunk/shared';
 import {API_URI} from '../../lib/constants';

-interface VerifyEmailResponse {
-  email: string;
-  valid: boolean;
-  isDisposable: boolean;
-  isTypo: boolean;
-  isPlusAddressed: boolean;
-  domainExists: boolean;
-  hasMxRecords: boolean;
-  suggestedEmail?: string;
-  reasons: string[];
-}
-
 interface ErrorResponse {
  error: string;
 }

-export default async function handler(req: NextApiRequest, res: NextApiResponse<VerifyEmailResponse | ErrorResponse>) {
+export default async function handler(
+  req: NextApiRequest,
+  res: NextApiResponse<EmailVerificationResult | ErrorResponse>,
+) {
  // Only allow POST requests
  if (req.method !== 'POST') {
    return res.status(405).json({error: 'Method not allowed'});
@@ -4,7 +4,8 @@ import {DASHBOARD_URI} from '../../lib/constants';
 import React, {useState} from 'react';
 import {NextSeo} from 'next-seo';
 import {ArrowRight, CheckCircle, Loader2, Search} from 'lucide-react';
-import {type EmailVerificationResult as VerificationResult, verifyEmail} from '../../lib/emailVerification';
+import type {EmailVerificationResult as VerificationResult} from '@plunk/types';
+import {verifyEmail} from '../../lib/emailVerification';
 import {EmailVerificationResult} from '../../components/tools/EmailVerificationResult';
 import {Button, Input} from '@plunk/ui';
 import {EMAIL_VERIFICATION_FEATURES} from '../../lib/toolsContent';
@@ -596,7 +596,7 @@
      "post": {
        "tags": ["Public API"],
        "summary": "Verify email address",
-        "description": "Verify an email address for validity, check if it's from a disposable domain, verify MX records, and detect potential typos with suggestions.",
+        "description": "Verify an email address for validity, check if it's from a disposable domain or personal email provider, verify MX records, and detect potential typos with suggestions.",
        "operationId": "verifyEmail",
        "requestBody": {
          "required": true,
@@ -676,9 +676,17 @@
                          "type": "boolean",
                          "description": "Whether the email uses plus addressing (contains a + in the local part)"
                        },
+                        "isPersonalEmail": {
+                          "type": "boolean",
+                          "description": "Whether the email is from a personal/free email provider (Gmail, Hotmail, Yahoo, etc.)"
+                        },
                        "domainExists": {
                          "type": "boolean",
-                          "description": "Whether the domain exists (has DNS A or AAAA records)"
+                          "description": "Whether the domain exists in DNS (has NS records)"
+                        },
+                        "hasWebsite": {
+                          "type": "boolean",
+                          "description": "Whether the domain has a website (has DNS A or AAAA records) - informational only"
                        },
                        "hasMxRecords": {
                          "type": "boolean",
@@ -705,7 +713,9 @@
                        "isAlias",
                        "isTypo",
                        "isPlusAddressed",
+                        "isPersonalEmail",
                        "domainExists",
+                        "hasWebsite",
                        "hasMxRecords",
                        "reasons"
                      ]
@@ -724,7 +734,9 @@
                        "isAlias": false,
                        "isTypo": false,
                        "isPlusAddressed": false,
+                        "isPersonalEmail": true,
                        "domainExists": true,
+                        "hasWebsite": true,
                        "hasMxRecords": true,
                        "reasons": ["Email appears to be valid"]
                      }
@@ -741,10 +753,15 @@
                        "isAlias": false,
                        "isTypo": true,
                        "isPlusAddressed": false,
+                        "isPersonalEmail": false,
                        "domainExists": false,
+                        "hasWebsite": false,
                        "hasMxRecords": false,
                        "suggestedEmail": "user@gmail.com",
-                        "reasons": ["Possible typo detected, did you mean gmail.com?", "Domain does not exist"]
+                        "reasons": [
+                          "Possible typo detected, did you mean gmail.com?",
+                          "Domain does not exist (no nameservers found)"
+                        ]
                      }
                    }
                  },
@@ -759,7 +776,9 @@
                        "isAlias": false,
                        "isTypo": false,
                        "isPlusAddressed": false,
+                        "isPersonalEmail": false,
                        "domainExists": true,
+                        "hasWebsite": true,
                        "hasMxRecords": true,
                        "reasons": ["Email appears to be valid"]
                      }