feat: Add additional checks for website, NS records and personal emails

apps/api/src/services/EmailVerificationService.ts

@@ -6,6 +6,9 @@ import {redis} from '../database/redis.js';
 const DISPOSABLE_DOMAINS_URL =
   'https://raw.githubusercontent.com/disposable-email-domains/disposable-email-domains/main/disposable_email_blocklist.conf';
 const DISPOSABLE_DOMAINS_CACHE_KEY = 'email:disposable_domains';
+const PERSONAL_DOMAINS_URL =
+  'https://gist.githubusercontent.com/ammarshah/f5c2624d767f91a7cbdc4e54db8dd0bf/raw/660fd949eba09c0b86574d9d3aa0f2137161fc7c/all_email_provider_domains.txt';
+const PERSONAL_DOMAINS_CACHE_KEY = 'email:personal_domains';
 const CACHE_TTL_SECONDS = 24 * 60 * 60; // 24 hours (list updates daily)
 
 // Known email forwarding/alias services
@@ -39,12 +42,15 @@ const FORWARDING_DOMAINS = new Set([
 
 export class EmailVerificationService {
   private static disposableDomainsSet: Set<string> | null = null;
+  private static personalDomainsSet: Set<string> | null = null;
 
   /**
    * Verify an email address
+   * - Checks for NS records (proves domain exists in DNS)
    * - Checks for MX records (required for receiving email)
-   * - Checks if domain exists (DNS A/AAAA records) - informational only
+   * - Checks for A/AAAA records (informational - indicates if domain has a website)
    * - Detects disposable email addresses
+   * - Detects personal/free email providers (Gmail, Hotmail, etc.)
    * - Detects forwarding/alias email addresses
    * - Suggests corrections for common typos
    */
@@ -56,7 +62,9 @@ export class EmailVerificationService {
       isAlias: false,
       isTypo: false,
       isPlusAddressed: false,
+      isPersonalEmail: false,
       domainExists: false,
+      hasWebsite: false,
       hasMxRecords: false,
       reasons: [],
     };
@@ -74,6 +82,9 @@ export class EmailVerificationService {
     // Check if email is from a disposable domain using GitHub list
     result.isDisposable = await this.isDisposableDomain(domain);
 
+    // Check if email is from a personal/free email provider
+    result.isPersonalEmail = await this.isPersonalEmailDomain(domain);
+
     // Check if email is from a known forwarding/alias service
     result.isAlias = this.isForwardingDomain(domain);
 
@@ -88,34 +99,46 @@ export class EmailVerificationService {
       result.isTypo = true;
     }
 
-    // Check MX records first - this is what matters for email delivery
-    // A domain can receive email with only MX records, no A/AAAA records needed
+    // Step 1: Check NS records - proves the domain exists in DNS
+    try {
+      const nsRecords = await dns.resolveNs(domain);
+      result.domainExists = nsRecords && nsRecords.length > 0;
+    } catch {
+      result.domainExists = false;
+      result.valid = false;
+      result.reasons.push('Domain does not exist (no nameservers found)');
+      // If domain doesn't exist, no point checking MX/A records
+      return result;
+    }
+
+    // Step 2: Check MX records - required for receiving email
     try {
       const mxRecords = await dns.resolveMx(domain);
       result.hasMxRecords = mxRecords && mxRecords.length > 0;
       if (!result.hasMxRecords) {
         result.valid = false;
-        result.reasons.push('No MX records found for domain');
+        result.reasons.push('Domain cannot receive email (no MX records found)');
       }
     } catch {
       result.hasMxRecords = false;
       result.valid = false;
-      result.reasons.push('No MX records found for domain');
+      result.reasons.push('Domain cannot receive email (no MX records found)');
     }
 
-    // Check if domain exists (has A/AAAA records) - informational only
-    // This doesn't affect validity since email delivery only requires MX records
+    // Step 3: Check if domain has A/AAAA records - informational only
+    // This indicates if the domain has a website/web server
+    // Doesn't affect email validity since email only requires MX records
     try {
       await dns.resolve(domain, 'A');
-      result.domainExists = true;
+      result.hasWebsite = true;
     } catch {
       // Try AAAA records if A records fail
       try {
         await dns.resolve(domain, 'AAAA');
-        result.domainExists = true;
+        result.hasWebsite = true;
       } catch {
-        // Domain doesn't have A/AAAA records, but this is OK if it has MX records
-        result.domainExists = false;
+        // Domain doesn't have A/AAAA records (no website), but this is OK for email
+        result.hasWebsite = false;
       }
     }
 
@@ -186,4 +209,57 @@ export class EmailVerificationService {
   private static isForwardingDomain(domain: string): boolean {
     return FORWARDING_DOMAINS.has(domain.toLowerCase());
   }
+
+  /**
+   * Fetch and cache the personal email domains list from GitHub
+   * Uses Redis for caching with 24-hour TTL
+   * Falls back to in-memory cache if Redis fails
+   */
+  private static async getPersonalEmailDomains(): Promise<Set<string>> {
+    // Return in-memory cache if available
+    if (this.personalDomainsSet) {
+      return this.personalDomainsSet;
+    }
+
+    try {
+      // Try to get from Redis cache first
+      const cached = await redis.get(PERSONAL_DOMAINS_CACHE_KEY);
+      if (cached) {
+        const domains = JSON.parse(cached) as string[];
+        this.personalDomainsSet = new Set(domains);
+        return this.personalDomainsSet;
+      }
+
+      // Fetch from GitHub if not in cache
+      const response = await fetch(PERSONAL_DOMAINS_URL);
+      if (!response.ok) {
+        throw new Error(`Failed to fetch personal email domains: ${response.statusText}`);
+      }
+
+      const text = await response.text();
+      const domains = text
+        .split('\n')
+        .map(line => line.trim())
+        .filter(line => line && !line.startsWith('#')); // Filter empty lines and comments
+
+      // Cache in Redis
+      await redis.set(PERSONAL_DOMAINS_CACHE_KEY, JSON.stringify(domains), 'EX', CACHE_TTL_SECONDS);
+
+      // Cache in memory
+      this.personalDomainsSet = new Set(domains);
+      return this.personalDomainsSet;
+    } catch (error) {
+      console.error('Error fetching personal email domains:', error);
+      // Return empty set as fallback - don't block email verification
+      return new Set<string>();
+    }
+  }
+
+  /**
+   * Check if a domain is a personal/free email provider
+   */
+  private static async isPersonalEmailDomain(domain: string): Promise<boolean> {
+    const personalDomains = await this.getPersonalEmailDomains();
+    return personalDomains.has(domain.toLowerCase());
+  }
 }