* feat: add Cloudflare URL Scanner integration for malicious URL detection - Add MALICIOUS_URL_IN_WORKFLOW to LockReason enum - Add URL_SCANNING_ENABLED constant for feature flag - Create urlScanner.ts utility for Cloudflare Radar URL Scanner API - Create scanWorkflowUrls task for async URL scanning with polling - Integrate URL scanning into scanWorkflowBody task - Add URL scanning for event type redirect URLs - Lock user accounts when malicious URLs are detected - Fix pre-existing lint issues (parseInt radix, optional chaining) Co-Authored-By: peer@cal.com <peer@cal.com> * fix: address biome lint warnings and TypeScript iterator errors - Wrap iterators with Array.from() to fix TS2802 errors - Add biome-ignore comments for process.env usage - Extract helper functions to reduce function length - Move exports to end of file per useExportsLast rule - Remove problematic imports that cause TypeScript errors Co-Authored-By: peer@cal.com <peer@cal.com> * fix: address cubic-dev-ai review comments for URL scanning - Fix P0: Re-fetch workflow steps before scheduling notifications to use actual verifiedAt values from database instead of overriding with new Date() - Fix P1: Mark workflow step as verified in submitWorkflowStepForUrlScanning when URL scanning is disabled or no URLs found - Fix P1: Add whitelistWorkflows parameter to submitUrlForUrlScanning for consistency - Fix P2: Preserve URL context in error results in urlScanner.ts scanUrls function Co-Authored-By: peer@cal.com <peer@cal.com> * fix: add select clause to Prisma query for workflow steps Address cubic-dev-ai P2 comment: Use select to fetch only the required fields (id, action, sendTo, emailSubject, reminderBody, template, sender, verifiedAt) instead of fetching all columns from workflowStep table. Co-Authored-By: peer@cal.com <peer@cal.com> * fix: add select clause to Prisma query in scanWorkflowBody.ts Address cubic-dev-ai P2 review comment: Use select to fetch only the required fields (id, action, sendTo, emailSubject, reminderBody, template, sender, verifiedAt) instead of fetching all columns. Co-Authored-By: peer@cal.com <peer@cal.com> * test: add unit tests for URL scanning functionality - Add tests for urlScanner.ts (extractUrlsFromHtml, isUrlScanningEnabled) - Add tests for scanWorkflowUrls.ts (happy/unhappy paths for URL scanning task) - Add tests for scanWorkflowBody.ts (happy/unhappy paths for workflow body scanning) Tests cover: - URL extraction from HTML content - URL normalization and deduplication - Handling of malicious URLs and user locking - Fail-open behavior for API errors - Whitelisted user handling - Iffy spam detection integration Co-Authored-By: peer@cal.com <peer@cal.com> * test: remove incomplete test that provides no value Removed the 'should mark all steps as verified when neither Iffy nor URL scanning is enabled' test as it used vi.doMock() which doesn't work after module import, had no assertions, and gave false confidence in test coverage. Co-Authored-By: peer@cal.com <peer@cal.com> * refactor: use Cloudflare bulk scanning endpoint to reduce API quota usage - Added submitUrlsForBulkScanning function that uses /urlscanner/v2/bulk endpoint - Updated scanUrls to use bulk submission instead of individual URL submissions - Bulk endpoint accepts up to 100 URLs per request, batching is handled automatically - Reduces API quota usage as suggested by keithwillcode Co-Authored-By: peer@cal.com <peer@cal.com> * Update packages/features/tasker/tasks/scanWorkflowUrls.ts Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com> * fix: address cubic-dev-ai review comments - P1: Sanitize URLs before logging to prevent exposing sensitive query parameters - P2: Extract handleUrlScanningForStep helper function to reduce code duplication - P2: Use vi.stubGlobal for fetch mock in tests for proper cleanup Co-Authored-By: peer@cal.com <peer@cal.com> * fix: address volnei review comments on PR #26387 - Move vi.unstubAllGlobals() to afterEach hook in iffyScanBody tests - Restore updateMany optimization when URL scanning is disabled Co-Authored-By: peer@cal.com <peer@cal.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Volnei Munhoz <volnei@cal.com> Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com>
278 lines
8.0 KiB
TypeScript
278 lines
8.0 KiB
TypeScript
import z from "zod";
|
|
|
|
import { LockReason, lockUser } from "@calcom/features/ee/api-keys/lib/autoLock";
|
|
import {
|
|
extractUrlsFromHtml,
|
|
getScanResult,
|
|
isUrlScanningEnabled,
|
|
submitUrlForScanning,
|
|
} from "@calcom/features/ee/workflows/lib/urlScanner";
|
|
import tasker from "@calcom/features/tasker";
|
|
import logger from "@calcom/lib/logger";
|
|
import prisma from "@calcom/prisma";
|
|
|
|
export const scanWorkflowUrlsSchema = z.object({
|
|
userId: z.number(),
|
|
workflowStepId: z.number().optional(),
|
|
eventTypeId: z.number().optional(),
|
|
// URLs to scan (extracted from workflow body or event type redirect URL)
|
|
urls: z.array(z.string()).optional(),
|
|
// Scan IDs from Cloudflare (for polling results)
|
|
pendingScans: z
|
|
.array(
|
|
z.object({
|
|
url: z.string(),
|
|
scanId: z.string(),
|
|
})
|
|
)
|
|
.optional(),
|
|
// Number of poll attempts made
|
|
pollAttempts: z.number().optional(),
|
|
createdAt: z.string().optional(),
|
|
whitelistWorkflows: z.boolean().optional(),
|
|
});
|
|
|
|
const log = logger.getSubLogger({ prefix: ["[tasker] scanWorkflowUrls"] });
|
|
|
|
/**
|
|
* Sanitizes a URL for logging by removing query parameters that may contain sensitive data.
|
|
*/
|
|
function sanitizeUrlForLogging(url: string): string {
|
|
try {
|
|
const parsed = new URL(url);
|
|
// biome-ignore lint/nursery/noTernary: Simple ternary for conditional suffix
|
|
return `${parsed.protocol}//${parsed.host}${parsed.pathname}${parsed.search ? "[query_params_redacted]" : ""}`;
|
|
} catch {
|
|
return "[invalid_url]";
|
|
}
|
|
}
|
|
|
|
const MAX_POLL_ATTEMPTS = 10;
|
|
const POLL_DELAY_MS = 15000; // 15 seconds
|
|
|
|
export async function scanWorkflowUrls(payload: string) {
|
|
const parsed = scanWorkflowUrlsSchema.parse(JSON.parse(payload));
|
|
const {
|
|
userId,
|
|
workflowStepId,
|
|
eventTypeId,
|
|
urls,
|
|
pendingScans,
|
|
pollAttempts = 0,
|
|
whitelistWorkflows,
|
|
} = parsed;
|
|
|
|
if (!isUrlScanningEnabled()) {
|
|
log.info("URL scanning is not enabled, skipping");
|
|
// Mark workflow step as verified if this was for a workflow
|
|
if (workflowStepId) {
|
|
await markWorkflowStepVerified(workflowStepId);
|
|
}
|
|
return;
|
|
}
|
|
|
|
// Phase 1: Submit URLs for scanning
|
|
if (urls && urls.length > 0 && !pendingScans) {
|
|
log.info(`Submitting ${urls.length} URLs for scanning`, { userId, workflowStepId, eventTypeId });
|
|
|
|
const newPendingScans: Array<{ url: string; scanId: string }> = [];
|
|
const failedUrls: string[] = [];
|
|
|
|
for (const url of urls) {
|
|
const result = await submitUrlForScanning(url);
|
|
if ("error" in result) {
|
|
log.error(`Failed to submit URL for scanning: ${result.error}`, { url: sanitizeUrlForLogging(url) });
|
|
failedUrls.push(url);
|
|
} else {
|
|
newPendingScans.push({ url, scanId: result.scanId });
|
|
}
|
|
}
|
|
|
|
if (newPendingScans.length === 0) {
|
|
// All submissions failed, mark as verified (fail-open for submission errors)
|
|
log.warn("All URL submissions failed, marking as verified", { userId, workflowStepId, eventTypeId });
|
|
if (workflowStepId) {
|
|
await markWorkflowStepVerified(workflowStepId);
|
|
}
|
|
return;
|
|
}
|
|
|
|
// Schedule follow-up task to poll for results
|
|
const scheduledAt = new Date(Date.now() + POLL_DELAY_MS);
|
|
await tasker.create(
|
|
"scanWorkflowUrls",
|
|
{
|
|
userId,
|
|
workflowStepId,
|
|
eventTypeId,
|
|
pendingScans: newPendingScans,
|
|
pollAttempts: 0,
|
|
whitelistWorkflows,
|
|
},
|
|
{ scheduledAt }
|
|
);
|
|
|
|
return;
|
|
}
|
|
|
|
// Phase 2: Poll for scan results
|
|
if (pendingScans && pendingScans.length > 0) {
|
|
if (pollAttempts >= MAX_POLL_ATTEMPTS) {
|
|
log.warn("Max poll attempts reached, marking as verified", {
|
|
userId,
|
|
workflowStepId,
|
|
eventTypeId,
|
|
pendingScans,
|
|
});
|
|
// Fail-open: mark as verified if we can't get results
|
|
if (workflowStepId) {
|
|
await markWorkflowStepVerified(workflowStepId);
|
|
}
|
|
return;
|
|
}
|
|
|
|
const stillPending: Array<{ url: string; scanId: string }> = [];
|
|
const maliciousUrls: string[] = [];
|
|
|
|
for (const { url, scanId } of pendingScans) {
|
|
const result = await getScanResult(scanId);
|
|
|
|
if (result === null) {
|
|
// Still pending
|
|
stillPending.push({ url, scanId });
|
|
} else if (result.status === "error") {
|
|
log.error(`Error getting scan result: ${result.error}`, { url: sanitizeUrlForLogging(url), scanId });
|
|
// Don't add to stillPending, treat as non-malicious (fail-open for errors)
|
|
} else if (result.malicious) {
|
|
maliciousUrls.push(url);
|
|
log.warn(`Malicious URL detected`, {
|
|
url: sanitizeUrlForLogging(url),
|
|
scanId,
|
|
categories: result.categories,
|
|
userId,
|
|
workflowStepId,
|
|
eventTypeId,
|
|
});
|
|
}
|
|
}
|
|
|
|
// If malicious URLs found, lock the user (unless whitelisted)
|
|
if (maliciousUrls.length > 0) {
|
|
if (whitelistWorkflows) {
|
|
log.warn(`Skipping lock for whitelisted user with malicious URLs`, {
|
|
userId,
|
|
maliciousUrlCount: maliciousUrls.length,
|
|
workflowStepId,
|
|
eventTypeId,
|
|
});
|
|
} else {
|
|
log.warn(`Locking user due to malicious URLs`, {
|
|
userId,
|
|
maliciousUrlCount: maliciousUrls.length,
|
|
workflowStepId,
|
|
eventTypeId,
|
|
});
|
|
await lockUser("userId", String(userId), LockReason.MALICIOUS_URL_IN_WORKFLOW);
|
|
}
|
|
// Don't mark as verified - the workflow step should not be sent
|
|
return;
|
|
}
|
|
|
|
// If still pending, schedule another poll
|
|
if (stillPending.length > 0) {
|
|
const scheduledAt = new Date(Date.now() + POLL_DELAY_MS);
|
|
await tasker.create(
|
|
"scanWorkflowUrls",
|
|
{
|
|
userId,
|
|
workflowStepId,
|
|
eventTypeId,
|
|
pendingScans: stillPending,
|
|
pollAttempts: pollAttempts + 1,
|
|
whitelistWorkflows,
|
|
},
|
|
{ scheduledAt }
|
|
);
|
|
return;
|
|
}
|
|
|
|
// All scans completed and no malicious URLs found
|
|
log.info("All URL scans completed, no malicious URLs found", { userId, workflowStepId, eventTypeId });
|
|
if (workflowStepId) {
|
|
await markWorkflowStepVerified(workflowStepId);
|
|
}
|
|
}
|
|
}
|
|
|
|
async function markWorkflowStepVerified(workflowStepId: number) {
|
|
await prisma.workflowStep.update({
|
|
where: { id: workflowStepId },
|
|
data: { verifiedAt: new Date() },
|
|
});
|
|
log.info(`Marked workflow step as verified`, { workflowStepId });
|
|
}
|
|
|
|
/**
|
|
* Helper function to extract URLs from workflow step body and submit for scanning.
|
|
* Called from scanWorkflowBody task or directly from workflow update handler.
|
|
*/
|
|
export async function submitWorkflowStepForUrlScanning(
|
|
workflowStepId: number,
|
|
reminderBody: string,
|
|
userId: number,
|
|
whitelistWorkflows?: boolean
|
|
): Promise<void> {
|
|
if (!isUrlScanningEnabled()) {
|
|
// URL scanning is disabled, mark as verified since there's nothing to scan
|
|
await markWorkflowStepVerified(workflowStepId);
|
|
return;
|
|
}
|
|
|
|
const urls = extractUrlsFromHtml(reminderBody);
|
|
if (urls.length === 0) {
|
|
// No URLs found, mark as verified since there's nothing to scan
|
|
log.info("No URLs found in workflow step body, marking as verified", { workflowStepId });
|
|
await markWorkflowStepVerified(workflowStepId);
|
|
return;
|
|
}
|
|
|
|
log.info(`Found ${urls.length} URLs in workflow step body, submitting for scanning`, {
|
|
workflowStepId,
|
|
urlCount: urls.length,
|
|
});
|
|
|
|
await tasker.create("scanWorkflowUrls", {
|
|
userId,
|
|
workflowStepId,
|
|
urls,
|
|
whitelistWorkflows,
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Helper function to scan a single URL (e.g., event type redirect URL).
|
|
*/
|
|
export async function submitUrlForUrlScanning(
|
|
url: string,
|
|
userId: number,
|
|
eventTypeId: number,
|
|
whitelistWorkflows?: boolean
|
|
): Promise<void> {
|
|
if (!isUrlScanningEnabled()) {
|
|
return;
|
|
}
|
|
|
|
log.info(`Submitting event type redirect URL for scanning`, {
|
|
url: sanitizeUrlForLogging(url),
|
|
userId,
|
|
eventTypeId,
|
|
});
|
|
|
|
await tasker.create("scanWorkflowUrls", {
|
|
userId,
|
|
eventTypeId,
|
|
urls: [url],
|
|
whitelistWorkflows,
|
|
});
|
|
}
|