fix: Handle invalid non-delegation credential in cron (#21270)

Co-authored-by: Benny Joo <sldisek783@gmail.com>
Co-authored-by: Omar López <zomars@me.com>

apps/web/app/api/cron/credentials/__tests__/cron.test.ts

@@ -0,0 +1,201 @@
+import prismock from "../../../../../../../tests/libs/__mocks__/prisma";
+
+import { describe, it, expect, beforeEach } from "vitest";
+
+import type { MembershipRole } from "@calcom/prisma/enums";
+
+import { handleCreateCredentials } from "../route";
+
+type OrgParams = { id: number };
+const createOrg = async ({ id }: OrgParams) =>
+  await prismock.team.create({ data: { id, name: `Org${id}`, isOrganization: true } });
+
+type UserParams = { id: number; email: string };
+const createUser = async ({ id, email }: UserParams) => await prismock.user.create({ data: { id, email } });
+
+type WorkspacePlatformParams = { id: number; name?: string; slug?: string; description?: string };
+const createWorkspacePlatform = async ({ id, name, slug, description }: WorkspacePlatformParams) =>
+  await prismock.workspacePlatform.create({
+    data: {
+      id,
+      name: name ?? "Google",
+      slug: slug ?? "google",
+      description: description ?? "Test platform",
+      defaultServiceAccountKey: {},
+    },
+  });
+
+type DelegationCredentialParams = {
+  id: string;
+  orgId: number;
+  domain: string;
+  workspacePlatformId: number;
+};
+
+const createDelegationCredential = async ({
+  id,
+  orgId,
+  domain,
+  workspacePlatformId,
+}: DelegationCredentialParams) =>
+  await prismock.delegationCredential.create({
+    data: {
+      id,
+      enabled: true,
+      domain,
+      organizationId: orgId,
+      workspacePlatformId,
+      serviceAccountKey: {
+        client_email: "svc@example.com",
+        private_key: "pk",
+        client_id: "cid",
+      },
+    },
+  });
+
+type CredentialParams = { id?: number; userId?: number; delegationCredentialId?: string; key?: object };
+const createCredential = async ({
+  id = 1,
+  userId = 1,
+  delegationCredentialId = "delegation-credential-1",
+  key = {},
+}: CredentialParams = {}) =>
+  await prismock.credential.create({
+    data: {
+      id,
+      type: "google_calendar",
+      key,
+      userId,
+      appId: "google_calendar",
+      delegationCredentialId,
+    },
+  });
+
+type MembershipParams = { teamId: number; userId: number; role?: MembershipRole };
+const createMembership = async ({ teamId, userId, role = "MEMBER" }: MembershipParams) =>
+  await prismock.membership.create({ data: { teamId, userId, accepted: true, role } });
+
+// Helper to assert credentials in DB match expected (partial, order-insensitive)
+type PartialCredential = Partial<Awaited<ReturnType<typeof prismock.credential.create>>>;
+async function expectCredentials(expected: PartialCredential[]) {
+  const actual = await prismock.credential.findMany({});
+  expect(actual.length).toBe(expected.length);
+  expect(actual).toEqual(expect.arrayContaining(expected.map((exp) => expect.objectContaining(exp))));
+}
+
+describe("Delegation Credentials: Organization-wide Google Calendar Access", () => {
+  beforeEach(() => {
+    prismock.delegationCredential.deleteMany();
+    prismock.credential.deleteMany();
+    prismock.user.deleteMany();
+    prismock.team.deleteMany();
+    prismock.workspacePlatform.deleteMany();
+    prismock.membership.deleteMany();
+  });
+
+  it("shows a helpful message when no Delegation Credentials are enabled", async () => {
+    const result = await handleCreateCredentials();
+    expect(result).toEqual({
+      message: "No enabled delegation credentials found",
+      success: 0,
+      failures: 0,
+    });
+    await expectCredentials([]);
+  });
+
+  it("creates credentials for all eligible organization members when Delegation Credential is enabled", async () => {
+    const org = await createOrg({ id: 1 });
+    const workspacePlatform = await createWorkspacePlatform({ id: 1 });
+    const user1 = await createUser({ id: 1, email: "alice@example.com" });
+    const user2 = await createUser({ id: 2, email: "bob@example.com" });
+    await createMembership({ teamId: org.id, userId: user1.id });
+    await createMembership({ teamId: org.id, userId: user2.id });
+    await createDelegationCredential({
+      id: "delegation-credential-1",
+      orgId: org.id,
+      domain: "example.com",
+      workspacePlatformId: workspacePlatform.id,
+    });
+
+    const result = await handleCreateCredentials();
+    expect(result.success).toBe(2);
+    expect(result.failures).toBe(0);
+    await expectCredentials([
+      { userId: user1.id, delegationCredentialId: "delegation-credential-1", type: "google_calendar" },
+      { userId: user2.id, delegationCredentialId: "delegation-credential-1", type: "google_calendar" },
+    ]);
+  });
+
+  it("skips credential creation for unsupported workspace platforms", async () => {
+    const org = await createOrg({ id: 1 });
+    // Create a non-google workspace platform
+    const outlookPlatform = await createWorkspacePlatform({
+      id: 2,
+      name: "Outlook",
+      slug: "outlook",
+      description: "Test",
+    });
+    const user = await createUser({ id: 1, email: "user1@outlook.com" });
+    await createMembership({ teamId: org.id, userId: user.id });
+    await createDelegationCredential({
+      id: "delegation-credential-2",
+      orgId: org.id,
+      domain: "outlook.com",
+      workspacePlatformId: outlookPlatform.id,
+    });
+    const result = await handleCreateCredentials();
+    expect(result.success).toBe(0);
+    await expectCredentials([]);
+  });
+
+  it("does not create duplicate credentials for users who already have one", async () => {
+    const org = await createOrg({ id: 1 });
+    const workspacePlatform = await createWorkspacePlatform({ id: 1 });
+    const user = await createUser({ id: 1, email: "user1@example.com" });
+    await createMembership({ teamId: org.id, userId: user.id });
+    await createDelegationCredential({
+      id: "delegation-credential-1",
+      orgId: org.id,
+      domain: "example.com",
+      workspacePlatformId: workspacePlatform.id,
+    });
+    await createCredential({ id: 1, userId: user.id, delegationCredentialId: "delegation-credential-1" });
+    const result = await handleCreateCredentials();
+    expect(result.success).toBe(0);
+    await expectCredentials([
+      { userId: user.id, delegationCredentialId: "delegation-credential-1", type: "google_calendar" },
+    ]);
+  });
+
+  it("processes only a batch of members at a time if there are more than the batch size", async () => {
+    const org = await createOrg({ id: 1 });
+    const workspacePlatform = await createWorkspacePlatform({
+      id: 1,
+      name: "Google",
+      slug: "google",
+      description: "Test platform",
+    });
+    // Simulate more than 100 delegated members
+    const users = [];
+    for (let i = 1; i <= 120; i++) {
+      const user = await createUser({ id: i, email: `user${i}@example.com` });
+      users.push(user);
+      await createMembership({ teamId: org.id, userId: user.id });
+    }
+    await createDelegationCredential({
+      id: "delegation-credential-1",
+      orgId: org.id,
+      domain: "example.com",
+      workspacePlatformId: workspacePlatform.id,
+    });
+    const result = await handleCreateCredentials();
+    expect(result.success).toBe(100); // Only batch size processed
+    await expectCredentials(
+      users.slice(0, 100).map((u) => ({
+        userId: u.id,
+        delegationCredentialId: "delegation-credential-1",
+        type: "google_calendar",
+      }))
+    );
+  });
+});

apps/web/app/api/cron/credentials/route.ts

@@ -14,7 +14,7 @@ import { DelegationCredentialRepository } from "@calcom/lib/server/repository/de
 
 import { defaultResponderForAppDir } from "../../defaultResponderForAppDir";
 
-const log = logger.getSubLogger({ prefix: ["CreateCredentials"] });
+const moduleLogger = logger.getSubLogger({ prefix: ["[api]", "[delegation]", "[credentials/cron]"] });
 const batchSizeToCreateCredentials = 100;
 const validateRequest = (req: NextRequest) => {
   const url = new URL(req.url);
@@ -24,9 +24,12 @@ const validateRequest = (req: NextRequest) => {
   }
 };
 
-async function handleCreateCredentials() {
+export async function handleCreateCredentials() {
+  const log = moduleLogger.getSubLogger({ prefix: ["CreateCredentials"] });
   const delegationCredentials = await DelegationCredentialRepository.findAllEnabledIncludeDelegatedMembers();
 
+  log.info(`Found ${delegationCredentials.length} enabled delegation credentials to create credentials for`);
+
   if (!delegationCredentials.length) {
     return {
       message: "No enabled delegation credentials found",
@@ -107,9 +110,12 @@ async function handleCreateCredentials() {
 }
 
 async function handleDeleteCredentials() {
+  const log = moduleLogger.getSubLogger({ prefix: ["DeleteCredentials"] });
   const delegationCredentials =
     await DelegationCredentialRepository.findAllDisabledAndIncludeNextBatchOfMembersToProcess();
 
+  log.info(`Found ${delegationCredentials.length} delegation credentials to delete credentials for`);
+
   if (!delegationCredentials.length) {
     return {
       message: "No disabled delegation credentials found",

apps/web/app/api/cron/selected-calendars/__tests__/cron.test.ts

@@ -0,0 +1,276 @@
+import prismock from "../../../../../../../tests/libs/__mocks__/prisma";
+import "@calcom/lib/server/__mocks__/serviceAccountKey";
+
+import { describe, it, expect, beforeEach, vi } from "vitest";
+
+import { CalendarAppDelegationCredentialInvalidGrantError } from "@calcom/lib/CalendarAppError";
+
+import { handleCreateSelectedCalendars } from "../route";
+
+// Mock GoogleCalendarService
+const fetchPrimaryCalendarMock = vi.fn();
+vi.mock("@calcom/app-store/googlecalendar/lib/CalendarService", () => {
+  return {
+    default: vi.fn().mockImplementation(() => ({
+      fetchPrimaryCalendar: fetchPrimaryCalendarMock,
+    })),
+  };
+});
+
+type OrgParams = { id?: number };
+const createOrg = async ({ id = 1 }: OrgParams = {}) =>
+  await prismock.team.create({ data: { id, name: `Org${id}`, isOrganization: true } });
+
+type UserParams = { id?: number; email?: string };
+const createUser = async ({ id = 1, email = `user${id}@example.com` }: UserParams = {}) =>
+  await prismock.user.create({ data: { id, email } });
+
+type WorkspacePlatformParams = { id?: number };
+const createWorkspacePlatform = async ({ id = 1 }: WorkspacePlatformParams = {}) =>
+  await prismock.workspacePlatform.create({
+    data: {
+      id,
+      name: "Google",
+      slug: "google",
+      description: "Test platform",
+      defaultServiceAccountKey: {},
+    },
+  });
+
+type DelegationCredentialParams = { id: string; orgId: number; domain: string };
+const createDelegationCredential = async ({ id, orgId, domain }: DelegationCredentialParams) =>
+  await prismock.delegationCredential.create({
+    data: {
+      id,
+      enabled: true,
+      domain: domain ?? "example.com",
+      organizationId: orgId ?? 1,
+      workspacePlatformId: 1,
+      serviceAccountKey: {
+        client_email: "svc@example.com",
+        private_key: "pk",
+        client_id: "cid",
+      },
+    },
+  });
+
+type CredentialParams = {
+  id: number;
+  userId: number;
+  delegationCredentialId: string | null;
+  key?: object;
+};
+const createCredential = async ({ id, userId, delegationCredentialId, key = {} }: CredentialParams) =>
+  await prismock.credential.create({
+    data: {
+      id,
+      type: "google_calendar",
+      key,
+      userId,
+      appId: "google_calendar",
+      delegationCredentialId,
+    },
+  });
+
+type SelectedCalendarParams = {
+  userId?: number;
+  externalId?: string;
+  error?: string | null;
+  id?: string;
+  delegationCredentialId?: string | null;
+  credentialId?: number;
+};
+const createSelectedCalendar = async ({
+  userId = 1,
+  externalId = "user1@example.com",
+  error = null,
+  id = `${userId}-sc`,
+  delegationCredentialId = null,
+  credentialId = 1,
+}: SelectedCalendarParams = {}) =>
+  await prismock.selectedCalendar.create({
+    data: {
+      id,
+      integration: "google_calendar",
+      externalId,
+      userId,
+      delegationCredentialId,
+      credentialId,
+      error,
+    },
+  });
+
+// Helper to assert selected calendars in DB match expected (partial, order-insensitive)
+type PartialSelectedCalendar = Partial<Awaited<ReturnType<typeof prismock.selectedCalendar.create>>>;
+async function expectSelectedCalendars(expected: PartialSelectedCalendar[]) {
+  const actual = await prismock.selectedCalendar.findMany({});
+  expect(actual.length).toBe(expected.length);
+  expect(actual).toEqual(expect.arrayContaining(expected.map((exp) => expect.objectContaining(exp))));
+}
+
+describe("handleCreateSelectedCalendars integration", () => {
+  beforeEach(() => {
+    prismock.delegationCredential.deleteMany();
+    prismock.credential.deleteMany();
+    prismock.selectedCalendar.deleteMany();
+    prismock.user.deleteMany();
+    prismock.team.deleteMany();
+    prismock.workspacePlatform.deleteMany();
+    fetchPrimaryCalendarMock.mockReset();
+  });
+
+  it("shows a helpful message when no Delegation Credentials are set up in the system", async () => {
+    const result = await handleCreateSelectedCalendars();
+    expect(result).toEqual({
+      message: "No delegation credentials found",
+      success: 0,
+      failures: 0,
+    });
+  });
+
+  it("automatically creates a Selected Calendar for every eligible user when Delegation Credential is enabled and user matches domain", async () => {
+    await createOrg({ id: 1 });
+    await createWorkspacePlatform({ id: 1 });
+    const user = await createUser({ id: 1, email: "user1@example.com" });
+    const delegationCredential = await createDelegationCredential({
+      id: "delegation-credential-1",
+      orgId: 1,
+      domain: "example.com",
+    });
+    await createCredential({ id: 1, userId: user.id, delegationCredentialId: delegationCredential.id });
+    fetchPrimaryCalendarMock.mockResolvedValue({ id: "user1@example.com" });
+
+    const result = await handleCreateSelectedCalendars();
+    expect(result.success).toBe(1);
+    expect(result.failures).toBe(0);
+    await expectSelectedCalendars([
+      { userId: user.id, externalId: "user1@example.com", delegationCredentialId: delegationCredential.id },
+    ]);
+  });
+
+  it("creates a Selected Calendar even if the user's primary calendar ID does not match their email address", async () => {
+    await createOrg({ id: 1 });
+    await createWorkspacePlatform({ id: 1 });
+    const user = await createUser({ id: 1, email: "user1@example.com" });
+    const delegationCredential = await createDelegationCredential({
+      id: "delegation-credential-1",
+      orgId: 1,
+      domain: "example.com",
+    });
+    await createCredential({ id: 1, userId: user.id, delegationCredentialId: delegationCredential.id });
+    fetchPrimaryCalendarMock.mockResolvedValue({ id: "notuser@example.com" });
+
+    const result = await handleCreateSelectedCalendars();
+    expect(result.success).toBe(1);
+    await expectSelectedCalendars([
+      { userId: user.id, externalId: "notuser@example.com", delegationCredentialId: delegationCredential.id },
+    ]);
+  });
+
+  it("does not create duplicate Selected Calendars for users who already have a valid one", async () => {
+    await createOrg({ id: 1 });
+    await createWorkspacePlatform({ id: 1 });
+    const user = await createUser({ id: 1, email: "user1@example.com" });
+    const delegationCredential = await createDelegationCredential({
+      id: "delegation-credential-1",
+      orgId: 1,
+      domain: "example.com",
+    });
+    await createCredential({ id: 1, userId: user.id, delegationCredentialId: delegationCredential.id });
+    await createSelectedCalendar({
+      userId: user.id,
+      externalId: "user1@example.com",
+      delegationCredentialId: delegationCredential.id,
+      credentialId: 1,
+    });
+    fetchPrimaryCalendarMock.mockResolvedValue({ id: "user1@example.com" });
+
+    const result = await handleCreateSelectedCalendars();
+    expect(result.success).toBe(0);
+    await expectSelectedCalendars([
+      { userId: user.id, externalId: "user1@example.com", delegationCredentialId: delegationCredential.id },
+    ]);
+  });
+
+  it("replaces a Selected Calendar if the existing one has an error, ensuring users always have a working calendar connection", async () => {
+    await createOrg({ id: 1 });
+    await createWorkspacePlatform({ id: 1 });
+    const user = await createUser({ id: 1, email: "user1@example.com" });
+    const delegationCredential = await createDelegationCredential({
+      id: "delegation-credential-1",
+      orgId: 1,
+      domain: "example.com",
+    });
+    // Create a Delegation User Credential
+    await createCredential({
+      id: 2,
+      userId: user.id,
+      delegationCredentialId: delegationCredential.id,
+    });
+    // Create a Regular User Credential
+    const regularCredential = await createCredential({
+      id: 1,
+      userId: user.id,
+      delegationCredentialId: null,
+    });
+    // Create a SelectedCalendar attached to regular credential
+    await createSelectedCalendar({
+      userId: user.id,
+      externalId: "user1@example.com",
+      error: "some error" as unknown as null,
+      credentialId: regularCredential.id,
+    });
+    fetchPrimaryCalendarMock.mockResolvedValue({ id: "user1@example.com" });
+
+    const result = await handleCreateSelectedCalendars();
+    expect(result.success).toBe(1);
+    await expectSelectedCalendars([
+      {
+        userId: user.id,
+        externalId: "user1@example.com",
+        delegationCredentialId: delegationCredential.id,
+        error: null,
+      },
+    ]);
+  });
+
+  describe("when GoogleCalendarService throws error", () => {
+    it("creates SelectedCalendar with user.email when CalendarAppDelegationCredentialInvalidGrantError is thrown", async () => {
+      await createOrg({ id: 1 });
+      await createWorkspacePlatform({ id: 1 });
+      const user = await createUser({ id: 1, email: "user1@example.com" });
+      const delegationCredential = await createDelegationCredential({
+        id: "delegation-credential-1",
+        orgId: 1,
+        domain: "example.com",
+      });
+      await createCredential({ id: 1, userId: user.id, delegationCredentialId: delegationCredential.id });
+      fetchPrimaryCalendarMock.mockRejectedValue(
+        new CalendarAppDelegationCredentialInvalidGrantError("some error")
+      );
+
+      const result = await handleCreateSelectedCalendars();
+      expect(result.success).toBe(1);
+      await expectSelectedCalendars([
+        { userId: user.id, externalId: "user1@example.com", delegationCredentialId: delegationCredential.id },
+      ]);
+    });
+
+    it("does not create SelectedCalendar when some other error than CalendarAppDelegationCredentialInvalidGrantError is thrown", async () => {
+      await createOrg({ id: 1 });
+      await createWorkspacePlatform({ id: 1 });
+      const user = await createUser({ id: 1, email: "user1@example.com" });
+      const delegationCredential = await createDelegationCredential({
+        id: "delegation-credential-1",
+        orgId: 1,
+        domain: "example.com",
+      });
+      await createCredential({ id: 1, userId: user.id, delegationCredentialId: delegationCredential.id });
+      fetchPrimaryCalendarMock.mockRejectedValue(new Error("some error"));
+
+      const result = await handleCreateSelectedCalendars();
+      expect(result.success).toBe(0);
+      await expectSelectedCalendars([]);
+    });
+  });
+});

apps/web/app/api/cron/selected-calendars/route.ts

@@ -7,6 +7,7 @@ import type { NextRequest } from "next/server";
 import { NextResponse } from "next/server";
 
 import GoogleCalendarService from "@calcom/app-store/googlecalendar/lib/CalendarService";
+import { CalendarAppDelegationCredentialInvalidGrantError } from "@calcom/lib/CalendarAppError";
 import { findUniqueDelegationCalendarCredential } from "@calcom/lib/delegationCredential/server";
 import { HttpError } from "@calcom/lib/http-error";
 import logger from "@calcom/lib/logger";
@@ -19,7 +20,7 @@ import type { Ensure } from "@calcom/types/utils";
 import { defaultResponderForAppDir } from "../../defaultResponderForAppDir";
 
 const limitOnQueryingGoogleCalendar = 50;
-const log = logger.getSubLogger({ prefix: ["[api] selected-calendars/cron"] });
+const log = logger.getSubLogger({ prefix: ["[api]", "[delegation]", "[selected-calendars/cron]"] });
 const validateRequest = (req: NextRequest) => {
   const url = new URL(req.url);
   const apiKey = req.headers.get("authorization") || url.searchParams.get("apiKey");
@@ -28,13 +29,191 @@ const validateRequest = (req: NextRequest) => {
   }
 };
 
-async function handleCreateSelectedCalendars() {
+type DelegationUserCredential = Awaited<
+  ReturnType<typeof CredentialRepository.findAllDelegationByTypeIncludeUserAndTake>
+>[number];
+
+type DelegationUserCredentialWithEnsuredUser = Ensure<DelegationUserCredential, "user">;
+
+async function getDelegationUserCredentialsToProcess(delegationUserCredentials: DelegationUserCredential[]) {
+  const delegationUserCredentialsToProcess = await Promise.all(
+    delegationUserCredentials
+      .filter(
+        (delegationUserCredential): delegationUserCredential is DelegationUserCredentialWithEnsuredUser =>
+          delegationUserCredential.user !== null
+      )
+      .map(async (delegationUserCredential) => {
+        const userEmail = delegationUserCredential.user.email;
+        // Consider existing SelectedCalendars for this user, regardless of credentialId as we want to reuse the existing SelectedCalendar(which has their googleChannel related column set) after Delegation Credential has been enabled
+        // It is also important because SelectedCalendar has composite unique constraint on userId, externalId and integration and thus we couldn't have the same values for different credentialId
+        const selectedCalendar = await SelectedCalendarRepository.findFirst({
+          where: {
+            userId: delegationUserCredential.user.id,
+            externalId: userEmail,
+            integration: "google_calendar",
+          },
+        });
+
+        if (selectedCalendar) {
+          if (selectedCalendar.delegationCredentialId === delegationUserCredential.delegationCredentialId) {
+            // In case this SelectedCalendar has an error, we would let CalendarCache cron handle it
+            log.info(
+              `Found an existing SelectedCalendar for userId: ${delegationUserCredential.user.id} and delegationCredentialId: ${delegationUserCredential.delegationCredentialId}`
+            );
+            return null;
+          }
+          if (!selectedCalendar.error) {
+            log.info(`Found a reusable SelectedCalendar for userId: ${delegationUserCredential.user.id}`);
+            return null;
+          } else {
+            log.error(
+              `Found reusable Non-Delegation SelectedCalendar for ${delegationUserCredential.user.id} but it has an error: '${selectedCalendar.error}', so deleting it, new one will be created`
+            );
+
+            // We found a SelectedCalendar attached to non-delegation credential, so we shouldn't reuse it as the non-delegation credential attached to it could be invalid
+            // So, we delete it so that SelectedCalendarRepository allow creating a similar one with Delegation Credential
+            // Also, we don't delete the possibly invalid credential due to false alarm and we would need that intact in case we disable Delegation Credential
+            // Because credential isn't deleted, the corresponding CalendarCache entry would still exist and thus we will end up having two CalendarCache entries for same key and userId with different credentialId
+            await SelectedCalendarRepository.deleteById({ id: selectedCalendar.id });
+            return delegationUserCredential;
+          }
+        }
+        return delegationUserCredential;
+      })
+  );
+
+  return delegationUserCredentialsToProcess.filter(
+    (credential): credential is DelegationUserCredentialWithEnsuredUser => credential !== null
+  );
+}
+
+async function getCalendarService(delegationUserCredential: DelegationUserCredentialWithEnsuredUser) {
+  const credentialForCalendarService = await findUniqueDelegationCalendarCredential({
+    userId: delegationUserCredential.user.id,
+    delegationCredentialId: delegationUserCredential.delegationCredentialId,
+  });
+
+  if (!credentialForCalendarService) {
+    log.error(
+      `Credential not found for delegationCredentialId: ${delegationUserCredential.delegationCredentialId} and userId: ${delegationUserCredential.user.id}`
+    );
+    return null;
+  }
+
+  if (
+    !credentialForCalendarService.delegatedTo ||
+    !credentialForCalendarService.delegatedTo.serviceAccountKey ||
+    !credentialForCalendarService.delegatedTo.serviceAccountKey.client_email
+  ) {
+    log.error(
+      `Invalid delegatedTo for delegationCredentialId: ${delegationUserCredential.delegationCredentialId}`,
+      safeStringify({
+        delegatedToSet: !!credentialForCalendarService.delegatedTo,
+        serviceAccountKeySet: !!credentialForCalendarService.delegatedTo?.serviceAccountKey,
+        clientEmailSet: !!credentialForCalendarService.delegatedTo?.serviceAccountKey?.client_email,
+      })
+    );
+    return null;
+  }
+
+  const googleCalendarService = new GoogleCalendarService(
+    credentialForCalendarService as CredentialForCalendarServiceWithEmail
+  );
+
+  return googleCalendarService;
+}
+
+async function fetchPrimaryCalendarId({
+  googleCalendarService,
+  delegationUserCredential,
+}: {
+  googleCalendarService: GoogleCalendarService;
+  delegationUserCredential: DelegationUserCredentialWithEnsuredUser;
+}) {
+  let primaryCalendarId;
+  const userEmail = delegationUserCredential.user.email;
+
+  try {
+    const primaryCalendar = await googleCalendarService.fetchPrimaryCalendar();
+    primaryCalendarId = primaryCalendar?.id;
+  } catch (error) {
+    log.error(
+      `Error fetching primary calendar for delegationCredentialId: ${delegationUserCredential.delegationCredentialId}, credentialId: ${delegationUserCredential.id} and userId: ${delegationUserCredential.userId}`,
+      safeStringify(error)
+    );
+
+    if (error instanceof CalendarAppDelegationCredentialInvalidGrantError) {
+      // It is known to happen in case john+test@acme.com is the user email and the actual email is john@acme.com. Google rejects the authorization for john+test@acme.com
+      log.info(
+        `Error seems to be that the email isn't present in the third party workspace, further attempts would fail too, so we assume the user's email as the primary calendar id to skip this user in future cron runs`
+      );
+      // If we are unable to fetch the primary calendar, we assume the user's email as the primary calendar id
+      // It ensures that this calendar is skipped in the future cron runs
+      primaryCalendarId = userEmail;
+    }
+  }
+
+  return primaryCalendarId;
+}
+
+async function processDelegationUserCredential(
+  delegationUserCredential: DelegationUserCredentialWithEnsuredUser
+) {
+  try {
+    const userEmail = delegationUserCredential.user.email;
+    const googleCalendarService = await getCalendarService(delegationUserCredential);
+    if (!googleCalendarService) {
+      return;
+    }
+
+    const externalCalendarId = await fetchPrimaryCalendarId({
+      googleCalendarService,
+      delegationUserCredential,
+    });
+
+    if (!externalCalendarId) {
+      log.error(
+        `External calendar not found for delegationCredentialId: ${delegationUserCredential.delegationCredentialId}, credentialId: ${delegationUserCredential.id} and userId: ${delegationUserCredential.userId}`
+      );
+      throw new Error("externalCalendarId could not be determined");
+    }
+
+    if (externalCalendarId !== userEmail) {
+      // We don't know the scenario when Delegation Credential allows access to an email that is not the primary calendar email
+      // So log a warning for further investigation
+      log.warn(
+        `External calendar id mismatch for delegationCredentialId: ${delegationUserCredential.delegationCredentialId}, credentialId: ${delegationUserCredential.id} and userId: ${delegationUserCredential.userId}`,
+        safeStringify({
+          externalCalendarId,
+          userEmail,
+        })
+      );
+    }
+
+    await SelectedCalendarRepository.create({
+      integration: "google_calendar",
+      externalId: externalCalendarId,
+      userId: delegationUserCredential.user.id,
+      delegationCredentialId: delegationUserCredential.delegationCredentialId,
+      credentialId: delegationUserCredential.id,
+    });
+
+    log.debug(`Created SelectedCalendar for user ${delegationUserCredential.userId}`);
+  } catch (error) {
+    log.error(`Error processing credential ${delegationUserCredential.id}:`, safeStringify(error));
+    throw error;
+  }
+}
+
+export async function handleCreateSelectedCalendars() {
   // These are in DB delegation user credentials in contrast to in-memory delegation user credentials that are used elsewhere
   const allDelegationUserCredentials = await CredentialRepository.findAllDelegationByTypeIncludeUserAndTake({
     type: "google_calendar",
     take: limitOnQueryingGoogleCalendar,
   });
 
+  log.info(`Found ${allDelegationUserCredentials.length} delegation user credentials to process`);
+
   if (!allDelegationUserCredentials.length) {
     const message = "No delegation credentials found";
     log.info(message);
@@ -57,106 +236,16 @@ async function handleCreateSelectedCalendars() {
   for (const [delegationCredentialId, delegationUserCredentials] of Object.entries(
     groupedDelegationUserCredentials
   )) {
-    // First, create an array of promises that check if each credential should be processed
-    const checkedDelegationUserCredentials = await Promise.all(
-      delegationUserCredentials.map(async (delegationUserCredential) => {
-        if (!delegationUserCredential.user) {
-          log.error(`Credential ${delegationUserCredential.id} has no user`);
-          return null;
-        }
-
-        // Get all selectedCalendars for this user, regardless of credentialId as we want to reuse the existing SelectedCalendar(which has their googleChannel related column set) after Delegation Credential has been enabled
-        const hasSelectedCalendarForUserEmail = !!(await SelectedCalendarRepository.findFirst({
-          where: {
-            userId: delegationUserCredential.user.id,
-            externalId: delegationUserCredential.user.email,
-          },
-        }));
-
-        if (hasSelectedCalendarForUserEmail) {
-          log.info(`Found a reusable SelectedCalendar for ${delegationUserCredential.user.id}`);
-          return null;
-        }
-        return delegationUserCredential;
-      })
+    log.info(`Processing delegation user credentials for delegationCredentialId: ${delegationCredentialId}`);
+    const delegationUserCredentialsToProcess = await getDelegationUserCredentialsToProcess(
+      delegationUserCredentials
     );
-
-    const delegationUserCredentialsToProcess = checkedDelegationUserCredentials.filter(
-      (credential): credential is Ensure<(typeof delegationUserCredentials)[number], "user"> =>
-        credential !== null
-    );
-
     log.info(
       `Found ${delegationUserCredentialsToProcess.length} delegationUserCredentials to process for delegationCredentialId: ${delegationCredentialId}`
     );
 
     const results = await Promise.allSettled(
-      delegationUserCredentialsToProcess.map(async (delegationUserCredential) => {
-        try {
-          const credentialForCalendarService = await findUniqueDelegationCalendarCredential({
-            userId: delegationUserCredential.user.id,
-            delegationCredentialId: delegationUserCredential.delegationCredentialId,
-          });
-
-          if (!credentialForCalendarService) {
-            log.error(
-              `Credential not found for delegationCredentialId: ${delegationUserCredential.delegationCredentialId} and userId: ${delegationUserCredential.user.id}`
-            );
-            return;
-          }
-
-          if (
-            !credentialForCalendarService.delegatedTo ||
-            !credentialForCalendarService.delegatedTo.serviceAccountKey ||
-            !credentialForCalendarService.delegatedTo.serviceAccountKey.client_email
-          ) {
-            log.error(
-              `Invalid delegatedTo for delegationCredentialId: ${delegationUserCredential.delegationCredentialId}`,
-              safeStringify({
-                delegatedToSet: !!credentialForCalendarService.delegatedTo,
-                serviceAccountKeySet: !!credentialForCalendarService.delegatedTo?.serviceAccountKey,
-                clientEmailSet: !!credentialForCalendarService.delegatedTo?.serviceAccountKey?.client_email,
-              })
-            );
-            return;
-          }
-
-          const googleCalendarService = new GoogleCalendarService(
-            credentialForCalendarService as CredentialForCalendarServiceWithEmail
-          );
-
-          let primaryCalendar;
-          try {
-            primaryCalendar = await googleCalendarService.fetchPrimaryCalendar();
-          } catch (error) {
-            log.error(
-              `Error fetching primary calendar for delegationCredentialId: ${delegationUserCredential.delegationCredentialId} and userId: ${delegationUserCredential.userId}`,
-              safeStringify(error)
-            );
-            return;
-          }
-
-          if (!primaryCalendar || !primaryCalendar.id) {
-            log.error(
-              `Primary calendar not found for delegationCredentialId: ${delegationUserCredential.delegationCredentialId} and userId: ${delegationUserCredential.userId}`
-            );
-            return;
-          }
-
-          await SelectedCalendarRepository.create({
-            integration: "google_calendar",
-            externalId: primaryCalendar.id,
-            userId: delegationUserCredential.user.id,
-            delegationCredentialId: delegationUserCredential.delegationCredentialId,
-            credentialId: delegationUserCredential.id,
-          });
-
-          log.debug(`Created SelectedCalendar for user ${delegationUserCredential.userId}`);
-        } catch (error) {
-          log.error(`Error processing credential ${delegationUserCredential.id}:`, safeStringify(error));
-          throw error;
-        }
-      })
+      delegationUserCredentialsToProcess.map(processDelegationUserCredential)
     );
     const successCount = results.filter((r) => r.status === "fulfilled").length;
     const failureCount = results.filter((r) => r.status === "rejected").length;

packages/app-store/googlecalendar/lib/CalendarService.ts

@@ -974,15 +974,15 @@ export default class GoogleCalendarService implements Calendar {
           expiration: otherCalendarsWithSameSubscription[0].googleChannelExpiration,
         }
       : {};
-    let error: string | undefined;
+    let error: string | null = null;
 
     if (!otherCalendarsWithSameSubscription.length) {
       try {
         googleChannelProps = await this.startWatchingCalendarsInGoogle({ calendarId });
-      } catch (error) {
-        this.log.error(`Failed to watch calendar ${calendarId}`, error);
+      } catch (_error) {
+        this.log.error(`Failed to watch calendar ${calendarId}`, _error);
         // We set error to prevent attempting to watch on next cron run
-        error = error instanceof Error ? error.message : "Unknown error";
+        error = _error instanceof Error ? _error.message : "Unknown error";
       }
     } else {
       logger.info(

packages/features/calendar-cache/api/cron.ts

@@ -2,6 +2,7 @@ import type { NextApiRequest } from "next";
 
 import { HttpError } from "@calcom/lib/http-error";
 import logger from "@calcom/lib/logger";
+import { safeStringify } from "@calcom/lib/safeStringify";
 import { defaultHandler } from "@calcom/lib/server/defaultHandler";
 import { defaultResponder } from "@calcom/lib/server/defaultResponder";
 import { SelectedCalendarRepository } from "@calcom/lib/server/repository/selectedCalendar";
@@ -82,6 +83,13 @@ const handleCalendarsToUnwatch = async () => {
           if (error instanceof Error) {
             errorMessage = error.message;
           }
+          log.error(
+            "Error unwatching calendar: ",
+            safeStringify({
+              selectedCalendarId: id,
+              error: errorMessage,
+            })
+          );
           await SelectedCalendarRepository.updateById(id, {
             error: `Error unwatching calendar: ${errorMessage}`,
           });
@@ -117,6 +125,13 @@ const handleCalendarsToWatch = async () => {
           if (error instanceof Error) {
             errorMessage = error.message;
           }
+          log.error(
+            "Error watching calendar: ",
+            safeStringify({
+              selectedCalendarId: id,
+              error: errorMessage,
+            })
+          );
           await SelectedCalendarRepository.updateById(id, {
             error: `Error watching calendar: ${errorMessage}`,
           });

packages/features/calendar-cache/calendar-cache.repository.ts

@@ -91,7 +91,9 @@ export class CalendarCacheRepository implements ICalendarCacheRepository {
     log.debug("Getting cached availability", safeStringify({ credentialId, userId, args }));
     const key = parseKeyForCache(args);
     let cached;
+    let usedInMemoryDelegationCredential = false;
     if (isInMemoryDelegationCredential({ credentialId })) {
+      usedInMemoryDelegationCredential = true;
       if (!userId) {
         log.warn("userId is not available when querying cache for in-memory delegation credential");
         return null;
@@ -109,6 +111,11 @@ export class CalendarCacheRepository implements ICalendarCacheRepository {
           key,
           expiresAt: { gte: new Date(Date.now()) },
         },
+        orderBy: {
+          // In case of multiple entries for same key and userId, we prefer the one with highest expiry, which will be the most updated one
+          // TODO: For better tracking we could also want to use updatedAt directly which doesn't exist yet in CalendarCache table
+          expiresAt: "desc",
+        },
       });
     } else {
       cached = await prisma.calendarCache.findUnique({
@@ -121,7 +128,10 @@ export class CalendarCacheRepository implements ICalendarCacheRepository {
         },
       });
     }
-    log.info("Got cached availability", safeStringify({ key, cached }));
+    log.info(
+      "Got cached availability",
+      safeStringify({ key, cached, credentialId, usedInMemoryDelegationCredential })
+    );
     return cached;
   }
   async upsertCachedAvailability({

packages/features/delegation-credentials/README.md

@@ -96,6 +96,8 @@ Step 6: Enable Delegation Credential(To Be taken By Cal.com organization Owner/A
 
 - Delegation Credential: A Delegation Credential service account key along with user's email becomes the Delegation Credential which is an alternative to regular Credential in DB.
 - Delegation User Credential: A Delegation User Credential is a Credential record in DB that uses DelegationCredential record to actually access the user's calendar. A Credential record with delegationCredentialId set is a Delegation User Credential.
+- In-DB Delegation Credential: Another name for Delegation User Credential. This is used to build the CalendarCache records.
+- In-Memory Delegation Credential: It is a Credential like object but only in-memory and has id=-1. This is used to to connect with the third party Calendar. We might want to move away from In-Memory Delegation Credential to use In-DB Delegation Credential in future.
 
 ### How Delegation Credential works
 
@@ -109,7 +111,7 @@ Step 6: Enable Delegation Credential(To Be taken By Cal.com organization Owner/A
 
 ### Cron Jobs
 
-Cron jobs ensure that for each and every member of the organization that has Delegation Credential enabled, corresponding SelectedCalendar records are there. These crons currently run every 5 minutes, look at vercel.json for the up-to-date schedule.
+Cron jobs ensure that for each and every member of the organization that has Delegation Credential enabled, corresponding SelectedCalendar records are there. These crons currently run every 5 minutes and process a batch in one run to avoid overloading the DB and third party CalendarAPIs, look at vercel.json for the up-to-date schedule.
 
 - `credentials` cron job creates Delegation User Credential records for all the members of the organization who don't have Delegation User Credentials yet. It also ensures that on disabling Delegation Credential, the Delegation User Credentials are deleted which automatically deletes the SelectedCalendars through DB cascade.
 - `selected-calendars` cron job creates SelectedCalendar records for all the Delegation User Credentials of the organization who don't have Selected Calendars yet.
@@ -137,6 +139,7 @@ Disabling effectively stops generating in-memory delegation user credentials. So
 ### Impact of enabling Delegation Credential
 - Existing calendar-cache records are re-used as we identify the relevant record by userId and key of CalendarCache record.
   - Any updates to those calendar-cache records keep on working by using the non-delegation credential attached with the SelectedCalendar record.
+  - In case there is an error while watching the SelectedCalendar using non-delegation credential, we will delete the SelectedCalendar record and create a new one using Delegation User Credential.
 - For any new members, we create Credential records and SelectedCalendar records through cron jobs and thus their calendar-cache records will also be created.
 
 ### Notes when testing locally
@@ -146,8 +149,3 @@ Disabling effectively stops generating in-memory delegation user credentials. So
 - Make sure to change the email of the user above to your workspace owner's email(other member's email might also work). This is necessary otherwise you won't be able to enable Delegation Credential for the organization.
   - Note: After changing the email, you would have to logout and login again
 
-
-
-## TODO
-- Test what happens when credential expires that was used in CalendarCache/SelectedCalendar
-  - It seems that if refresh token is valid then it would still be refreshed but if it becomes invalid then it ends up causing the calendar-cache updates to break because it isn't able to renew the access token. How do you fix it?

packages/features/delegation-credentials/scripts/rate-limit-test-google-service-account-impersonation.js

@@ -0,0 +1,53 @@
+/**
+ * How to verify that rate limit is different for different impersonated users?
+ * Open two terminals
+ * Terminal 1: Run `node rate-limit-test-google-service-account-impersonation.js email1@cal.com 1000` - This makes email1 hit the limit. 1000 is much more than 600, so it gives more reliability in reaching the limit.
+ * Terminal 1: Run `node rate-limit-test-google-service-account-impersonation.js email1@cal.com 100` - Verify that email1 is still hitting the limit
+ * // Run this immediately after the above commands
+ * Terminal 2: Run `node rate-limit-test-google-service-account-impersonation.js email2@cal.com 100` - Check for email2's limit, it should not hit the limit
+ */
+const { calendar_v3 } = require("@googleapis/calendar");
+const { JWT } = require("googleapis-common");
+
+// client_email in the service account key json file
+const serviceAccountClientEmail = "";
+// private_key in the service account key json file
+const serviceAccountPrivateKey = ``;
+
+if (!serviceAccountClientEmail || !serviceAccountPrivateKey) {
+  throw new Error("serviceAccountClientEmail and serviceAccountPrivateKey must be set");
+}
+
+async function sendRequest(calendar) {
+  const response = await calendar.calendarList.list({
+    fields: `items(id),nextPageToken`,
+  });
+  console.log(response.data);
+}
+const args = process.argv.slice(2);
+const emailToImpersonate = args[0];
+const totalRequestsArg = args[1];
+(async function () {
+  console.log({ emailToImpersonate, totalRequestsArg });
+  const authClient = new JWT({
+    email: serviceAccountClientEmail,
+    key: serviceAccountPrivateKey,
+    scopes: ["https://www.googleapis.com/auth/calendar"],
+    subject: emailToImpersonate,
+  });
+
+  await authClient.authorize();
+  const calendar = new calendar_v3.Calendar({
+    auth: authClient,
+  });
+
+  // 600 requests per minute is the API limit per user
+  const totalRequests = parseInt(totalRequestsArg) || 600;
+  const promises = [];
+  for (let i = 0; i < totalRequests; i++) {
+    console.log(`Queuing request ${i + 1}/${totalRequests}`);
+    promises.push(sendRequest(calendar));
+  }
+  await Promise.all(promises);
+  console.log("All requests completed");
+})();

packages/lib/getCalendarsEvents.ts

@@ -52,7 +52,9 @@ export const getCalendarsEventsWithTimezones = async (
       if (!isADelegationCredential) {
         // It was done to fix the secondary calendar connections from always checking the conflicts even if intentional no calendars are selected.
         // https://github.com/calcom/cal.com/issues/8929
-        log.error("No selected calendars for non DWD credential: Skipping getAvailability call");
+        log.error(
+          `No selected calendars for non DWD credential: Skipping getAvailability call for credential ${credential?.id}`
+        );
         return [];
       }
       // For delegation credential, we should allow getAvailability even without any selected calendars. It ensures that enabling Delegation Credential at Organization level always ensure one selected calendar for conflicts checking, without requiring any manual action from organization members
@@ -118,7 +120,9 @@ const getCalendarsEvents = async (
       if (!isADelegationCredential) {
         // It was done to fix the secondary calendar connections from always checking the conflicts even if intentional no calendars are selected.
         // https://github.com/calcom/cal.com/issues/8929
-        log.error("No selected calendars for non DWD credential: Skipping getAvailability call");
+        log.error(
+          `No selected calendars for non DWD credential: Skipping getAvailability call for credential ${credential?.id}`
+        );
         return [];
       }
       // For delegation credential, we should allow getAvailability even without any selected calendars. It ensures that enabling Delegation Credential at Organization level always ensure one selected calendar for conflicts checking, without requiring any manual action from organization members

packages/lib/server/__mocks__/serviceAccountKey.ts

@@ -0,0 +1,21 @@
+import { vi } from "vitest";
+
+vi.mock("@calcom/lib/server/serviceAccountKey", async (importOriginal) => {
+  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+  // @ts-ignore
+  const actual = await importOriginal<any>();
+  return {
+    ...actual,
+    decryptServiceAccountKey: vi.fn((input) => {
+      // If input is a string, parse it; otherwise, return as is
+      if (typeof input === "string") {
+        try {
+          return JSON.parse(input);
+        } catch {
+          return input;
+        }
+      }
+      return input;
+    }),
+  };
+});

packages/lib/server/repository/selectedCalendar.ts

@@ -67,7 +67,9 @@ export class SelectedCalendarRepository {
     const conflictingCalendar = await SelectedCalendarRepository.findConflicting(data);
 
     if (conflictingCalendar) {
-      throw new Error("Selected calendar already exists");
+      throw new Error(
+        `Selected calendar already exists for userId: ${data.userId}, integration: ${data.integration}, externalId: ${data.externalId}, eventTypeId: ${data.eventTypeId}`
+      );
     }
 
     return await prisma.selectedCalendar.create({
@@ -125,6 +127,14 @@ export class SelectedCalendarRepository {
     });
   }
 
+  static async deleteById({ id }: { id: string }) {
+    return await prisma.selectedCalendar.delete({
+      where: {
+        id,
+      },
+    });
+  }
+
   static async createIfNotExists(data: Prisma.SelectedCalendarUncheckedCreateInput) {
     const conflictingCalendar = await SelectedCalendarRepository.findConflicting(data);