diff --git a/apps/api/v1/lib/validations/selected-calendar.ts b/apps/api/v1/lib/validations/selected-calendar.ts index e012efcb6a..1e1c2a5261 100644 --- a/apps/api/v1/lib/validations/selected-calendar.ts +++ b/apps/api/v1/lib/validations/selected-calendar.ts @@ -18,6 +18,11 @@ export const schemaSelectedCalendarBodyParams = schemaSelectedCalendarBaseBodyPa id: true, // No eventTypeId support in API v1 eventTypeId: true, + + /** No watch related fields support in API v1 */ + watchAttempts: true, + unwatchAttempts: true, + maxAttempts: true, }); export const schemaSelectedCalendarUpdateBodyParams = schemaSelectedCalendarBaseBodyParams diff --git a/apps/api/v1/test/lib/selected-calendars/_post.test.ts b/apps/api/v1/test/lib/selected-calendars/_post.test.ts index 424dee1fea..e872b116ce 100644 --- a/apps/api/v1/test/lib/selected-calendars/_post.test.ts +++ b/apps/api/v1/test/lib/selected-calendars/_post.test.ts @@ -89,6 +89,10 @@ describe("POST /api/selected-calendars", () => { googleChannelResourceUri: null, googleChannelExpiration: null, error: null, + lastErrorAt: null, + watchAttempts: 0, + maxAttempts: 3, + unwatchAttempts: 0, }); await handler(req, res); @@ -128,6 +132,10 @@ describe("POST /api/selected-calendars", () => { domainWideDelegationCredentialId: null, eventTypeId: null, error: null, + lastErrorAt: null, + watchAttempts: 0, + maxAttempts: 3, + unwatchAttempts: 0, }); await handler(req, res); diff --git a/packages/app-store/googlecalendar/lib/CalendarService.ts b/packages/app-store/googlecalendar/lib/CalendarService.ts index c66d67763a..3e4be621fe 100644 --- a/packages/app-store/googlecalendar/lib/CalendarService.ts +++ b/packages/app-store/googlecalendar/lib/CalendarService.ts @@ -974,15 +974,13 @@ export default class GoogleCalendarService implements Calendar { expiration: otherCalendarsWithSameSubscription[0].googleChannelExpiration, } : {}; - let error: string | null = null; if (!otherCalendarsWithSameSubscription.length) { try { googleChannelProps = await this.startWatchingCalendarsInGoogle({ calendarId }); - } catch (_error) { - this.log.error(`Failed to watch calendar ${calendarId}`, _error); - // We set error to prevent attempting to watch on next cron run - error = _error instanceof Error ? _error.message : "Unknown error"; + } catch (error) { + this.log.error(`Failed to watch calendar ${calendarId}`, safeStringify(error)); + throw error; } } else { logger.info( @@ -1000,7 +998,6 @@ export default class GoogleCalendarService implements Calendar { googleChannelResourceId: googleChannelProps.resourceId, googleChannelResourceUri: googleChannelProps.resourceUri, googleChannelExpiration: googleChannelProps.expiration, - error, }, eventTypeIds ); diff --git a/packages/features/calendar-cache/api/cron.ts b/packages/features/calendar-cache/api/cron.ts index d021cf8dc8..39abff8a12 100644 --- a/packages/features/calendar-cache/api/cron.ts +++ b/packages/features/calendar-cache/api/cron.ts @@ -68,7 +68,8 @@ const handleCalendarsToUnwatch = async () => { // So we don't retry on next cron run // FIXME: There could actually be multiple calendars with the same externalId and thus we need to technically update error for all of them - await SelectedCalendarRepository.updateById(id, { + await SelectedCalendarRepository.setErrorInUnwatching({ + id, error: "Missing credentialId", }); log.error("no credentialId for SelectedCalendar: ", id); @@ -78,26 +79,30 @@ const handleCalendarsToUnwatch = async () => { try { const cc = await CalendarCache.initFromCredentialId(credentialId); await cc.unwatchCalendar({ calendarId: externalId, eventTypeIds }); + await SelectedCalendarRepository.removeUnwatchingError({ id }); } catch (error) { let errorMessage = "Unknown error"; if (error instanceof Error) { errorMessage = error.message; } log.error( - "Error unwatching calendar: ", + `Error unwatching calendar ${externalId}`, safeStringify({ selectedCalendarId: id, error: errorMessage, }) ); - await SelectedCalendarRepository.updateById(id, { - error: `Error unwatching calendar: ${errorMessage}`, + await SelectedCalendarRepository.setErrorInUnwatching({ + id, + error: `${errorMessage}`, }); } } ) ); + log.info(`Processed ${result.length} calendars for unwatching`); + result.forEach(logRejected); return result; }; @@ -110,9 +115,7 @@ const handleCalendarsToWatch = async () => { async ([externalId, { credentialId, eventTypeIds, id }]) => { if (!credentialId) { // So we don't retry on next cron run - await SelectedCalendarRepository.updateById(id, { - error: "Missing credentialId", - }); + await SelectedCalendarRepository.setErrorInWatching({ id, error: "Missing credentialId" }); log.error("no credentialId for SelectedCalendar: ", id); return; } @@ -120,25 +123,28 @@ const handleCalendarsToWatch = async () => { try { const cc = await CalendarCache.initFromCredentialId(credentialId); await cc.watchCalendar({ calendarId: externalId, eventTypeIds }); + await SelectedCalendarRepository.removeWatchingError({ id }); } catch (error) { let errorMessage = "Unknown error"; if (error instanceof Error) { errorMessage = error.message; } log.error( - "Error watching calendar: ", + `Error watching calendar ${externalId}`, safeStringify({ selectedCalendarId: id, error: errorMessage, }) ); - await SelectedCalendarRepository.updateById(id, { - error: `Error watching calendar: ${errorMessage}`, + await SelectedCalendarRepository.setErrorInWatching({ + id, + error: `${errorMessage}`, }); } } ) ); + log.info(`Processed ${result.length} calendars for watching`); result.forEach(logRejected); return result; }; diff --git a/packages/lib/server/repository/selectedCalendar.ts b/packages/lib/server/repository/selectedCalendar.ts index 2f5fc04230..ee89bf90c4 100644 --- a/packages/lib/server/repository/selectedCalendar.ts +++ b/packages/lib/server/repository/selectedCalendar.ts @@ -167,13 +167,35 @@ export class SelectedCalendarRepository { }, // RN we only support google calendar subscriptions for now integration: "google_calendar", - // We skip retrying calendars that have errored - error: null, - OR: [ - // Either is a calendar pending to be watched - { googleChannelExpiration: null }, - // Or is a calendar that is about to expire - { googleChannelExpiration: { lt: tomorrowTimestamp } }, + AND: [ + { + OR: [ + // Either is a calendar that has not errored + { error: null }, + // Or is a calendar that has errored but has not reached max attempts + { + error: { not: null }, + watchAttempts: { + lt: { + // Using ts-ignore instead of ts-expect-error because I am seeing conflicting errors in CI. In one case ts-expect-error fails with `Unused '@ts-expect-error' directive.` + // Removing ts-expect-error fails in another case that _ref isn't defined + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + _ref: "maxAttempts", + _container: "SelectedCalendar", + }, + }, + }, + ], + }, + { + OR: [ + // Either is a calendar pending to be watched + { googleChannelExpiration: null }, + // Or is a calendar that is about to expire + { googleChannelExpiration: { lt: tomorrowTimestamp } }, + ], + }, ], }, }); @@ -188,19 +210,43 @@ export class SelectedCalendarRepository { // RN we only support google calendar subscriptions for now integration: "google_calendar", googleChannelExpiration: { not: null }, - user: { - teams: { - every: { - team: { - features: { - none: { - featureId: "calendar-cache", + AND: [ + { + OR: [ + // Either is a calendar that has not errored during unwatch + { error: null }, + // Or is a calendar that has errored during unwatch but has not reached max attempts + { + error: { not: null }, + unwatchAttempts: { + lt: { + // Using ts-ignore instead of ts-expect-error because I am seeing conflicting errors in CI. In one case ts-expect-error fails with `Unused '@ts-expect-error' directive.` + // Removing ts-expect-error fails in another case that _ref isn't defined + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + _ref: "maxAttempts", + _container: "SelectedCalendar", + }, + }, + }, + ], + }, + { + user: { + teams: { + every: { + team: { + features: { + none: { + featureId: "calendar-cache", + }, + }, }, }, }, }, }, - }, + ], }; // If calendar cache is disabled globally, we skip team features and unwatch all subscriptions const nextBatch = await prisma.selectedCalendar.findMany({ @@ -351,4 +397,36 @@ export class SelectedCalendarRepository { data, }); } + + static async setErrorInWatching({ id, error }: { id: string; error: string }) { + await SelectedCalendarRepository.updateById(id, { + error, + lastErrorAt: new Date(), + watchAttempts: { increment: 1 }, + }); + } + + static async setErrorInUnwatching({ id, error }: { id: string; error: string }) { + await SelectedCalendarRepository.updateById(id, { + error, + lastErrorAt: new Date(), + unwatchAttempts: { increment: 1 }, + }); + } + + static async removeWatchingError({ id }: { id: string }) { + await SelectedCalendarRepository.updateById(id, { + error: null, + lastErrorAt: null, + watchAttempts: 0, + }); + } + + static async removeUnwatchingError({ id }: { id: string }) { + await SelectedCalendarRepository.updateById(id, { + error: null, + lastErrorAt: null, + unwatchAttempts: 0, + }); + } } diff --git a/packages/prisma/migrations/20250520072457_add_more_error_tracking_fields_selected_calendar/migration.sql b/packages/prisma/migrations/20250520072457_add_more_error_tracking_fields_selected_calendar/migration.sql new file mode 100644 index 0000000000..1f2c52dadf --- /dev/null +++ b/packages/prisma/migrations/20250520072457_add_more_error_tracking_fields_selected_calendar/migration.sql @@ -0,0 +1,14 @@ +-- DropIndex +DROP INDEX "SelectedCalendar_integration_idx"; + +-- AlterTable +ALTER TABLE "SelectedCalendar" ADD COLUMN "lastErrorAt" TIMESTAMP(3), +ADD COLUMN "maxAttempts" INTEGER NOT NULL DEFAULT 3, +ADD COLUMN "unwatchAttempts" INTEGER NOT NULL DEFAULT 0, +ADD COLUMN "watchAttempts" INTEGER NOT NULL DEFAULT 0; + +-- CreateIndex +CREATE INDEX "SelectedCalendar_watch_idx" ON "SelectedCalendar"("integration", "googleChannelExpiration", "error", "watchAttempts", "maxAttempts"); + +-- CreateIndex +CREATE INDEX "SelectedCalendar_unwatch_idx" ON "SelectedCalendar"("integration", "googleChannelExpiration", "error", "unwatchAttempts", "maxAttempts"); diff --git a/packages/prisma/schema.prisma b/packages/prisma/schema.prisma index e497b0d9e1..e017f97a98 100644 --- a/packages/prisma/schema.prisma +++ b/packages/prisma/schema.prisma @@ -831,6 +831,10 @@ model SelectedCalendar { domainWideDelegationCredential DomainWideDelegation? @relation(fields: [domainWideDelegationCredentialId], references: [id], onDelete: Cascade) domainWideDelegationCredentialId String? error String? + lastErrorAt DateTime? + watchAttempts Int @default(0) + unwatchAttempts Int @default(0) + maxAttempts Int @default(3) eventTypeId Int? eventType EventType? @relation(fields: [eventTypeId], references: [id]) @@ -841,10 +845,12 @@ model SelectedCalendar { @@unique([userId, integration, externalId, eventTypeId]) @@unique([googleChannelId, eventTypeId]) @@index([userId]) - @@index([integration]) @@index([externalId]) @@index([eventTypeId]) @@index([credentialId]) + // Composite indices to optimize calendar-cache queries + @@index([integration, googleChannelExpiration, error, watchAttempts, maxAttempts], name: "SelectedCalendar_watch_idx") + @@index([integration, googleChannelExpiration, error, unwatchAttempts, maxAttempts], name: "SelectedCalendar_unwatch_idx") } enum EventTypeCustomInputType {