Clean up and enhance logging for messaging and calendar (#17498)

This PR reduces noise to signal ratio for messaging and calendar logging
in production
Impact would be faster queries and debugging

packages/twenty-server/src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service.ts

@@ -88,7 +88,7 @@ export class MessagingMessageCleanerService {
           continue;
         }
 
-        this.logger.log(
+        this.logger.debug(
           `WorkspaceId: ${workspaceId} Deleting ${orphanMessages.length} orphan messages`,
         );
 
@@ -109,7 +109,7 @@ export class MessagingMessageCleanerService {
           continue;
         }
 
-        this.logger.log(
+        this.logger.debug(
           `WorkspaceId: ${workspaceId} Deleting ${orphanMessageThreads.length} orphan message threads`,
         );
 
@@ -171,7 +171,7 @@ export class MessagingMessageCleanerService {
               workspaceId: string,
               transactionManager?: WorkspaceEntityManager,
             ) => {
-              this.logger.log(
+              this.logger.debug(
                 `WorkspaceId: ${workspaceId} Deleting ${ids.length} messages from message cleaner`,
               );
               await messageRepository.delete(ids, transactionManager);

packages/twenty-server/src/modules/messaging/message-folder-manager/drivers/gmail/services/gmail-get-all-folders.service.ts

@@ -100,7 +100,7 @@ export class GmailGetAllFoldersService implements MessageFolderDriver {
         });
       }
 
-      this.logger.log(
+      this.logger.debug(
         `Found ${folders.length} folders for Gmail account ${connectedAccount.handle}`,
       );
 

packages/twenty-server/src/modules/messaging/message-folder-manager/drivers/microsoft/services/microsoft-get-all-folders.service.ts

@@ -99,7 +99,7 @@ export class MicrosoftGetAllFoldersService implements MessageFolderDriver {
         });
       }
 
-      this.logger.log(
+      this.logger.debug(
         `Found ${folderInfos.length} folders for Microsoft account ${connectedAccount.handle}`,
       );
 

packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-messages.service.ts

@@ -107,7 +107,7 @@ export class ImapGetMessagesService {
     client: ImapFlow,
     connectedAccount: ConnectedAccount,
   ): Promise<MessageWithParticipants[]> {
-    this.logger.log(
+    this.logger.debug(
       `Fetching ${messageUids.length} messages from ${folderPath}`,
     );
     const startTime = Date.now();
@@ -146,7 +146,7 @@ export class ImapGetMessagesService {
       );
     }
 
-    this.logger.log(
+    this.logger.debug(
       `Parsed ${messages.length}/${results.length} messages from ${folderPath} in ${Date.now() - startTime}ms`,
     );
 

packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-sync.service.ts

@@ -66,7 +66,7 @@ export class ImapSyncService {
     const { maxUid } = mailboxState;
 
     if (canUseQresync(client, previousCursor, mailboxState)) {
-      this.logger.log(`Using QRESYNC for folder ${folderPath}`);
+      this.logger.debug(`Using QRESYNC for folder ${folderPath}`);
 
       try {
         return await this.fetchWithQresync(
@@ -81,7 +81,7 @@ export class ImapSyncService {
       }
     }
 
-    this.logger.log(`Using UID range fetch for folder ${folderPath}`);
+    this.logger.debug(`Using UID range fetch for folder ${folderPath}`);
 
     return this.fetchWithUidRange(client, lastSyncedUid, maxUid);
   }
@@ -122,7 +122,7 @@ export class ImapSyncService {
       return [];
     }
 
-    this.logger.log(`QRESYNC found ${uids.length} new/modified messages`);
+    this.logger.debug(`QRESYNC found ${uids.length} new/modified messages`);
 
     return uids;
   }

packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-message-list-fetch-error-handler.service.ts

@@ -15,7 +15,7 @@ export class MicrosoftMessageListFetchErrorHandler {
 
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   public handleError(error: any): void {
-    this.logger.log(`Error fetching message list: ${JSON.stringify(error)}`);
+    this.logger.error(`Error fetching message list: ${JSON.stringify(error)}`);
 
     const networkError = this.microsoftNetworkErrorHandler.handleError(error);
 

packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-messages-import-error-handler.service.ts

@@ -15,7 +15,7 @@ export class MicrosoftMessagesImportErrorHandler {
 
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   public handleError(error: any): void {
-    this.logger.log(`Error fetching messages: ${JSON.stringify(error)}`);
+    this.logger.error(`Error fetching messages: ${JSON.stringify(error)}`);
 
     const networkError = this.microsoftNetworkErrorHandler.handleError(error);
 

packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-delete-folder-messages.service.ts

@@ -62,7 +62,7 @@ export class MessagingDeleteFolderMessagesService {
 
         totalDeletedCount += validExternalIds.length;
 
-        this.logger.log(
+        this.logger.debug(
           `WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id}, FolderId: ${messageFolder.id} - Processed ${validExternalIds.length} message deletions`,
         );
       }

packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-delete-group-email-messages.service.ts

@@ -55,7 +55,7 @@ export class MessagingDeleteGroupEmailMessagesService {
           });
 
         if (!firstRecord) {
-          this.logger.log(
+          this.logger.debug(
             `WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannelId} - No message associations found`,
           );
 
@@ -125,7 +125,7 @@ export class MessagingDeleteGroupEmailMessagesService {
 
                 totalDeletedCount += messageExternalIdsChunk.length;
 
-                this.logger.log(
+                this.logger.debug(
                   `WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannelId} - Deleted ${messageExternalIdsChunk.length} group email messages`,
                 );
               }

packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-import-exception-handler.service.ts

@@ -134,12 +134,14 @@ export class MessageImportExceptionHandlerService {
       this.exceptionHandlerService.captureExceptions(
         [
           new Error(
-            `Temporary error occurred ${MESSAGING_THROTTLE_MAX_ATTEMPTS} times while importing messages for message channel ${messageChannel.id.slice(0, 5)}... in workspace ${workspaceId}: ${exception?.message}`,
+            `Temporary error occurred ${MESSAGING_THROTTLE_MAX_ATTEMPTS} times while importing messages for message channel ${messageChannel.id} in workspace ${workspaceId}: ${exception?.message}`,
           ),
         ],
         {
           additionalData: {
             messageChannelId: messageChannel.id,
+            syncStep,
+            throttleFailureCount: messageChannel.throttleFailureCount,
           },
           workspace: { id: workspaceId },
         },
@@ -247,6 +249,7 @@ export class MessageImportExceptionHandlerService {
         {
           additionalData: {
             messageChannelId: messageChannel.id,
+            syncStep,
           },
           workspace: { id: workspaceId },
         },

packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-list-fetch.service.ts

@@ -75,7 +75,7 @@ export class MessagingMessageListFetchService {
         );
 
         this.logger.log(
-          `messageChannelId: ${messageChannel.id} Processing message list fetch`,
+          `WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Processing message list fetch`,
         );
 
         const messageChannelRepository =
@@ -96,7 +96,7 @@ export class MessagingMessageListFetchService {
 
         if (!isDefined(freshMessageChannel)) {
           this.logger.error(
-            `error processing message list fetch: messageChannelId: ${messageChannel.id} Message channel not found`,
+            `WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Message channel not found`,
           );
 
           return;
@@ -157,11 +157,7 @@ export class MessagingMessageListFetchService {
         let totalMessagesToImportCount = 0;
 
         this.logger.log(
-          `messageChannelId: ${freshMessageChannel.id} Is full sync: ${isFullSync} and toImportCount: ${messageExternalIds.length}, toDeleteCount: ${messageExternalIdsToDelete.length}, cursors: ${messageLists.map(
-            (messageList) => {
-              messageList.nextSyncCursor;
-            },
-          )}`,
+          `WorkspaceId: ${workspaceId}, MessageChannelId: ${freshMessageChannel.id} - Is full sync: ${isFullSync}, toImportCount: ${messageExternalIds.length}, toDeleteCount: ${messageExternalIdsToDelete.length}`,
         );
 
         const messageChannelMessageAssociationRepository =
@@ -198,7 +194,7 @@ export class MessagingMessageListFetchService {
           );
 
           if (messageExternalIdsToImport.length) {
-            this.logger.log(
+            this.logger.debug(
               `messageChannelId: ${freshMessageChannel.id} Adding ${messageExternalIdsToImport.length} message external ids to import in batch ${index + 1}`,
             );
 
@@ -241,13 +237,13 @@ export class MessagingMessageListFetchService {
 
         if (allMessageExternalIdsToDelete.length) {
           this.logger.log(
-            `messageChannelId: ${freshMessageChannel.id} Deleting ${allMessageExternalIdsToDelete.length} message channel message associations`,
+            `WorkspaceId: ${workspaceId}, MessageChannelId: ${freshMessageChannel.id} - Deleting ${allMessageExternalIdsToDelete.length} message channel message associations`,
           );
 
           const toDeleteChunks = chunk(allMessageExternalIdsToDelete, 200);
 
           for (const [index, toDeleteChunk] of toDeleteChunks.entries()) {
-            this.logger.log(
+            this.logger.debug(
               `messageChannelId: ${freshMessageChannel.id} Deleting ${toDeleteChunk.length} message channel message associations in batch ${index + 1}`,
             );
 
@@ -264,7 +260,7 @@ export class MessagingMessageListFetchService {
         }
 
         this.logger.log(
-          `messageChannelId: ${freshMessageChannel.id} Total messages to import count: ${totalMessagesToImportCount}`,
+          `WorkspaceId: ${workspaceId}, MessageChannelId: ${freshMessageChannel.id} - Total messages to import count: ${totalMessagesToImportCount}`,
         );
 
         if (totalMessagesToImportCount === 0) {
@@ -276,7 +272,7 @@ export class MessagingMessageListFetchService {
           return;
         }
 
-        this.logger.log(
+        this.logger.debug(
           `messageChannelId: ${freshMessageChannel.id} Scheduling direct messages import`,
         );
 

packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-messages-import.service.ts

@@ -198,7 +198,7 @@ export class MessagingMessagesImportService {
         );
       } catch (error) {
         this.logger.error(
-          `Error (${error.code}) importing messages for workspace ${workspaceId.slice(0, 8)} and account ${connectedAccount.id.slice(0, 8)}: ${error.message} - ${error.body}`,
+          `WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Error (${error.code}) importing messages: ${error.message}`,
         );
         await this.cacheStorage.setAdd(
           `messages-to-import:${workspaceId}:${messageChannel.id}`,

packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-process-folder-actions.service.ts

@@ -49,7 +49,7 @@ export class MessagingProcessFolderActionsService {
 
     for (const folder of foldersWithPendingActions) {
       try {
-        this.logger.log(
+        this.logger.debug(
           `WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id}, FolderId: ${folder.id} - Processing folder action: ${folder.pendingSyncAction}`,
         );
 
@@ -65,7 +65,7 @@ export class MessagingProcessFolderActionsService {
 
           folderIdsToDelete.push(folder.id);
 
-          this.logger.log(
+          this.logger.debug(
             `WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id}, FolderId: ${folder.id} - Completed FOLDER_DELETION action`,
           );
         }
@@ -109,7 +109,7 @@ export class MessagingProcessFolderActionsService {
                   transactionManager,
                 );
 
-                this.logger.log(
+                this.logger.debug(
                   `WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Reset pendingSyncAction to NONE for ${processedFolderIds.length} folders`,
                 );
               }

packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-process-group-email-actions.service.ts

@@ -42,7 +42,7 @@ export class MessagingProcessGroupEmailActionsService {
         { pendingGroupEmailsAction },
       );
 
-      this.logger.log(
+      this.logger.debug(
         `WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Marked message channel as pending group emails action: ${pendingGroupEmailsAction}`,
       );
     }, authContext);
@@ -61,7 +61,7 @@ export class MessagingProcessGroupEmailActionsService {
       return;
     }
 
-    this.logger.log(
+    this.logger.debug(
       `WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Processing group email action: ${pendingGroupEmailsAction}`,
     );
 
@@ -106,7 +106,7 @@ export class MessagingProcessGroupEmailActionsService {
               transactionManager,
             );
 
-            this.logger.log(
+            this.logger.debug(
               `WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Reset pendingGroupEmailsAction to NONE`,
             );
           } catch (error) {
@@ -137,7 +137,7 @@ export class MessagingProcessGroupEmailActionsService {
       transactionManager,
     });
 
-    this.logger.log(
+    this.logger.debug(
       `WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannelId} - Completed GROUP_EMAILS_DELETION action`,
     );
   }
@@ -153,7 +153,7 @@ export class MessagingProcessGroupEmailActionsService {
       transactionManager,
     });
 
-    this.logger.log(
+    this.logger.debug(
       `WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannelId} - Completed GROUP_EMAILS_IMPORT action`,
     );
   }