Messaging cleaning fixes (#14345)

In this PR, I'm solving several issues:
1) We were not checking if the currentWorkspaceMember was owning the
message in the thread. It kind of worked before because the case of
shared threads (with shared threads visibility restriction) was not
happening that often. It seems that the bug has always been there
2) Re-implement orphan messages and threads deletion on messageChannel
deletion. We used to brutally look for all orphans, we disabled it last
week because it was too heavy on db. I've re-implemented it more
carefully and "surgically"
3) Gmail sync was not handling folder synced correctly. It was
leveraging labelIds which it shouldn't do (this is a AND AND parameter)
in full sync
4) Added a command to clean orphan message threads manually if needed.
Usually this is done when you remove a messageChannel, or change
blocklist rules but it can be useful to have it to debug

packages/twenty-server/src/command/command.module.ts

@@ -8,6 +8,7 @@ import { WorkspaceCleanerModule } from 'src/engine/workspace-manager/workspace-c
 import { WorkspaceHealthCommandModule } from 'src/engine/workspace-manager/workspace-health/commands/workspace-health-command.module';
 import { WorkspaceMigrationRunnerCommandsModule } from 'src/engine/workspace-manager/workspace-migration-runner/commands/workspace-migration-runner-commands.module';
 import { WorkspaceSyncMetadataCommandsModule } from 'src/engine/workspace-manager/workspace-sync-metadata/commands/workspace-sync-metadata-commands.module';
+import { MessagingMessageCleanerModule } from 'src/modules/messaging/message-cleaner/messaging-message-cleaner.module';
 
 @Module({
   imports: [
@@ -16,6 +17,7 @@ import { WorkspaceSyncMetadataCommandsModule } from 'src/engine/workspace-manage
     DatabaseCommandModule,
     WorkspaceCleanerModule,
     WorkspaceHealthCommandModule,
+    MessagingMessageCleanerModule,
     WorkspaceMigrationRunnerCommandsModule,
     ObjectMetadataModule,
     FieldMetadataModule,

packages/twenty-server/src/engine/core-modules/messaging/services/timeline-messaging.service.ts

@@ -1,6 +1,6 @@
 import { Injectable } from '@nestjs/common';
 
-import { In, Not } from 'typeorm';
+import { In } from 'typeorm';
 
 import { type TimelineThread } from 'src/engine/core-modules/messaging/dtos/timeline-thread.dto';
 import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
@@ -94,6 +94,7 @@ export class TimelineMessagingService {
       await this.twentyORMManager.getRepository<MessageParticipantWorkspaceEntity>(
         'messageParticipant',
       );
+
     const threadParticipants = await messageParticipantRepository
       .createQueryBuilder()
       .select('messageParticipant')
@@ -183,32 +184,11 @@ export class TimelineMessagingService {
         'messageThread',
       );
 
-    const threadsWithoutWorkspaceMember = await messageThreadRepository.find({
-      select: {
-        id: true,
-      },
-      where: {
-        id: In(messageThreadIds),
-        messages: {
-          messageChannelMessageAssociations: {
-            messageChannel: {
-              connectedAccount: {
-                accountOwnerId: Not(workspaceMemberId),
-              },
-            },
-          },
-        },
-      },
-    });
-
-    const threadIdsWithoutWorkspaceMember = threadsWithoutWorkspaceMember.map(
-      (thread) => thread.id,
-    );
-
     const threadVisibility = await messageThreadRepository
       .createQueryBuilder()
       .select('messageThread.id', 'id')
       .addSelect('messageChannel.visibility', 'visibility')
+      .addSelect('connectedAccount.accountOwnerId', 'accountOwnerId')
       .leftJoin('messageThread.messages', 'message')
       .leftJoin(
         'message.messageChannelMessageAssociations',
@@ -218,46 +198,34 @@ export class TimelineMessagingService {
         'messageChannelMessageAssociation.messageChannel',
         'messageChannel',
       )
+      .leftJoin('messageChannel.connectedAccount', 'connectedAccount')
       .where('messageThread.id = ANY(:messageThreadIds)', {
-        messageThreadIds: threadIdsWithoutWorkspaceMember,
+        messageThreadIds: messageThreadIds,
       })
       .getRawMany();
 
     const visibilityValues = Object.values(MessageChannelVisibility);
 
-    const threadVisibilityByThreadIdForWhichWorkspaceMemberIsNotOwner:
-      | {
-          [key: string]: MessageChannelVisibility;
-        }
-      | undefined = threadVisibility?.reduce(
-      (threadVisibilityAcc, threadVisibility) => {
-        threadVisibilityAcc[threadVisibility.id] =
-          visibilityValues[
-            Math.max(
-              visibilityValues.indexOf(threadVisibility.visibility),
-              visibilityValues.indexOf(
-                threadVisibilityAcc[threadVisibility.id] ??
-                  MessageChannelVisibility.METADATA,
-              ),
-            )
-          ];
-
-        return threadVisibilityAcc;
-      },
-      {},
-    );
-
     const threadVisibilityByThreadId: {
       [key: string]: MessageChannelVisibility;
-    } = messageThreadIds.reduce((threadVisibilityAcc, messageThreadId) => {
-      // If the workspace member is not the owner of the thread, use the visibility value from the query
-      // @ts-expect-error legacy noImplicitAny
-      threadVisibilityAcc[messageThreadId] =
-        threadIdsWithoutWorkspaceMember.includes(messageThreadId)
-          ? (threadVisibilityByThreadIdForWhichWorkspaceMemberIsNotOwner?.[
-              messageThreadId
-            ] ?? MessageChannelVisibility.METADATA)
-          : MessageChannelVisibility.SHARE_EVERYTHING;
+    } = threadVisibility.reduce((threadVisibilityAcc, threadVisibility) => {
+      if (threadVisibility.accountOwnerId === workspaceMemberId) {
+        threadVisibilityAcc[threadVisibility.id] =
+          MessageChannelVisibility.SHARE_EVERYTHING;
+
+        return threadVisibilityAcc;
+      }
+
+      threadVisibilityAcc[threadVisibility.id] =
+        visibilityValues[
+          Math.max(
+            visibilityValues.indexOf(threadVisibility.visibility),
+            visibilityValues.indexOf(
+              threadVisibilityAcc[threadVisibility.id] ??
+                MessageChannelVisibility.METADATA,
+            ),
+          )
+        ];
 
       return threadVisibilityAcc;
     }, {});

packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job.ts

@@ -140,6 +140,6 @@ export class BlocklistItemDeleteMessagesJob {
       }
     }
 
-    await this.threadCleanerService.cleanWorkspaceThreads(workspaceId);
+    await this.threadCleanerService.cleanOrphanMessagesAndThreads(workspaceId);
   }
 }

packages/twenty-server/src/modules/messaging/message-cleaner/commands/messaging-message-clearner-remove-orphans.command.ts

@@ -0,0 +1,39 @@
+import { InjectRepository } from '@nestjs/typeorm';
+
+import { Command } from 'nest-commander';
+import { Repository } from 'typeorm';
+
+import {
+  ActiveOrSuspendedWorkspacesMigrationCommandRunner,
+  type RunOnWorkspaceArgs,
+} from 'src/database/commands/command-runners/active-or-suspended-workspaces-migration.command-runner';
+import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
+import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
+import { MessagingMessageCleanerService } from 'src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service';
+
+@Command({
+  name: 'messaging:message-cleaner-remove-orphans',
+  description: 'Remove orphan message and threads from messaging',
+})
+export class MessagingMessageCleanerRemoveOrphansCommand extends ActiveOrSuspendedWorkspacesMigrationCommandRunner {
+  constructor(
+    @InjectRepository(Workspace)
+    protected readonly workspaceRepository: Repository<Workspace>,
+    protected readonly twentyORMGlobalManager: TwentyORMGlobalManager,
+    private readonly messagingMessageCleanerService: MessagingMessageCleanerService,
+  ) {
+    super(workspaceRepository, twentyORMGlobalManager);
+  }
+
+  override async runOnWorkspace({
+    workspaceId,
+  }: RunOnWorkspaceArgs): Promise<void> {
+    try {
+      await this.messagingMessageCleanerService.cleanOrphanMessagesAndThreads(
+        workspaceId,
+      );
+    } catch (error) {
+      this.logger.error('Error while deleting workflowRun', error);
+    }
+  }
+}

packages/twenty-server/src/modules/messaging/message-cleaner/jobs/messaging-connected-account-deletion-cleanup.job.ts

@@ -23,6 +23,8 @@ export class MessagingConnectedAccountDeletionCleanupJob {
   async handle(
     data: MessagingConnectedAccountDeletionCleanupJobData,
   ): Promise<void> {
-    await this.messageCleanerService.cleanWorkspaceThreads(data.workspaceId);
+    await this.messageCleanerService.cleanOrphanMessagesAndThreads(
+      data.workspaceId,
+    );
   }
 }

packages/twenty-server/src/modules/messaging/message-cleaner/messaging-message-cleaner.module.ts

@@ -1,15 +1,19 @@
 import { Module } from '@nestjs/common';
+import { TypeOrmModule } from '@nestjs/typeorm';
 
+import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity';
+import { MessagingMessageCleanerRemoveOrphansCommand } from 'src/modules/messaging/message-cleaner/commands/messaging-message-clearner-remove-orphans.command';
 import { MessagingConnectedAccountDeletionCleanupJob } from 'src/modules/messaging/message-cleaner/jobs/messaging-connected-account-deletion-cleanup.job';
 import { MessagingMessageCleanerConnectedAccountListener } from 'src/modules/messaging/message-cleaner/listeners/messaging-message-cleaner-connected-account.listener';
 import { MessagingMessageCleanerService } from 'src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service';
 
 @Module({
-  imports: [],
+  imports: [TypeOrmModule.forFeature([Workspace])],
   providers: [
     MessagingMessageCleanerService,
     MessagingConnectedAccountDeletionCleanupJob,
     MessagingMessageCleanerConnectedAccountListener,
+    MessagingMessageCleanerRemoveOrphansCommand,
   ],
   exports: [MessagingMessageCleanerService],
 })

packages/twenty-server/src/modules/messaging/message-cleaner/services/messaging-message-cleaner.service.ts

@@ -1,9 +1,11 @@
 import { Injectable, Logger } from '@nestjs/common';
 
-import { IsNull } from 'typeorm';
+import chunk from 'lodash.chunk';
+import { In, IsNull } from 'typeorm';
 
 import { type WorkspaceEntityManager } from 'src/engine/twenty-orm/entity-manager/workspace-entity-manager';
-import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager';
+import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager';
+import { MessageChannelMessageAssociationWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel-message-association.workspace-entity';
 import { type MessageThreadWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-thread.workspace-entity';
 import { type MessageWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message.workspace-entity';
 import { deleteUsingPagination } from 'src/modules/messaging/message-cleaner/utils/delete-using-pagination.util';
@@ -11,20 +13,123 @@ import { deleteUsingPagination } from 'src/modules/messaging/message-cleaner/uti
 @Injectable()
 export class MessagingMessageCleanerService {
   private readonly logger = new Logger(MessagingMessageCleanerService.name);
-  constructor(private readonly twentyORMManager: TwentyORMManager) {}
+  constructor(
+    private readonly twentyORMGlobalManager: TwentyORMGlobalManager,
+  ) {}
+
+  async deleteMessagesChannelMessageAssociationsAndRelatedOrphans({
+    workspaceId,
+    messageExternalIds,
+    messageChannelId,
+  }: {
+    workspaceId: string;
+    messageExternalIds: string[];
+    messageChannelId: string;
+  }) {
+    const messageRepository =
+      await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageWorkspaceEntity>(
+        workspaceId,
+        'message',
+      );
+
+    const messageChannelMessageAssociationRepository =
+      await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageChannelMessageAssociationWorkspaceEntity>(
+        workspaceId,
+        'messageChannelMessageAssociation',
+      );
 
-  public async cleanWorkspaceThreads(workspaceId: string) {
     const messageThreadRepository =
-      await this.twentyORMManager.getRepository<MessageThreadWorkspaceEntity>(
+      await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageThreadWorkspaceEntity>(
+        workspaceId,
+        'messageThread',
+      );
+
+    const messageExternalIdsChunks = chunk(messageExternalIds, 500);
+
+    for (const messageExternalIdsChunk of messageExternalIdsChunks) {
+      const messageChannelMessageAssociationsToDelete =
+        await messageChannelMessageAssociationRepository.find({
+          where: {
+            messageExternalId: In(messageExternalIdsChunk),
+            messageChannelId,
+          },
+        });
+
+      if (messageChannelMessageAssociationsToDelete.length <= 0) {
+        continue;
+      }
+
+      await messageChannelMessageAssociationRepository.delete(
+        messageChannelMessageAssociationsToDelete.map(({ id }) => id),
+      );
+
+      this.logger.log(
+        `WorkspaceId: ${workspaceId} Deleting ${messageChannelMessageAssociationsToDelete.length} message channel message associations`,
+      );
+
+      const orphanMessages = await messageRepository.find({
+        where: {
+          id: In(
+            messageChannelMessageAssociationsToDelete.map(
+              ({ messageId }) => messageId,
+            ),
+          ),
+          messageChannelMessageAssociations: {
+            id: IsNull(),
+          },
+        },
+      });
+
+      if (orphanMessages.length <= 0) {
+        continue;
+      }
+
+      this.logger.log(
+        `WorkspaceId: ${workspaceId} Deleting ${orphanMessages.length} orphan messages`,
+      );
+
+      await messageRepository.delete(orphanMessages.map(({ id }) => id));
+
+      const orphanMessageThreads = await messageThreadRepository.find({
+        where: {
+          id: In(orphanMessages.map(({ messageThreadId }) => messageThreadId)),
+          messages: {
+            id: IsNull(),
+          },
+        },
+      });
+
+      if (orphanMessageThreads.length <= 0) {
+        continue;
+      }
+
+      this.logger.log(
+        `WorkspaceId: ${workspaceId} Deleting ${orphanMessageThreads.length} orphan message threads`,
+      );
+
+      await messageThreadRepository.delete(
+        orphanMessageThreads.map(({ id }) => id),
+      );
+    }
+  }
+
+  public async cleanOrphanMessagesAndThreads(workspaceId: string) {
+    const messageThreadRepository =
+      await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageThreadWorkspaceEntity>(
+        workspaceId,
         'messageThread',
       );
 
     const messageRepository =
-      await this.twentyORMManager.getRepository<MessageWorkspaceEntity>(
+      await this.twentyORMGlobalManager.getRepositoryForWorkspace<MessageWorkspaceEntity>(
+        workspaceId,
         'message',
       );
 
-    const workspaceDataSource = await this.twentyORMManager.getDatasource();
+    const workspaceDataSource =
+      await this.twentyORMGlobalManager.getDataSourceForWorkspace({
+        workspaceId,
+      });
 
     await workspaceDataSource.transaction(
       async (transactionManager: WorkspaceEntityManager) => {

packages/twenty-server/src/modules/messaging/message-folder-manager/drivers/gmail/gmail-get-all-folders.service.ts

@@ -1,18 +1,14 @@
 import { Injectable, Logger } from '@nestjs/common';
 
-import { gmail_v1 } from 'googleapis';
-import { isDefined } from 'twenty-shared/utils';
-
 import {
   MessageFolder,
   MessageFolderDriver,
 } from 'src/modules/messaging/message-folder-manager/interfaces/message-folder-driver.interface';
 
 import { type ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity';
-import { MESSAGING_GMAIL_EXCLUDED_CATEGORIES } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-excluded-categories';
+import { MESSAGING_GMAIL_DEFAULT_NOT_SYNCED_LABELS } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-default-not-synced-labels';
 import { GmailClientProvider } from 'src/modules/messaging/message-import-manager/drivers/gmail/providers/gmail-client.provider';
 import { GmailHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-handle-error.service';
-import { computeGmailCategoryLabelId } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-label-id.util';
 
 @Injectable()
 export class GmailGetAllFoldersService implements MessageFolderDriver {
@@ -23,24 +19,8 @@ export class GmailGetAllFoldersService implements MessageFolderDriver {
     private readonly gmailHandleErrorService: GmailHandleErrorService,
   ) {}
 
-  private isExcludedCategoryFolder(labelId: string): boolean {
-    const excludedCategoryIds = MESSAGING_GMAIL_EXCLUDED_CATEGORIES.map(
-      (category) => computeGmailCategoryLabelId(category),
-    );
-
-    return excludedCategoryIds.includes(labelId);
-  }
-
-  private isIncludedFolder(label: gmail_v1.Schema$Label): boolean {
-    if (!isDefined(label.id)) {
-      return false;
-    }
-
-    const isTargetSystemFolder =
-      label.type === 'system' && (label.id === 'INBOX' || label.id === 'SENT');
-    const isUserFolder = label.type === 'user';
-
-    return isTargetSystemFolder || isUserFolder;
+  private isSyncedByDefault(labelId: string): boolean {
+    return !MESSAGING_GMAIL_DEFAULT_NOT_SYNCED_LABELS.includes(labelId);
   }
 
   async getAllMessageFolders(
@@ -74,21 +54,12 @@ export class GmailGetAllFoldersService implements MessageFolderDriver {
           continue;
         }
 
-        if (this.isExcludedCategoryFolder(label.id)) {
-          continue;
-        }
-
-        if (!this.isIncludedFolder(label)) {
-          continue;
-        }
-
         const isSentFolder = label.id === 'SENT';
-        const isSyncedByDefault = label.id === 'INBOX' || label.id === 'SENT';
 
         folders.push({
           externalId: label.id,
           name: label.name,
-          isSynced: isSyncedByDefault,
+          isSynced: this.isSyncedByDefault(label.id),
           isSentFolder,
         });
       }

packages/twenty-server/src/modules/messaging/message-folder-manager/services/sync-message-folders.service.ts

@@ -69,6 +69,7 @@ export class SyncMessageFoldersService {
       messageFolderRepository,
     });
 
+    // TODO: we should delete folders that are not in the list anymore
     for (const folder of folders) {
       const existingFolder = this.findExistingFolderInMap(
         existingFolderMap,
@@ -80,8 +81,6 @@ export class SyncMessageFoldersService {
           existingFolder.id,
           {
             name: folder.name,
-            isSynced: folder.isSynced,
-            isSentFolder: folder.isSentFolder,
             externalId: folder.externalId,
           },
           manager,

packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-default-not-synced-labels.ts

@@ -0,0 +1,10 @@
+export const MESSAGING_GMAIL_DEFAULT_NOT_SYNCED_LABELS = [
+  'CATEGORY_PROMOTIONS',
+  'CATEGORY_SOCIAL',
+  'CATEGORY_FORUMS',
+  'CATEGORY_UPDATES',
+  'TRASH',
+  'SPAM',
+  'DRAFT',
+  'CHAT',
+];

packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-excluded-categories.ts

@@ -1,8 +0,0 @@
-import { GmailDefaultMessageCategory } from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-default-message-category.type';
-
-export const MESSAGING_GMAIL_EXCLUDED_CATEGORIES = [
-  GmailDefaultMessageCategory.promotions,
-  GmailDefaultMessageCategory.forums,
-  GmailDefaultMessageCategory.social,
-  GmailDefaultMessageCategory.updates,
-];

packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-message-list.service.ts

@@ -10,14 +10,11 @@ import {
   MessageImportDriverException,
   MessageImportDriverExceptionCode,
 } from 'src/modules/messaging/message-import-manager/drivers/exceptions/message-import-driver.exception';
-import { MESSAGING_GMAIL_EXCLUDED_CATEGORIES } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-excluded-categories';
 import { MESSAGING_GMAIL_USERS_MESSAGES_LIST_MAX_RESULT } from 'src/modules/messaging/message-import-manager/drivers/gmail/constants/messaging-gmail-users-messages-list-max-result.constant';
 import { GmailClientProvider } from 'src/modules/messaging/message-import-manager/drivers/gmail/providers/gmail-client.provider';
 import { GmailGetHistoryService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-get-history.service';
 import { GmailHandleErrorService } from 'src/modules/messaging/message-import-manager/drivers/gmail/services/gmail-handle-error.service';
-import { computeGmailCategoryExcludeSearchFilter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-excude-search-filter.util';
-import { computeGmailCategoryLabelId } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-label-id.util';
-import { mapGmailDefaultFolderToCategoryOrUndefined } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/map-gmail-default-folder-to-category';
+import { computeGmailExcludeSearchFilter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-exclude-search-filter.util';
 import { type GetMessageListsArgs } from 'src/modules/messaging/message-import-manager/types/get-message-lists-args.type';
 import { type GetMessageListsResponse } from 'src/modules/messaging/message-import-manager/types/get-message-lists-response.type';
 import { assertNotNull } from 'src/utils/assert';
@@ -48,7 +45,9 @@ export class GmailGetMessageListService {
     let hasMoreMessages = true;
 
     const messageExternalIds: string[] = [];
-    const excludedCategories = this.computeExcludedCategories(messageFolders);
+
+    const excludedSearchFilter =
+      computeGmailExcludeSearchFilter(messageFolders);
 
     while (hasMoreMessages) {
       const messageList = await gmailClient.users.messages
@@ -56,8 +55,7 @@ export class GmailGetMessageListService {
           userId: 'me',
           maxResults: MESSAGING_GMAIL_USERS_MESSAGES_LIST_MAX_RESULT,
           pageToken,
-          q: computeGmailCategoryExcludeSearchFilter(excludedCategories),
-          labelIds: this.getCustomLabelIds(messageFolders),
+          q: excludedSearchFilter,
         })
         .catch((error) => {
           this.logger.error(
@@ -157,7 +155,7 @@ export class GmailGetMessageListService {
     const { messagesAdded, messagesDeleted } =
       await this.gmailGetHistoryService.getMessageIdsFromHistory(history);
 
-    const messageIdsToFilter = await this.getEmailIdsFromExcludedCategories(
+    const messageIdsToFilter = await this.getEmailIdsFromExcludedFolders(
       gmailClient,
       messageChannel.syncCursor,
       messageFolders,
@@ -185,36 +183,30 @@ export class GmailGetMessageListService {
     ];
   }
 
-  private computeExcludedCategories(
-    messageFolders: Pick<MessageFolderWorkspaceEntity, 'name' | 'externalId'>[],
-  ) {
-    const includedDefaultCategories = messageFolders
-      .map((messageFolder) =>
-        mapGmailDefaultFolderToCategoryOrUndefined(messageFolder.name),
-      )
-      .filter(isDefined);
-
-    return MESSAGING_GMAIL_EXCLUDED_CATEGORIES.filter(
-      (excludedCategory) =>
-        !includedDefaultCategories.includes(excludedCategory),
-    );
-  }
-
-  private async getEmailIdsFromExcludedCategories(
+  private async getEmailIdsFromExcludedFolders(
     gmailClient: gmailV1.Gmail,
     lastSyncHistoryId: string,
-    messageFolders: Pick<MessageFolderWorkspaceEntity, 'name' | 'externalId'>[],
+    messageFolders: Pick<
+      MessageFolderWorkspaceEntity,
+      'name' | 'externalId' | 'isSynced'
+    >[],
   ): Promise<string[]> {
     const emailIds: string[] = [];
 
-    const excludedCategories = this.computeExcludedCategories(messageFolders);
+    const toBeExcludedFolders = messageFolders.filter(
+      (folder) => !folder.isSynced,
+    );
+
+    for (const folder of toBeExcludedFolders) {
+      if (!isDefined(folder.externalId)) {
+        continue;
+      }
 
-    for (const category of excludedCategories) {
       const { history } = await this.gmailGetHistoryService.getHistory(
         gmailClient,
         lastSyncHistoryId,
         ['messageAdded'],
-        computeGmailCategoryLabelId(category),
+        folder.externalId,
       );
 
       const emailIdsFromCategory = history
@@ -229,23 +221,4 @@ export class GmailGetMessageListService {
 
     return emailIds;
   }
-
-  private getCustomLabelIds(
-    messageFolders: Pick<
-      MessageFolderWorkspaceEntity,
-      'name' | 'externalId' | 'isSynced'
-    >[],
-  ): string[] | undefined {
-    const customLabelIds = messageFolders
-      .filter(
-        (folder) =>
-          folder.externalId &&
-          folder.isSynced &&
-          !mapGmailDefaultFolderToCategoryOrUndefined(folder.name),
-      )
-      .map((folder) => folder.externalId)
-      .filter((id): id is string => !!id);
-
-    return customLabelIds.length > 0 ? customLabelIds : undefined;
-  }
 }

packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-default-message-category.type.ts

@@ -1,6 +0,0 @@
-export enum GmailDefaultMessageCategory {
-  promotions = 'promotions',
-  social = 'social',
-  forums = 'forums',
-  updates = 'updates',
-}

packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-category-excude-search-filter.spec.ts

@@ -1,27 +0,0 @@
-import { computeGmailCategoryExcludeSearchFilter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-excude-search-filter.util';
-
-describe('computeGmailCategoryExcludeSearchFilter', () => {
-  it('should return correct exclude search filter with empty category array', () => {
-    const result = computeGmailCategoryExcludeSearchFilter([]);
-
-    expect(result).toBe('');
-  });
-
-  it('should return correct exclude search filter with one category', () => {
-    const result = computeGmailCategoryExcludeSearchFilter(['CATEGORY1']);
-
-    expect(result).toBe('-category:CATEGORY1');
-  });
-
-  it('should return correct exclude search filter with multiple categories', () => {
-    const result = computeGmailCategoryExcludeSearchFilter([
-      'CATEGORY1',
-      'CATEGORY2',
-      'CATEGORY3',
-    ]);
-
-    expect(result).toBe(
-      '-category:CATEGORY1 -category:CATEGORY2 -category:CATEGORY3',
-    );
-  });
-});

packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-category-label-id.spec.ts

@@ -1,9 +0,0 @@
-import { computeGmailCategoryLabelId } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-label-id.util';
-
-describe('computeGmailCategoryLabelId', () => {
-  it('should return correct category label id', () => {
-    const result = computeGmailCategoryLabelId('CATEGORY1');
-
-    expect(result).toBe('CATEGORY_CATEGORY1');
-  });
-});

packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/__tests__/compute-gmail-exclude-search-filter.spec.ts

@@ -0,0 +1,69 @@
+import { computeGmailExcludeSearchFilter } from 'src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-exclude-search-filter.util';
+
+describe('computeGmailExcludeSearchFilter', () => {
+  it('should return correct exclude search filter with empty label array', () => {
+    const result = computeGmailExcludeSearchFilter([]);
+
+    expect(result).toBe('');
+  });
+
+  it('should return correct exclude search filter with one label', () => {
+    const result = computeGmailExcludeSearchFilter([
+      {
+        externalId: 'LABEL1',
+        isSynced: false,
+      },
+    ]);
+
+    expect(result).toBe('-label:LABEL1');
+  });
+
+  it('should return correct exclude search filter with multiple categories', () => {
+    const result = computeGmailExcludeSearchFilter([
+      {
+        externalId: 'LABEL1',
+        isSynced: false,
+      },
+      {
+        externalId: 'LABEL2',
+        isSynced: false,
+      },
+      {
+        externalId: 'LABEL3',
+        isSynced: false,
+      },
+    ]);
+
+    expect(result).toBe('-label:LABEL1 -label:LABEL2 -label:LABEL3');
+  });
+
+  it('should return correct exclude search filter with one label that is synced', () => {
+    const result = computeGmailExcludeSearchFilter([
+      {
+        externalId: 'LABEL1',
+        isSynced: true,
+      },
+    ]);
+
+    expect(result).toBe('');
+  });
+
+  it('should return correct exclude search filter with multiple categories that are synced', () => {
+    const result = computeGmailExcludeSearchFilter([
+      {
+        externalId: 'LABEL1',
+        isSynced: true,
+      },
+      {
+        externalId: 'LABEL2',
+        isSynced: false,
+      },
+      {
+        externalId: 'LABEL3',
+        isSynced: true,
+      },
+    ]);
+
+    expect(result).toBe('-label:LABEL2');
+  });
+});

packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-excude-search-filter.util.ts

@@ -1,3 +0,0 @@
-export const computeGmailCategoryExcludeSearchFilter = (
-  excludedCategories: string[],
-) => excludedCategories.map((category) => `-category:${category}`).join(' ');

packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-category-label-id.util.ts

@@ -1,2 +0,0 @@
-export const computeGmailCategoryLabelId = (category: string) =>
-  `CATEGORY_${category.toUpperCase()}`;

packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/compute-gmail-exclude-search-filter.util.ts

@@ -0,0 +1,12 @@
+import { type MessageFolderWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-folder.workspace-entity';
+
+export const computeGmailExcludeSearchFilter = (
+  messageFolders: Pick<
+    MessageFolderWorkspaceEntity,
+    'externalId' | 'isSynced'
+  >[],
+) =>
+  messageFolders
+    .filter((folder) => !folder.isSynced)
+    .map((folder) => `-label:${folder.externalId}`)
+    .join(' ');

packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/utils/map-gmail-default-folder-to-category.ts

@@ -1,37 +0,0 @@
-import { isDefined } from 'twenty-shared/utils';
-
-import { GmailDefaultMessageCategory } from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-default-message-category.type';
-import { GmailDefaultMessageFolder } from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-default-message-folder.type';
-
-const DEFAULT_FOLDER_NAME_TO_CATEGORY_MAPPING = [
-  {
-    folderName: GmailDefaultMessageFolder.CATEGORY_FORUMS,
-    category: GmailDefaultMessageCategory.forums,
-  },
-  {
-    folderName: GmailDefaultMessageFolder.CATEGORY_PROMOTIONS,
-    category: GmailDefaultMessageCategory.promotions,
-  },
-  {
-    folderName: GmailDefaultMessageFolder.CATEGORY_SOCIAL,
-    category: GmailDefaultMessageCategory.social,
-  },
-  {
-    folderName: GmailDefaultMessageFolder.CATEGORY_UPDATES,
-    category: GmailDefaultMessageCategory.updates,
-  },
-];
-
-export const mapGmailDefaultFolderToCategoryOrUndefined = (
-  messageFolderName: string,
-) => {
-  const mapping = DEFAULT_FOLDER_NAME_TO_CATEGORY_MAPPING.find(
-    ({ folderName }) => folderName === messageFolderName,
-  );
-
-  if (!isDefined(mapping)) {
-    return undefined;
-  }
-
-  return mapping.category;
-};

packages/twenty-server/src/modules/messaging/message-import-manager/drivers/imap/services/imap-get-message-list.service.ts

@@ -33,12 +33,15 @@ export class ImapGetMessageListService {
     messageFolders,
   }: GetMessageListsArgs): Promise<GetMessageListsResponse> {
     let client: ImapFlow | null = null;
+    const syncedMessageFolders = messageFolders.filter(
+      (folder) => folder.isSynced,
+    );
 
     try {
       client = await this.imapClientProvider.getClient(connectedAccount);
       const result: GetMessageListsResponse = [];
 
-      for (const folder of messageFolders) {
+      for (const folder of syncedMessageFolders) {
         this.logger.log(`Processing folder: ${folder.name}`);
 
         try {
@@ -76,7 +79,7 @@ export class ImapGetMessageListService {
 
       this.imapHandleErrorService.handleImapMessageListFetchError(error);
 
-      return messageFolders.map((folder) => ({
+      return syncedMessageFolders.map((folder) => ({
         messageExternalIds: [],
         nextSyncCursor: folder.syncCursor || '',
         previousSyncCursor: folder.syncCursor,

packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.ts

@@ -47,7 +47,11 @@ export class MicrosoftGetMessageListService {
       );
     }
 
-    for (const folder of messageFolders) {
+    const syncedMessageFolders = messageFolders.filter(
+      (folder) => folder.isSynced,
+    );
+
+    for (const folder of syncedMessageFolders) {
       const response = await this.getMessageList(connectedAccount, folder);
 
       result.push({

packages/twenty-server/src/modules/messaging/message-import-manager/services/messaging-message-list-fetch.service.ts

@@ -91,17 +91,16 @@ export class MessagingMessageListFetchService {
           'messageFolder',
         );
 
-      const messageFoldersToSync = await messageFolderRepository.find({
+      const messageFolders = await messageFolderRepository.find({
         where: {
           messageChannelId: messageChannel.id,
-          isSynced: true,
         },
       });
 
       const messageLists =
         await this.messagingGetMessageListService.getMessageLists(
           messageChannelWithFreshTokens,
-          messageFoldersToSync,
+          messageFolders,
         );
 
       await this.cacheStorage.del(
@@ -124,7 +123,11 @@ export class MessagingMessageListFetchService {
       let totalMessagesToImportCount = 0;
 
       this.logger.log(
-        `messageChannelId: ${messageChannel.id} Is full sync: ${isFullSync} and toImportCount: ${messageExternalIds.length}, toDeleteCount: ${messageExternalIdsToDelete.length}`,
+        `messageChannelId: ${messageChannel.id} Is full sync: ${isFullSync} and toImportCount: ${messageExternalIds.length}, toDeleteCount: ${messageExternalIdsToDelete.length}, cursors: ${messageLists.map(
+          (messageList) => {
+            messageList.nextSyncCursor;
+          },
+        )}`,
       );
 
       const messageChannelMessageAssociationRepository =
@@ -207,26 +210,22 @@ export class MessagingMessageListFetchService {
         const toDeleteChunks = chunk(allMessageExternalIdsToDelete, 200);
 
         for (const [index, toDeleteChunk] of toDeleteChunks.entries()) {
-          await messageChannelMessageAssociationRepository.delete({
-            messageChannelId: messageChannelWithFreshTokens.id,
-            messageExternalId: In(toDeleteChunk),
-          });
-
           this.logger.log(
-            `messageChannelId: ${messageChannel.id} Deleted ${toDeleteChunk.length} message channel message associations in batch ${index + 1}`,
+            `messageChannelId: ${messageChannel.id} Deleting ${toDeleteChunk.length} message channel message associations in batch ${index + 1}`,
+          );
+
+          await this.messagingMessageCleanerService.deleteMessagesChannelMessageAssociationsAndRelatedOrphans(
+            {
+              workspaceId,
+              messageExternalIds: toDeleteChunk.filter((messageExternalId) =>
+                isNonEmptyString(messageExternalId),
+              ),
+              messageChannelId: messageChannelWithFreshTokens.id,
+            },
           );
         }
       }
 
-      this.logger.log(
-        `messageChannelId: ${messageChannel.id} launching workspace thread cleanup`,
-      );
-
-      // TODO: Re-enable this after we have a way to clean up threads without impacting the message import
-      // await this.messagingMessageCleanerService.cleanWorkspaceThreads(
-      //   workspaceId,
-      // );
-
       this.logger.log(
         `messageChannelId: ${messageChannel.id} Total messages to import count: ${totalMessagesToImportCount}`,
       );