feat(redis): add support for dedicated queue client configuration (#14840)

Fix https://github.com/twentyhq/core-team-issues/issues/452
Fix https://github.com/twentyhq/core-team-issues/issues/923

---------

Co-authored-by: Félix Malfait <felix@twenty.com>

packages/twenty-server/src/engine/core-modules/message-queue/drivers/pg-boss.driver.ts

@@ -1,120 +0,0 @@
-import { type OnModuleDestroy, type OnModuleInit } from '@nestjs/common';
-
-import PgBoss from 'pg-boss';
-
-import {
-  type QueueCronJobOptions,
-  type QueueJobOptions,
-} from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';
-import { type MessageQueueJob } from 'src/engine/core-modules/message-queue/interfaces/message-queue-job.interface';
-import { type MessageQueueWorkerOptions } from 'src/engine/core-modules/message-queue/interfaces/message-queue-worker-options.interface';
-import { type MessageQueueDriver } from 'src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface';
-
-import { type MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
-import { getJobKey } from 'src/engine/core-modules/message-queue/utils/get-job-key.util';
-
-export type PgBossDriverOptions = PgBoss.ConstructorOptions;
-
-const DEFAULT_PG_BOSS_CRON_PATTERN_WHEN_NOT_PROVIDED = '*/1 * * * *';
-
-export class PgBossDriver
-  implements MessageQueueDriver, OnModuleInit, OnModuleDestroy
-{
-  private pgBoss: PgBoss;
-
-  constructor(options: PgBossDriverOptions) {
-    this.pgBoss = new PgBoss(options);
-  }
-
-  async onModuleInit() {
-    await this.pgBoss.start();
-  }
-
-  async onModuleDestroy() {
-    await this.pgBoss.stop();
-  }
-
-  async work<T>(
-    queueName: string,
-    handler: (job: MessageQueueJob<T>) => Promise<void>,
-    options?: MessageQueueWorkerOptions,
-  ) {
-    return this.pgBoss.work<T>(
-      `${queueName}.*`,
-      options?.concurrency
-        ? {
-            teamConcurrency: options.concurrency,
-          }
-        : {},
-      async (job) => {
-        // PGBoss work with wildcard job name
-        const jobName = job.name.split('.')?.[1];
-
-        if (!jobName) {
-          throw new Error('Job name could not be splited from the job.');
-        }
-
-        await handler({
-          data: job.data,
-          id: job.id,
-          name: jobName,
-        });
-      },
-    );
-  }
-
-  async addCron<T>({
-    queueName,
-    jobName,
-    data,
-    options,
-    jobId,
-  }: {
-    queueName: MessageQueue;
-    jobName: string;
-    data: T;
-    options: QueueCronJobOptions;
-    jobId?: string;
-  }): Promise<void> {
-    const name = `${queueName}.${getJobKey({ jobName, jobId })}`;
-
-    await this.pgBoss.schedule(
-      name,
-      options.repeat.pattern ?? DEFAULT_PG_BOSS_CRON_PATTERN_WHEN_NOT_PROVIDED,
-      data as object,
-    );
-  }
-
-  async removeCron({
-    queueName,
-    jobName,
-    jobId,
-  }: {
-    queueName: MessageQueue;
-    jobName: string;
-    jobId?: string;
-  }): Promise<void> {
-    const name = `${queueName}.${getJobKey({ jobName, jobId })}`;
-
-    await this.pgBoss.unschedule(name);
-  }
-
-  async add<T>(
-    queueName: MessageQueue,
-    jobName: string,
-    data: T,
-    options?: QueueJobOptions,
-  ): Promise<void> {
-    await this.pgBoss.send(
-      `${queueName}.${jobName}`,
-      data as object,
-      options
-        ? {
-            ...options,
-            singletonKey: options?.id,
-            useSingletonQueue: true, // When used with singletonKey, ensures only one job can be queued. See https://logsnag.com/blog/deep-dive-into-background-jobs-with-pg-boss-and-typescript
-          }
-        : {},
-    );
-  }
-}

packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-module-options.interface.ts

@@ -1,17 +1,10 @@
 import { type BullMQDriverOptions } from 'src/engine/core-modules/message-queue/drivers/bullmq.driver';
-import { type PgBossDriverOptions } from 'src/engine/core-modules/message-queue/drivers/pg-boss.driver';
 
 export enum MessageQueueDriverType {
-  PgBoss = 'pg-boss',
   BullMQ = 'bull-mq',
   Sync = 'sync',
 }
 
-export interface PgBossDriverFactoryOptions {
-  type: MessageQueueDriverType.PgBoss;
-  options: PgBossDriverOptions;
-}
-
 export interface BullMQDriverFactoryOptions {
   type: MessageQueueDriverType.BullMQ;
   options: BullMQDriverOptions;
@@ -24,6 +17,5 @@ export interface SyncDriverFactoryOptions {
 }
 
 export type MessageQueueModuleOptions =
-  | PgBossDriverFactoryOptions
   | BullMQDriverFactoryOptions
   | SyncDriverFactoryOptions;

packages/twenty-server/src/engine/core-modules/message-queue/message-queue-core.module.ts

@@ -8,21 +8,20 @@ import {
 
 import { type MessageQueueDriver } from 'src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface';
 
+import { BullMQDriver } from 'src/engine/core-modules/message-queue/drivers/bullmq.driver';
+import { SyncDriver } from 'src/engine/core-modules/message-queue/drivers/sync.driver';
 import { MessageQueueDriverType } from 'src/engine/core-modules/message-queue/interfaces';
 import {
   MessageQueue,
   QUEUE_DRIVER,
 } from 'src/engine/core-modules/message-queue/message-queue.constants';
-import { PgBossDriver } from 'src/engine/core-modules/message-queue/drivers/pg-boss.driver';
-import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
-import { BullMQDriver } from 'src/engine/core-modules/message-queue/drivers/bullmq.driver';
-import { SyncDriver } from 'src/engine/core-modules/message-queue/drivers/sync.driver';
-import { getQueueToken } from 'src/engine/core-modules/message-queue/utils/get-queue-token.util';
 import {
   type ASYNC_OPTIONS_TYPE,
   ConfigurableModuleClass,
   type OPTIONS_TYPE,
 } from 'src/engine/core-modules/message-queue/message-queue.module-definition';
+import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
+import { getQueueToken } from 'src/engine/core-modules/message-queue/utils/get-queue-token.util';
 
 @Global()
 @Module({})
@@ -94,9 +93,6 @@ export class MessageQueueCoreModule extends ConfigurableModuleClass {
 
   static async createDriver({ type, options }: typeof OPTIONS_TYPE) {
     switch (type) {
-      case MessageQueueDriverType.PgBoss: {
-        return new PgBossDriver(options);
-      }
       case MessageQueueDriverType.BullMQ: {
         return new BullMQDriver(options);
       }

packages/twenty-server/src/engine/core-modules/message-queue/message-queue.module-factory.ts

@@ -18,28 +18,11 @@ export const messageQueueModuleFactory = async (
   const driverType = MessageQueueDriverType.BullMQ;
 
   switch (driverType) {
-    /* 
-    case MessageQueueDriverType.Sync: {
-      return {
-        type: MessageQueueDriverType.Sync,
-        options: {},
-      } satisfies SyncDriverFactoryOptions;
-    }
-    case MessageQueueDriverType.PgBoss: {
-      const connectionString = twentyConfigService.get('PG_DATABASE_URL');
-
-      return {
-        type: MessageQueueDriverType.PgBoss,
-        options: {
-          connectionString,
-        },
-      } satisfies PgBossDriverFactoryOptions;
-    }*/
     case MessageQueueDriverType.BullMQ: {
       return {
         type: MessageQueueDriverType.BullMQ,
         options: {
-          connection: redisClientService.getClient(),
+          connection: redisClientService.getQueueClient(),
         },
       } satisfies BullMQDriverFactoryOptions;
     }