diff --git a/packages/twenty-server/src/engine/core-modules/view/flat-view/flat-view.module.ts b/packages/twenty-server/src/engine/core-modules/view/flat-view/flat-view.module.ts new file mode 100644 index 00000000000..56129938f35 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/view/flat-view/flat-view.module.ts @@ -0,0 +1,12 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { ViewEntity } from 'src/engine/core-modules/view/entities/view.entity'; +import { WorkspaceFlatViewMapCacheService } from 'src/engine/core-modules/view/flat-view/services/workspace-flat-view-map-cache.service'; + +@Module({ + imports: [TypeOrmModule.forFeature([ViewEntity])], + providers: [WorkspaceFlatViewMapCacheService], + exports: [WorkspaceFlatViewMapCacheService], +}) +export class FlatViewModule {} diff --git a/packages/twenty-server/src/engine/core-modules/view/flat-view/services/workspace-flat-view-map-cache.service.ts b/packages/twenty-server/src/engine/core-modules/view/flat-view/services/workspace-flat-view-map-cache.service.ts new file mode 100644 index 00000000000..5d6bfe4afcf --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/view/flat-view/services/workspace-flat-view-map-cache.service.ts @@ -0,0 +1,46 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; + +import { Repository } from 'typeorm'; + +import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator'; +import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service'; +import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum'; +import { ViewEntity } from 'src/engine/core-modules/view/entities/view.entity'; +import { type FlatViewMaps } from 'src/engine/core-modules/view/flat-view/types/flat-view-maps.type'; +import { generateFlatViewMaps } from 'src/engine/core-modules/view/flat-view/utils/generate-flat-view-maps.util'; +import { WorkspaceFlatMapCache } from 'src/engine/workspace-flat-map-cache/decorators/workspace-flat-map-cache.decorator'; +import { WorkspaceFlatMapCacheService } from 'src/engine/workspace-flat-map-cache/services/workspace-flat-map-cache.service'; + +@Injectable() +@WorkspaceFlatMapCache('view') +export class WorkspaceFlatViewMapCacheService extends WorkspaceFlatMapCacheService { + constructor( + @InjectCacheStorage(CacheStorageNamespace.EngineWorkspace) + cacheStorageService: CacheStorageService, + @InjectRepository(ViewEntity) + private readonly viewRepository: Repository, + ) { + super(cacheStorageService); + } + + protected async computeFlatMap({ + workspaceId, + }: { + workspaceId: string; + }): Promise { + const views = await this.viewRepository.find({ + where: { + workspaceId, + }, + relations: ['viewFields'], + select: { + viewFields: { + id: true, + }, + }, + }); + + return generateFlatViewMaps(views); + } +} diff --git a/packages/twenty-server/src/engine/core-modules/view/view.module.ts b/packages/twenty-server/src/engine/core-modules/view/view.module.ts index f656005029e..0d6776be52a 100644 --- a/packages/twenty-server/src/engine/core-modules/view/view.module.ts +++ b/packages/twenty-server/src/engine/core-modules/view/view.module.ts @@ -15,6 +15,7 @@ import { ViewFilterEntity } from 'src/engine/core-modules/view/entities/view-fil import { ViewGroupEntity } from 'src/engine/core-modules/view/entities/view-group.entity'; import { ViewSortEntity } from 'src/engine/core-modules/view/entities/view-sort.entity'; import { ViewEntity } from 'src/engine/core-modules/view/entities/view.entity'; +import { FlatViewModule } from 'src/engine/core-modules/view/flat-view/flat-view.module'; import { ViewFieldResolver } from 'src/engine/core-modules/view/resolvers/view-field.resolver'; import { ViewFilterGroupResolver } from 'src/engine/core-modules/view/resolvers/view-filter-group.resolver'; import { ViewFilterResolver } from 'src/engine/core-modules/view/resolvers/view-filter.resolver'; @@ -48,6 +49,7 @@ import { WorkspaceMigrationV2Module } from 'src/engine/workspace-manager/workspa WorkspaceMetadataCacheModule, WorkspaceMigrationV2Module, ViewCacheModule, + FlatViewModule, ], controllers: [ ViewController, diff --git a/packages/twenty-server/src/engine/workspace-flat-map-cache/decorators/workspace-flat-map-cache.decorator.ts b/packages/twenty-server/src/engine/workspace-flat-map-cache/decorators/workspace-flat-map-cache.decorator.ts new file mode 100644 index 00000000000..f213492078a --- /dev/null +++ b/packages/twenty-server/src/engine/workspace-flat-map-cache/decorators/workspace-flat-map-cache.decorator.ts @@ -0,0 +1,6 @@ +import { SetMetadata } from '@nestjs/common'; + +export const WORKSPACE_FLAT_MAP_CACHE_KEY = 'workspaceFlatMapCacheKey'; + +export const WorkspaceFlatMapCache = (cacheKey: string) => + SetMetadata(WORKSPACE_FLAT_MAP_CACHE_KEY, cacheKey); diff --git a/packages/twenty-server/src/engine/workspace-flat-map-cache/exceptions/workspace-flat-map-cache.exception.ts b/packages/twenty-server/src/engine/workspace-flat-map-cache/exceptions/workspace-flat-map-cache.exception.ts new file mode 100644 index 00000000000..22ccf033bd5 --- /dev/null +++ b/packages/twenty-server/src/engine/workspace-flat-map-cache/exceptions/workspace-flat-map-cache.exception.ts @@ -0,0 +1,12 @@ +import { + appendCommonExceptionCode, + CustomException, +} from 'src/utils/custom-exception'; + +export class WorkspaceFlatMapCacheException extends CustomException< + keyof typeof WorkspaceFlatMapCacheExceptionCode +> {} + +export const WorkspaceFlatMapCacheExceptionCode = appendCommonExceptionCode({ + MISSING_DECORATOR: 'MISSING_DECORATOR', +} as const); diff --git a/packages/twenty-server/src/engine/workspace-flat-map-cache/services/workspace-flat-map-cache.service.ts b/packages/twenty-server/src/engine/workspace-flat-map-cache/services/workspace-flat-map-cache.service.ts new file mode 100644 index 00000000000..ba62a699336 --- /dev/null +++ b/packages/twenty-server/src/engine/workspace-flat-map-cache/services/workspace-flat-map-cache.service.ts @@ -0,0 +1,188 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { Reflector } from '@nestjs/core'; + +import crypto from 'crypto'; + +import { isDefined } from 'twenty-shared/utils'; + +import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator'; +import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service'; +import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum'; +import { type FlatEntityMaps } from 'src/engine/core-modules/common/types/flat-entity-maps.type'; +import { type FlatEntity } from 'src/engine/core-modules/common/types/flat-entity.type'; +import { WORKSPACE_FLAT_MAP_CACHE_KEY } from 'src/engine/workspace-flat-map-cache/decorators/workspace-flat-map-cache.decorator'; +import { + WorkspaceFlatMapCacheException, + WorkspaceFlatMapCacheExceptionCode, +} from 'src/engine/workspace-flat-map-cache/exceptions/workspace-flat-map-cache.exception'; + +@Injectable() +export abstract class WorkspaceFlatMapCacheService< + T extends FlatEntityMaps, +> { + protected readonly logger = new Logger(this.constructor.name); + + private readonly localCacheFlatMaps = new Map(); + private readonly localCacheHashes = new Map(); + private readonly reflector = new Reflector(); + + constructor( + @InjectCacheStorage(CacheStorageNamespace.EngineWorkspace) + private readonly cacheStorageService: CacheStorageService, + ) {} + + protected abstract computeFlatMap({ + workspaceId, + }: { + workspaceId: string; + }): Promise; + + async getExistingOrRecomputeFlatMaps({ + workspaceId, + }: { + workspaceId: string; + }): Promise { + const localCacheHash = this.localCacheHashes.get(workspaceId); + const remoteCacheHash = await this.getHashFromRemoteCache({ workspaceId }); + + if ( + isDefined(localCacheHash) && + isDefined(remoteCacheHash) && + localCacheHash === remoteCacheHash + ) { + const localCacheFlatMap = this.localCacheFlatMaps.get(workspaceId); + + if (localCacheFlatMap) { + return localCacheFlatMap; + } + } + + if (remoteCacheHash) { + const remoteCacheFlatMap = await this.getFlatMapFromRemoteCache({ + workspaceId, + }); + + if (remoteCacheFlatMap) { + this.localCacheFlatMaps.set(workspaceId, remoteCacheFlatMap); + this.localCacheHashes.set(workspaceId, remoteCacheHash); + + return remoteCacheFlatMap; + } + } + + const freshFlatMap = await this.recomputeAndStoreInCache({ + workspaceId, + }); + + return freshFlatMap; + } + + async recomputeAndStoreInCache({ + workspaceId, + }: { + workspaceId: string; + }): Promise { + const freshFlatMap = await this.computeFlatMap({ workspaceId }); + const newHash = this.generateHash({ flatMap: freshFlatMap }); + + await this.setFlatMapInRemoteCache({ workspaceId, flatMap: freshFlatMap }); + await this.setHashInRemoteCache({ workspaceId, hash: newHash }); + + this.localCacheFlatMaps.set(workspaceId, freshFlatMap); + this.localCacheHashes.set(workspaceId, newHash); + + return freshFlatMap; + } + + async invalidateCache({ + workspaceId, + }: { + workspaceId: string; + }): Promise { + const { flatMapKey, hashKey } = this.buildRemoteCacheKeys({ workspaceId }); + + await this.cacheStorageService.del(flatMapKey); + await this.cacheStorageService.del(hashKey); + + this.localCacheFlatMaps.delete(workspaceId); + this.localCacheHashes.delete(workspaceId); + + await this.recomputeAndStoreInCache({ workspaceId }); + } + + private getFlatMapCacheKey(): string { + const cacheKey = this.reflector.get( + WORKSPACE_FLAT_MAP_CACHE_KEY, + this.constructor, + ); + + if (!cacheKey) { + throw new WorkspaceFlatMapCacheException( + `${this.constructor.name} must be decorated with @WorkspaceFlatMapCache('cacheKey')`, + WorkspaceFlatMapCacheExceptionCode.MISSING_DECORATOR, + ); + } + + return cacheKey; + } + + private buildRemoteCacheKeys({ workspaceId }: { workspaceId: string }) { + const cacheKey = this.getFlatMapCacheKey(); + + return { + flatMapKey: `flat-maps:${cacheKey}:${workspaceId}:flat-map`, + hashKey: `flat-maps:${cacheKey}:${workspaceId}:hash`, + }; + } + + private generateHash({ flatMap }: { flatMap: T }): string { + return crypto + .createHash('sha256') + .update(JSON.stringify(flatMap)) + .digest('hex'); + } + + private async getHashFromRemoteCache({ + workspaceId, + }: { + workspaceId: string; + }): Promise { + const { hashKey } = this.buildRemoteCacheKeys({ workspaceId }); + + return this.cacheStorageService.get(hashKey); + } + + private async setHashInRemoteCache({ + workspaceId, + hash, + }: { + workspaceId: string; + hash: string; + }): Promise { + const { hashKey } = this.buildRemoteCacheKeys({ workspaceId }); + + await this.cacheStorageService.set(hashKey, hash); + } + + private async getFlatMapFromRemoteCache({ + workspaceId, + }: { + workspaceId: string; + }): Promise { + const { flatMapKey } = this.buildRemoteCacheKeys({ workspaceId }); + + return this.cacheStorageService.get(flatMapKey); + } + + private async setFlatMapInRemoteCache({ + workspaceId, + flatMap, + }: { + workspaceId: string; + flatMap: T; + }): Promise { + const { flatMapKey } = this.buildRemoteCacheKeys({ workspaceId }); + + await this.cacheStorageService.set(flatMapKey, flatMap); + } +} diff --git a/packages/twenty-server/src/engine/workspace-flat-map-cache/workspace-flat-map-cache.module.ts b/packages/twenty-server/src/engine/workspace-flat-map-cache/workspace-flat-map-cache.module.ts new file mode 100644 index 00000000000..55bd84c1908 --- /dev/null +++ b/packages/twenty-server/src/engine/workspace-flat-map-cache/workspace-flat-map-cache.module.ts @@ -0,0 +1,10 @@ +import { Module } from '@nestjs/common'; + +import { CacheStorageModule } from 'src/engine/core-modules/cache-storage/cache-storage.module'; + +@Module({ + imports: [CacheStorageModule], + providers: [], + exports: [], +}) +export class WorkspaceFlatMapCacheModule {}