diff --git a/packages/twenty-server/src/app.module.ts b/packages/twenty-server/src/app.module.ts index 671905d9d73..64e8c7fc3ff 100644 --- a/packages/twenty-server/src/app.module.ts +++ b/packages/twenty-server/src/app.module.ts @@ -23,7 +23,7 @@ import { RestApiModule } from 'src/engine/api/rest/rest-api.module'; import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module'; import { DataloaderModule } from 'src/engine/dataloaders/dataloader.module'; import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; -import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; +import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { GraphQLHydrateRequestFromTokenMiddleware } from 'src/engine/middlewares/graphql-hydrate-request-from-token.middleware'; import { MiddlewareModule } from 'src/engine/middlewares/middleware.module'; import { RestCoreMiddleware } from 'src/engine/middlewares/rest-core.middleware'; @@ -71,7 +71,7 @@ const MIGRATED_REST_METHODS = [ McpModule, DataSourceModule, MiddlewareModule, - WorkspaceMetadataCacheModule, + WorkspaceMetadataVersionModule, // I18n module for translations I18nModule, // Conditional modules diff --git a/packages/twenty-server/src/engine/api/graphql/__tests__/workspace.factory.spec.ts b/packages/twenty-server/src/engine/api/graphql/__tests__/workspace.factory.spec.ts index 085f9cfe6b5..b18c2afb611 100644 --- a/packages/twenty-server/src/engine/api/graphql/__tests__/workspace.factory.spec.ts +++ b/packages/twenty-server/src/engine/api/graphql/__tests__/workspace.factory.spec.ts @@ -8,7 +8,7 @@ import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/service import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; import { ObjectMetadataServiceV2 } from 'src/engine/metadata-modules/object-metadata/object-metadata-v2.service'; -import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service'; +import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service'; import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; describe('WorkspaceSchemaFactory', () => { @@ -43,7 +43,7 @@ describe('WorkspaceSchemaFactory', () => { useValue: {}, }, { - provide: WorkspaceMetadataCacheService, + provide: WorkspaceManyOrAllFlatEntityMapsCacheService, useValue: {}, }, { diff --git a/packages/twenty-server/src/engine/api/graphql/core-graphql-api.module.ts b/packages/twenty-server/src/engine/api/graphql/core-graphql-api.module.ts index 28fafb88587..fe05df7aa95 100644 --- a/packages/twenty-server/src/engine/api/graphql/core-graphql-api.module.ts +++ b/packages/twenty-server/src/engine/api/graphql/core-graphql-api.module.ts @@ -5,7 +5,7 @@ import { WorkspaceResolverBuilderModule } from 'src/engine/api/graphql/workspace import { WorkspaceSchemaBuilderModule } from 'src/engine/api/graphql/workspace-schema-builder/workspace-schema-builder.module'; import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module'; import { MetadataEngineModule } from 'src/engine/metadata-modules/metadata-engine.module'; -import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; +import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module'; import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; import { WorkspaceSchemaFactory } from './workspace-schema.factory'; @@ -16,7 +16,7 @@ import { WorkspaceSchemaFactory } from './workspace-schema.factory'; WorkspaceSchemaBuilderModule, WorkspaceResolverBuilderModule, WorkspaceCacheStorageModule, - WorkspaceMetadataCacheModule, + WorkspaceManyOrAllFlatEntityMapsCacheModule, FeatureFlagModule, ], providers: [WorkspaceSchemaFactory, ScalarsExplorerService], diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts b/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts index c8994b63750..91cf818db5e 100644 --- a/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts +++ b/packages/twenty-server/src/engine/api/graphql/workspace-schema.factory.ts @@ -17,10 +17,13 @@ import { type AuthContext } from 'src/engine/core-modules/auth/types/auth-contex import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; import { - WorkspaceMetadataCacheException, - WorkspaceMetadataCacheExceptionCode, -} from 'src/engine/metadata-modules/workspace-metadata-cache/exceptions/workspace-metadata-cache.exception'; -import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service'; + FlatEntityMapsException, + FlatEntityMapsExceptionCode, +} from 'src/engine/metadata-modules/flat-entity/exceptions/flat-entity-maps.exception'; +import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service'; +import { buildObjectIdByNameMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-id-by-name-maps.util'; +import { buildObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-metadata-item-with-field-maps.util'; +import { type ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; import { standardObjectMetadataDefinitions } from 'src/engine/workspace-manager/workspace-sync-metadata/standard-objects'; import { shouldExcludeFromWorkspaceApi } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/should-exclude-from-workspace-api.util'; @@ -33,7 +36,7 @@ export class WorkspaceSchemaFactory { private readonly workspaceGraphQLSchemaGenerator: WorkspaceGraphQLSchemaGenerator, private readonly workspaceResolverFactory: WorkspaceResolverFactory, private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, - private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService, + private readonly workspaceManyOrAllFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService, private readonly featureFlagService: FeatureFlagService, ) {} @@ -51,20 +54,6 @@ export class WorkspaceSchemaFactory { return new GraphQLSchema({}); } - const { objectMetadataMaps, metadataVersion } = - await this.workspaceMetadataCacheService.getExistingOrRecomputeMetadataMaps( - { - workspaceId: authContext.workspace.id, - }, - ); - - if (!objectMetadataMaps) { - throw new WorkspaceMetadataCacheException( - 'Object metadata collection not found', - WorkspaceMetadataCacheExceptionCode.OBJECT_METADATA_COLLECTION_NOT_FOUND, - ); - } - const workspaceId = authContext.workspace.id; if (!workspaceId) { @@ -74,9 +63,57 @@ export class WorkspaceSchemaFactory { ); } + const { flatObjectMetadataMaps, flatFieldMetadataMaps, flatIndexMaps } = + await this.workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( + { + workspaceId, + flatMapsKeys: [ + 'flatObjectMetadataMaps', + 'flatFieldMetadataMaps', + 'flatIndexMaps', + ], + }, + ); + + if (!isDefined(flatObjectMetadataMaps)) { + throw new FlatEntityMapsException( + 'Object metadata collection not found', + FlatEntityMapsExceptionCode.ENTITY_NOT_FOUND, + ); + } + const workspaceFeatureFlagsMap = await this.featureFlagService.getWorkspaceFeatureFlagsMap(workspaceId); + let metadataVersion = + await this.workspaceCacheStorageService.getMetadataVersion(workspaceId); + + if (!isDefined(metadataVersion)) { + metadataVersion = authContext.workspace.metadataVersion ?? 0; + await this.workspaceCacheStorageService.setMetadataVersion( + workspaceId, + metadataVersion, + ); + } + + const { idByNameSingular } = buildObjectIdByNameMaps( + flatObjectMetadataMaps, + ); + const objectMetadataMaps: ObjectMetadataMaps = { + byId: {}, + idByNameSingular, + }; + + for (const [id, flatObj] of Object.entries(flatObjectMetadataMaps.byId)) { + if (isDefined(flatObj)) { + objectMetadataMaps.byId[id] = buildObjectMetadataItemWithFieldMaps( + flatObj, + flatFieldMetadataMaps, + flatIndexMaps, + ); + } + } + const objectMetadataCollection = Object.values(objectMetadataMaps.byId) .filter(isDefined) .map((objectMetadataItem) => ({ diff --git a/packages/twenty-server/src/engine/api/rest/core/handlers/rest-api-base.handler.ts b/packages/twenty-server/src/engine/api/rest/core/handlers/rest-api-base.handler.ts index e25031f2a75..800d9a9c3ec 100644 --- a/packages/twenty-server/src/engine/api/rest/core/handlers/rest-api-base.handler.ts +++ b/packages/twenty-server/src/engine/api/rest/core/handlers/rest-api-base.handler.ts @@ -21,6 +21,9 @@ import { AccessTokenService } from 'src/engine/core-modules/auth/token/services/ import { WorkspaceDomainsService } from 'src/engine/core-modules/domain/workspace-domains/services/workspace-domains.service'; import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service'; import { WorkspaceNotFoundDefaultError } from 'src/engine/core-modules/workspace/workspace.exception'; +import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service'; +import { buildObjectIdByNameMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-id-by-name-maps.util'; +import { buildObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-metadata-item-with-field-maps.util'; import { PermissionsException, PermissionsExceptionCode, @@ -29,9 +32,6 @@ import { import { type ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps'; import { type ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; import { UserRoleService } from 'src/engine/metadata-modules/user-role/user-role.service'; -import { getObjectMetadataMapItemByNamePlural } from 'src/engine/metadata-modules/utils/get-object-metadata-map-item-by-name-plural.util'; -import { getObjectMetadataMapItemByNameSingular } from 'src/engine/metadata-modules/utils/get-object-metadata-map-item-by-name-singular.util'; -import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service'; import { WorkspacePermissionsCacheService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.service'; import { TwentyORMManager } from 'src/engine/twenty-orm/twenty-orm.manager'; import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; @@ -63,7 +63,7 @@ export abstract class RestApiBaseHandler { @Inject() protected readonly workspaceCacheStorageService: WorkspaceCacheStorageService; @Inject() - protected readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService; + protected readonly workspaceManyOrAllFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService; @Inject() protected readonly apiKeyRoleService: ApiKeyRoleService; @Inject() @@ -178,19 +178,31 @@ export abstract class RestApiBaseHandler { await this.workspaceCacheStorageService.getMetadataVersion(workspace.id); if (currentCacheVersion === undefined) { - await this.workspaceMetadataCacheService.recomputeMetadataCache({ - workspaceId: workspace.id, - }); - - throw new BadRequestException('Metadata cache version not found'); + if (isDefined(workspace.metadataVersion)) { + await this.workspaceCacheStorageService.setMetadataVersion( + workspace.id, + workspace.metadataVersion, + ); + } else { + throw new BadRequestException( + 'Workspace metadata version not found in database', + ); + } } - const objectMetadataMaps = - await this.workspaceCacheStorageService.getObjectMetadataMaps( - workspace.id, - currentCacheVersion, + + const { flatObjectMetadataMaps, flatFieldMetadataMaps, flatIndexMaps } = + await this.workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( + { + workspaceId: workspace.id, + flatMapsKeys: [ + 'flatObjectMetadataMaps', + 'flatFieldMetadataMaps', + 'flatIndexMaps', + ], + }, ); - if (!objectMetadataMaps) { + if (!isDefined(flatObjectMetadataMaps)) { throw new BadRequestException( `No object was found for the workspace associated with this API key. You may generate a new one here ${this.workspaceDomainsService .buildWorkspaceURL({ @@ -201,21 +213,25 @@ export abstract class RestApiBaseHandler { ); } - const objectMetadataItem = getObjectMetadataMapItemByNamePlural( - objectMetadataMaps, - parsedObject, + const { idByNameSingular, idByNamePlural } = buildObjectIdByNameMaps( + flatObjectMetadataMaps, ); - if (!objectMetadataItem) { - const wrongObjectMetadataItem = getObjectMetadataMapItemByNameSingular( - objectMetadataMaps, - parsedObject, - ); + let objectId = idByNamePlural[parsedObject]; + let flatObjectMetadataItem = objectId + ? flatObjectMetadataMaps.byId[objectId] + : undefined; + + if (!flatObjectMetadataItem) { + const wrongObjectId = idByNameSingular[parsedObject]; + const wrongFlatObjectMetadataItem = wrongObjectId + ? flatObjectMetadataMaps.byId[wrongObjectId] + : undefined; let hint = 'eg: companies'; - if (wrongObjectMetadataItem) { - hint = `Did you mean '${wrongObjectMetadataItem.namePlural}'?`; + if (wrongFlatObjectMetadataItem) { + hint = `Did you mean '${wrongFlatObjectMetadataItem.namePlural}'?`; } throw new BadRequestException( @@ -226,6 +242,12 @@ export abstract class RestApiBaseHandler { const workspaceFeatureFlagsMap = await this.featureFlagService.getWorkspaceFeatureFlagsMap(workspace.id); + const objectMetadataItem = buildObjectMetadataItemWithFieldMaps( + flatObjectMetadataItem, + flatFieldMetadataMaps, + flatIndexMaps, + ); + // Check if this entity is workspace-gated and should be blocked from workspace API if ( shouldExcludeFromWorkspaceApi( @@ -239,6 +261,21 @@ export abstract class RestApiBaseHandler { ); } + const objectMetadataMaps: ObjectMetadataMaps = { + byId: {}, + idByNameSingular, + }; + + for (const [id, flatObj] of Object.entries(flatObjectMetadataMaps.byId)) { + if (isDefined(flatObj)) { + objectMetadataMaps.byId[id] = buildObjectMetadataItemWithFieldMaps( + flatObj, + flatFieldMetadataMaps, + flatIndexMaps, + ); + } + } + return { objectMetadataMaps, objectMetadataMapItem: objectMetadataItem, diff --git a/packages/twenty-server/src/engine/api/rest/core/rest-api-core.module.ts b/packages/twenty-server/src/engine/api/rest/core/rest-api-core.module.ts index e0d0e7e96cf..e4822c1d8ad 100644 --- a/packages/twenty-server/src/engine/api/rest/core/rest-api-core.module.ts +++ b/packages/twenty-server/src/engine/api/rest/core/rest-api-core.module.ts @@ -27,11 +27,12 @@ import { AuthModule } from 'src/engine/core-modules/auth/auth.module'; import { WorkspaceDomainsModule } from 'src/engine/core-modules/domain/workspace-domains/workspace-domains.module'; import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module'; import { RecordTransformerModule } from 'src/engine/core-modules/record-transformer/record-transformer.module'; +import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module'; import { UserRoleModule } from 'src/engine/metadata-modules/user-role/user-role.module'; -import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; import { WorkspacePermissionsCacheModule } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.module'; import { TwentyORMModule } from 'src/engine/twenty-orm/twenty-orm.module'; import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; +import { WorkspaceFlatMapCacheModule } from 'src/engine/workspace-flat-map-cache/workspace-flat-map-cache.module'; const restApiCoreResolvers = [ RestApiCreateOneHandler, @@ -62,7 +63,8 @@ const restApiCoreResolvers = [ TwentyORMModule, RecordTransformerModule, WorkspacePermissionsCacheModule, - WorkspaceMetadataCacheModule, + WorkspaceManyOrAllFlatEntityMapsCacheModule, + WorkspaceFlatMapCacheModule, ActorModule, FeatureFlagModule, CoreCommonApiModule, diff --git a/packages/twenty-server/src/engine/core-modules/actor/actor.module.ts b/packages/twenty-server/src/engine/core-modules/actor/actor.module.ts index a0daea55d76..1b686f7c38c 100644 --- a/packages/twenty-server/src/engine/core-modules/actor/actor.module.ts +++ b/packages/twenty-server/src/engine/core-modules/actor/actor.module.ts @@ -4,14 +4,14 @@ import { TypeOrmModule } from '@nestjs/typeorm'; import { CreatedByCreateManyPreQueryHook } from 'src/engine/core-modules/actor/query-hooks/created-by.create-many.pre-query-hook'; import { CreatedByCreateOnePreQueryHook } from 'src/engine/core-modules/actor/query-hooks/created-by.create-one.pre-query-hook'; import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; -import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; +import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module'; import { CreatedByFromAuthContextService } from './services/created-by-from-auth-context.service'; @Module({ imports: [ TypeOrmModule.forFeature([FieldMetadataEntity]), - WorkspaceMetadataCacheModule, + WorkspaceManyOrAllFlatEntityMapsCacheModule, ], providers: [ CreatedByCreateManyPreQueryHook, diff --git a/packages/twenty-server/src/engine/core-modules/actor/services/__tests__/created-by-from-auth-context.service.spec.ts b/packages/twenty-server/src/engine/core-modules/actor/services/__tests__/created-by-from-auth-context.service.spec.ts index 0590ad1d1ad..16dfa9a136d 100644 --- a/packages/twenty-server/src/engine/core-modules/actor/services/__tests__/created-by-from-auth-context.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/actor/services/__tests__/created-by-from-auth-context.service.spec.ts @@ -13,7 +13,7 @@ import { type AuthContext } from 'src/engine/core-modules/auth/types/auth-contex import { type UserEntity } from 'src/engine/core-modules/user/user.entity'; import { type WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; -import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service'; +import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service'; import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; import { type WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity'; @@ -59,29 +59,30 @@ describe('CreatedByFromAuthContextService', () => { }, }, { - provide: WorkspaceMetadataCacheService, + provide: WorkspaceManyOrAllFlatEntityMapsCacheService, useValue: { - getExistingOrRecomputeMetadataMaps: jest.fn().mockResolvedValue({ - objectMetadataMaps: { + getOrRecomputeManyOrAllFlatEntityMaps: jest.fn().mockResolvedValue({ + flatObjectMetadataMaps: { byId: { 'person-id': { id: 'person-id', nameSingular: 'person', - fieldsById: { - createdBy: { - id: 'createdBy-id', - name: 'createdBy', - }, - }, - fieldIdByName: { - createdBy: 'createdBy-id', - }, + fieldMetadataIds: ['createdBy-id'], }, }, - idByNameSingular: { - person: 'person-id', + }, + flatFieldMetadataMaps: { + byId: { + 'createdBy-id': { + id: 'createdBy-id', + name: 'createdBy', + objectMetadataId: 'person-id', + }, }, }, + flatIndexMaps: { + byId: {}, + }, }), }, }, diff --git a/packages/twenty-server/src/engine/core-modules/actor/services/created-by-from-auth-context.service.ts b/packages/twenty-server/src/engine/core-modules/actor/services/created-by-from-auth-context.service.ts index 01ac1c171b2..e7e23715fce 100644 --- a/packages/twenty-server/src/engine/core-modules/actor/services/created-by-from-auth-context.service.ts +++ b/packages/twenty-server/src/engine/core-modules/actor/services/created-by-from-auth-context.service.ts @@ -1,17 +1,15 @@ import { Injectable, Logger } from '@nestjs/common'; -import { InjectRepository } from '@nestjs/typeorm'; -import { assertIsDefinedOrThrow, isDefined } from 'twenty-shared/utils'; -import { Repository } from 'typeorm'; import { type ActorMetadata } from 'twenty-shared/types'; +import { assertIsDefinedOrThrow, isDefined } from 'twenty-shared/utils'; import { buildCreatedByFromApiKey } from 'src/engine/core-modules/actor/utils/build-created-by-from-api-key.util'; import { buildCreatedByFromFullNameMetadata } from 'src/engine/core-modules/actor/utils/build-created-by-from-full-name-metadata.util'; import { type AuthContext } from 'src/engine/core-modules/auth/types/auth-context.type'; import { WorkspaceNotFoundDefaultError } from 'src/engine/core-modules/workspace/workspace.exception'; -import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; -import { getObjectMetadataMapItemByNameSingular } from 'src/engine/metadata-modules/utils/get-object-metadata-map-item-by-name-singular.util'; -import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service'; +import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service'; +import { buildFieldMapsForObject } from 'src/engine/metadata-modules/flat-field-metadata/utils/build-field-maps-for-object.util'; +import { buildObjectIdByNameMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-id-by-name-maps.util'; import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; import { WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity'; @@ -23,10 +21,8 @@ export class CreatedByFromAuthContextService { private readonly logger = new Logger(CreatedByFromAuthContextService.name); constructor( - @InjectRepository(FieldMetadataEntity) - private readonly fieldMetadataRepository: Repository, private readonly twentyORMGlobalManager: TwentyORMGlobalManager, - private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService, + private readonly flatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService, ) {} async injectCreatedBy( @@ -38,10 +34,11 @@ export class CreatedByFromAuthContextService { assertIsDefinedOrThrow(workspace, WorkspaceNotFoundDefaultError); - const { objectMetadataMaps } = - await this.workspaceMetadataCacheService.getExistingOrRecomputeMetadataMaps( + const { flatObjectMetadataMaps, flatFieldMetadataMaps } = + await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId: workspace.id, + flatMapsKeys: ['flatObjectMetadataMaps', 'flatFieldMetadataMaps'], }, ); @@ -49,16 +46,24 @@ export class CreatedByFromAuthContextService { `Injecting createdBy from auth context for object ${objectMetadataNameSingular} and workspace ${workspace.id}`, ); - const objectMetadata = getObjectMetadataMapItemByNameSingular( - objectMetadataMaps, - objectMetadataNameSingular, + const { idByNameSingular } = buildObjectIdByNameMaps( + flatObjectMetadataMaps, ); + const objectId = idByNameSingular[objectMetadataNameSingular]; + const objectMetadata = objectId + ? flatObjectMetadataMaps.byId[objectId] + : undefined; + + const fieldIdByName = objectMetadata + ? buildFieldMapsForObject(flatFieldMetadataMaps, objectMetadata.id) + .fieldIdByName + : {}; this.logger.log( - `Object metadata found with fields: ${Object.keys(objectMetadata?.fieldIdByName ?? {})}`, + `Object metadata found with fields: ${Object.keys(fieldIdByName)}`, ); - if (!isDefined(objectMetadata?.fieldIdByName['createdBy'])) { + if (!isDefined(fieldIdByName['createdBy'])) { this.logger.log( `CreatedBy field not found in object metadata, skipping injection`, ); diff --git a/packages/twenty-server/src/engine/core-modules/search/search.module.ts b/packages/twenty-server/src/engine/core-modules/search/search.module.ts index 8f14cdd26e4..2c8aa75407c 100644 --- a/packages/twenty-server/src/engine/core-modules/search/search.module.ts +++ b/packages/twenty-server/src/engine/core-modules/search/search.module.ts @@ -3,10 +3,10 @@ import { Module } from '@nestjs/common'; import { FileModule } from 'src/engine/core-modules/file/file.module'; import { SearchResolver } from 'src/engine/core-modules/search/search.resolver'; import { SearchService } from 'src/engine/core-modules/search/services/search.service'; -import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; +import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module'; @Module({ - imports: [FileModule, WorkspaceCacheStorageModule], + imports: [FileModule, WorkspaceManyOrAllFlatEntityMapsCacheModule], providers: [SearchResolver, SearchService], }) export class SearchModule {} diff --git a/packages/twenty-server/src/engine/core-modules/search/search.resolver.ts b/packages/twenty-server/src/engine/core-modules/search/search.resolver.ts index b9ed43df212..2ec10e1211f 100644 --- a/packages/twenty-server/src/engine/core-modules/search/search.resolver.ts +++ b/packages/twenty-server/src/engine/core-modules/search/search.resolver.ts @@ -13,7 +13,10 @@ import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.ent import { AuthWorkspace } from 'src/engine/decorators/auth/auth-workspace.decorator'; import { CustomPermissionGuard } from 'src/engine/guards/custom-permission.guard'; import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard'; -import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; +import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service'; +import { buildObjectIdByNameMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-id-by-name-maps.util'; +import { buildObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-metadata-item-with-field-maps.util'; +import { type ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; @Resolver() @UseFilters(SearchApiExceptionFilter, PreventNestToAutoLogGraphqlErrorsFilter) @@ -22,7 +25,7 @@ import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage export class SearchResolver { constructor( private readonly searchService: SearchService, - private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, + private readonly workspaceManyOrAllFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService, ) {} @Query(() => SearchResultConnectionDTO) @@ -38,11 +41,36 @@ export class SearchResolver { after, }: SearchArgs, ) { - const objectMetadataMaps = - await this.workspaceCacheStorageService.getObjectMetadataMapsOrThrow( - workspace.id, + const { flatObjectMetadataMaps, flatFieldMetadataMaps, flatIndexMaps } = + await this.workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( + { + workspaceId: workspace.id, + flatMapsKeys: [ + 'flatObjectMetadataMaps', + 'flatFieldMetadataMaps', + 'flatIndexMaps', + ], + }, ); + const { idByNameSingular } = buildObjectIdByNameMaps( + flatObjectMetadataMaps, + ); + const objectMetadataMaps: ObjectMetadataMaps = { + byId: {}, + idByNameSingular, + }; + + for (const [id, flatObj] of Object.entries(flatObjectMetadataMaps.byId)) { + if (isDefined(flatObj)) { + objectMetadataMaps.byId[id] = buildObjectMetadataItemWithFieldMaps( + flatObj, + flatFieldMetadataMaps, + flatIndexMaps, + ); + } + } + const filteredObjectMetadataItems = this.searchService.filterObjectMetadataItems({ objectMetadataItemWithFieldMaps: Object.values( diff --git a/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts b/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts index 2139ec97616..fcb3432f03c 100644 --- a/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts +++ b/packages/twenty-server/src/engine/core-modules/workspace/workspace.module.ts @@ -35,7 +35,6 @@ import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata import { PermissionsModule } from 'src/engine/metadata-modules/permissions/permissions.module'; import { RoleModule } from 'src/engine/metadata-modules/role/role.module'; import { ViewModule } from 'src/engine/metadata-modules/view/view.module'; -import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-manager.module'; @@ -50,7 +49,6 @@ import { WorkspaceManagerModule } from 'src/engine/workspace-manager/workspace-m FileModule, TokenModule, FileUploadModule, - WorkspaceMetadataCacheModule, NestjsQueryTypeOrmModule.forFeature([ UserEntity, WorkspaceEntity, diff --git a/packages/twenty-server/src/engine/dataloaders/dataloader.interface.ts b/packages/twenty-server/src/engine/dataloaders/dataloader.interface.ts index 3f9c3acde2d..2fb1ff98f2c 100644 --- a/packages/twenty-server/src/engine/dataloaders/dataloader.interface.ts +++ b/packages/twenty-server/src/engine/dataloaders/dataloader.interface.ts @@ -12,7 +12,7 @@ import { type FieldMetadataDTO } from 'src/engine/metadata-modules/field-metadat import { type RelationDTO } from 'src/engine/metadata-modules/field-metadata/dtos/relation.dto'; import { type IndexFieldMetadataDTO } from 'src/engine/metadata-modules/index-metadata/dtos/index-field-metadata.dto'; import { type IndexMetadataDTO } from 'src/engine/metadata-modules/index-metadata/dtos/index-metadata.dto'; -import { type ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps'; +import { type ObjectMetadataDTO } from 'src/engine/metadata-modules/object-metadata/dtos/object-metadata.dto'; export interface IDataloaders { relationLoader: DataLoader; @@ -39,6 +39,6 @@ export interface IDataloaders { objectMetadataLoader: DataLoader< ObjectMetadataLoaderPayload, - ObjectMetadataItemWithFieldMaps | null + ObjectMetadataDTO | null >; } diff --git a/packages/twenty-server/src/engine/dataloaders/dataloader.module.ts b/packages/twenty-server/src/engine/dataloaders/dataloader.module.ts index a968b5640bc..bf28db69508 100644 --- a/packages/twenty-server/src/engine/dataloaders/dataloader.module.ts +++ b/packages/twenty-server/src/engine/dataloaders/dataloader.module.ts @@ -3,14 +3,9 @@ import { Module } from '@nestjs/common'; import { DataloaderService } from 'src/engine/dataloaders/dataloader.service'; import { FieldMetadataModule } from 'src/engine/metadata-modules/field-metadata/field-metadata.module'; import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module'; -import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; @Module({ - imports: [ - FieldMetadataModule, - WorkspaceMetadataCacheModule, - WorkspaceManyOrAllFlatEntityMapsCacheModule, - ], + imports: [FieldMetadataModule, WorkspaceManyOrAllFlatEntityMapsCacheModule], providers: [DataloaderService], exports: [DataloaderService], }) diff --git a/packages/twenty-server/src/engine/dataloaders/dataloader.service.ts b/packages/twenty-server/src/engine/dataloaders/dataloader.service.ts index 1ce3177958f..f2ada927b1f 100644 --- a/packages/twenty-server/src/engine/dataloaders/dataloader.service.ts +++ b/packages/twenty-server/src/engine/dataloaders/dataloader.service.ts @@ -23,12 +23,12 @@ import { fromFlatFieldMetadataToFieldMetadataDto } from 'src/engine/metadata-mod import { fromMorphOrRelationFlatFieldMetadataToRelationDto } from 'src/engine/metadata-modules/flat-field-metadata/utils/from-morph-or-relation-flat-field-metadata-to-relation-dto.util'; import { isFlatFieldMetadataOfType } from 'src/engine/metadata-modules/flat-field-metadata/utils/is-flat-field-metadata-of-type.util'; import { isMorphOrRelationFlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/utils/is-morph-or-relation-flat-field-metadata.util'; +import { fromFlatObjectMetadataToObjectMetadataDto } from 'src/engine/metadata-modules/flat-object-metadata/utils/from-flat-object-metadata-to-object-metadata-dto.util'; import { getMorphNameFromMorphFieldMetadataName } from 'src/engine/metadata-modules/flat-object-metadata/utils/get-morph-name-from-morph-field-metadata-name.util'; import { type IndexFieldMetadataDTO } from 'src/engine/metadata-modules/index-metadata/dtos/index-field-metadata.dto'; import { type IndexMetadataDTO } from 'src/engine/metadata-modules/index-metadata/dtos/index-metadata.dto'; +import { ObjectMetadataDTO } from 'src/engine/metadata-modules/object-metadata/dtos/object-metadata.dto'; import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; -import { type ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps'; -import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service'; export type RelationMetadataLoaderPayload = { workspaceId: string; @@ -74,7 +74,6 @@ export class DataloaderService { constructor( private readonly i18nService: I18nService, private readonly flatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService, - private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService, ) {} createLoaders(): IDataloaders { @@ -302,32 +301,40 @@ export class DataloaderService { (dataLoaderParam) => dataLoaderParam.objectMetadata.id, ); - const { objectMetadataMaps } = - await this.workspaceMetadataCacheService.getExistingOrRecomputeMetadataMaps( - { workspaceId }, - ); - - const indexMetadataCollection = objectMetadataIds.map((id) => { - const objectMetadata = objectMetadataMaps.byId[id]; - - if (!isDefined(objectMetadata)) { - return []; - } - - return Object.values(objectMetadata.indexMetadatas).map( - (indexMetadata) => { - return { - ...indexMetadata, - createdAt: new Date(indexMetadata.createdAt), - updatedAt: new Date(indexMetadata.updatedAt), - id: indexMetadata.id, - indexWhereClause: indexMetadata.indexWhereClause ?? undefined, - objectMetadataId: id, - workspaceId: workspaceId, - }; + const { flatIndexMaps, flatObjectMetadataMaps } = + await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( + { + workspaceId, + flatMapsKeys: ['flatIndexMaps', 'flatObjectMetadataMaps'], }, ); - }); + + const indexMetadataCollection = objectMetadataIds.map( + (objectMetadataId) => { + const flatObjectMetadata = + findFlatEntityByIdInFlatEntityMapsOrThrow({ + flatEntityId: objectMetadataId, + flatEntityMaps: flatObjectMetadataMaps, + }); + + const indexMetadatas = + findManyFlatEntityByIdInFlatEntityMapsOrThrow({ + flatEntityIds: flatObjectMetadata.indexMetadataIds, + flatEntityMaps: flatIndexMaps, + }); + + return indexMetadatas.map((indexMetadata) => ({ + ...indexMetadata, + indexFieldMetadatas: indexMetadata.flatIndexFieldMetadatas, + createdAt: new Date(indexMetadata.createdAt), + updatedAt: new Date(indexMetadata.updatedAt), + id: indexMetadata.id, + indexWhereClause: indexMetadata.indexWhereClause ?? undefined, + objectMetadataId, + workspaceId, + })); + }, + ); return indexMetadataCollection; }, @@ -363,8 +370,8 @@ export class DataloaderService { }); const objectFlatFieldMetadatas = findManyFlatEntityByIdInFlatEntityMapsOrThrow({ - flatEntityMaps: flatFieldMetadataMaps, flatEntityIds: flatObjectMetadata.fieldMetadataIds, + flatEntityMaps: flatFieldMetadataMaps, }); const overriddenFieldMetadataEntities = @@ -441,31 +448,23 @@ export class DataloaderService { >(async (dataLoaderParams: IndexFieldMetadataLoaderPayload[]) => { const workspaceId = dataLoaderParams[0].workspaceId; - const { objectMetadataMaps } = - await this.workspaceMetadataCacheService.getExistingOrRecomputeMetadataMaps( - { workspaceId }, + const { flatIndexMaps } = + await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( + { + workspaceId, + flatMapsKeys: ['flatIndexMaps'], + }, ); return dataLoaderParams.map( - ({ - objectMetadata: { id: objectMetadataId }, - indexMetadata: { id: indexMetadataId }, - }) => { - const objectMetadata = objectMetadataMaps.byId[objectMetadataId]; - - if (!isDefined(objectMetadata)) { - return []; - } - - const indexMetadataEntity = objectMetadata.indexMetadatas.find( - (indexMetadata) => indexMetadata.id === indexMetadataId, - ); + ({ indexMetadata: { id: indexMetadataId } }) => { + const indexMetadataEntity = flatIndexMaps.byId[indexMetadataId]; if (!isDefined(indexMetadataEntity)) { return []; } - return indexMetadataEntity.indexFieldMetadatas.map( + return indexMetadataEntity.flatIndexFieldMetadatas.map( (indexFieldMetadata) => { return { id: indexFieldMetadata.id, @@ -486,20 +485,27 @@ export class DataloaderService { private createObjectMetadataLoader() { return new DataLoader< ObjectMetadataLoaderPayload, - ObjectMetadataItemWithFieldMaps | null + ObjectMetadataDTO | null >(async (dataLoaderParams: ObjectMetadataLoaderPayload[]) => { const workspaceId = dataLoaderParams[0].workspaceId; - const objectMetadataIds = dataLoaderParams.map( - (dataLoaderParam) => dataLoaderParam.objectMetadataId, - ); - const { objectMetadataMaps } = - await this.workspaceMetadataCacheService.getExistingOrRecomputeMetadataMaps( - { workspaceId }, + const { flatObjectMetadataMaps } = + await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( + { + workspaceId, + flatMapsKeys: ['flatObjectMetadataMaps'], + }, ); - return objectMetadataIds.map((objectMetadataId) => { - return objectMetadataMaps.byId[objectMetadataId] || null; + return dataLoaderParams.map((dataLoaderParam) => { + const flatObjectMetadata = + flatObjectMetadataMaps.byId[dataLoaderParam.objectMetadataId]; + + if (!isDefined(flatObjectMetadata)) { + return null; + } + + return fromFlatObjectMetadataToObjectMetadataDto(flatObjectMetadata); }); }); } diff --git a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.module.ts b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.module.ts index 4b7bc6194c9..47b50a5d9ad 100644 --- a/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.module.ts +++ b/packages/twenty-server/src/engine/metadata-modules/field-metadata/field-metadata.module.ts @@ -33,7 +33,6 @@ import { ViewFieldModule } from 'src/engine/metadata-modules/view-field/view-fie import { ViewFilterModule } from 'src/engine/metadata-modules/view-filter/view-filter.module'; import { ViewGroupModule } from 'src/engine/metadata-modules/view-group/view-group.module'; import { ViewModule } from 'src/engine/metadata-modules/view/view.module'; -import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module'; import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; @@ -68,7 +67,6 @@ import { UpdateFieldInput } from './dtos/update-field.input'; ViewFilterModule, ViewGroupModule, PermissionsModule, - WorkspaceMetadataCacheModule, WorkspaceMigrationV2Module, FlatFieldMetadataModule, IndexMetadataModule, diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/build-field-maps-for-object.util.ts b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/build-field-maps-for-object.util.ts new file mode 100644 index 00000000000..9962cdde57e --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/flat-field-metadata/utils/build-field-maps-for-object.util.ts @@ -0,0 +1,45 @@ +import { isDefined } from 'twenty-shared/utils'; + +import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type'; +import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type'; +import { isMorphOrRelationFlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/utils/is-morph-or-relation-flat-field-metadata.util'; + +export type FieldMapsForObject = { + fieldIdByName: Record; + fieldIdByJoinColumnName: Record; +}; + +export const buildFieldMapsForObject = ( + flatFieldMetadataMaps: FlatEntityMaps, + objectMetadataId: string, +): FieldMapsForObject => { + const fieldIdByName: Record = {}; + const fieldIdByJoinColumnName: Record = {}; + + for (const [fieldId, field] of Object.entries(flatFieldMetadataMaps.byId)) { + if (!isDefined(field)) { + continue; + } + + if (field.objectMetadataId !== objectMetadataId) { + continue; + } + + fieldIdByName[field.name] = fieldId; + + if (isMorphOrRelationFlatFieldMetadata(field)) { + const joinColumnName = ( + field.settings as { joinColumnName?: string } | undefined + )?.joinColumnName; + + if (joinColumnName) { + fieldIdByJoinColumnName[joinColumnName] = fieldId; + } + } + } + + return { + fieldIdByName, + fieldIdByJoinColumnName, + }; +}; diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-object-metadata/utils/build-object-id-by-name-maps.util.ts b/packages/twenty-server/src/engine/metadata-modules/flat-object-metadata/utils/build-object-id-by-name-maps.util.ts new file mode 100644 index 00000000000..e4d30b47293 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/flat-object-metadata/utils/build-object-id-by-name-maps.util.ts @@ -0,0 +1,27 @@ +import { isDefined } from 'twenty-shared/utils'; + +import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type'; +import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type'; + +export const buildObjectIdByNameMaps = ( + flatObjectMetadataMaps: FlatEntityMaps, +): { + idByNameSingular: Record; + idByNamePlural: Record; +} => { + const idByNameSingular: Record = {}; + const idByNamePlural: Record = {}; + + for (const [objectId, objectMetadata] of Object.entries( + flatObjectMetadataMaps.byId, + )) { + if (!isDefined(objectMetadata)) { + continue; + } + + idByNameSingular[objectMetadata.nameSingular] = objectId; + idByNamePlural[objectMetadata.namePlural] = objectId; + } + + return { idByNameSingular, idByNamePlural }; +}; diff --git a/packages/twenty-server/src/engine/metadata-modules/flat-object-metadata/utils/build-object-metadata-item-with-field-maps.util.ts b/packages/twenty-server/src/engine/metadata-modules/flat-object-metadata/utils/build-object-metadata-item-with-field-maps.util.ts new file mode 100644 index 00000000000..fc6dfd7ca92 --- /dev/null +++ b/packages/twenty-server/src/engine/metadata-modules/flat-object-metadata/utils/build-object-metadata-item-with-field-maps.util.ts @@ -0,0 +1,45 @@ +import { isDefined } from 'twenty-shared/utils'; + +import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type'; +import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type'; +import { buildFieldMapsForObject } from 'src/engine/metadata-modules/flat-field-metadata/utils/build-field-maps-for-object.util'; +import { type FlatIndexMetadata } from 'src/engine/metadata-modules/flat-index-metadata/types/flat-index-metadata.type'; +import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type'; +import { type ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps'; + +export const buildObjectMetadataItemWithFieldMaps = ( + flatObjectMetadata: FlatObjectMetadata, + flatFieldMetadataMaps: FlatEntityMaps, + flatIndexMaps: FlatEntityMaps, +): ObjectMetadataItemWithFieldMaps => { + const fieldsById: Record = {}; + + for (const fieldId of flatObjectMetadata.fieldMetadataIds) { + const field = flatFieldMetadataMaps.byId[fieldId]; + + if (isDefined(field)) { + fieldsById[fieldId] = field; + } + } + + const { fieldIdByName, fieldIdByJoinColumnName } = buildFieldMapsForObject( + flatFieldMetadataMaps, + flatObjectMetadata.id, + ); + + const indexMetadatas = flatObjectMetadata.indexMetadataIds + .map((indexId) => flatIndexMaps.byId[indexId]) + .filter(isDefined) + .map((flatIndex) => ({ + ...flatIndex, + indexFieldMetadatas: flatIndex.flatIndexFieldMetadatas, + })); + + return { + ...flatObjectMetadata, + fieldsById, + fieldIdByName, + fieldIdByJoinColumnName, + indexMetadatas, + } as unknown as ObjectMetadataItemWithFieldMaps; +}; diff --git a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts index f3c48b2d6b4..4cb1c2514c8 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-metadata/object-metadata.module.ts @@ -31,7 +31,6 @@ import { RemoteTableRelationsModule } from 'src/engine/metadata-modules/remote-s import { ViewFieldModule } from 'src/engine/metadata-modules/view-field/view-field.module'; import { ViewEntity } from 'src/engine/metadata-modules/view/entities/view.entity'; import { ViewModule } from 'src/engine/metadata-modules/view/view.module'; -import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { WorkspaceMigrationModule } from 'src/engine/metadata-modules/workspace-migration/workspace-migration.module'; import { WorkspacePermissionsCacheModule } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.module'; @@ -61,7 +60,6 @@ import { WorkspaceMigrationV2Module } from 'src/engine/workspace-manager/workspa PermissionsModule, WorkspacePermissionsCacheModule, WorkspaceCacheStorageModule, - WorkspaceMetadataCacheModule, WorkspaceDataSourceModule, FeatureFlagModule, WorkspaceMigrationV2Module, diff --git a/packages/twenty-server/src/engine/metadata-modules/object-permission/__tests__/object-permission.service.spec.ts b/packages/twenty-server/src/engine/metadata-modules/object-permission/__tests__/object-permission.service.spec.ts index e2ec516fb6f..e4346e7b322 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-permission/__tests__/object-permission.service.spec.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-permission/__tests__/object-permission.service.spec.ts @@ -3,6 +3,7 @@ import { getRepositoryToken } from '@nestjs/typeorm'; import { type Repository } from 'typeorm'; +import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { type UpsertObjectPermissionsInput } from 'src/engine/metadata-modules/object-permission/dtos/upsert-object-permissions.input'; import { ObjectPermissionEntity } from 'src/engine/metadata-modules/object-permission/object-permission.entity'; @@ -13,9 +14,7 @@ import { PermissionsExceptionMessage, } from 'src/engine/metadata-modules/permissions/permissions.exception'; import { RoleEntity } from 'src/engine/metadata-modules/role/role.entity'; -import { type ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps'; import { WorkspacePermissionsCacheService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.service'; -import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; describe('ObjectPermissionService', () => { let service: ObjectPermissionService; @@ -24,7 +23,7 @@ describe('ObjectPermissionService', () => { >; let roleRepository: jest.Mocked>; let workspacePermissionsCacheService: jest.Mocked; - let workspaceCacheStorageService: jest.Mocked; + let workspaceManyOrAllFlatEntityMapsCacheService: jest.Mocked; beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ @@ -56,9 +55,9 @@ describe('ObjectPermissionService', () => { }, }, { - provide: WorkspaceCacheStorageService, + provide: WorkspaceManyOrAllFlatEntityMapsCacheService, useValue: { - getObjectMetadataMapsOrThrow: jest.fn(), + getOrRecomputeManyOrAllFlatEntityMaps: jest.fn(), }, }, ], @@ -72,7 +71,9 @@ describe('ObjectPermissionService', () => { workspacePermissionsCacheService = module.get( WorkspacePermissionsCacheService, ); - workspaceCacheStorageService = module.get(WorkspaceCacheStorageService); + workspaceManyOrAllFlatEntityMapsCacheService = module.get( + WorkspaceManyOrAllFlatEntityMapsCacheService, + ); }); describe('upsertObjectPermissions', () => { @@ -106,18 +107,26 @@ describe('ObjectPermissionService', () => { ], }; - // Mock object metadata maps with a system object - workspaceCacheStorageService.getObjectMetadataMapsOrThrow.mockResolvedValue( + // Mock flat object metadata maps with a system object + workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps.mockResolvedValue( { - byId: { - [systemObjectMetadataId]: { - id: systemObjectMetadataId, - isSystem: true, - workspaceId, - } as ObjectMetadataItemWithFieldMaps, + flatObjectMetadataMaps: { + byId: { + [systemObjectMetadataId]: { + id: systemObjectMetadataId, + isSystem: true, + workspaceId, + fieldMetadataIds: [], + indexMetadataIds: [], + viewIds: [], + universalIdentifier: systemObjectMetadataId, + applicationId: null, + } as any, + }, + idByUniversalIdentifier: {}, + universalIdentifiersByApplicationId: {}, }, - idByNameSingular: {}, - }, + } as any, ); // Act & Assert @@ -155,18 +164,26 @@ describe('ObjectPermissionService', () => { ], }; - // Mock object metadata maps with a custom object - workspaceCacheStorageService.getObjectMetadataMapsOrThrow.mockResolvedValue( + // Mock flat object metadata maps with a custom object + workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps.mockResolvedValue( { - byId: { - [customObjectMetadataId]: { - id: customObjectMetadataId, - isSystem: false, - workspaceId, - } as ObjectMetadataItemWithFieldMaps, + flatObjectMetadataMaps: { + byId: { + [customObjectMetadataId]: { + id: customObjectMetadataId, + isSystem: false, + workspaceId, + fieldMetadataIds: [], + indexMetadataIds: [], + viewIds: [], + universalIdentifier: customObjectMetadataId, + applicationId: null, + } as any, + }, + idByUniversalIdentifier: {}, + universalIdentifiersByApplicationId: {}, }, - idByNameSingular: {}, - }, + } as any, ); // Mock successful upsert @@ -236,12 +253,15 @@ describe('ObjectPermissionService', () => { ], }; - // Mock empty object metadata maps - workspaceCacheStorageService.getObjectMetadataMapsOrThrow.mockResolvedValue( + // Mock empty flat object metadata maps + workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps.mockResolvedValue( { - byId: {}, - idByNameSingular: {}, - }, + flatObjectMetadataMaps: { + byId: {}, + idByUniversalIdentifier: {}, + universalIdentifiersByApplicationId: {}, + }, + } as any, ); // Act & Assert diff --git a/packages/twenty-server/src/engine/metadata-modules/object-permission/field-permission/__tests__/field-permissions.service.spec.ts b/packages/twenty-server/src/engine/metadata-modules/object-permission/field-permission/__tests__/field-permissions.service.spec.ts index 2ad09c459f3..3d15684fba4 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-permission/field-permission/__tests__/field-permissions.service.spec.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-permission/field-permission/__tests__/field-permissions.service.spec.ts @@ -10,6 +10,7 @@ import { objectMetadataItemMock, } from 'src/engine/api/__mocks__/object-metadata-item.mock'; import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service'; import { type UpsertFieldPermissionsInput } from 'src/engine/metadata-modules/object-permission/dtos/upsert-field-permissions.input'; import { FieldPermissionEntity } from 'src/engine/metadata-modules/object-permission/field-permission/field-permission.entity'; import { FieldPermissionService } from 'src/engine/metadata-modules/object-permission/field-permission/field-permission.service'; @@ -20,7 +21,6 @@ import { } from 'src/engine/metadata-modules/permissions/permissions.exception'; import { RoleEntity } from 'src/engine/metadata-modules/role/role.entity'; import { WorkspacePermissionsCacheService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.service'; -import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; import { getMockFieldMetadataEntity } from 'src/utils/__test__/get-field-metadata-entity.mock'; describe('FieldPermissionService', () => { @@ -31,7 +31,7 @@ describe('FieldPermissionService', () => { let roleRepository: jest.Mocked>; let fieldMetadataRepository: jest.Mocked>; let workspacePermissionsCacheService: jest.Mocked; - let workspaceCacheStorageService: jest.Mocked; + let workspaceManyOrAllFlatEntityMapsCacheService: jest.Mocked; const testWorkspaceId = '20202020-0000-0000-0000-000000000000'; const testRoleId = '20202020-0000-0000-0000-000000000001'; @@ -98,9 +98,9 @@ describe('FieldPermissionService', () => { }, }, { - provide: WorkspaceCacheStorageService, + provide: WorkspaceManyOrAllFlatEntityMapsCacheService, useValue: { - getObjectMetadataMapsOrThrow: jest.fn(), + getOrRecomputeManyOrAllFlatEntityMaps: jest.fn(), }, }, { @@ -123,7 +123,9 @@ describe('FieldPermissionService', () => { workspacePermissionsCacheService = module.get( WorkspacePermissionsCacheService, ); - workspaceCacheStorageService = module.get(WorkspaceCacheStorageService); + workspaceManyOrAllFlatEntityMapsCacheService = module.get( + WorkspaceManyOrAllFlatEntityMapsCacheService, + ); // Setup default mocks roleRepository.findOne.mockResolvedValue(mockRole); @@ -137,39 +139,49 @@ describe('FieldPermissionService', () => { data: mockRolesPermissions, }, ); - workspaceCacheStorageService.getObjectMetadataMapsOrThrow.mockResolvedValue( + const testFieldMetadata = getMockFieldMetadataEntity({ + ...fieldTextMock, + label: 'Test Field', + objectMetadataId: testObjectMetadataId, + workspaceId: testWorkspaceId, + id: testFieldMetadataId, + }); + + workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps.mockResolvedValue( { - byId: { - [testObjectMetadataId]: { - ...objectMetadataItemMock, - fieldsById: { - [fieldTextMock.id]: getMockFieldMetadataEntity({ - ...fieldTextMock, - label: 'Test Field', - objectMetadataId: testObjectMetadataId, - workspaceId: testWorkspaceId, - id: '20202020-0000-0000-0000-000000000003', - }), - }, - fieldIdByJoinColumnName: {}, - fieldIdByName: {}, - indexMetadatas: [], + flatObjectMetadataMaps: { + byId: { + [testObjectMetadataId]: { + ...objectMetadataItemMock, + id: testObjectMetadataId, + fieldMetadataIds: [testFieldMetadataId], + indexMetadataIds: [], + viewIds: [], + universalIdentifier: testObjectMetadataId, + applicationId: null, + } as any, + [fieldRelationMock.objectMetadataId]: { + ...objectMetadataItemMock, + id: fieldRelationMock.objectMetadataId, + fieldMetadataIds: [fieldRelationMock.id], + indexMetadataIds: [], + viewIds: [], + universalIdentifier: fieldRelationMock.objectMetadataId, + applicationId: null, + } as any, }, - [fieldRelationMock.objectMetadataId]: { - ...objectMetadataItemMock, - id: fieldRelationMock.objectMetadataId, - fieldsById: { - [fieldRelationMock.id]: fieldRelationMock, - }, - fieldIdByJoinColumnName: {}, - fieldIdByName: {}, - indexMetadatas: [], + idByUniversalIdentifier: {}, + universalIdentifiersByApplicationId: {}, + }, + flatFieldMetadataMaps: { + byId: { + [testFieldMetadataId]: testFieldMetadata as any, + [fieldRelationMock.id]: fieldRelationMock as any, }, + idByUniversalIdentifier: {}, + universalIdentifiersByApplicationId: {}, }, - idByNameSingular: { - testObject: testObjectMetadataId, - }, - }, + } as any, ); fieldPermissionsRepository.find.mockResolvedValue([]); fieldPermissionsRepository.upsert.mockResolvedValue({} as any); @@ -386,11 +398,19 @@ describe('FieldPermissionService', () => { }); it('should throw error when object metadata is not found', async () => { - workspaceCacheStorageService.getObjectMetadataMapsOrThrow.mockResolvedValue( + workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps.mockResolvedValue( { - byId: {}, - idByNameSingular: {}, - }, + flatObjectMetadataMaps: { + byId: {}, + idByUniversalIdentifier: {}, + universalIdentifiersByApplicationId: {}, + }, + flatFieldMetadataMaps: { + byId: {}, + idByUniversalIdentifier: {}, + universalIdentifiersByApplicationId: {}, + }, + } as any, ); const input = createUpsertInput([ @@ -420,21 +440,29 @@ describe('FieldPermissionService', () => { isSystem: true, }; - workspaceCacheStorageService.getObjectMetadataMapsOrThrow.mockResolvedValue( + workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps.mockResolvedValue( { - byId: { - [testObjectMetadataId]: { - ...systemObjectMetadata, - fieldsById: {}, - fieldIdByJoinColumnName: {}, - fieldIdByName: {}, - indexMetadatas: [], + flatObjectMetadataMaps: { + byId: { + [testObjectMetadataId]: { + ...systemObjectMetadata, + id: testObjectMetadataId, + fieldMetadataIds: [], + indexMetadataIds: [], + viewIds: [], + universalIdentifier: testObjectMetadataId, + applicationId: null, + } as any, }, + idByUniversalIdentifier: {}, + universalIdentifiersByApplicationId: {}, }, - idByNameSingular: { - testObject: testObjectMetadataId, + flatFieldMetadataMaps: { + byId: {}, + idByUniversalIdentifier: {}, + universalIdentifiersByApplicationId: {}, }, - }, + } as any, ); const input = createUpsertInput([ @@ -458,23 +486,29 @@ describe('FieldPermissionService', () => { }); it('should throw error when field metadata is not found', async () => { - const objectMetadataWithoutField = { - ...objectMetadataItemMock, - fieldsById: {}, - fieldIdByJoinColumnName: {}, - fieldIdByName: {}, - indexMetadatas: [], - }; - - workspaceCacheStorageService.getObjectMetadataMapsOrThrow.mockResolvedValue( + workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps.mockResolvedValue( { - byId: { - [testObjectMetadataId]: objectMetadataWithoutField, + flatObjectMetadataMaps: { + byId: { + [testObjectMetadataId]: { + ...objectMetadataItemMock, + id: testObjectMetadataId, + fieldMetadataIds: [], + indexMetadataIds: [], + viewIds: [], + universalIdentifier: testObjectMetadataId, + applicationId: null, + } as any, + }, + idByUniversalIdentifier: {}, + universalIdentifiersByApplicationId: {}, }, - idByNameSingular: { - testObject: testObjectMetadataId, + flatFieldMetadataMaps: { + byId: {}, + idByUniversalIdentifier: {}, + universalIdentifiersByApplicationId: {}, }, - }, + } as any, ); const input = createUpsertInput([ diff --git a/packages/twenty-server/src/engine/metadata-modules/object-permission/field-permission/field-permission.service.ts b/packages/twenty-server/src/engine/metadata-modules/object-permission/field-permission/field-permission.service.ts index 0049616339d..df02f811a3e 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-permission/field-permission/field-permission.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-permission/field-permission/field-permission.service.ts @@ -14,6 +14,9 @@ import { } from 'src/engine/core-modules/graphql/utils/graphql-errors.util'; import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; import { isFieldMetadataTypeRelation } from 'src/engine/metadata-modules/field-metadata/utils/is-field-metadata-type-relation.util'; +import { createEmptyFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/constant/create-empty-flat-entity-maps.constant'; +import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service'; +import { buildObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-metadata-item-with-field-maps.util'; import { type UpsertFieldPermissionsInput } from 'src/engine/metadata-modules/object-permission/dtos/upsert-field-permissions.input'; import { FieldPermissionEntity } from 'src/engine/metadata-modules/object-permission/field-permission/field-permission.entity'; import { @@ -24,7 +27,6 @@ import { import { RoleEntity } from 'src/engine/metadata-modules/role/role.entity'; import { type ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; import { WorkspacePermissionsCacheService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.service'; -import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; @Injectable() export class FieldPermissionService { @@ -36,7 +38,7 @@ export class FieldPermissionService { @InjectRepository(FieldPermissionEntity) private readonly fieldPermissionsRepository: Repository, private readonly workspacePermissionsCacheService: WorkspacePermissionsCacheService, - private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, + private readonly workspaceManyOrAllFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService, ) {} public async upsertFieldPermissions({ @@ -60,11 +62,26 @@ export class FieldPermissionService { role, }); - const { byId: objectMetadataMapsById } = - await this.workspaceCacheStorageService.getObjectMetadataMapsOrThrow( - workspaceId, + const { flatObjectMetadataMaps, flatFieldMetadataMaps } = + await this.workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( + { + workspaceId, + flatMapsKeys: ['flatObjectMetadataMaps', 'flatFieldMetadataMaps'], + }, ); + const objectMetadataMapsById: ObjectMetadataMaps['byId'] = {}; + + for (const [id, flatObj] of Object.entries(flatObjectMetadataMaps.byId)) { + if (isDefined(flatObj)) { + objectMetadataMapsById[id] = buildObjectMetadataItemWithFieldMaps( + flatObj, + flatFieldMetadataMaps, + createEmptyFlatEntityMaps(), + ); + } + } + const existingFieldPermissions = await this.fieldPermissionsRepository.find( { where: { diff --git a/packages/twenty-server/src/engine/metadata-modules/object-permission/object-permission.module.ts b/packages/twenty-server/src/engine/metadata-modules/object-permission/object-permission.module.ts index 75a4b8018ea..78df3012cde 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-permission/object-permission.module.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-permission/object-permission.module.ts @@ -2,6 +2,7 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; import { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity'; +import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { FieldPermissionEntity } from 'src/engine/metadata-modules/object-permission/field-permission/field-permission.entity'; import { FieldPermissionService } from 'src/engine/metadata-modules/object-permission/field-permission/field-permission.service'; @@ -9,7 +10,6 @@ import { ObjectPermissionEntity } from 'src/engine/metadata-modules/object-permi import { ObjectPermissionService } from 'src/engine/metadata-modules/object-permission/object-permission.service'; import { RoleEntity } from 'src/engine/metadata-modules/role/role.entity'; import { WorkspacePermissionsCacheModule } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.module'; -import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; @Module({ imports: [ @@ -20,8 +20,8 @@ import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/ FieldPermissionEntity, FieldMetadataEntity, ]), - WorkspaceCacheStorageModule, WorkspacePermissionsCacheModule, + WorkspaceManyOrAllFlatEntityMapsCacheModule, ], providers: [ObjectPermissionService, FieldPermissionService], exports: [ObjectPermissionService, FieldPermissionService], diff --git a/packages/twenty-server/src/engine/metadata-modules/object-permission/object-permission.service.ts b/packages/twenty-server/src/engine/metadata-modules/object-permission/object-permission.service.ts index 77e44ff61a4..7f281a903de 100644 --- a/packages/twenty-server/src/engine/metadata-modules/object-permission/object-permission.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/object-permission/object-permission.service.ts @@ -4,6 +4,7 @@ import { msg } from '@lingui/core/macro'; import { isDefined } from 'twenty-shared/utils'; import { In, Repository } from 'typeorm'; +import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { type ObjectPermissionInput, @@ -17,7 +18,6 @@ import { } from 'src/engine/metadata-modules/permissions/permissions.exception'; import { RoleEntity } from 'src/engine/metadata-modules/role/role.entity'; import { WorkspacePermissionsCacheService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.service'; -import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; export class ObjectPermissionService { constructor( @@ -28,7 +28,7 @@ export class ObjectPermissionService { @InjectRepository(ObjectMetadataEntity) private readonly objectMetadataRepository: Repository, private readonly workspacePermissionsCacheService: WorkspacePermissionsCacheService, - private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, + private readonly workspaceManyOrAllFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService, ) {} public async upsertObjectPermissions({ @@ -53,14 +53,17 @@ export class ObjectPermissionService { roleWithObjectPermissions: role, }); - const { byId: objectMetadataMapsById } = - await this.workspaceCacheStorageService.getObjectMetadataMapsOrThrow( - workspaceId, + const { flatObjectMetadataMaps } = + await this.workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( + { + workspaceId, + flatMapsKeys: ['flatObjectMetadataMaps'], + }, ); input.objectPermissions.forEach((objectPermission) => { const objectMetadataForObjectPermission = - objectMetadataMapsById[objectPermission.objectMetadataId]; + flatObjectMetadataMaps.byId[objectPermission.objectMetadataId]; if (!isDefined(objectMetadataForObjectPermission)) { throw new PermissionsException( diff --git a/packages/twenty-server/src/engine/metadata-modules/view/controllers/view.controller.ts b/packages/twenty-server/src/engine/metadata-modules/view/controllers/view.controller.ts index 4ff69437665..062c737783e 100644 --- a/packages/twenty-server/src/engine/metadata-modules/view/controllers/view.controller.ts +++ b/packages/twenty-server/src/engine/metadata-modules/view/controllers/view.controller.ts @@ -24,6 +24,7 @@ import { RequestLocale } from 'src/engine/decorators/locale/request-locale.decor import { CustomPermissionGuard } from 'src/engine/guards/custom-permission.guard'; import { NoPermissionGuard } from 'src/engine/guards/no-permission.guard'; import { WorkspaceAuthGuard } from 'src/engine/guards/workspace-auth.guard'; +import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service'; import { resolveObjectMetadataStandardOverride } from 'src/engine/metadata-modules/object-metadata/utils/resolve-object-metadata-standard-override.util'; import { CreateViewPermissionGuard } from 'src/engine/metadata-modules/view-permissions/guards/create-view-permission.guard'; import { DeleteViewPermissionGuard } from 'src/engine/metadata-modules/view-permissions/guards/delete-view-permission.guard'; @@ -41,7 +42,6 @@ import { import { ViewRestApiExceptionFilter } from 'src/engine/metadata-modules/view/filters/view-rest-api-exception.filter'; import { ViewV2Service } from 'src/engine/metadata-modules/view/services/view-v2.service'; import { ViewService } from 'src/engine/metadata-modules/view/services/view.service'; -import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service'; @Controller('rest/metadata/views') @UseGuards(WorkspaceAuthGuard) @@ -51,7 +51,7 @@ export class ViewController { private readonly viewService: ViewService, private readonly viewV2Service: ViewV2Service, private readonly featureFlagService: FeatureFlagService, - private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService, + private readonly flatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService, private readonly i18nService: I18nService, ) {} @@ -226,16 +226,20 @@ export class ViewController { return views; } - const { objectMetadataMaps } = - await this.workspaceMetadataCacheService.getExistingOrRecomputeMetadataMaps( - { workspaceId }, + const { flatObjectMetadataMaps } = + await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( + { + workspaceId, + flatMapsKeys: ['flatObjectMetadataMaps'], + }, ); return views.map((view) => { let processedName = view.name; if (view.name.includes('{objectLabelPlural}')) { - const objectMetadata = objectMetadataMaps.byId[view.objectMetadataId]; + const objectMetadata = + flatObjectMetadataMaps.byId[view.objectMetadataId]; if (objectMetadata) { const i18n = this.i18nService.getI18nInstance(locale ?? 'en'); diff --git a/packages/twenty-server/src/engine/metadata-modules/view/resolvers/view.resolver.ts b/packages/twenty-server/src/engine/metadata-modules/view/resolvers/view.resolver.ts index d1b78cfc5aa..57fca08119c 100644 --- a/packages/twenty-server/src/engine/metadata-modules/view/resolvers/view.resolver.ts +++ b/packages/twenty-server/src/engine/metadata-modules/view/resolvers/view.resolver.ts @@ -76,7 +76,7 @@ export class ViewResolver { workspaceId: workspace.id, }); - if (objectMetadata) { + if (isDefined(objectMetadata)) { const i18n = this.i18nService.getI18nInstance(context.req.locale); const translatedObjectLabel = resolveObjectMetadataStandardOverride( { diff --git a/packages/twenty-server/src/engine/metadata-modules/view/view.module.ts b/packages/twenty-server/src/engine/metadata-modules/view/view.module.ts index d42dbca7d00..87836574b51 100644 --- a/packages/twenty-server/src/engine/metadata-modules/view/view.module.ts +++ b/packages/twenty-server/src/engine/metadata-modules/view/view.module.ts @@ -19,7 +19,6 @@ import { ViewEntity } from 'src/engine/metadata-modules/view/entities/view.entit import { ViewResolver } from 'src/engine/metadata-modules/view/resolvers/view.resolver'; import { ViewV2Service } from 'src/engine/metadata-modules/view/services/view-v2.service'; import { ViewService } from 'src/engine/metadata-modules/view/services/view.service'; -import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; import { WorkspaceMigrationV2Module } from 'src/engine/workspace-manager/workspace-migration-v2/workspace-migration-v2.module'; @@ -37,7 +36,6 @@ import { WorkspaceMigrationV2Module } from 'src/engine/workspace-manager/workspa FeatureFlagModule, PermissionsModule, UserRoleModule, - WorkspaceMetadataCacheModule, WorkspaceCacheStorageModule, WorkspaceMigrationV2Module, FlatViewModule, diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/exceptions/workspace-metadata-cache.exception.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/exceptions/workspace-metadata-cache.exception.ts deleted file mode 100644 index ce35e87be12..00000000000 --- a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/exceptions/workspace-metadata-cache.exception.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { CustomException } from 'src/utils/custom-exception'; - -export class WorkspaceMetadataCacheException extends CustomException {} - -export enum WorkspaceMetadataCacheExceptionCode { - OBJECT_METADATA_MAP_NOT_FOUND = 'Object Metadata map not found', - FIELD_METADATA_NOT_FOUND = 'Field Metadata not found', - FIELD_METADATA_INVALID = 'Field Metadata is invalid', - OBJECT_METADATA_COLLECTION_NOT_FOUND = 'Object Metadata collection not found', -} diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service.ts deleted file mode 100644 index a41a24aa7b4..00000000000 --- a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service.ts +++ /dev/null @@ -1,162 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { InjectRepository } from '@nestjs/typeorm'; - -import { isDefined } from 'twenty-shared/utils'; -import { In, Repository } from 'typeorm'; - -import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; -import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; -import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; -import { type ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; -import { generateObjectMetadataMaps } from 'src/engine/metadata-modules/utils/generate-object-metadata-maps.util'; -import { - WorkspaceMetadataVersionException, - WorkspaceMetadataVersionExceptionCode, -} from 'src/engine/metadata-modules/workspace-metadata-version/exceptions/workspace-metadata-version.exception'; -import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; - -type GetExistingOrRecomputeMetadataMapsResult = { - objectMetadataMaps: ObjectMetadataMaps; - metadataVersion: number; -}; - -@Injectable() -export class WorkspaceMetadataCacheService { - logger = new Logger(WorkspaceMetadataCacheService.name); - - constructor( - @InjectRepository(WorkspaceEntity) - private readonly workspaceRepository: Repository, - private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, - @InjectRepository(ObjectMetadataEntity) - private readonly objectMetadataRepository: Repository, - @InjectRepository(IndexMetadataEntity) - private readonly indexMetadataRepository: Repository, - ) {} - - async getExistingOrRecomputeMetadataMaps({ - workspaceId, - }: { - workspaceId: string; - }): Promise { - const currentCacheVersion = - await this.getMetadataVersionFromCache(workspaceId); - - const currentDatabaseVersion = - await this.getMetadataVersionFromDatabase(workspaceId); - - if (!isDefined(currentDatabaseVersion)) { - throw new WorkspaceMetadataVersionException( - 'Metadata version not found in the database', - WorkspaceMetadataVersionExceptionCode.METADATA_VERSION_NOT_FOUND, - ); - } - - const shouldRecompute = - !isDefined(currentCacheVersion) || - currentCacheVersion !== currentDatabaseVersion; - - const existingObjectMetadataMaps = - await this.workspaceCacheStorageService.getObjectMetadataMaps( - workspaceId, - currentDatabaseVersion, - ); - - if (isDefined(existingObjectMetadataMaps) && !shouldRecompute) { - return { - objectMetadataMaps: existingObjectMetadataMaps, - metadataVersion: currentDatabaseVersion, - }; - } - - const { objectMetadataMaps, metadataVersion } = - await this.recomputeMetadataCache({ - workspaceId, - }); - - return { - objectMetadataMaps, - metadataVersion, - }; - } - - async recomputeMetadataCache({ - workspaceId, - }: { - workspaceId: string; - }): Promise { - const currentDatabaseVersion = - await this.getMetadataVersionFromDatabase(workspaceId); - - if (!isDefined(currentDatabaseVersion)) { - throw new WorkspaceMetadataVersionException( - 'Metadata version not found in the database', - WorkspaceMetadataVersionExceptionCode.METADATA_VERSION_NOT_FOUND, - ); - } - - await this.workspaceCacheStorageService.flushVersionedMetadata(workspaceId); - - const objectMetadataItems = await this.objectMetadataRepository.find({ - where: { workspaceId }, - relations: ['fields'], - }); - - const objectMetadataItemsIds = objectMetadataItems.map( - (objectMetadataItem) => objectMetadataItem.id, - ); - - const indexMetadataItems = await this.indexMetadataRepository.find({ - where: { objectMetadataId: In(objectMetadataItemsIds) }, - relations: ['indexFieldMetadatas'], - }); - - const objectMetadataItemsWithIndexMetadatas = objectMetadataItems.map( - (objectMetadataItem) => ({ - ...objectMetadataItem, - indexMetadatas: indexMetadataItems.filter( - (indexMetadataItem) => - indexMetadataItem.objectMetadataId === objectMetadataItem.id, - ), - }), - ); - - const freshObjectMetadataMaps = generateObjectMetadataMaps( - objectMetadataItemsWithIndexMetadatas, - ); - - await this.workspaceCacheStorageService.setObjectMetadataMaps( - workspaceId, - currentDatabaseVersion, - freshObjectMetadataMaps, - ); - - await this.workspaceCacheStorageService.setMetadataVersion( - workspaceId, - currentDatabaseVersion, - ); - - return { - objectMetadataMaps: freshObjectMetadataMaps, - metadataVersion: currentDatabaseVersion, - }; - } - - private async getMetadataVersionFromDatabase( - workspaceId: string, - ): Promise { - const workspace = await this.workspaceRepository.findOne({ - where: { id: workspaceId }, - }); - - return workspace?.metadataVersion; - } - - private async getMetadataVersionFromCache( - workspaceId: string, - ): Promise { - return await this.workspaceCacheStorageService.getMetadataVersion( - workspaceId, - ); - } -} diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module.ts deleted file mode 100644 index c2d4636fd82..00000000000 --- a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { Module } from '@nestjs/common'; -import { TypeOrmModule } from '@nestjs/typeorm'; - -import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; -import { IndexMetadataEntity } from 'src/engine/metadata-modules/index-metadata/index-metadata.entity'; -import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; -import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service'; -import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; - -@Module({ - imports: [ - TypeOrmModule.forFeature([ - WorkspaceEntity, - ObjectMetadataEntity, - IndexMetadataEntity, - ]), - WorkspaceCacheStorageModule, - ], - exports: [WorkspaceMetadataCacheService], - providers: [WorkspaceMetadataCacheService], -}) -export class WorkspaceMetadataCacheModule {} diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service.ts index 723f2a43b66..26b52a1bc28 100644 --- a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service.ts +++ b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service.ts @@ -5,20 +5,22 @@ import { isDefined } from 'twenty-shared/utils'; import { Repository } from 'typeorm'; import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; -import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service'; +import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service'; import { WorkspaceMetadataVersionException, WorkspaceMetadataVersionExceptionCode, } from 'src/engine/metadata-modules/workspace-metadata-version/exceptions/workspace-metadata-version.exception'; +import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; @Injectable() export class WorkspaceMetadataVersionService { - logger = new Logger(WorkspaceMetadataCacheService.name); + logger = new Logger(WorkspaceMetadataVersionService.name); constructor( @InjectRepository(WorkspaceEntity) private readonly workspaceRepository: Repository, - private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService, + private readonly flatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService, + private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, ) {} async incrementMetadataVersion(workspaceId: string): Promise { @@ -42,8 +44,18 @@ export class WorkspaceMetadataVersionService { { metadataVersion: newMetadataVersion }, ); - await this.workspaceMetadataCacheService.recomputeMetadataCache({ + await this.workspaceCacheStorageService.setMetadataVersion( workspaceId, + newMetadataVersion, + ); + + await this.flatEntityMapsCacheService.invalidateFlatEntityMaps({ + workspaceId, + flatMapsKeys: [ + 'flatObjectMetadataMaps', + 'flatFieldMetadataMaps', + 'flatIndexMaps', + ], }); } } diff --git a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module.ts b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module.ts index ccce4a5a68e..889d59e5b7d 100644 --- a/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module.ts +++ b/packages/twenty-server/src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module.ts @@ -2,7 +2,7 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; -import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; +import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module'; import { WorkspaceMetadataVersionService } from 'src/engine/metadata-modules/workspace-metadata-version/services/workspace-metadata-version.service'; import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; @@ -10,7 +10,7 @@ import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/ imports: [ TypeOrmModule.forFeature([WorkspaceEntity]), WorkspaceCacheStorageModule, - WorkspaceMetadataCacheModule, + WorkspaceManyOrAllFlatEntityMapsCacheModule, ], exports: [WorkspaceMetadataVersionService], providers: [WorkspaceMetadataVersionService], diff --git a/packages/twenty-server/src/engine/middlewares/middleware.module.ts b/packages/twenty-server/src/engine/middlewares/middleware.module.ts index eba2ce51142..d5a22a96b76 100644 --- a/packages/twenty-server/src/engine/middlewares/middleware.module.ts +++ b/packages/twenty-server/src/engine/middlewares/middleware.module.ts @@ -3,7 +3,7 @@ import { Module } from '@nestjs/common'; import { TokenModule } from 'src/engine/core-modules/auth/token/token.module'; import { JwtModule } from 'src/engine/core-modules/jwt/jwt.module'; import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; -import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; +import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module'; import { MiddlewareService } from 'src/engine/middlewares/middleware.service'; import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; @@ -11,7 +11,7 @@ import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/ imports: [ DataSourceModule, WorkspaceCacheStorageModule, - WorkspaceMetadataCacheModule, + WorkspaceManyOrAllFlatEntityMapsCacheModule, TokenModule, JwtModule, ], diff --git a/packages/twenty-server/src/engine/middlewares/middleware.service.ts b/packages/twenty-server/src/engine/middlewares/middleware.service.ts index 5e76ade0fc0..f0020284b4a 100644 --- a/packages/twenty-server/src/engine/middlewares/middleware.service.ts +++ b/packages/twenty-server/src/engine/middlewares/middleware.service.ts @@ -13,7 +13,7 @@ import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handl import { ErrorCode } from 'src/engine/core-modules/graphql/utils/graphql-errors.util'; import { JwtWrapperService } from 'src/engine/core-modules/jwt/services/jwt-wrapper.service'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; -import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service'; +import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service'; import { INTERNAL_SERVER_ERROR } from 'src/engine/middlewares/constants/default-error-message.constant'; import { handleException, @@ -27,7 +27,7 @@ export class MiddlewareService { constructor( private readonly accessTokenService: AccessTokenService, private readonly workspaceStorageCacheService: WorkspaceCacheStorageService, - private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService, + private readonly flatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService, private readonly dataSourceService: DataSourceService, private readonly exceptionHandlerService: ExceptionHandlerService, private readonly jwtWrapperService: JwtWrapperService, @@ -107,8 +107,13 @@ export class MiddlewareService { : undefined; if (metadataVersion === undefined && isDefined(data.workspace)) { - await this.workspaceMetadataCacheService.recomputeMetadataCache({ + await this.flatEntityMapsCacheService.invalidateFlatEntityMaps({ workspaceId: data.workspace.id, + flatMapsKeys: [ + 'flatObjectMetadataMaps', + 'flatFieldMetadataMaps', + 'flatIndexMaps', + ], }); throw new Error('Metadata cache version not found'); } diff --git a/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts b/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts index 3e72aacb3ca..b1231623bf7 100644 --- a/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts +++ b/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts @@ -10,8 +10,11 @@ import { type FeatureFlagMap } from 'src/engine/core-modules/feature-flag/interf import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service'; +import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service'; +import { buildObjectIdByNameMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-id-by-name-maps.util'; +import { buildObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-metadata-item-with-field-maps.util'; +import { type ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; import { WorkspaceFeatureFlagsMapCacheService } from 'src/engine/metadata-modules/workspace-feature-flags-map-cache/workspace-feature-flags-map-cache.service'; -import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service'; import { WorkspacePermissionsCacheStorageService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache-storage.service'; import { ROLES_PERMISSIONS, @@ -45,7 +48,7 @@ export class WorkspaceDatasourceFactory { private readonly dataSourceService: DataSourceService, private readonly twentyConfigService: TwentyConfigService, private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, - private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService, + private readonly workspaceManyOrAllFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService, private readonly entitySchemaFactory: EntitySchemaFactory, private readonly workspacePermissionsCacheService: WorkspacePermissionsCacheService, private readonly workspacePermissionsCacheStorageService: WorkspacePermissionsCacheStorageService, @@ -126,15 +129,47 @@ export class WorkspaceDatasourceFactory { let cachedEntitySchemas: EntitySchema[]; const { - objectMetadataMaps: cachedObjectMetadataMaps, - metadataVersion: metadataVersionForFinalUpToDateCheck, + flatObjectMetadataMaps, + flatFieldMetadataMaps, + flatIndexMaps, } = - await this.workspaceMetadataCacheService.getExistingOrRecomputeMetadataMaps( + await this.workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, + flatMapsKeys: [ + 'flatObjectMetadataMaps', + 'flatFieldMetadataMaps', + 'flatIndexMaps', + ], }, ); + const { idByNameSingular } = buildObjectIdByNameMaps( + flatObjectMetadataMaps, + ); + const cachedObjectMetadataMaps: ObjectMetadataMaps = { + byId: {}, + idByNameSingular, + }; + + for (const [id, flatObj] of Object.entries( + flatObjectMetadataMaps.byId, + )) { + if (isDefined(flatObj)) { + cachedObjectMetadataMaps.byId[id] = + buildObjectMetadataItemWithFieldMaps( + flatObj, + flatFieldMetadataMaps, + flatIndexMaps, + ); + } + } + + const metadataVersionForFinalUpToDateCheck = + await this.workspaceCacheStorageService.getMetadataVersion( + workspaceId, + ); + if ( metadataVersionForFinalUpToDateCheck !== dataSourceMetadataVersion ) { diff --git a/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.module.ts b/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.module.ts index f8917e41969..beb3297d038 100644 --- a/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.module.ts +++ b/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.module.ts @@ -5,8 +5,8 @@ import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature- import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty-config.module'; import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; +import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module'; import { WorkspaceFeatureFlagsMapCacheModule } from 'src/engine/metadata-modules/workspace-feature-flags-map-cache/workspace-feature-flags-map-cache.module'; -import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; import { WorkspacePermissionsCacheModule } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.module'; import { EntitySchemaColumnFactory } from 'src/engine/twenty-orm/factories/entity-schema-column.factory'; import { EntitySchemaRelationFactory } from 'src/engine/twenty-orm/factories/entity-schema-relation.factory'; @@ -22,7 +22,7 @@ import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/ TypeOrmModule.forFeature([WorkspaceEntity]), DataSourceModule, WorkspaceCacheStorageModule, - WorkspaceMetadataCacheModule, + WorkspaceManyOrAllFlatEntityMapsCacheModule, WorkspacePermissionsCacheModule, WorkspaceFeatureFlagsMapCacheModule, FeatureFlagModule, diff --git a/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager.ts b/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager.ts index 1a35d1f049a..cd4bbbde8c7 100644 --- a/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager.ts +++ b/packages/twenty-server/src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager.ts @@ -1,13 +1,15 @@ import { Injectable, type Type } from '@nestjs/common'; import { isDefined } from 'twenty-shared/utils'; -import { EntitySchema, type ObjectLiteral } from 'typeorm'; +import { EntitySchema, ObjectLiteral } from 'typeorm'; import { WorkspaceAuthContext } from 'src/engine/api/common/interfaces/workspace-auth-context.interface'; +import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service'; +import { buildObjectIdByNameMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-id-by-name-maps.util'; +import { buildObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-metadata-item-with-field-maps.util'; import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; import { WorkspaceFeatureFlagsMapCacheService } from 'src/engine/metadata-modules/workspace-feature-flags-map-cache/workspace-feature-flags-map-cache.service'; -import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service'; import { WorkspacePermissionsCacheService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.service'; import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory'; import { GlobalWorkspaceDataSourceService } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-datasource.service'; @@ -24,10 +26,10 @@ import { convertClassNameToObjectMetadataName } from 'src/engine/workspace-manag export class GlobalWorkspaceOrmManager { constructor( private readonly globalWorkspaceDataSourceService: GlobalWorkspaceDataSourceService, - private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService, + private readonly workspaceManyOrAllFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService, + private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, private readonly workspaceFeatureFlagsMapCacheService: WorkspaceFeatureFlagsMapCacheService, private readonly workspacePermissionsCacheService: WorkspacePermissionsCacheService, - private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, private readonly entitySchemaFactory: EntitySchemaFactory, ) {} @@ -84,13 +86,46 @@ export class GlobalWorkspaceOrmManager { authContext: WorkspaceAuthContext, ): Promise { const workspaceId = authContext.workspace.id; - const { objectMetadataMaps, metadataVersion } = - await this.workspaceMetadataCacheService.getExistingOrRecomputeMetadataMaps( + + const { flatObjectMetadataMaps, flatFieldMetadataMaps, flatIndexMaps } = + await this.workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, + flatMapsKeys: [ + 'flatObjectMetadataMaps', + 'flatFieldMetadataMaps', + 'flatIndexMaps', + ], }, ); + const { idByNameSingular } = buildObjectIdByNameMaps( + flatObjectMetadataMaps, + ); + const objectMetadataMaps: ObjectMetadataMaps = { + byId: {}, + idByNameSingular, + }; + + for (const [id, flatObj] of Object.entries(flatObjectMetadataMaps.byId)) { + if (isDefined(flatObj)) { + objectMetadataMaps.byId[id] = buildObjectMetadataItemWithFieldMaps( + flatObj, + flatFieldMetadataMaps, + flatIndexMaps, + ); + } + } + + const metadataVersion = + await this.workspaceCacheStorageService.getMetadataVersion(workspaceId); + + if (!isDefined(metadataVersion)) { + throw new Error( + `Metadata version not found for workspace ${workspaceId}`, + ); + } + const { data: featureFlagsMap } = await this.workspaceFeatureFlagsMapCacheService.getWorkspaceFeatureFlagsMapAndVersion( { workspaceId }, diff --git a/packages/twenty-server/src/engine/twenty-orm/twenty-orm.module.ts b/packages/twenty-server/src/engine/twenty-orm/twenty-orm.module.ts index f4a428802b3..92c29eb2553 100644 --- a/packages/twenty-server/src/engine/twenty-orm/twenty-orm.module.ts +++ b/packages/twenty-server/src/engine/twenty-orm/twenty-orm.module.ts @@ -5,11 +5,11 @@ import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature- import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty-config.module'; import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity'; import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; +import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { PermissionsModule } from 'src/engine/metadata-modules/permissions/permissions.module'; import { RoleTargetsEntity } from 'src/engine/metadata-modules/role/role-targets.entity'; import { WorkspaceFeatureFlagsMapCacheModule } from 'src/engine/metadata-modules/workspace-feature-flags-map-cache/workspace-feature-flags-map-cache.module'; -import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; import { WorkspacePermissionsCacheModule } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.module'; import { entitySchemaFactories } from 'src/engine/twenty-orm/factories'; import { EntitySchemaFactory } from 'src/engine/twenty-orm/factories/entity-schema.factory'; @@ -30,7 +30,7 @@ import { PgPoolSharedModule } from './pg-shared-pool/pg-shared-pool.module'; ]), DataSourceModule, WorkspaceCacheStorageModule, - WorkspaceMetadataCacheModule, + WorkspaceManyOrAllFlatEntityMapsCacheModule, PermissionsModule, WorkspaceFeatureFlagsMapCacheModule, WorkspacePermissionsCacheModule, diff --git a/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts b/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts index 7c0d3fd0fab..b6bbdfbb45e 100644 --- a/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts +++ b/packages/twenty-server/src/engine/workspace-cache-storage/workspace-cache-storage.service.ts @@ -10,15 +10,6 @@ import { type FeatureFlagMap } from 'src/engine/core-modules/feature-flag/interf import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator'; import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service'; import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum'; -import { type ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; -import { - WorkspaceMetadataCacheException, - WorkspaceMetadataCacheExceptionCode, -} from 'src/engine/metadata-modules/workspace-metadata-cache/exceptions/workspace-metadata-cache.exception'; -import { - WorkspaceMetadataVersionException, - WorkspaceMetadataVersionExceptionCode, -} from 'src/engine/metadata-modules/workspace-metadata-version/exceptions/workspace-metadata-version.exception'; export const METADATA_VERSIONED_WORKSPACE_CACHE_KEY = { GraphQLTypeDefs: 'graphql:type-defs', @@ -95,52 +86,6 @@ export class WorkspaceCacheStorageService { ); } - setObjectMetadataMaps( - workspaceId: string, - metadataVersion: number, - objectMetadataMaps: ObjectMetadataMaps, - ) { - return this.cacheStorageService.set( - `${METADATA_VERSIONED_WORKSPACE_CACHE_KEY.MetadataObjectMetadataMaps}:${workspaceId}:${metadataVersion}`, - objectMetadataMaps, - TTL_ONE_WEEK, - ); - } - - getObjectMetadataMaps( - workspaceId: string, - metadataVersion: number, - ): Promise { - return this.cacheStorageService.get( - `${METADATA_VERSIONED_WORKSPACE_CACHE_KEY.MetadataObjectMetadataMaps}:${workspaceId}:${metadataVersion}`, - ); - } - - async getObjectMetadataMapsOrThrow(workspaceId: string) { - const currentCacheVersion = await this.getMetadataVersion(workspaceId); - - if (currentCacheVersion === undefined) { - throw new WorkspaceMetadataVersionException( - `Metadata version not found for workspace ${workspaceId}`, - WorkspaceMetadataVersionExceptionCode.METADATA_VERSION_NOT_FOUND, - ); - } - - const objectMetadataMaps = await this.getObjectMetadataMaps( - workspaceId, - currentCacheVersion, - ); - - if (!objectMetadataMaps) { - throw new WorkspaceMetadataCacheException( - `Object metadata map not found for workspace ${workspaceId} and metadata version ${currentCacheVersion}`, - WorkspaceMetadataCacheExceptionCode.OBJECT_METADATA_MAP_NOT_FOUND, - ); - } - - return objectMetadataMaps; - } - setGraphQLTypeDefs( workspaceId: string, metadataVersion: number, diff --git a/packages/twenty-server/src/engine/workspace-flat-map-cache/workspace-flat-map-cache.module.ts b/packages/twenty-server/src/engine/workspace-flat-map-cache/workspace-flat-map-cache.module.ts index 710b0cae806..ddf85bf04ea 100644 --- a/packages/twenty-server/src/engine/workspace-flat-map-cache/workspace-flat-map-cache.module.ts +++ b/packages/twenty-server/src/engine/workspace-flat-map-cache/workspace-flat-map-cache.module.ts @@ -17,16 +17,12 @@ import { ViewFieldEntity } from 'src/engine/metadata-modules/view-field/entities import { ViewFilterEntity } from 'src/engine/metadata-modules/view-filter/entities/view-filter.entity'; import { ViewGroupEntity } from 'src/engine/metadata-modules/view-group/entities/view-group.entity'; import { ViewEntity } from 'src/engine/metadata-modules/view/entities/view.entity'; -import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; -import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { WorkspacePermissionsCacheModule } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.module'; import { WorkspaceFlatMapCacheRegistryService } from 'src/engine/workspace-flat-map-cache/services/workspace-flat-map-cache-registry.service'; @Module({ imports: [ DiscoveryModule, - WorkspaceMetadataCacheModule, - WorkspaceMetadataVersionModule, WorkspacePermissionsCacheModule, TypeOrmModule.forFeature([ ViewEntity, diff --git a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/dev-seeder.module.ts b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/dev-seeder.module.ts index f8ae737cb02..beb9839f0b6 100644 --- a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/dev-seeder.module.ts +++ b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/dev-seeder.module.ts @@ -15,7 +15,6 @@ import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadat import { ObjectPermissionModule } from 'src/engine/metadata-modules/object-permission/object-permission.module'; import { RoleModule } from 'src/engine/metadata-modules/role/role.module'; import { UserRoleModule } from 'src/engine/metadata-modules/user-role/user-role.module'; -import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; import { WorkspacePermissionsCacheModule } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.module'; import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; import { WorkspaceDataSourceModule } from 'src/engine/workspace-datasource/workspace-datasource.module'; @@ -45,7 +44,6 @@ import { WorkspaceSyncMetadataModule } from 'src/engine/workspace-manager/worksp ObjectPermissionModule, WorkspacePermissionsCacheModule, WorkspaceManyOrAllFlatEntityMapsCacheModule, - WorkspaceMetadataCacheModule, ], exports: [DevSeederService], providers: [ diff --git a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/metadata/services/dev-seeder-metadata.service.ts b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/metadata/services/dev-seeder-metadata.service.ts index b9e2e52510a..383e0b9f103 100644 --- a/packages/twenty-server/src/engine/workspace-manager/dev-seeder/metadata/services/dev-seeder-metadata.service.ts +++ b/packages/twenty-server/src/engine/workspace-manager/dev-seeder/metadata/services/dev-seeder-metadata.service.ts @@ -8,9 +8,9 @@ import { FlatApplication } from 'src/engine/core-modules/application/types/flat- import { type DataSourceEntity } from 'src/engine/metadata-modules/data-source/data-source.entity'; import { CreateFieldInput } from 'src/engine/metadata-modules/field-metadata/dtos/create-field.input'; import { FieldMetadataServiceV2 } from 'src/engine/metadata-modules/field-metadata/services/field-metadata.service-v2'; +import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service'; +import { buildObjectIdByNameMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-id-by-name-maps.util'; import { ObjectMetadataServiceV2 } from 'src/engine/metadata-modules/object-metadata/object-metadata-v2.service'; -import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; -import { WorkspaceMetadataCacheService } from 'src/engine/metadata-modules/workspace-metadata-cache/services/workspace-metadata-cache.service'; import { SEED_APPLE_WORKSPACE_ID, SEED_YCOMBINATOR_WORKSPACE_ID, @@ -32,7 +32,7 @@ export class DevSeederMetadataService { constructor( private readonly objectMetadataServiceV2: ObjectMetadataServiceV2, private readonly fieldMetadataServiceV2: FieldMetadataServiceV2, - private readonly workspaceMetadataCacheService: WorkspaceMetadataCacheService, + private readonly flatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService, @InjectDataSource() private readonly coreDataSource: DataSource, ) {} @@ -237,16 +237,20 @@ export class DevSeederMetadataService { seeds: (FieldMetadataSeed & { targetObjectMetadataNames: string[] })[]; }; }): Promise { - const { objectMetadataMaps } = - await this.workspaceMetadataCacheService.getExistingOrRecomputeMetadataMaps( + const { flatObjectMetadataMaps } = + await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( { workspaceId, + flatMapsKeys: ['flatObjectMetadataMaps'], }, ); + const { idByNameSingular: objectIdByNameSingular } = + buildObjectIdByNameMaps(flatObjectMetadataMaps); + const createFieldInputs = this.createFieldInputs({ relation, - objectMetadataMaps, + objectIdByNameSingular, }); await this.fieldMetadataServiceV2.createManyFields({ @@ -257,16 +261,15 @@ export class DevSeederMetadataService { private createFieldInputs({ relation, - objectMetadataMaps, + objectIdByNameSingular, }: { relation: { objectName: string; seeds: (FieldMetadataSeed & { targetObjectMetadataNames: string[] })[]; }; - objectMetadataMaps: ObjectMetadataMaps; + objectIdByNameSingular: Record; }): Omit[] { - const objectMetadataId = - objectMetadataMaps.idByNameSingular[relation.objectName]; + const objectMetadataId = objectIdByNameSingular[relation.objectName]; if (!isDefined(objectMetadataId)) { throw new Error( @@ -282,7 +285,7 @@ export class DevSeederMetadataService { morphRelationsCreationPayload: seed.targetObjectMetadataNames.map( (targetObjectMetadataName) => { const targetObjectMetadataId = - objectMetadataMaps.idByNameSingular[targetObjectMetadataName]; + objectIdByNameSingular[targetObjectMetadataName]; if (!isDefined(targetObjectMetadataId)) { throw new Error( diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-v2/workspace-migration-runner-v2/workspace-migration-runner-v2.module.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-v2/workspace-migration-runner-v2/workspace-migration-runner-v2.module.ts index 45b46385430..4d2382661c5 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-v2/workspace-migration-runner-v2/workspace-migration-runner-v2.module.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-v2/workspace-migration-runner-v2/workspace-migration-runner-v2.module.ts @@ -5,7 +5,6 @@ import { TypeORMModule } from 'src/database/typeorm/typeorm.module'; import { FeatureFlagModule } from 'src/engine/core-modules/feature-flag/feature-flag.module'; import { DataSourceModule } from 'src/engine/metadata-modules/data-source/data-source.module'; import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module'; -import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; import { WorkspaceMetadataVersionModule } from 'src/engine/metadata-modules/workspace-metadata-version/workspace-metadata-version.module'; import { WorkspacePermissionsCacheModule } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.module'; import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; @@ -20,7 +19,6 @@ import { WorkspaceMigrationRunnerV2Service } from 'src/engine/workspace-manager/ DataSourceModule, WorkspaceMetadataVersionModule, WorkspacePermissionsCacheModule, - WorkspaceMetadataCacheModule, WorkspaceSchemaMigrationRunnerActionHandlersModule, WorkspaceManyOrAllFlatEntityMapsCacheModule, DiscoveryModule, diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-v2/workspace-migration-v2.module.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-v2/workspace-migration-v2.module.ts index b65ec657fa5..5616ce2fd14 100644 --- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration-v2/workspace-migration-v2.module.ts +++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration-v2/workspace-migration-v2.module.ts @@ -1,6 +1,5 @@ import { Module } from '@nestjs/common'; -import { WorkspaceMetadataCacheModule } from 'src/engine/metadata-modules/workspace-metadata-cache/workspace-metadata-cache.module'; import { WorkspaceMigrationBuildOrchestratorService } from 'src/engine/workspace-manager/workspace-migration-v2/services/workspace-migration-build-orchestrator.service'; import { WorkspaceMigrationValidateBuildAndRunService } from 'src/engine/workspace-manager/workspace-migration-v2/services/workspace-migration-validate-build-and-run-service'; import { WorkspaceMigrationBuilderV2Module } from 'src/engine/workspace-manager/workspace-migration-v2/workspace-migration-builder-v2/workspace-migration-builder-v2.module'; @@ -10,7 +9,6 @@ import { WorkspaceMigrationRunnerV2Module } from 'src/engine/workspace-manager/w imports: [ WorkspaceMigrationBuilderV2Module, WorkspaceMigrationRunnerV2Module, - WorkspaceMetadataCacheModule, ], providers: [ WorkspaceMigrationValidateBuildAndRunService, diff --git a/packages/twenty-server/src/modules/workflow/common/query-hooks/workflow-query-hook.module.ts b/packages/twenty-server/src/modules/workflow/common/query-hooks/workflow-query-hook.module.ts index 7b91dbdea63..195bd7ec2d1 100644 --- a/packages/twenty-server/src/modules/workflow/common/query-hooks/workflow-query-hook.module.ts +++ b/packages/twenty-server/src/modules/workflow/common/query-hooks/workflow-query-hook.module.ts @@ -3,10 +3,10 @@ import { Module } from '@nestjs/common'; import { NestjsQueryTypeOrmModule } from '@ptc-org/nestjs-query-typeorm'; import { RecordPositionModule } from 'src/engine/core-modules/record-position/record-position.module'; +import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module'; import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module'; import { ServerlessFunctionModule } from 'src/engine/metadata-modules/serverless-function/serverless-function.module'; -import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; import { WorkflowCreateManyPostQueryHook } from 'src/modules/workflow/common/query-hooks/workflow-create-many.post-query.hook'; import { WorkflowCreateManyPreQueryHook } from 'src/modules/workflow/common/query-hooks/workflow-create-many.pre-query.hook'; import { WorkflowCreateOnePostQueryHook } from 'src/modules/workflow/common/query-hooks/workflow-create-one.post-query.hook'; @@ -39,7 +39,7 @@ import { WorkflowVersionValidationWorkspaceService } from 'src/modules/workflow/ NestjsQueryTypeOrmModule.forFeature([ObjectMetadataEntity]), ServerlessFunctionModule, RecordPositionModule, - WorkspaceCacheStorageModule, + WorkspaceManyOrAllFlatEntityMapsCacheModule, ObjectMetadataModule, ], providers: [ diff --git a/packages/twenty-server/src/modules/workflow/common/workflow-common.module.ts b/packages/twenty-server/src/modules/workflow/common/workflow-common.module.ts index 55111383d28..26372a268d5 100644 --- a/packages/twenty-server/src/modules/workflow/common/workflow-common.module.ts +++ b/packages/twenty-server/src/modules/workflow/common/workflow-common.module.ts @@ -1,7 +1,7 @@ import { Module } from '@nestjs/common'; +import { WorkspaceManyOrAllFlatEntityMapsCacheModule } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.module'; import { ServerlessFunctionModule } from 'src/engine/metadata-modules/serverless-function/serverless-function.module'; -import { WorkspaceCacheStorageModule } from 'src/engine/workspace-cache-storage/workspace-cache-storage.module'; import { WorkflowQueryHookModule } from 'src/modules/workflow/common/query-hooks/workflow-query-hook.module'; import { WorkflowCommonWorkspaceService } from 'src/modules/workflow/common/workspace-services/workflow-common.workspace-service'; @@ -9,7 +9,7 @@ import { WorkflowCommonWorkspaceService } from 'src/modules/workflow/common/work imports: [ WorkflowQueryHookModule, ServerlessFunctionModule, - WorkspaceCacheStorageModule, + WorkspaceManyOrAllFlatEntityMapsCacheModule, ], providers: [WorkflowCommonWorkspaceService], exports: [WorkflowCommonWorkspaceService], diff --git a/packages/twenty-server/src/modules/workflow/common/workspace-services/workflow-common.workspace-service.ts b/packages/twenty-server/src/modules/workflow/common/workspace-services/workflow-common.workspace-service.ts index 767f3d4f27a..822a6ca4aed 100644 --- a/packages/twenty-server/src/modules/workflow/common/workspace-services/workflow-common.workspace-service.ts +++ b/packages/twenty-server/src/modules/workflow/common/workspace-services/workflow-common.workspace-service.ts @@ -1,12 +1,16 @@ import { Injectable } from '@nestjs/common'; +import { isDefined } from 'twenty-shared/utils'; + +import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service'; +import { buildObjectIdByNameMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-id-by-name-maps.util'; +import { buildObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/flat-object-metadata/utils/build-object-metadata-item-with-field-maps.util'; import { ServerlessFunctionService } from 'src/engine/metadata-modules/serverless-function/serverless-function.service'; import { type ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps'; import { type ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; import { getObjectMetadataMapItemByNameSingular } from 'src/engine/metadata-modules/utils/get-object-metadata-map-item-by-name-singular.util'; import { type WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository'; import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; -import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; import { WorkflowCommonException, WorkflowCommonExceptionCode, @@ -37,7 +41,7 @@ export class WorkflowCommonWorkspaceService { constructor( private readonly twentyORMGlobalManager: TwentyORMGlobalManager, private readonly serverlessFunctionService: ServerlessFunctionService, - private readonly workspaceCacheStorageService: WorkspaceCacheStorageService, + private readonly workspaceManyOrAllFlatEntityMapsCacheService: WorkspaceManyOrAllFlatEntityMapsCacheService, ) {} async getWorkflowVersionOrFail({ @@ -94,12 +98,36 @@ export class WorkflowCommonWorkspaceService { async getObjectMetadataMaps( workspaceId: string, ): Promise { - // TODO: replace this with the new cache service - const objectMetadataMaps = - await this.workspaceCacheStorageService.getObjectMetadataMapsOrThrow( - workspaceId, + const { flatObjectMetadataMaps, flatFieldMetadataMaps, flatIndexMaps } = + await this.workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps( + { + workspaceId, + flatMapsKeys: [ + 'flatObjectMetadataMaps', + 'flatFieldMetadataMaps', + 'flatIndexMaps', + ], + }, ); + const { idByNameSingular } = buildObjectIdByNameMaps( + flatObjectMetadataMaps, + ); + const objectMetadataMaps: ObjectMetadataMaps = { + byId: {}, + idByNameSingular, + }; + + for (const [id, flatObj] of Object.entries(flatObjectMetadataMaps.byId)) { + if (isDefined(flatObj)) { + objectMetadataMaps.byId[id] = buildObjectMetadataItemWithFieldMaps( + flatObj, + flatFieldMetadataMaps, + flatIndexMaps, + ); + } + } + return objectMetadataMaps; } diff --git a/packages/twenty-server/test/integration/metadata/suites/agent/utils/agent-tool-test-utils.ts b/packages/twenty-server/test/integration/metadata/suites/agent/utils/agent-tool-test-utils.ts index 2fa3be10fe1..34eb8f61603 100644 --- a/packages/twenty-server/test/integration/metadata/suites/agent/utils/agent-tool-test-utils.ts +++ b/packages/twenty-server/test/integration/metadata/suites/agent/utils/agent-tool-test-utils.ts @@ -20,6 +20,7 @@ import { AgentHandoffService } from 'src/engine/metadata-modules/agent/agent-han import { AgentToolGeneratorService } from 'src/engine/metadata-modules/agent/agent-tool-generator.service'; import { AgentEntity } from 'src/engine/metadata-modules/agent/agent.entity'; import { AgentService } from 'src/engine/metadata-modules/agent/agent.service'; +import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service'; import { ObjectMetadataServiceV2 } from 'src/engine/metadata-modules/object-metadata/object-metadata-v2.service'; import { type ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadata/object-metadata.entity'; import { PermissionsService } from 'src/engine/metadata-modules/permissions/permissions.service'; @@ -27,7 +28,6 @@ import { RoleEntity } from 'src/engine/metadata-modules/role/role.entity'; import { WorkspacePermissionsCacheService } from 'src/engine/metadata-modules/workspace-permissions-cache/workspace-permissions-cache.service'; import { ScopedWorkspaceContextFactory } from 'src/engine/twenty-orm/factories/scoped-workspace-context.factory'; import { TwentyORMGlobalManager } from 'src/engine/twenty-orm/twenty-orm-global.manager'; -import { WorkspaceCacheStorageService } from 'src/engine/workspace-cache-storage/workspace-cache-storage.service'; import { MessagingSendMessageService } from 'src/modules/messaging/message-import-manager/services/messaging-send-message.service'; import { WorkflowToolWorkspaceService } from 'src/modules/workflow/workflow-tools/services/workflow-tool.workspace-service'; import { getMockObjectMetadataEntity } from 'src/utils/__test__/get-object-metadata-entity.mock'; @@ -135,9 +135,9 @@ export const createAgentToolTestModule = }, }, { - provide: WorkspaceCacheStorageService, + provide: WorkspaceManyOrAllFlatEntityMapsCacheService, useValue: { - getObjectMetadataMapsOrThrow: jest.fn(), + getOrRecomputeManyOrAllFlatEntityMaps: jest.fn(), }, }, { @@ -300,27 +300,36 @@ export const createAgentToolTestModule = }); // Ensure ToolService input transformation has access to minimal metadata maps - const workspaceCacheStorageService = - module.get(WorkspaceCacheStorageService); + const workspaceManyOrAllFlatEntityMapsCacheService = + module.get( + WorkspaceManyOrAllFlatEntityMapsCacheService, + ); - // Return a barebones object metadata map where fields are unknown (so transformer is a no-op) + // Return a barebones flat object metadata map where fields are unknown (so transformer is a no-op) const getMapsMock = - workspaceCacheStorageService.getObjectMetadataMapsOrThrow as jest.Mock; + workspaceManyOrAllFlatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps as jest.Mock; getMapsMock.mockResolvedValue({ - byId: { - [testObjectMetadata.id]: { - ...testObjectMetadata, - fieldsById: {}, - fieldIdByJoinColumnName: {}, - fieldIdByName: {}, - indexMetadatas: [], + flatObjectMetadataMaps: { + byId: { + [testObjectMetadata.id]: { + ...testObjectMetadata, + fieldMetadataIds: [], + indexMetadataIds: [], + viewIds: [], + universalIdentifier: testObjectMetadata.id, + applicationId: null, + } as any, }, + idByUniversalIdentifier: {}, + universalIdentifiersByApplicationId: {}, }, - idByNameSingular: { - [testObjectMetadata.nameSingular]: testObjectMetadata.id, + flatFieldMetadataMaps: { + byId: {}, + idByUniversalIdentifier: {}, + universalIdentifiersByApplicationId: {}, }, - }); + } as any); return { module,