Deprecate object metadata maps in favor of flat entities (#16080)

## Context Deprecating the old objectMetadataMap type in favour of split flat entities to match with our new caching. In the long run, trying to achieve: - Better performance through caching - Consistent data access patterns across the codebase - Reduced database queries Now that everything is based on flat entities, which are cached, we can finish the refactoring of workspace context cache which should already improve performances. Then the last step will be to consume that new cache in the new global datasource to get rid of the many workspace datasources stored in the server
2025-11-27 13:43:34 +01:00
parent 978c0acb90
commit 1607aebcc6
281 changed files with 6510 additions and 6627 deletions
@@ -1,11 +1,17 @@
-import { mockPersonObjectMetadataWithFieldMaps } from 'src/engine/api/graphql/graphql-query-runner/__mocks__/mockPersonObjectMetadata';
+import {
+  mockPersonFlatFieldMetadataMaps,
+  mockPersonFlatObjectMetadata,
+  mockPersonFlatObjectMetadataMaps,
+} from 'src/engine/api/graphql/graphql-query-runner/__mocks__/mockPersonObjectMetadata';
 import { mockPersonRecords } from 'src/engine/api/graphql/graphql-query-runner/__mocks__/mockPersonRecords';
 import { buildDuplicateConditions } from 'src/engine/api/utils/build-duplicate-conditions.utils';

 describe('buildDuplicateConditions', () => {
  it('should build conditions based on duplicate criteria from composite field', () => {
    const duplicateConditons = buildDuplicateConditions(
-      mockPersonObjectMetadataWithFieldMaps([['emailsPrimaryEmail']]),
+      mockPersonFlatObjectMetadata([['emailsPrimaryEmail']]),
+      mockPersonFlatObjectMetadataMaps([['emailsPrimaryEmail']]),
+      mockPersonFlatFieldMetadataMaps(),
      mockPersonRecords,
      'recordId',
    );
@@ -28,7 +34,9 @@ describe('buildDuplicateConditions', () => {

  it('should build conditions based on duplicate criteria from basic field', () => {
    const duplicateConditons = buildDuplicateConditions(
-      mockPersonObjectMetadataWithFieldMaps([['jobTitle']]),
+      mockPersonFlatObjectMetadata([['jobTitle']]),
+      mockPersonFlatObjectMetadataMaps([['jobTitle']]),
+      mockPersonFlatFieldMetadataMaps(),
      mockPersonRecords,
      'recordId',
    );
@@ -49,7 +57,9 @@ describe('buildDuplicateConditions', () => {

  it('should not build conditions based on duplicate criteria if record value is null or too small', () => {
    const duplicateConditons = buildDuplicateConditions(
-      mockPersonObjectMetadataWithFieldMaps([['linkedinLinkPrimaryLinkUrl']]),
+      mockPersonFlatObjectMetadata([['linkedinLinkPrimaryLinkUrl']]),
+      mockPersonFlatObjectMetadataMaps([['linkedinLinkPrimaryLinkUrl']]),
+      mockPersonFlatFieldMetadataMaps(),
      mockPersonRecords,
      'recordId',
    );
@@ -59,7 +69,9 @@ describe('buildDuplicateConditions', () => {

  it('should build conditions based on duplicate criteria and without recordId filter', () => {
    const duplicateConditons = buildDuplicateConditions(
-      mockPersonObjectMetadataWithFieldMaps([['jobTitle']]),
+      mockPersonFlatObjectMetadata([['jobTitle']]),
+      mockPersonFlatObjectMetadataMaps([['jobTitle']]),
+      mockPersonFlatFieldMetadataMaps(),
      mockPersonRecords,
    );

@@ -2,79 +2,119 @@ import { FieldMetadataType, OrderByDirection } from 'twenty-shared/types';

 import { GraphqlQueryRunnerException } from 'src/engine/api/graphql/graphql-query-runner/errors/graphql-query-runner.exception';
 import { computeCursorArgFilter } from 'src/engine/api/utils/compute-cursor-arg-filter.utils';
-import { getMockFieldMetadataEntity } from 'src/utils/__test__/get-field-metadata-entity.mock';
-import { getMockObjectMetadataItemWithFieldsMaps } from 'src/utils/__test__/get-object-metadata-item-with-fields-maps.mock';
+import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
+import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
+import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';

 describe('computeCursorArgFilter', () => {
-  const objectMetadataItemWithFieldMaps =
-    getMockObjectMetadataItemWithFieldsMaps({
-      id: 'object-id',
-      workspaceId: 'workspace-id',
-      nameSingular: 'person',
-      namePlural: 'people',
-      isCustom: false,
-      isRemote: false,
-      labelSingular: 'Person',
-      labelPlural: 'People',
-      targetTableName: 'person',
-      indexMetadatas: [],
-      isSystem: false,
-      isActive: true,
-      isAuditLogged: false,
-      isSearchable: false,
-      fieldIdByJoinColumnName: {},
-      icon: 'Icon123',
-      fieldIdByName: {
-        name: 'name-id',
-        age: 'age-id',
-        fullName: 'fullname-id',
+  const workspaceId = 'workspace-id';
+  const objectMetadataId = 'object-id';
+
+  const createMockField = (
+    overrides: Partial<FlatFieldMetadata> & {
+      id: string;
+      name: string;
+      type: FieldMetadataType;
+    },
+  ): FlatFieldMetadata =>
+    ({
+      workspaceId,
+      objectMetadataId,
+      isNullable: true,
+      isLabelSyncedWithName: true,
+      createdAt: new Date(),
+      updatedAt: new Date(),
+      universalIdentifier: overrides.id,
+      viewFieldIds: [],
+      viewFilterIds: [],
+      viewGroupIds: [],
+      kanbanAggregateOperationViewIds: [],
+      calendarViewIds: [],
+      applicationId: null,
+      label: overrides.name,
+      ...overrides,
+    }) as FlatFieldMetadata;
+
+  const nameField = createMockField({
+    id: 'name-id',
+    type: FieldMetadataType.TEXT,
+    name: 'name',
+    label: 'Name',
+  });
+
+  const ageField = createMockField({
+    id: 'age-id',
+    type: FieldMetadataType.NUMBER,
+    name: 'age',
+    label: 'Age',
+  });
+
+  const fullNameField = createMockField({
+    id: 'fullname-id',
+    type: FieldMetadataType.FULL_NAME,
+    name: 'fullName',
+    label: 'Full Name',
+  });
+
+  const buildFlatFieldMetadataMaps = (
+    fields: FlatFieldMetadata[],
+  ): FlatEntityMaps<FlatFieldMetadata> => ({
+    byId: fields.reduce(
+      (acc, field) => {
+        acc[field.id] = field;
+
+        return acc;
      },
-      fieldsById: {
-        'name-id': getMockFieldMetadataEntity({
-          workspaceId: 'workspace-id',
-          objectMetadataId: 'object-id',
-          id: 'name-id',
-          type: FieldMetadataType.TEXT,
-          name: 'name',
-          label: 'Name',
-          isLabelSyncedWithName: true,
-          isNullable: true,
-          createdAt: new Date(),
-          updatedAt: new Date(),
-        }),
-        'age-id': getMockFieldMetadataEntity({
-          workspaceId: 'workspace-id',
-          objectMetadataId: 'object-id',
-          id: 'age-id',
-          type: FieldMetadataType.NUMBER,
-          name: 'age',
-          label: 'Age',
-          isLabelSyncedWithName: true,
-          isNullable: true,
-          createdAt: new Date(),
-          updatedAt: new Date(),
-        }),
-        'fullname-id': getMockFieldMetadataEntity({
-          workspaceId: 'workspace-id',
-          objectMetadataId: 'object-id',
-          id: 'fullname-id',
-          type: FieldMetadataType.FULL_NAME,
-          name: 'fullName',
-          label: 'Full Name',
-          isLabelSyncedWithName: true,
-          isNullable: true,
-          createdAt: new Date(),
-          updatedAt: new Date(),
-        }),
+      {} as Record<string, FlatFieldMetadata>,
+    ),
+    idByUniversalIdentifier: fields.reduce(
+      (acc, field) => {
+        acc[field.universalIdentifier] = field.id;
+
+        return acc;
      },
-    });
+      {} as Record<string, string>,
+    ),
+    universalIdentifiersByApplicationId: {},
+  });
+
+  const flatFieldMetadataMaps = buildFlatFieldMetadataMaps([
+    nameField,
+    ageField,
+    fullNameField,
+  ]);
+
+  const flatObjectMetadata: FlatObjectMetadata = {
+    id: objectMetadataId,
+    workspaceId,
+    nameSingular: 'person',
+    namePlural: 'people',
+    labelSingular: 'Person',
+    labelPlural: 'People',
+    targetTableName: 'person',
+    isCustom: false,
+    isRemote: false,
+    isActive: true,
+    isSystem: false,
+    isAuditLogged: false,
+    isSearchable: false,
+    icon: 'Icon123',
+    createdAt: new Date(),
+    updatedAt: new Date(),
+    universalIdentifier: objectMetadataId,
+    fieldMetadataIds: ['name-id', 'age-id', 'fullname-id'],
+    indexMetadataIds: [],
+    viewIds: [],
+    applicationId: null,
+  } as unknown as FlatObjectMetadata;

  describe('basic cursor filtering', () => {
    it('should return empty array when cursor is empty', () => {
      const result = computeCursorArgFilter(
        {},
        [],
-        objectMetadataItemWithFieldMaps,
+        flatObjectMetadata,
+        flatFieldMetadataMaps,
        true,
      );

@@ -88,7 +128,8 @@ describe('computeCursorArgFilter', () => {
      const result = computeCursorArgFilter(
        cursor,
        orderBy,
-        objectMetadataItemWithFieldMaps,
+        flatObjectMetadata,
+        flatFieldMetadataMaps,
        true,
      );

@@ -102,7 +143,8 @@ describe('computeCursorArgFilter', () => {
      const result = computeCursorArgFilter(
        cursor,
        orderBy,
-        objectMetadataItemWithFieldMaps,
+        flatObjectMetadata,
+        flatFieldMetadataMaps,
        false,
      );

@@ -121,7 +163,8 @@ describe('computeCursorArgFilter', () => {
      const result = computeCursorArgFilter(
        cursor,
        orderBy,
-        objectMetadataItemWithFieldMaps,
+        flatObjectMetadata,
+        flatFieldMetadataMaps,
        true,
      );

@@ -149,7 +192,8 @@ describe('computeCursorArgFilter', () => {
      const result = computeCursorArgFilter(
        cursor,
        orderBy,
-        objectMetadataItemWithFieldMaps,
+        flatObjectMetadata,
+        flatFieldMetadataMaps,
        true,
      );

@@ -195,7 +239,8 @@ describe('computeCursorArgFilter', () => {
      const result = computeCursorArgFilter(
        cursor,
        orderBy,
-        objectMetadataItemWithFieldMaps,
+        flatObjectMetadata,
+        flatFieldMetadataMaps,
        true,
      );

@@ -224,7 +269,8 @@ describe('computeCursorArgFilter', () => {
      const result = computeCursorArgFilter(
        cursor,
        orderBy,
-        objectMetadataItemWithFieldMaps,
+        flatObjectMetadata,
+        flatFieldMetadataMaps,
        false,
      );

@@ -265,7 +311,8 @@ describe('computeCursorArgFilter', () => {
        computeCursorArgFilter(
          cursor,
          orderBy,
-          objectMetadataItemWithFieldMaps,
+          flatObjectMetadata,
+          flatFieldMetadataMaps,
          true,
        ),
      ).toThrow(GraphqlQueryRunnerException);
@@ -279,7 +326,8 @@ describe('computeCursorArgFilter', () => {
        computeCursorArgFilter(
          cursor,
          orderBy,
-          objectMetadataItemWithFieldMaps,
+          flatObjectMetadata,
+          flatFieldMetadataMaps,
          true,
        ),
      ).toThrow(GraphqlQueryRunnerException);
@@ -16,14 +16,19 @@ import { computeOperator } from 'src/engine/api/utils/compute-operator.utils';
 import { isAscendingOrder } from 'src/engine/api/utils/is-ascending-order.utils';
 import { validateAndGetOrderByForScalarField } from 'src/engine/api/utils/validate-and-get-order-by.utils';
 import { isCompositeFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/utils/is-composite-field-metadata-type.util';
-import { type ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
+import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
+import { findFlatEntityByIdInFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-id-in-flat-entity-maps.util';
+import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
+import { buildFieldMapsFromFlatObjectMetadata } from 'src/engine/metadata-modules/flat-field-metadata/utils/build-field-maps-from-flat-object-metadata.util';
+import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';

 type BuildCursorWhereConditionParams = {
  cursorKey: keyof ObjectRecord;
  cursorValue:
    | ObjectRecordCursorLeafScalarValue
    | ObjectRecordCursorLeafCompositeValue;
-  objectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps;
+  flatObjectMetadata: FlatObjectMetadata;
+  flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>;
  orderBy: ObjectRecordOrderBy;
  isForwardPagination: boolean;
  isEqualityCondition?: boolean;
@@ -32,15 +37,23 @@ type BuildCursorWhereConditionParams = {
 export const buildCursorWhereCondition = ({
  cursorKey,
  cursorValue,
-  objectMetadataItemWithFieldMaps,
+  flatObjectMetadata,
+  flatFieldMetadataMaps,
  orderBy,
  isForwardPagination,
  isEqualityCondition = false,
 }: BuildCursorWhereConditionParams): Record<string, unknown> => {
-  const fieldMetadataId =
-    objectMetadataItemWithFieldMaps.fieldIdByName[cursorKey];
-  const fieldMetadata =
-    objectMetadataItemWithFieldMaps.fieldsById[fieldMetadataId];
+  const { fieldIdByName } = buildFieldMapsFromFlatObjectMetadata(
+    flatFieldMetadataMaps,
+    flatObjectMetadata,
+  );
+
+  const fieldMetadataId = fieldIdByName[cursorKey];
+
+  const fieldMetadata = findFlatEntityByIdInFlatEntityMaps({
+    flatEntityMaps: flatFieldMetadataMaps,
+    flatEntityId: fieldMetadataId,
+  });

  if (!fieldMetadata) {
    throw new GraphqlQueryRunnerException(
@@ -4,12 +4,16 @@ import { type ObjectRecord } from 'twenty-shared/types';
 import { type ObjectRecordFilter } from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface';

 import { settings } from 'src/engine/constants/settings';
-import { type ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
+import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
+import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
+import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
 import { formatData } from 'src/engine/twenty-orm/utils/format-data.util';
 import { getCompositeFieldMetadataMap } from 'src/engine/twenty-orm/utils/format-result.util';

 export const buildDuplicateConditions = (
-  objectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps,
+  flatObjectMetadata: FlatObjectMetadata,
+  flatObjectMetadataMaps: FlatEntityMaps<FlatObjectMetadata>,
+  flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>,
  records?: Partial<ObjectRecord>[] | undefined,
  filteringByExistingRecordId?: string,
 ): Partial<ObjectRecordFilter> => {
@@ -17,13 +21,17 @@ export const buildDuplicateConditions = (
    return {};
  }

-  const criteriaCollection =
-    objectMetadataItemWithFieldMaps.duplicateCriteria || [];
+  const criteriaCollection = flatObjectMetadata.duplicateCriteria || [];

-  const formattedRecords = formatData(records, objectMetadataItemWithFieldMaps);
+  const formattedRecords = formatData(
+    records,
+    flatObjectMetadata,
+    flatFieldMetadataMaps,
+  );

  const compositeFieldMetadataMap = getCompositeFieldMetadataMap(
-    objectMetadataItemWithFieldMaps,
+    flatObjectMetadata,
+    flatFieldMetadataMaps,
  );

  const conditions = formattedRecords.flatMap((record) => {
@@ -10,12 +10,15 @@ import {

 import { buildCursorCumulativeWhereCondition } from 'src/engine/api/utils/build-cursor-cumulative-where-conditions.utils';
 import { buildCursorWhereCondition } from 'src/engine/api/utils/build-cursor-where-condition.utils';
-import { type ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
+import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
+import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
+import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';

 export const computeCursorArgFilter = (
  cursor: ObjectRecordCursor,
  orderBy: ObjectRecordOrderBy,
-  objectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps,
+  flatObjectMetadata: FlatObjectMetadata,
+  flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>,
  isForwardPagination = true,
 ): ObjectRecordFilter[] => {
  const cursorEntries = Object.entries(cursor)
@@ -42,7 +45,8 @@ export const computeCursorArgFilter = (
      buildCursorWhereCondition({
        cursorKey,
        cursorValue,
-        objectMetadataItemWithFieldMaps,
+        flatObjectMetadata,
+        flatFieldMetadataMaps,
        orderBy,
        isForwardPagination: true,
        isEqualityCondition: true,
@@ -51,7 +55,8 @@ export const computeCursorArgFilter = (
      buildCursorWhereCondition({
        cursorKey,
        cursorValue,
-        objectMetadataItemWithFieldMaps,
+        flatObjectMetadata,
+        flatFieldMetadataMaps,
        orderBy,
        isForwardPagination,
      }),
@@ -1,12 +1,17 @@
 import { type RestrictedFieldsPermissions } from 'twenty-shared/types';
 import { isDefined } from 'twenty-shared/utils';

-import { type ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps';
+import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
+import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
+import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
 import { getFieldMetadataIdToColumnNamesMap } from 'src/engine/twenty-orm/utils/get-field-metadata-id-to-column-names-map.util';

 type GetAllSelectableFieldsArgs = {
  restrictedFields: RestrictedFieldsPermissions;
-  objectMetadata: { objectMetadataMapItem: ObjectMetadataItemWithFieldMaps };
+  objectMetadata: {
+    objectMetadataMapItem: FlatObjectMetadata;
+    flatFieldMetadataMaps: FlatEntityMaps<FlatFieldMetadata>;
+  };
 };

 export const getAllSelectableColumnNames = ({
@@ -19,6 +24,7 @@ export const getAllSelectableColumnNames = ({

  const fieldMetadataIdToColumnNamesMap = getFieldMetadataIdToColumnNamesMap(
    objectMetadata.objectMetadataMapItem,
+    objectMetadata.flatFieldMetadataMaps,
  );

  const restrictedFieldsColumnNames: string[] = restrictedFieldsIds