Extract field metadata build out of object metadata build (#14763)

# Introduction

Extracting the legacy fields build and dispatch out of the object one to
follow the generic flat entity build, also update caches entries for
object

## Main tasks
- flat field map cache
- flat field builder
- refactored the dispatch matrix to return flat entity maps instead of
flat entity arrays
- orchestrator aggregator
- removing legacy code
- making universal identifier aka standardId of standard field for
custom object deterministically dynamic
- perfs debug logs for v2

## TODO
- [x] Refactor the generic entity builder to be dependency flat maps in
order to main foreign keys list in flat parent
- [x] Refactor the flat object metadata to contain the array of related
fields and avoid costy find object fields
- [ ] Improve the create field handler to handle multiple field at once
- [ ] Refactor the dispatch to embbed the comparison
- [ ] Improve perf by extracting from elements out of existing
- [ ] Fix the labelIdentifierId validators on object before field
creation ( integ tests are in failing mode )

## Debug logs snippet

```ts
[EntityBuilder fieldMetadata] matrix computation: 0.027ms
[EntityBuilder fieldMetadata] creation validation: 0.001ms
[EntityBuilder fieldMetadata] deletion validation: 0.293ms
[EntityBuilder fieldMetadata] update validation: 0.006ms
[EntityBuilder fieldMetadata] entity processing: 0.363ms
[EntityBuilder fieldMetadata] validateAndBuild: 0.455ms
[EntityBuilder index] matrix computation: 0.005ms
[EntityBuilder index] creation validation: 0.001ms
[EntityBuilder index] deletion validation: 0.146ms
[EntityBuilder index] update validation: 0.004ms
[EntityBuilder index] entity processing: 0.199ms
[EntityBuilder index] validateAndBuild: 0.228ms
[Runner] Initial cache retrieval: 0.549ms
[BaseWorkspaceMigrationRunnerActionHandlerService] delete_index executeForWorkspaceSchema: 11.665ms
[BaseWorkspaceMigrationRunnerActionHandlerService] delete_index executeForMetadata: 12.864ms
[BaseWorkspaceMigrationRunnerActionHandlerService] delete_field executeForWorkspaceSchema: 1.476ms
[BaseWorkspaceMigrationRunnerActionHandlerService] delete_field executeForMetadata: 6.816ms
[BaseWorkspaceMigrationRunnerActionHandlerService] delete_field executeForWorkspaceSchema: 0.062ms
[BaseWorkspaceMigrationRunnerActionHandlerService] delete_field executeForMetadata: 0.889ms
[Runner] Transaction execution: 23.434ms
[Runner] Cache invalidation: 316.662ms
[Runner] Total execution: 340.767ms
```
As you can see cache invalidation is way to long, we could replace the
cache by the optimistic in the end
This commit is contained in:
Paul Rastoin
2025-10-02 16:07:34 +00:00
committed by GitHub
parent d4b83df3f5
commit 170f6d27c5
240 changed files with 11385 additions and 17744 deletions
@@ -7,6 +7,7 @@ import {
BASE_OBJECT_STANDARD_FIELD_IDS,
CUSTOM_OBJECT_STANDARD_FIELD_IDS,
} from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids';
import { createDeterministicUuid } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/create-deterministic-uuid.util';
import { getTsVectorColumnExpressionFromFields } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/get-ts-vector-column-expression.util';
type BuildDefaultFlatFieldMetadataForCustomObjectArgs = {
@@ -29,7 +30,10 @@ export const buildDefaultFlatFieldMetadatasForCustomObject = ({
isLabelSyncedWithName: false,
isUnique: true,
objectMetadataId,
universalIdentifier: BASE_OBJECT_STANDARD_FIELD_IDS.id,
universalIdentifier: createDeterministicUuid([
objectMetadataId,
BASE_OBJECT_STANDARD_FIELD_IDS.id,
]),
workspaceId,
standardId: BASE_OBJECT_STANDARD_FIELD_IDS.id,
name: 'id',
@@ -45,8 +49,6 @@ export const buildDefaultFlatFieldMetadatasForCustomObject = ({
createdAt,
updatedAt: createdAt,
flatRelationTargetFieldMetadata: null,
flatRelationTargetObjectMetadata: null,
options: null,
standardOverrides: null,
relationTargetFieldMetadataId: null,
@@ -61,7 +63,10 @@ export const buildDefaultFlatFieldMetadatasForCustomObject = ({
isLabelSyncedWithName: false,
isUnique: false,
objectMetadataId,
universalIdentifier: CUSTOM_OBJECT_STANDARD_FIELD_IDS.name,
universalIdentifier: createDeterministicUuid([
objectMetadataId,
CUSTOM_OBJECT_STANDARD_FIELD_IDS.name,
]),
workspaceId,
standardId: CUSTOM_OBJECT_STANDARD_FIELD_IDS.name,
name: 'name',
@@ -77,8 +82,6 @@ export const buildDefaultFlatFieldMetadatasForCustomObject = ({
createdAt,
updatedAt: createdAt,
flatRelationTargetFieldMetadata: null,
flatRelationTargetObjectMetadata: null,
options: null,
standardOverrides: null,
relationTargetFieldMetadataId: null,
@@ -93,7 +96,10 @@ export const buildDefaultFlatFieldMetadatasForCustomObject = ({
isLabelSyncedWithName: false,
isUnique: false,
objectMetadataId,
universalIdentifier: BASE_OBJECT_STANDARD_FIELD_IDS.createdAt,
universalIdentifier: createDeterministicUuid([
objectMetadataId,
BASE_OBJECT_STANDARD_FIELD_IDS.createdAt,
]),
workspaceId,
standardId: BASE_OBJECT_STANDARD_FIELD_IDS.createdAt,
name: 'createdAt',
@@ -109,8 +115,6 @@ export const buildDefaultFlatFieldMetadatasForCustomObject = ({
createdAt,
updatedAt: createdAt,
flatRelationTargetFieldMetadata: null,
flatRelationTargetObjectMetadata: null,
options: null,
standardOverrides: null,
relationTargetFieldMetadataId: null,
@@ -125,7 +129,10 @@ export const buildDefaultFlatFieldMetadatasForCustomObject = ({
isLabelSyncedWithName: false,
isUnique: false,
objectMetadataId,
universalIdentifier: BASE_OBJECT_STANDARD_FIELD_IDS.updatedAt,
universalIdentifier: createDeterministicUuid([
objectMetadataId,
BASE_OBJECT_STANDARD_FIELD_IDS.updatedAt,
]),
workspaceId,
standardId: BASE_OBJECT_STANDARD_FIELD_IDS.updatedAt,
name: 'updatedAt',
@@ -141,8 +148,6 @@ export const buildDefaultFlatFieldMetadatasForCustomObject = ({
createdAt,
updatedAt: createdAt,
flatRelationTargetFieldMetadata: null,
flatRelationTargetObjectMetadata: null,
options: null,
standardOverrides: null,
relationTargetFieldMetadataId: null,
@@ -157,7 +162,10 @@ export const buildDefaultFlatFieldMetadatasForCustomObject = ({
isLabelSyncedWithName: false,
isUnique: false,
objectMetadataId,
universalIdentifier: BASE_OBJECT_STANDARD_FIELD_IDS.deletedAt,
universalIdentifier: createDeterministicUuid([
objectMetadataId,
BASE_OBJECT_STANDARD_FIELD_IDS.deletedAt,
]),
workspaceId,
standardId: BASE_OBJECT_STANDARD_FIELD_IDS.deletedAt,
name: 'deletedAt',
@@ -173,8 +181,6 @@ export const buildDefaultFlatFieldMetadatasForCustomObject = ({
createdAt,
updatedAt: createdAt,
flatRelationTargetFieldMetadata: null,
flatRelationTargetObjectMetadata: null,
options: null,
standardOverrides: null,
relationTargetFieldMetadataId: null,
@@ -189,7 +195,10 @@ export const buildDefaultFlatFieldMetadatasForCustomObject = ({
isLabelSyncedWithName: false,
isUnique: false,
objectMetadataId,
universalIdentifier: CUSTOM_OBJECT_STANDARD_FIELD_IDS.createdBy,
universalIdentifier: createDeterministicUuid([
objectMetadataId,
CUSTOM_OBJECT_STANDARD_FIELD_IDS.createdBy,
]),
workspaceId,
standardId: CUSTOM_OBJECT_STANDARD_FIELD_IDS.createdBy,
name: 'createdBy',
@@ -205,8 +214,6 @@ export const buildDefaultFlatFieldMetadatasForCustomObject = ({
createdAt,
updatedAt: createdAt,
flatRelationTargetFieldMetadata: null,
flatRelationTargetObjectMetadata: null,
options: null,
standardOverrides: null,
relationTargetFieldMetadataId: null,
@@ -221,7 +228,10 @@ export const buildDefaultFlatFieldMetadatasForCustomObject = ({
isLabelSyncedWithName: false,
isUnique: false,
objectMetadataId,
universalIdentifier: CUSTOM_OBJECT_STANDARD_FIELD_IDS.position,
universalIdentifier: createDeterministicUuid([
objectMetadataId,
CUSTOM_OBJECT_STANDARD_FIELD_IDS.position,
]),
workspaceId,
standardId: CUSTOM_OBJECT_STANDARD_FIELD_IDS.position,
name: 'position',
@@ -237,8 +247,6 @@ export const buildDefaultFlatFieldMetadatasForCustomObject = ({
createdAt,
updatedAt: createdAt,
flatRelationTargetFieldMetadata: null,
flatRelationTargetObjectMetadata: null,
options: null,
standardOverrides: null,
relationTargetFieldMetadataId: null,
@@ -253,7 +261,10 @@ export const buildDefaultFlatFieldMetadatasForCustomObject = ({
isLabelSyncedWithName: false,
isUnique: false,
objectMetadataId,
universalIdentifier: CUSTOM_OBJECT_STANDARD_FIELD_IDS.searchVector,
universalIdentifier: createDeterministicUuid([
objectMetadataId,
CUSTOM_OBJECT_STANDARD_FIELD_IDS.searchVector,
]),
workspaceId,
standardId: CUSTOM_OBJECT_STANDARD_FIELD_IDS.searchVector,
name: 'searchVector',
@@ -269,8 +280,6 @@ export const buildDefaultFlatFieldMetadatasForCustomObject = ({
createdAt,
updatedAt: createdAt,
flatRelationTargetFieldMetadata: null,
flatRelationTargetObjectMetadata: null,
options: null,
standardOverrides: null,
relationTargetFieldMetadataId: null,
@@ -1,5 +1,6 @@
import { v4 } from 'uuid';
import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
import { type FlatIndexMetadata } from 'src/engine/metadata-modules/flat-index-metadata/types/flat-index-metadata.type';
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
import { IndexType } from 'src/engine/metadata-modules/index-metadata/types/indexType.types';
@@ -10,14 +11,17 @@ export const buildDefaultIndexesForCustomObject = ({
workspaceId,
flatObjectMetadata,
defaultFlatFieldForCustomObjectMaps,
objectFlatFieldMetadatas,
}: {
workspaceId: string;
flatObjectMetadata: FlatObjectMetadata;
objectFlatFieldMetadatas: FlatFieldMetadata[];
defaultFlatFieldForCustomObjectMaps: DefaultFlatFieldForCustomObjectMaps;
}) => {
const tsFlatVectorIndexId = v4();
const createdAt = new Date();
const tsVectorFlatIndex = generateFlatIndexMetadataWithNameOrThrow({
objectFlatFieldMetadatas,
flatIndex: {
createdAt,
flatIndexFieldMetadatas: [
@@ -4,9 +4,10 @@ import { v4 } from 'uuid';
import { RelationType } from 'src/engine/metadata-modules/field-metadata/interfaces/relation-type.interface';
import { type FlatEntityMaps } from 'src/engine/core-modules/common/types/flat-entity-maps.type';
import { findFlatEntityByIdInFlatEntityMaps } from 'src/engine/core-modules/common/utils/find-flat-entity-by-id-in-flat-entity-maps.util';
import { type MorphOrRelationFieldMetadataType } from 'src/engine/metadata-modules/field-metadata/types/morph-or-relation-field-metadata-type.type';
import { type FlatFieldMetadata } from 'src/engine/metadata-modules/flat-field-metadata/types/flat-field-metadata.type';
import { type FlatObjectMetadataMaps } from 'src/engine/metadata-modules/flat-object-metadata-maps/types/flat-object-metadata-maps.type';
import { findFlatObjectMetadataInFlatObjectMetadataMaps } from 'src/engine/metadata-modules/flat-object-metadata-maps/utils/find-flat-object-metadata-in-flat-object-metadata-maps.util';
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
import {
ObjectMetadataException,
@@ -15,19 +16,16 @@ import {
import { buildDescriptionForRelationFieldMetadataOnFromField } from 'src/engine/metadata-modules/object-metadata/utils/build-description-for-relation-field-on-from-field.util';
import { buildDescriptionForRelationFieldMetadataOnToField } from 'src/engine/metadata-modules/object-metadata/utils/build-description-for-relation-field-on-to-field.util';
import { RelationOnDeleteAction } from 'src/engine/metadata-modules/relation-metadata/relation-on-delete-action.type';
import { fromFlatObjectMetadataToFlatObjectMetadataWithoutFields } from 'src/engine/workspace-manager/workspace-migration-v2/workspace-migration-builder-v2/utils/from-flat-object-metadata-to-flat-object-metadata-without-fields.util';
import {
CUSTOM_OBJECT_STANDARD_FIELD_IDS,
STANDARD_OBJECT_FIELD_IDS,
} from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-field-ids';
import { STANDARD_OBJECT_ICONS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-object-icons';
import { type STANDARD_OBJECT_IDS } from 'src/engine/workspace-manager/workspace-sync-metadata/constants/standard-object-ids';
import { createRelationDeterministicUuid } from 'src/engine/workspace-manager/workspace-sync-metadata/utils/create-deterministic-uuid.util';
type FlatFieldMetadataRelationWithoutRelations = Omit<
FlatFieldMetadata<FieldMetadataType.RELATION>,
'flatRelationTargetFieldMetadata' | 'flatRelationTargetObjectMetadata'
>;
import {
createDeterministicUuid,
createRelationDeterministicUuid,
} from 'src/engine/workspace-manager/workspace-sync-metadata/utils/create-deterministic-uuid.util';
const generateSourceFlatFieldMetadata = ({
workspaceId,
@@ -39,7 +37,7 @@ const generateSourceFlatFieldMetadata = ({
> & {
sourceFlatObjectMetadata: FlatObjectMetadata;
targetFlatObjectMetadata: FlatObjectMetadata;
}): FlatFieldMetadataRelationWithoutRelations => {
}): FlatFieldMetadata<FieldMetadataType.RELATION> => {
const { description } = buildDescriptionForRelationFieldMetadataOnFromField({
relationObjectMetadataNamePlural: targetFlatObjectMetadata.namePlural,
targetObjectLabelSingular: sourceFlatObjectMetadata.labelSingular,
@@ -90,7 +88,10 @@ const generateSourceFlatFieldMetadata = ({
standardId,
standardOverrides: null,
type: FieldMetadataType.RELATION,
universalIdentifier: standardId,
universalIdentifier: createDeterministicUuid([
sourceFlatObjectMetadata.id,
standardId,
]),
workspaceId,
morphId: null,
};
@@ -107,8 +108,8 @@ const generateTargetFlatFieldMetadata = ({
> & {
sourceFlatObjectMetadata: FlatObjectMetadata;
targetFlatObjectMetadata: FlatObjectMetadata;
sourceFlatFieldMetadata: FlatFieldMetadataRelationWithoutRelations;
}): FlatFieldMetadataRelationWithoutRelations => {
sourceFlatFieldMetadata: FlatFieldMetadata<MorphOrRelationFieldMetadataType>;
}): FlatFieldMetadata<FieldMetadataType.RELATION> => {
const customStandardFieldId =
STANDARD_OBJECT_FIELD_IDS[
targetFlatObjectMetadata.nameSingular as (typeof DEFAULT_RELATIONS_OBJECTS_STANDARD_IDS)[number]
@@ -154,7 +155,10 @@ const generateTargetFlatFieldMetadata = ({
relationTargetFieldMetadataId: sourceFlatFieldMetadata.id,
relationTargetObjectMetadataId: sourceFlatObjectMetadata.id,
standardOverrides: null,
universalIdentifier: standardId,
universalIdentifier: createDeterministicUuid([
targetFlatObjectMetadata.id,
standardId,
]),
};
};
@@ -167,7 +171,7 @@ const DEFAULT_RELATIONS_OBJECTS_STANDARD_IDS = [
] as const satisfies (keyof typeof STANDARD_OBJECT_IDS)[];
export type BuildDefaultRelationFieldsForCustomObjectArgs = {
existingFlatObjectMetadataMaps: FlatObjectMetadataMaps;
existingFlatObjectMetadataMaps: FlatEntityMaps<FlatObjectMetadata>;
workspaceId: string;
sourceFlatObjectMetadata: FlatObjectMetadata;
};
@@ -187,12 +191,23 @@ export const buildDefaultRelationFlatFieldMetadatasForCustomObject = ({
sourceFlatObjectMetadata,
workspaceId,
}: BuildDefaultRelationFieldsForCustomObjectArgs): SourceAndTargetFlatFieldMetadatasRecord => {
return DEFAULT_RELATIONS_OBJECTS_STANDARD_IDS.reduce(
const objectIdByNameSingular = Object.values(
existingFlatObjectMetadataMaps.byId,
).reduce<Record<string, string>>((acc, flatObject) => {
if (!isDefined(flatObject)) {
return acc;
}
return {
...acc,
[flatObject.nameSingular]: flatObject.id,
};
}, {});
return DEFAULT_RELATIONS_OBJECTS_STANDARD_IDS.reduce<SourceAndTargetFlatFieldMetadatasRecord>(
(sourceAndTargetFlatFieldMetadatasRecord, objectMetadataNameSingular) => {
const targetFlatObjectMetadataId =
existingFlatObjectMetadataMaps.idByNameSingular[
objectMetadataNameSingular
];
objectIdByNameSingular[objectMetadataNameSingular];
if (!isDefined(targetFlatObjectMetadataId)) {
throw new ObjectMetadataException(
@@ -201,11 +216,10 @@ export const buildDefaultRelationFlatFieldMetadatasForCustomObject = ({
);
}
const targetFlatObjectMetadata =
findFlatObjectMetadataInFlatObjectMetadataMaps({
flatObjectMetadataMaps: existingFlatObjectMetadataMaps,
objectMetadataId: targetFlatObjectMetadataId,
});
const targetFlatObjectMetadata = findFlatEntityByIdInFlatEntityMaps({
flatEntityMaps: existingFlatObjectMetadataMaps,
flatEntityId: targetFlatObjectMetadataId,
});
if (!isDefined(targetFlatObjectMetadata)) {
throw new ObjectMetadataException(
@@ -216,7 +230,7 @@ export const buildDefaultRelationFlatFieldMetadatasForCustomObject = ({
const sourceFlatFieldMetadata = generateSourceFlatFieldMetadata({
sourceFlatObjectMetadata,
targetFlatObjectMetadata: targetFlatObjectMetadata,
targetFlatObjectMetadata,
workspaceId,
});
@@ -230,33 +244,11 @@ export const buildDefaultRelationFlatFieldMetadatasForCustomObject = ({
return {
standardSourceFlatFieldMetadatas: [
...sourceAndTargetFlatFieldMetadatasRecord.standardSourceFlatFieldMetadatas,
{
...sourceFlatFieldMetadata,
flatRelationTargetFieldMetadata: {
...targetFlatFieldMetadata,
flatRelationTargetFieldMetadata: null,
flatRelationTargetObjectMetadata: null,
},
flatRelationTargetObjectMetadata:
fromFlatObjectMetadataToFlatObjectMetadataWithoutFields(
targetFlatObjectMetadata,
),
},
sourceFlatFieldMetadata,
],
standardTargetFlatFieldMetadatas: [
...sourceAndTargetFlatFieldMetadatasRecord.standardTargetFlatFieldMetadatas,
{
...targetFlatFieldMetadata,
flatRelationTargetFieldMetadata: {
...sourceFlatFieldMetadata,
flatRelationTargetFieldMetadata: null,
flatRelationTargetObjectMetadata: null,
},
flatRelationTargetObjectMetadata:
fromFlatObjectMetadataToFlatObjectMetadataWithoutFields(
sourceFlatObjectMetadata,
),
},
targetFlatFieldMetadata,
],
};
},