Improve view identification command perfs and reliability (#17161)
# Introduction Instead of comparing existing to expected comparing expected to existing Also improved logs and fallback use cases # Test Tested on a prod extract
This commit is contained in:
Paul Rastoin
+220
-199
@@ -1,7 +1,7 @@
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { Command } from 'nest-commander';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
import { capitalize, isDefined, uncapitalize } from 'twenty-shared/utils';
|
||||
import { WorkspaceActivationStatus } from 'twenty-shared/workspace';
|
||||
import { IsNull, Repository } from 'typeorm';
|
||||
import { v4 } from 'uuid';
|
||||
@@ -10,43 +10,34 @@ import {
|
||||
RunOnWorkspaceArgs,
|
||||
WorkspacesMigrationCommandRunner,
|
||||
} from 'src/database/commands/command-runners/workspaces-migration.command-runner';
|
||||
import { computeFormattedViewName } from 'src/database/commands/upgrade-version-command/1-16/utils/compute-formatted-view-name.util';
|
||||
import { ALL_ENTITY_VIEW_NAME } from 'src/database/commands/upgrade-version-command/1-16/utils/compute-formatted-view-name.util';
|
||||
import { ApplicationService } from 'src/engine/core-modules/application/application.service';
|
||||
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
|
||||
import { WorkspaceManyOrAllFlatEntityMapsCacheService } from 'src/engine/metadata-modules/flat-entity/services/workspace-many-or-all-flat-entity-maps-cache.service';
|
||||
import { type FlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/types/flat-entity-maps.type';
|
||||
import { findFlatEntityByUniversalIdentifier } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-universal-identifier.util';
|
||||
import { findManyFlatEntityByIdInFlatEntityMapsOrThrow } from 'src/engine/metadata-modules/flat-entity/utils/find-many-flat-entity-by-id-in-flat-entity-maps-or-throw.util';
|
||||
import { getMetadataFlatEntityMapsKey } from 'src/engine/metadata-modules/flat-entity/utils/get-metadata-flat-entity-maps-key.util';
|
||||
import { getMetadataRelatedMetadataNames } from 'src/engine/metadata-modules/flat-entity/utils/get-metadata-related-metadata-names.util';
|
||||
import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';
|
||||
import { type FlatView } from 'src/engine/metadata-modules/flat-view/types/flat-view.type';
|
||||
import { ViewEntity } from 'src/engine/metadata-modules/view/entities/view.entity';
|
||||
import { ViewKey } from 'src/engine/metadata-modules/view/enums/view-key.enum';
|
||||
import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspace-datasource/global-workspace-orm.manager';
|
||||
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
|
||||
import { STANDARD_OBJECTS } from 'src/engine/workspace-manager/twenty-standard-application/constants/standard-object.constant';
|
||||
|
||||
type CustomViewMetadata = {
|
||||
viewEntity: ViewEntity;
|
||||
fromStandard: boolean;
|
||||
};
|
||||
|
||||
type StandardViewMetadata = {
|
||||
viewEntity: ViewEntity;
|
||||
type StandardViewUpdate = {
|
||||
flatView: FlatView;
|
||||
universalIdentifier: string;
|
||||
objectNameSingular: string;
|
||||
};
|
||||
|
||||
type AllWarnings = 'unknown_object';
|
||||
|
||||
type ViewMetadataWarning = {
|
||||
viewEntity: ViewEntity;
|
||||
warning: AllWarnings;
|
||||
objectNameSingular?: string;
|
||||
};
|
||||
|
||||
type AllExceptions =
|
||||
| 'existing_universal_id_mismatch'
|
||||
| 'not_found_object'
|
||||
| 'unknown_standard_view_name';
|
||||
type AllExceptions = 'unknown_standard_view';
|
||||
|
||||
type ViewMetadataException = {
|
||||
viewEntity: ViewEntity;
|
||||
flatView: FlatView;
|
||||
exception: AllExceptions;
|
||||
objectNameSingular?: string;
|
||||
};
|
||||
@@ -87,7 +78,200 @@ export class IdentifyViewMetadataCommand extends WorkspacesMigrationCommandRunne
|
||||
{ workspaceId },
|
||||
);
|
||||
|
||||
const allViewEntities = await this.viewRepository.find({
|
||||
const { flatObjectMetadataMaps, flatViewMaps } =
|
||||
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
|
||||
{
|
||||
workspaceId,
|
||||
flatMapsKeys: ['flatObjectMetadataMaps', 'flatViewMaps'],
|
||||
},
|
||||
);
|
||||
|
||||
await this.identifyStandardViewsOrThrow({
|
||||
workspaceId,
|
||||
flatObjectMetadataMaps,
|
||||
flatViewMaps,
|
||||
twentyStandardApplicationId: twentyStandardFlatApplication.id,
|
||||
dryRun: options.dryRun ?? false,
|
||||
});
|
||||
|
||||
await this.identifyCustomViews({
|
||||
workspaceId,
|
||||
flatObjectMetadataMaps,
|
||||
workspaceCustomApplicationId: workspaceCustomFlatApplication.id,
|
||||
dryRun: options.dryRun ?? false,
|
||||
});
|
||||
|
||||
const relatedMetadataNames = getMetadataRelatedMetadataNames('view');
|
||||
const relatedCacheKeysToInvalidate = relatedMetadataNames.map(
|
||||
getMetadataFlatEntityMapsKey,
|
||||
);
|
||||
|
||||
this.logger.log(
|
||||
`Invalidating caches: ${relatedCacheKeysToInvalidate.join(' ')}`,
|
||||
);
|
||||
await this.workspaceCacheService.invalidateAndRecompute(workspaceId, [
|
||||
'flatViewMaps',
|
||||
...relatedCacheKeysToInvalidate,
|
||||
]);
|
||||
}
|
||||
|
||||
private async identifyStandardViewsOrThrow({
|
||||
workspaceId,
|
||||
flatObjectMetadataMaps,
|
||||
flatViewMaps,
|
||||
twentyStandardApplicationId,
|
||||
dryRun,
|
||||
}: {
|
||||
workspaceId: string;
|
||||
flatObjectMetadataMaps: FlatEntityMaps<FlatObjectMetadata>;
|
||||
flatViewMaps: FlatEntityMaps<FlatView>;
|
||||
twentyStandardApplicationId: string;
|
||||
dryRun: boolean;
|
||||
}): Promise<void> {
|
||||
const standardViewUpdates: StandardViewUpdate[] = [];
|
||||
const exceptions: ViewMetadataException[] = [];
|
||||
|
||||
for (const [objectNameSingular, objectConfig] of Object.entries(
|
||||
STANDARD_OBJECTS,
|
||||
)) {
|
||||
const objectViews =
|
||||
'views' in objectConfig
|
||||
? (objectConfig.views as Record<
|
||||
string,
|
||||
{ universalIdentifier: string } | undefined
|
||||
>)
|
||||
: null;
|
||||
|
||||
if (!isDefined(objectViews)) {
|
||||
continue;
|
||||
}
|
||||
const flatObjectMetadata = findFlatEntityByUniversalIdentifier({
|
||||
flatEntityMaps: flatObjectMetadataMaps,
|
||||
universalIdentifier: objectConfig.universalIdentifier,
|
||||
});
|
||||
|
||||
if (!isDefined(flatObjectMetadata)) {
|
||||
this.logger.error(
|
||||
`Standard object "${objectNameSingular}" not found in workspace, this needs investigation, skipping`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const relatedFlatViews = findManyFlatEntityByIdInFlatEntityMapsOrThrow({
|
||||
flatEntityIds: flatObjectMetadata.viewIds,
|
||||
flatEntityMaps: flatViewMaps,
|
||||
});
|
||||
|
||||
for (const flatView of relatedFlatViews) {
|
||||
// INDEX views -> forward to standard (if object has views config)
|
||||
if (
|
||||
flatView.key === ViewKey.INDEX &&
|
||||
flatView.name === ALL_ENTITY_VIEW_NAME
|
||||
) {
|
||||
const formattedViewName = `all${capitalize(flatObjectMetadata.namePlural)}`;
|
||||
|
||||
const viewConfig = objectViews[formattedViewName];
|
||||
const universalIdentifier = viewConfig?.universalIdentifier;
|
||||
|
||||
if (!isDefined(universalIdentifier)) {
|
||||
exceptions.push({
|
||||
exception: 'unknown_standard_view',
|
||||
flatView,
|
||||
objectNameSingular: flatObjectMetadata.nameSingular,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
standardViewUpdates.push({
|
||||
flatView,
|
||||
universalIdentifier:
|
||||
flatView.universalIdentifier ?? universalIdentifier,
|
||||
objectNameSingular: flatObjectMetadata.nameSingular,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Views with "All" name pattern but not INDEX -> forward to custom
|
||||
if (flatView.name.startsWith('All ')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Remaining views (like assignedToMe, byStatus, etc.)
|
||||
const formattedViewName = uncapitalize(
|
||||
flatView.name.split(' ').map(capitalize).join(''),
|
||||
);
|
||||
|
||||
const viewConfig = objectViews[formattedViewName];
|
||||
const universalIdentifier = viewConfig?.universalIdentifier;
|
||||
|
||||
if (!isDefined(universalIdentifier)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
standardViewUpdates.push({
|
||||
flatView,
|
||||
universalIdentifier:
|
||||
flatView.universalIdentifier ?? universalIdentifier,
|
||||
objectNameSingular: flatObjectMetadata.nameSingular,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (exceptions.length > 0) {
|
||||
this.logger.error(
|
||||
`Found ${exceptions.length} exception(s) while processing view metadata for workspace ${workspaceId}. No updates will be applied.`,
|
||||
);
|
||||
|
||||
for (const { flatView, exception, objectNameSingular } of exceptions) {
|
||||
this.logger.error(
|
||||
`Exception for view "${flatView.name}" on object "${objectNameSingular ?? 'unknown'}" (id=${flatView.id}): ${exception}`,
|
||||
);
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Aborting migration for workspace ${workspaceId} due to ${exceptions.length} exception(s). See logs above for details.`,
|
||||
);
|
||||
}
|
||||
|
||||
const standardUpdates = standardViewUpdates.map(
|
||||
({ flatView, universalIdentifier }) => ({
|
||||
id: flatView.id,
|
||||
universalIdentifier,
|
||||
applicationId: twentyStandardApplicationId,
|
||||
}),
|
||||
);
|
||||
|
||||
this.logger.log(
|
||||
`Found ${standardUpdates.length} standard view(s) to update for workspace ${workspaceId}`,
|
||||
);
|
||||
|
||||
for (const {
|
||||
flatView,
|
||||
universalIdentifier,
|
||||
objectNameSingular,
|
||||
} of standardViewUpdates) {
|
||||
this.logger.log(
|
||||
` - Standard view "${flatView.name}" on object "${objectNameSingular}" (id=${flatView.id}) -> universalIdentifier=${universalIdentifier}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!dryRun) {
|
||||
await this.viewRepository.save(standardUpdates);
|
||||
}
|
||||
}
|
||||
|
||||
private async identifyCustomViews({
|
||||
workspaceId,
|
||||
flatObjectMetadataMaps,
|
||||
workspaceCustomApplicationId,
|
||||
dryRun,
|
||||
}: {
|
||||
workspaceId: string;
|
||||
flatObjectMetadataMaps: FlatEntityMaps<FlatObjectMetadata>;
|
||||
workspaceCustomApplicationId: string;
|
||||
dryRun: boolean;
|
||||
}): Promise<void> {
|
||||
const remainingCustomViews = await this.viewRepository.find({
|
||||
select: {
|
||||
id: true,
|
||||
universalIdentifier: true,
|
||||
@@ -102,190 +286,27 @@ export class IdentifyViewMetadataCommand extends WorkspacesMigrationCommandRunne
|
||||
},
|
||||
});
|
||||
|
||||
const { flatObjectMetadataMaps } =
|
||||
await this.flatEntityMapsCacheService.getOrRecomputeManyOrAllFlatEntityMaps(
|
||||
{
|
||||
workspaceId,
|
||||
flatMapsKeys: ['flatObjectMetadataMaps'],
|
||||
},
|
||||
);
|
||||
const customUpdates = remainingCustomViews.map((viewEntity) => ({
|
||||
id: viewEntity.id,
|
||||
universalIdentifier: viewEntity.universalIdentifier ?? v4(),
|
||||
applicationId: workspaceCustomApplicationId,
|
||||
}));
|
||||
|
||||
const customViewMetadataEntities: CustomViewMetadata[] = [];
|
||||
const standardViewMetadataEntities: StandardViewMetadata[] = [];
|
||||
const warnings: ViewMetadataWarning[] = [];
|
||||
const exceptions: ViewMetadataException[] = [];
|
||||
|
||||
for (const viewEntity of allViewEntities) {
|
||||
// TODO double check that index view are not custom clearly not sure sure about that
|
||||
if (viewEntity.isCustom) {
|
||||
customViewMetadataEntities.push({
|
||||
viewEntity,
|
||||
fromStandard: false,
|
||||
});
|
||||
|
||||
continue;
|
||||
}
|
||||
this.logger.log(
|
||||
`Found ${customUpdates.length} custom view(s) to update for workspace ${workspaceId}`,
|
||||
);
|
||||
|
||||
for (const viewEntity of remainingCustomViews) {
|
||||
const flatObjectMetadata =
|
||||
flatObjectMetadataMaps.byId[viewEntity.objectMetadataId];
|
||||
|
||||
if (!isDefined(flatObjectMetadata)) {
|
||||
exceptions.push({
|
||||
viewEntity,
|
||||
exception: 'not_found_object',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const objectConfig =
|
||||
STANDARD_OBJECTS[
|
||||
flatObjectMetadata.nameSingular as keyof typeof STANDARD_OBJECTS
|
||||
];
|
||||
|
||||
if (!isDefined(objectConfig)) {
|
||||
warnings.push({
|
||||
viewEntity,
|
||||
warning: 'unknown_object',
|
||||
objectNameSingular: flatObjectMetadata.nameSingular,
|
||||
});
|
||||
customViewMetadataEntities.push({
|
||||
viewEntity,
|
||||
fromStandard: true,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const objectViews =
|
||||
'views' in objectConfig
|
||||
? (objectConfig.views as Record<
|
||||
string,
|
||||
{ universalIdentifier: string } | undefined
|
||||
>)
|
||||
: null;
|
||||
|
||||
if (!isDefined(objectViews)) {
|
||||
warnings.push({
|
||||
viewEntity,
|
||||
warning: 'unknown_object',
|
||||
objectNameSingular: flatObjectMetadata.nameSingular,
|
||||
});
|
||||
customViewMetadataEntities.push({
|
||||
viewEntity,
|
||||
fromStandard: true,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const formattedViewName = computeFormattedViewName({
|
||||
viewName: viewEntity.name,
|
||||
flatObjectMetadata,
|
||||
});
|
||||
|
||||
this.logger.log(formattedViewName);
|
||||
const viewConfig = objectViews[formattedViewName];
|
||||
const universalIdentifier = viewConfig?.universalIdentifier;
|
||||
|
||||
if (!isDefined(universalIdentifier)) {
|
||||
exceptions.push({
|
||||
viewEntity,
|
||||
exception: 'unknown_standard_view_name',
|
||||
objectNameSingular: flatObjectMetadata.nameSingular,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (
|
||||
isDefined(viewEntity.universalIdentifier) &&
|
||||
viewEntity.universalIdentifier !== universalIdentifier
|
||||
) {
|
||||
exceptions.push({
|
||||
viewEntity,
|
||||
exception: 'existing_universal_id_mismatch',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
standardViewMetadataEntities.push({
|
||||
viewEntity,
|
||||
universalIdentifier:
|
||||
viewEntity.universalIdentifier ?? universalIdentifier,
|
||||
});
|
||||
}
|
||||
|
||||
const totalUpdates =
|
||||
customViewMetadataEntities.length + standardViewMetadataEntities.length;
|
||||
|
||||
if (warnings.length > 0) {
|
||||
this.logger.warn(
|
||||
`Found ${warnings.length} warning(s) while processing view metadata for workspace ${workspaceId}. These views will become custom.`,
|
||||
);
|
||||
|
||||
for (const { viewEntity, warning, objectNameSingular } of warnings) {
|
||||
this.logger.warn(
|
||||
`Warning for view "${viewEntity.name}" on object "${objectNameSingular ?? 'unknown'}" (id=${viewEntity.id}): ${warning}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (exceptions.length > 0) {
|
||||
this.logger.error(
|
||||
`Found ${exceptions.length} exception(s) while processing view metadata for workspace ${workspaceId}. No updates will be applied.`,
|
||||
);
|
||||
|
||||
for (const { viewEntity, exception, objectNameSingular } of exceptions) {
|
||||
this.logger.error(
|
||||
`Exception for view "${viewEntity.name}" on object "${objectNameSingular ?? 'unknown'}" (id=${viewEntity.id}): ${exception}`,
|
||||
);
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Aborting migration for workspace ${workspaceId} due to ${exceptions.length} exception(s). See logs above for details.`,
|
||||
this.logger.log(
|
||||
` - Custom view "${viewEntity.name}" on object "${flatObjectMetadata?.nameSingular ?? 'unknown'}" (id=${viewEntity.id})`,
|
||||
);
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Successfully validated ${totalUpdates}/${allViewEntities.length} view metadata update(s) for workspace ${workspaceId} (${customViewMetadataEntities.length} custom, ${standardViewMetadataEntities.length} standard)`,
|
||||
);
|
||||
|
||||
if (!options.dryRun) {
|
||||
const customUpdates = customViewMetadataEntities.map(
|
||||
({ viewEntity }) => ({
|
||||
id: viewEntity.id,
|
||||
universalIdentifier: viewEntity.universalIdentifier ?? v4(),
|
||||
applicationId: workspaceCustomFlatApplication.id,
|
||||
}),
|
||||
);
|
||||
|
||||
const standardUpdates = standardViewMetadataEntities.map(
|
||||
({ viewEntity, universalIdentifier }) => ({
|
||||
id: viewEntity.id,
|
||||
universalIdentifier,
|
||||
applicationId: twentyStandardFlatApplication.id,
|
||||
}),
|
||||
);
|
||||
|
||||
await this.viewRepository.save([...customUpdates, ...standardUpdates]);
|
||||
|
||||
const relatedMetadataNames = getMetadataRelatedMetadataNames('view');
|
||||
const relatedCacheKeysToInvalidate = relatedMetadataNames.map(
|
||||
getMetadataFlatEntityMapsKey,
|
||||
);
|
||||
|
||||
this.logger.log(
|
||||
`Invalidating caches: ${relatedCacheKeysToInvalidate.join(' ')}`,
|
||||
);
|
||||
await this.workspaceCacheService.invalidateAndRecompute(workspaceId, [
|
||||
'flatViewMaps',
|
||||
...relatedCacheKeysToInvalidate,
|
||||
]);
|
||||
|
||||
this.logger.log(
|
||||
`Applied ${totalUpdates} view metadata update(s) for workspace ${workspaceId}`,
|
||||
);
|
||||
} else {
|
||||
this.logger.log(
|
||||
`Dry run: would apply ${totalUpdates} view metadata update(s) for workspace ${workspaceId}`,
|
||||
);
|
||||
if (!dryRun) {
|
||||
await this.viewRepository.save(customUpdates);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user