From d7bda9576f48b554fcaf65a36cae0d657de31194 Mon Sep 17 00:00:00 2001 From: Marie <51697796+ijreilly@users.noreply.github.com> Date: Fri, 24 Oct 2025 17:30:57 +0200 Subject: [PATCH] [groupBy] Load record within groups (without relations) (#15307) First step of https://github.com/twentyhq/core-team-issues/issues/1726. I will handle relations in another PR. Another ticket is planned to allow for sorting among the records. When querying records we have set the number of maximum groups to 50, and of maximum records par group to 10. image --- .../common-group-by-query-runner.service.ts | 125 +++++-- .../api/common/core-common-api.module.ts | 2 + .../types/common-group-by-output-item.type.ts | 10 +- .../common/types/common-query-args.type.ts | 2 + .../graphql-query-runner.module.ts | 2 - ...graphql-query-group-by-resolver.service.ts | 249 ------------- .../types/group-by-definition.types.ts | 8 + ...ult-with-group-by-dimension-values.util.ts | 104 ++++-- .../utils/parse-group-by-args.util.ts | 4 + .../services/group-by-with-records.service.ts | 168 +++++++++ .../factories/group-by-resolver.factory.ts | 73 ++-- .../handlers/rest-api-group-by.handler.ts | 38 +- .../core/interfaces/rest-api-base.handler.ts | 3 +- .../path-parsers/parse-core-path.utils.ts | 2 +- .../core/services/rest-api-core.service.ts | 10 +- ...nclude-records-sample-rest-request.util.ts | 13 + .../repository/permissions.utils.ts | 35 +- ... => group-by-resolver.integration-spec.ts} | 2 +- ...-with-records-resolver.integration-spec.ts | 352 ++++++++++++++++++ 19 files changed, 840 insertions(+), 362 deletions(-) delete mode 100644 packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/graphql-query-group-by-resolver.service.ts create mode 100644 packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/types/group-by-definition.types.ts create mode 100644 packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.service.ts create mode 100644 packages/twenty-server/src/engine/api/rest/input-request-parsers/group-by-with-records/parse-include-records-sample-rest-request.util.ts rename packages/twenty-server/test/integration/graphql/suites/{group-by-resolvers.integration-spec.ts => group-by-resolver.integration-spec.ts} (99%) create mode 100644 packages/twenty-server/test/integration/graphql/suites/group-by-with-records-resolver.integration-spec.ts diff --git a/packages/twenty-server/src/engine/api/common/common-query-runners/common-group-by-query-runner.service.ts b/packages/twenty-server/src/engine/api/common/common-query-runners/common-group-by-query-runner.service.ts index e3cea1ad652..2da54996c41 100644 --- a/packages/twenty-server/src/engine/api/common/common-query-runners/common-group-by-query-runner.service.ts +++ b/packages/twenty-server/src/engine/api/common/common-query-runners/common-group-by-query-runner.service.ts @@ -11,9 +11,9 @@ import { computeRecordGqlOperationFilter, convertViewFilterValueToString, getFilterTypeFromFieldType, - isDefined, turnAnyFieldFilterIntoRecordGqlFilter, } from 'twenty-shared/utils'; +import { ObjectLiteral } from 'typeorm'; import { WorkspaceAuthContext } from 'src/engine/api/common/interfaces/workspace-auth-context.interface'; import { ObjectRecordFilter } from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface'; @@ -28,11 +28,15 @@ import { CommonQueryNames, GroupByQueryArgs, } from 'src/engine/api/common/types/common-query-args.type'; +import { GraphqlQuerySelectedFieldsResult } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-selected-fields/graphql-selected-fields.parser'; +import { GraphqlQueryParser } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query.parser'; +import { GroupByDefinition } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/types/group-by-definition.types'; import { formatResultWithGroupByDimensionValues } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/format-result-with-group-by-dimension-values.util'; import { getGroupByExpression } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/get-group-by-expression.util'; import { isGroupByDateField } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/is-group-by-date-field.util'; import { parseGroupByArgs } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/parse-group-by-args.util'; import { removeQuotes } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/remove-quote.util'; +import { GroupByWithRecordsService } from 'src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.service'; import { ProcessAggregateHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-aggregate.helper'; import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps'; import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; @@ -40,6 +44,7 @@ import { ViewFilterGroupService } from 'src/engine/metadata-modules/view-filter- import { ViewFilterService } from 'src/engine/metadata-modules/view-filter/services/view-filter.service'; import { ViewEntity } from 'src/engine/metadata-modules/view/entities/view.entity'; import { ViewService } from 'src/engine/metadata-modules/view/services/view.service'; +import { WorkspaceSelectQueryBuilder } from 'src/engine/twenty-orm/repository/workspace-select-query-builder'; import { formatColumnNamesFromCompositeFieldAndSubfields } from 'src/engine/twenty-orm/utils/format-column-names-from-composite-field-and-subfield.util'; @Injectable() @@ -51,6 +56,7 @@ export class CommonGroupByQueryRunnerService extends CommonBaseQueryRunnerServic private readonly viewFilterService: ViewFilterService, private readonly viewFilterGroupService: ViewFilterGroupService, private readonly viewService: ViewService, + private readonly groupByWithRecordsService: GroupByWithRecordsService, ) { super(); } @@ -66,6 +72,7 @@ export class CommonGroupByQueryRunnerService extends CommonBaseQueryRunnerServic commonQueryParser, objectMetadataItemWithFieldMaps, authContext, + objectMetadataMaps, } = queryRunnerContext; const objectMetadataNameSingular = @@ -77,22 +84,14 @@ export class CommonGroupByQueryRunnerService extends CommonBaseQueryRunnerServic let appliedFilters = args.filter ?? ({} as ObjectRecordFilter); - if (isDefined(args.viewId)) { - appliedFilters = await this.addFiltersFromView({ - args, - authContext, - objectMetadataItemWithFieldMaps, - appliedFilters, - }); - } - - commonQueryParser.applyFilterToBuilder( - queryBuilder, - objectMetadataNameSingular, + await this.addFiltersToQueryBuilder({ + args, appliedFilters, - ); - - commonQueryParser.applyDeletedAtToBuilder(queryBuilder, appliedFilters); + queryBuilder, + objectMetadataItemWithFieldMaps, + workspaceId: authContext.workspace.id, + commonQueryParser, + }); ProcessAggregateHelper.addSelectedAggregatedFieldsQueriesToQueryBuilder({ selectedAggregatedFields: args.selectedFieldsResult.aggregate, @@ -147,13 +146,37 @@ export class CommonGroupByQueryRunnerService extends CommonBaseQueryRunnerServic groupByFields, ); - const result = await queryBuilder.getRawMany(); + const shouldIncludeRecords = args.includeRecords ?? false; - return formatResultWithGroupByDimensionValues( - result, + if (shouldIncludeRecords) { + const queryBuilderWithFiltersAndWithoutGroupBy = + repository.createQueryBuilder(objectMetadataNameSingular); + + await this.addFiltersToQueryBuilder({ + args, + objectMetadataItemWithFieldMaps, + workspaceId: authContext.workspace.id, + commonQueryParser, + appliedFilters, + queryBuilder: queryBuilderWithFiltersAndWithoutGroupBy, + }); + + return this.groupByWithRecordsService.resolveWithRecords({ + queryBuilderWithFiltersAndWithoutGroupBy, + queryBuilderWithGroupBy: queryBuilder, + groupByDefinitions, + objectMetadataItemWithFieldMaps, + objectMetadataMaps, + repository, + selectedFieldsResult: args.selectedFieldsResult, + }); + } + + return this.resolveWithoutRecords({ + queryBuilder, groupByDefinitions, - Object.keys(args.selectedFieldsResult.aggregate), - ); + selectedFieldsResult: args.selectedFieldsResult, + }); } async processQueryResult( @@ -169,17 +192,15 @@ export class CommonGroupByQueryRunnerService extends CommonBaseQueryRunnerServic args, objectMetadataItemWithFieldMaps, appliedFilters, - authContext, + workspaceId, }: { args: GroupByQueryArgs; - authContext: WorkspaceAuthContext; objectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps; appliedFilters: ObjectRecordFilter; + workspaceId: string; }): Promise { assertIsDefinedOrThrow(args.viewId); - const workspaceId = authContext.workspace.id; - const viewFilters = await this.viewFilterService.findByViewId( workspaceId, args.viewId, @@ -253,6 +274,60 @@ export class CommonGroupByQueryRunnerService extends CommonBaseQueryRunnerServic return appliedFilters; } + private async addFiltersToQueryBuilder({ + args, + appliedFilters, + queryBuilder, + objectMetadataItemWithFieldMaps, + workspaceId, + commonQueryParser, + }: { + args: GroupByQueryArgs; + appliedFilters: ObjectRecordFilter; + queryBuilder: WorkspaceSelectQueryBuilder; + objectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps; + workspaceId: string; + commonQueryParser: GraphqlQueryParser; + }): Promise { + const objectMetadataNameSingular = + objectMetadataItemWithFieldMaps.nameSingular; + + if (args.viewId) { + appliedFilters = await this.addFiltersFromView({ + args, + objectMetadataItemWithFieldMaps, + appliedFilters, + workspaceId, + }); + } + + commonQueryParser.applyFilterToBuilder( + queryBuilder, + objectMetadataNameSingular, + appliedFilters, + ); + + commonQueryParser.applyDeletedAtToBuilder(queryBuilder, appliedFilters); + } + + private async resolveWithoutRecords({ + queryBuilder, + groupByDefinitions, + selectedFieldsResult, + }: { + queryBuilder: WorkspaceSelectQueryBuilder; + groupByDefinitions: GroupByDefinition[]; + selectedFieldsResult: GraphqlQuerySelectedFieldsResult; + }): Promise { + const result = await queryBuilder.getRawMany(); + + return formatResultWithGroupByDimensionValues({ + groupsResult: result, + groupByDefinitions, + aggregateFieldNames: Object.keys(selectedFieldsResult.aggregate), + }); + } + async validate( _args: CommonInput, _queryRunnerContext: CommonBaseQueryRunnerContext, diff --git a/packages/twenty-server/src/engine/api/common/core-common-api.module.ts b/packages/twenty-server/src/engine/api/common/core-common-api.module.ts index 75097824a78..7bf6714df19 100644 --- a/packages/twenty-server/src/engine/api/common/core-common-api.module.ts +++ b/packages/twenty-server/src/engine/api/common/core-common-api.module.ts @@ -4,6 +4,7 @@ import { TypeOrmModule } from '@nestjs/typeorm'; import { CommonArgsHandlers } from 'src/engine/api/common/common-args-handlers/common-query-selected-fields/common-arg-handlers'; import { CommonQueryRunners } from 'src/engine/api/common/common-query-runners/common-query-runners'; import { CommonResultGettersService } from 'src/engine/api/common/common-result-getters/common-result-getters.service'; +import { GroupByWithRecordsService } from 'src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.service'; import { ProcessAggregateHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-aggregate.helper'; import { ProcessNestedRelationsV2Helper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-nested-relations-v2.helper'; import { ProcessNestedRelationsHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-nested-relations.helper'; @@ -40,6 +41,7 @@ import { WorkspacePermissionsCacheModule } from 'src/engine/metadata-modules/wor ProcessAggregateHelper, ...CommonQueryRunners, CommonResultGettersService, + GroupByWithRecordsService, ], exports: [...CommonQueryRunners], }) diff --git a/packages/twenty-server/src/engine/api/common/types/common-group-by-output-item.type.ts b/packages/twenty-server/src/engine/api/common/types/common-group-by-output-item.type.ts index ba2c8d8e70e..93eac372bc9 100644 --- a/packages/twenty-server/src/engine/api/common/types/common-group-by-output-item.type.ts +++ b/packages/twenty-server/src/engine/api/common/types/common-group-by-output-item.type.ts @@ -1,3 +1,5 @@ +import { type ObjectRecord } from 'twenty-shared/types'; + type AggregateValues = { [key: string]: string; }; @@ -6,4 +8,10 @@ type GroupByDimensionValues = { groupByDimensionValues: string[]; }; -export type CommonGroupByOutputItem = AggregateValues & GroupByDimensionValues; +type Records = { + records?: ObjectRecord[]; +}; + +export type CommonGroupByOutputItem = GroupByDimensionValues & + AggregateValues & + Records; diff --git a/packages/twenty-server/src/engine/api/common/types/common-query-args.type.ts b/packages/twenty-server/src/engine/api/common/types/common-query-args.type.ts index 2e4d8bfb24f..30b51d20cf8 100644 --- a/packages/twenty-server/src/engine/api/common/types/common-query-args.type.ts +++ b/packages/twenty-server/src/engine/api/common/types/common-query-args.type.ts @@ -66,6 +66,8 @@ export interface GroupByQueryArgs { orderBy?: OrderByWithGroupBy; groupBy: ObjectRecordGroupBy; viewId?: string; + includeRecords?: boolean; + selectedFields: CommonSelectedFields; } export interface DestroyOneQueryArgs { id: string; diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-runner.module.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-runner.module.ts index 7126f4e0a3e..ed6540e41d1 100644 --- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-runner.module.ts +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/graphql-query-runner.module.ts @@ -1,7 +1,6 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; -import { GraphqlQueryGroupByResolverService } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/graphql-query-group-by-resolver.service'; import { ProcessAggregateHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-aggregate.helper'; import { ProcessNestedRelationsV2Helper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-nested-relations-v2.helper'; import { ProcessNestedRelationsHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-nested-relations.helper'; @@ -39,7 +38,6 @@ const graphqlQueryResolvers = [ GraphqlQueryFindDuplicatesResolverService, GraphqlQueryFindManyResolverService, GraphqlQueryFindOneResolverService, - GraphqlQueryGroupByResolverService, GraphqlQueryMergeManyResolverService, GraphqlQueryRestoreManyResolverService, GraphqlQueryRestoreOneResolverService, diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/graphql-query-group-by-resolver.service.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/graphql-query-group-by-resolver.service.ts deleted file mode 100644 index 8668cfad0cc..00000000000 --- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/graphql-query-group-by-resolver.service.ts +++ /dev/null @@ -1,249 +0,0 @@ -import { Injectable } from '@nestjs/common'; - -import { - CompositeFieldSubFieldName, - ObjectRecord, - PartialFieldMetadataItemOption, - RecordFilterGroupLogicalOperator, -} from 'twenty-shared/types'; -import { - assertIsDefinedOrThrow, - combineFilters, - computeRecordGqlOperationFilter, - convertViewFilterValueToString, - getFilterTypeFromFieldType, - turnAnyFieldFilterIntoRecordGqlFilter, -} from 'twenty-shared/utils'; - -import { - GraphqlQueryBaseResolverService, - GraphqlQueryResolverExecutionArgs, -} from 'src/engine/api/graphql/graphql-query-runner/interfaces/base-resolver-service'; -import { ObjectRecordFilter } from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface'; -import { IEdge } from 'src/engine/api/graphql/workspace-query-runner/interfaces/edge.interface'; -import { IGroupByConnection } from 'src/engine/api/graphql/workspace-query-runner/interfaces/group-by-connection.interface'; -import { type WorkspaceQueryRunnerOptions } from 'src/engine/api/graphql/workspace-query-runner/interfaces/query-runner-option.interface'; -import { GroupByResolverArgs } from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolvers-builder.interface'; - -import { formatResultWithGroupByDimensionValues } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/format-result-with-group-by-dimension-values.util'; -import { getGroupByExpression } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/get-group-by-expression.util'; -import { isGroupByDateField } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/is-group-by-date-field.util'; -import { parseGroupByArgs } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/parse-group-by-args.util'; -import { removeQuotes } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/remove-quote.util'; -import { ProcessAggregateHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-aggregate.helper'; -import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum'; -import { WorkspaceNotFoundDefaultError } from 'src/engine/core-modules/workspace/workspace.exception'; -import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps'; -import { ViewFilterGroupService } from 'src/engine/metadata-modules/view-filter-group/services/view-filter-group.service'; -import { ViewFilterService } from 'src/engine/metadata-modules/view-filter/services/view-filter.service'; -import { ViewEntity } from 'src/engine/metadata-modules/view/entities/view.entity'; -import { ViewService } from 'src/engine/metadata-modules/view/services/view.service'; -import { formatColumnNamesFromCompositeFieldAndSubfields } from 'src/engine/twenty-orm/utils/format-column-names-from-composite-field-and-subfield.util'; - -@Injectable() -export class GraphqlQueryGroupByResolverService extends GraphqlQueryBaseResolverService< - GroupByResolverArgs, - IGroupByConnection>[] -> { - constructor( - private readonly viewFilterService: ViewFilterService, - private readonly viewFilterGroupService: ViewFilterGroupService, - private readonly viewService: ViewService, - ) { - super(); - } - - async resolve( - executionArgs: GraphqlQueryResolverExecutionArgs, - _featureFlagsMap: Record, - ): Promise>[]> { - const { objectMetadataItemWithFieldMaps } = executionArgs.options; - - const objectMetadataNameSingular = - objectMetadataItemWithFieldMaps.nameSingular; - - let queryBuilder = executionArgs.repository.createQueryBuilder( - objectMetadataNameSingular, - ); - - let appliedFilters = - executionArgs.args.filter ?? ({} as ObjectRecordFilter); - - if (executionArgs.args.viewId) { - appliedFilters = await this.addFiltersFromView({ - executionArgs, - objectMetadataItemWithFieldMaps, - appliedFilters, - }); - } - - executionArgs.graphqlQueryParser.applyFilterToBuilder( - queryBuilder, - objectMetadataNameSingular, - appliedFilters, - ); - - executionArgs.graphqlQueryParser.applyDeletedAtToBuilder( - queryBuilder, - appliedFilters, - ); - - ProcessAggregateHelper.addSelectedAggregatedFieldsQueriesToQueryBuilder({ - selectedAggregatedFields: - executionArgs.graphqlQuerySelectedFieldsResult.aggregate, - queryBuilder, - objectMetadataNameSingular, - }); - - const groupByFields = parseGroupByArgs( - executionArgs.args, - objectMetadataItemWithFieldMaps, - ); - - const groupByDefinitions = groupByFields.map((groupByField) => { - const columnNameWithQuotes = `"${ - formatColumnNamesFromCompositeFieldAndSubfields( - groupByField.fieldMetadata.name, - groupByField.subFieldName ? [groupByField.subFieldName] : undefined, - )[0] - }"`; - const alias = - removeQuotes(columnNameWithQuotes) + - (isGroupByDateField(groupByField) - ? `_${groupByField.dateGranularity}` - : ''); - - return { - columnNameWithQuotes, - expression: getGroupByExpression({ - groupByField, - columnNameWithQuotes, - }), - alias, - dateGranularity: isGroupByDateField(groupByField) - ? groupByField.dateGranularity - : undefined, - }; - }); - - groupByDefinitions.forEach((groupByColumn, index) => { - queryBuilder.addSelect(groupByColumn.expression, groupByColumn.alias); - - if (index === 0) { - queryBuilder.groupBy(groupByColumn.expression); - } else { - queryBuilder.addGroupBy(groupByColumn.expression); - } - }); - - executionArgs.graphqlQueryParser.applyGroupByOrderToBuilder( - queryBuilder, - executionArgs.args.orderBy ?? [], - groupByFields, - ); - - const result = await queryBuilder.getRawMany(); - - return formatResultWithGroupByDimensionValues( - result, - groupByDefinitions, - Object.keys(executionArgs.graphqlQuerySelectedFieldsResult.aggregate), - ); - } - - private async addFiltersFromView({ - executionArgs, - objectMetadataItemWithFieldMaps, - appliedFilters, - }: { - executionArgs: GraphqlQueryResolverExecutionArgs; - objectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps; - appliedFilters: ObjectRecordFilter; - }): Promise { - assertIsDefinedOrThrow(executionArgs.args.viewId); - - const workspaceId = executionArgs.options.authContext.workspace?.id; - - assertIsDefinedOrThrow(workspaceId, WorkspaceNotFoundDefaultError); - - const viewFilters = await this.viewFilterService.findByViewId( - workspaceId, - executionArgs.args.viewId, - ); - - const viewFilterGroups = await this.viewFilterGroupService.findByViewId( - workspaceId, - executionArgs.args.viewId, - ); - - const recordFilters = viewFilters.map((viewFilter) => { - const fieldMetadataItem = - objectMetadataItemWithFieldMaps.fieldsById[viewFilter.fieldMetadataId]; - - return { - id: viewFilter.id, - fieldMetadataId: viewFilter.fieldMetadataId, - value: convertViewFilterValueToString(viewFilter.value), - type: getFilterTypeFromFieldType(fieldMetadataItem.type), - operand: viewFilter.operand, - recordFilterGroupId: viewFilter.viewFilterGroupId, - positionInRecordFilterGroup: viewFilter.positionInViewFilterGroup, - subFieldName: viewFilter.subFieldName as CompositeFieldSubFieldName, - }; - }); - - const recordFilterGroups = viewFilterGroups.map((viewFilterGroup) => { - return { - id: viewFilterGroup.id, - logicalOperator: - viewFilterGroup.logicalOperator as unknown as RecordFilterGroupLogicalOperator, // TODO - https://github.com/twentyhq/twenty/issues/14746 - parentRecordFilterGroupId: viewFilterGroup.parentViewFilterGroupId, - }; - }); - - const fields = Object.values( - objectMetadataItemWithFieldMaps.fieldsById, - ).map((field) => ({ - id: field.id, - name: field.name, - type: field.type, - label: field.label, - options: field.options as PartialFieldMetadataItemOption[], - })); - - const filtersFromView = computeRecordGqlOperationFilter({ - recordFilters, - recordFilterGroups: recordFilterGroups, - fields, - filterValueDependencies: {}, - }); - - let view: ViewEntity | null = viewFilters[0]?.view; - - if (!view) { - view = await this.viewService.findById( - executionArgs.args.viewId, - workspaceId, - ); - } - - const { recordGqlOperationFilter: anyFieldFilter } = - turnAnyFieldFilterIntoRecordGqlFilter({ - fields, - filterValue: view?.anyFieldFilterValue ?? '', - }); - - appliedFilters = combineFilters([ - appliedFilters, - filtersFromView, - anyFieldFilter, - ]); - - return appliedFilters; - } - - async validate( - _args: GroupByResolverArgs, - _options: WorkspaceQueryRunnerOptions, - ): Promise {} -} diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/types/group-by-definition.types.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/types/group-by-definition.types.ts new file mode 100644 index 00000000000..6da7cc5c062 --- /dev/null +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/types/group-by-definition.types.ts @@ -0,0 +1,8 @@ +import { type ObjectRecordGroupByDateGranularity } from 'twenty-shared/types'; + +export type GroupByDefinition = { + columnNameWithQuotes: string; + expression: string; + alias: string; + dateGranularity?: ObjectRecordGroupByDateGranularity; +}; diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/format-result-with-group-by-dimension-values.util.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/format-result-with-group-by-dimension-values.util.ts index 561fdaf2ec5..281409ffd98 100644 --- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/format-result-with-group-by-dimension-values.util.ts +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/format-result-with-group-by-dimension-values.util.ts @@ -1,33 +1,61 @@ import { t } from '@lingui/core/macro'; -import { - ObjectRecordGroupByDateGranularity, - type ObjectRecord, -} from 'twenty-shared/types'; - -import { type IEdge } from 'src/engine/api/graphql/workspace-query-runner/interfaces/edge.interface'; -import { type IGroupByConnection } from 'src/engine/api/graphql/workspace-query-runner/interfaces/group-by-connection.interface'; +import { ObjectRecordGroupByDateGranularity } from 'twenty-shared/types'; +import { isDefined } from 'twenty-shared/utils'; import { type CommonGroupByOutputItem } from 'src/engine/api/common/types/common-group-by-output-item.type'; +import { type GroupByDefinition } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/types/group-by-definition.types'; +import { type ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps'; +import { type ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; +import { formatResult } from 'src/engine/twenty-orm/utils/format-result.util'; -export const formatResultWithGroupByDimensionValues = < - T extends - | IGroupByConnection> - | CommonGroupByOutputItem, ->( - result: Record[], - groupByColumnsWithQuotes: { - columnNameWithQuotes: string; - alias: string; - dateGranularity?: ObjectRecordGroupByDateGranularity; - }[], - aggregateFieldNames: string[], -): T[] => { - let formattedResult: T[] = []; +export const formatResultWithGroupByDimensionValues = ({ + groupsResult, + groupByDefinitions, + aggregateFieldNames, + recordsResult, + objectMetadataItemWithFieldMaps, + objectMetadataMaps, +}: { + groupsResult: Record[]; + groupByDefinitions: GroupByDefinition[]; + aggregateFieldNames: string[]; + recordsResult?: Array>; + objectMetadataItemWithFieldMaps?: ObjectMetadataItemWithFieldMaps; + objectMetadataMaps?: ObjectMetadataMaps; +}): CommonGroupByOutputItem[] => { + const formattedResult: CommonGroupByOutputItem[] = []; - result.forEach((group) => { - let dimensionValues: string[] = []; + const recordsByGroupKey = new Map>>(); - for (const groupByColumn of groupByColumnsWithQuotes) { + if (isDefined(recordsResult)) { + if ( + !isDefined(objectMetadataItemWithFieldMaps) || + !isDefined(objectMetadataMaps) + ) { + throw new Error('Metadata are required to format result'); + } + + recordsResult.forEach((entry) => { + const groupKey = createGroupKey(entry, groupByDefinitions); + + const records: Record[] = ( + (entry.records as Array>) ?? [] + ).map((record) => { + return formatResult( + record, + objectMetadataItemWithFieldMaps, + objectMetadataMaps, + ); + }); + + recordsByGroupKey.set(groupKey, records); + }); + } + + groupsResult.forEach((group) => { + const dimensionValues: unknown[] = []; + + for (const groupByColumn of groupByDefinitions) { dimensionValues.push( getTranslatedValueIfApplicable( group[groupByColumn.alias], @@ -44,23 +72,37 @@ export const formatResultWithGroupByDimensionValues = < return acc; }, - {} as Record, + {} as Record, ); + const groupKey = createGroupKey(group, groupByDefinitions); + const records = recordsByGroupKey.get(groupKey) || []; + formattedResult.push({ - groupByDimensionValues: dimensionValues, ...aggregateValues, - //TODO: Refacto-common - remove generic type - } as T); + groupByDimensionValues: dimensionValues, + ...(isDefined(recordsResult) ? { records } : {}), + } as CommonGroupByOutputItem); }); return formattedResult; }; -const getTranslatedValueIfApplicable = ( - value: string, - dateGranularity?: ObjectRecordGroupByDateGranularity, +const createGroupKey = ( + group: Record, + groupByDefinitions: GroupByDefinition[], ): string => { + return groupByDefinitions.map((def) => String(group[def.alias])).join('|'); +}; + +const getTranslatedValueIfApplicable = ( + value: T, + dateGranularity?: ObjectRecordGroupByDateGranularity, +): T | string => { + if (typeof value !== 'string') { + return value; + } + switch (dateGranularity) { case ObjectRecordGroupByDateGranularity.DAY_OF_THE_WEEK: switch (value) { diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/parse-group-by-args.util.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/parse-group-by-args.util.ts index 93431d3d859..7ebf07f566e 100644 --- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/parse-group-by-args.util.ts +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/parse-group-by-args.util.ts @@ -64,6 +64,10 @@ export const parseGroupByArgs = ( const fieldMetadata = objectMetadataItemWithFieldMaps.fieldsById[fieldMetadataId]; + if (!isDefined(fieldMetadata) || !isDefined(fieldMetadataId)) { + throw new Error(`Unidentified field in groupBy: ${fieldName}`); + } + if ( fieldMetadata.type === FieldMetadataType.DATE || fieldMetadata.type === FieldMetadataType.DATE_TIME diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.service.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.service.ts new file mode 100644 index 00000000000..c41c56d166c --- /dev/null +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.service.ts @@ -0,0 +1,168 @@ +import { isDefined } from 'class-validator'; +import { type ObjectLiteral } from 'typeorm'; + +import { type CommonGroupByOutputItem } from 'src/engine/api/common/types/common-group-by-output-item.type'; +import { type GraphqlQuerySelectedFieldsResult } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query-selected-fields/graphql-selected-fields.parser'; +import { type GroupByDefinition } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/types/group-by-definition.types'; +import { formatResultWithGroupByDimensionValues } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/format-result-with-group-by-dimension-values.util'; +import { buildColumnsToSelect } from 'src/engine/api/graphql/graphql-query-runner/utils/build-columns-to-select'; +import { type ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps'; +import { type ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; +import { type WorkspaceSelectQueryBuilder } from 'src/engine/twenty-orm/repository/workspace-select-query-builder'; +import { type WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository'; + +const GROUPS_LIMIT = 50; +const RECORDS_PER_GROUP_LIMIT = 10; +const SUB_QUERY_PREFIX = 'sub_query_'; + +export class GroupByWithRecordsService { + constructor() {} + + public async resolveWithRecords({ + queryBuilderWithGroupBy, + queryBuilderWithFiltersAndWithoutGroupBy, + groupByDefinitions, + objectMetadataItemWithFieldMaps, + selectedFieldsResult, + objectMetadataMaps, + repository, + }: { + queryBuilderWithGroupBy: WorkspaceSelectQueryBuilder; + queryBuilderWithFiltersAndWithoutGroupBy: WorkspaceSelectQueryBuilder; + groupByDefinitions: GroupByDefinition[]; + objectMetadataItemWithFieldMaps: ObjectMetadataItemWithFieldMaps; + selectedFieldsResult: GraphqlQuerySelectedFieldsResult; + objectMetadataMaps: ObjectMetadataMaps; + repository: WorkspaceRepository; + }): Promise { + const groupsResult = await queryBuilderWithGroupBy + .limit(GROUPS_LIMIT) + .getRawMany(); + + if (groupsResult.length === 0) { + return []; + } + + const columnsToSelect = buildColumnsToSelect({ + select: selectedFieldsResult.select, + relations: selectedFieldsResult.relations, // TODO - not handled for now + objectMetadataItemWithFieldMaps, + objectMetadataMaps: objectMetadataMaps, + }); + + const queryBuilderWithPartitionBy = this.addPartitionByToQueryBuilder({ + queryBuilderForSubQuery: queryBuilderWithFiltersAndWithoutGroupBy, + columnsToSelect, + groupsResult, + groupByDefinitions, + repository, + }); + + const recordsResult = await queryBuilderWithPartitionBy.getRawMany(); + + return formatResultWithGroupByDimensionValues({ + groupsResult, + recordsResult, + groupByDefinitions, + aggregateFieldNames: Object.keys(selectedFieldsResult.aggregate), + objectMetadataItemWithFieldMaps, + objectMetadataMaps, + }); + } + + private addPartitionByToQueryBuilder({ + queryBuilderForSubQuery, + columnsToSelect, + groupsResult, + groupByDefinitions, + repository, + }: { + queryBuilderForSubQuery: WorkspaceSelectQueryBuilder; + columnsToSelect: Record; + groupsResult: Array>; + groupByDefinitions: GroupByDefinition[]; + repository: WorkspaceRepository; + }): WorkspaceSelectQueryBuilder { + const groupByExpressions = groupByDefinitions + .map((def) => def.expression) + .join(', '); + + const groupByAliases = groupByDefinitions + .map((def) => `"${def.alias}"`) + .join(', '); + + const groupConditions = this.buildGroupConditions( + groupsResult, + groupByDefinitions, + queryBuilderForSubQuery, + ); + + const recordSelectWithAlias = Object.keys(columnsToSelect) + .map((col) => `"${col}" as "${SUB_QUERY_PREFIX}${col}"`) + .join(', '); + + const groupBySelectWithAlias = groupByDefinitions + .map((def) => `${def.expression} as "${def.alias}"`) + .join(', '); + + const subQuery = queryBuilderForSubQuery + .select(recordSelectWithAlias) + .addSelect(groupBySelectWithAlias) + .addSelect(`ROW_NUMBER() OVER (PARTITION BY ${groupByExpressions})`, 'rn') + .andWhere(groupConditions); + + let mainQueryQueryBuilder = repository.createQueryBuilder(); + + const mainQuery = mainQueryQueryBuilder + .from(`(${subQuery.getQuery()})`, 'ranked_records') + .setParameters(queryBuilderForSubQuery.expressionMap.parameters) + .select(groupByAliases) + .addSelect( + `JSON_AGG( + CASE WHEN rn <= ${RECORDS_PER_GROUP_LIMIT} THEN + JSON_BUILD_OBJECT( + ${[ + ...Object.keys(columnsToSelect).map( + (col) => `'${col}', "${SUB_QUERY_PREFIX}${col}"`, + ), + ...groupByDefinitions.map( + (def) => `'${def.alias}', "${def.alias}"`, + ), + ].join(',\n ')} + ) + END + ) FILTER (WHERE rn <= ${RECORDS_PER_GROUP_LIMIT})`, + 'records', + ) + .groupBy(groupByAliases); + + // Remove initial "from" condition (typeOrm limitation) + mainQuery.expressionMap.aliases = mainQuery.expressionMap.aliases.filter( + (alias) => isDefined(alias.subQuery), + ); + + return mainQuery as WorkspaceSelectQueryBuilder; + } + + private buildGroupConditions( + groupsResult: Array>, + groupByDefinitions: GroupByDefinition[], + queryBuilder: WorkspaceSelectQueryBuilder, + ): string { + const groupConditions = groupsResult.map((group, groupIndex) => { + const conditions = groupByDefinitions + .map((def, defIndex) => { + const paramName = `groupValue_${groupIndex}_${defIndex}`; + + queryBuilder.setParameter(paramName, group[def.alias]); + + return `${def.expression} = :${paramName}`; + }) + .join(' AND '); + + return `(${conditions})`; + }); + + return `(${groupConditions.join(' OR ')})`; + } +} diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-resolver-builder/factories/group-by-resolver.factory.ts b/packages/twenty-server/src/engine/api/graphql/workspace-resolver-builder/factories/group-by-resolver.factory.ts index fd1718e85fc..133edc6c706 100644 --- a/packages/twenty-server/src/engine/api/graphql/workspace-resolver-builder/factories/group-by-resolver.factory.ts +++ b/packages/twenty-server/src/engine/api/graphql/workspace-resolver-builder/factories/group-by-resolver.factory.ts @@ -1,8 +1,8 @@ import { Injectable } from '@nestjs/common'; +import { isDefined } from 'class-validator'; import graphqlFields from 'graphql-fields'; -import { type WorkspaceQueryRunnerOptions } from 'src/engine/api/graphql/workspace-query-runner/interfaces/query-runner-option.interface'; import { type WorkspaceResolverBuilderFactoryInterface } from 'src/engine/api/graphql/workspace-resolver-builder/interfaces/workspace-resolver-builder-factory.interface'; import { GroupByResolverArgs, @@ -11,11 +11,9 @@ import { import { WorkspaceSchemaBuilderContext } from 'src/engine/api/graphql/workspace-schema-builder/interfaces/workspace-schema-builder-context.interface'; import { CommonGroupByQueryRunnerService } from 'src/engine/api/common/common-query-runners/common-group-by-query-runner.service'; -import { GraphqlQueryGroupByResolverService } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/graphql-query-group-by-resolver.service'; +import { ObjectRecordsToGraphqlConnectionHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/object-records-to-graphql-connection.helper'; import { workspaceQueryRunnerGraphqlApiExceptionHandler } from 'src/engine/api/graphql/workspace-query-runner/utils/workspace-query-runner-graphql-api-exception-handler.util'; import { RESOLVER_METHOD_NAMES } from 'src/engine/api/graphql/workspace-resolver-builder/constants/resolver-method-names'; -import { FeatureFlagKey } from 'src/engine/core-modules/feature-flag/enums/feature-flag-key.enum'; -import { FeatureFlagService } from 'src/engine/core-modules/feature-flag/services/feature-flag.service'; @Injectable() export class GroupByResolverFactory @@ -24,9 +22,7 @@ export class GroupByResolverFactory public static methodName = RESOLVER_METHOD_NAMES.GROUP_BY; constructor( - private readonly graphqlQueryRunnerService: GraphqlQueryGroupByResolverService, private readonly commonGroupByQueryRunnerService: CommonGroupByQueryRunnerService, - private readonly featureFlagService: FeatureFlagService, ) {} create( @@ -35,37 +31,48 @@ export class GroupByResolverFactory const internalContext = context; return async (_source, args, _context, info) => { - const isCommonApiEnabled = await this.featureFlagService.isFeatureEnabled( - FeatureFlagKey.IS_COMMON_API_ENABLED, - internalContext.authContext.workspace?.id as string, - ); + const selectedFields = graphqlFields(info); - if (isCommonApiEnabled) { - const selectedFields = graphqlFields(info); + const shouldIncludeRecords = + isDefined(selectedFields.edges?.node) && + Object.keys(selectedFields.edges?.node).length > 0; - try { - return await this.commonGroupByQueryRunnerService.execute( - { ...args, selectedFields }, - internalContext, + try { + const typeORMObjectRecordsParser = + new ObjectRecordsToGraphqlConnectionHelper( + internalContext.objectMetadataMaps, ); - } catch (error) { - workspaceQueryRunnerGraphqlApiExceptionHandler(error); - } + + const results = await this.commonGroupByQueryRunnerService.execute( + { ...args, selectedFields, includeRecords: shouldIncludeRecords }, + internalContext, + ); + + const formattedResults = results.map((group) => { + const formattedRecords = typeORMObjectRecordsParser.createConnection({ + objectRecords: group.records ?? [], + objectName: + internalContext.objectMetadataItemWithFieldMaps.nameSingular, + objectRecordsAggregatedValues: {}, + selectedAggregatedFields: {}, + take: group.records?.length || 0, + totalCount: Number(group.totalCount ?? 0), + hasNextPage: false, + hasPreviousPage: false, + }); + + const { _records, ...groupWithoutRecords } = group; + + return { + ...groupWithoutRecords, + ...formattedRecords, + }; + }); + + return formattedResults; + } catch (error) { + workspaceQueryRunnerGraphqlApiExceptionHandler(error); } - - const options: WorkspaceQueryRunnerOptions = { - authContext: internalContext.authContext, - info, - objectMetadataMaps: internalContext.objectMetadataMaps, - objectMetadataItemWithFieldMaps: - internalContext.objectMetadataItemWithFieldMaps, - }; - - return await this.graphqlQueryRunnerService.execute( - args, - options, - GroupByResolverFactory.methodName, - ); }; } } diff --git a/packages/twenty-server/src/engine/api/rest/core/handlers/rest-api-group-by.handler.ts b/packages/twenty-server/src/engine/api/rest/core/handlers/rest-api-group-by.handler.ts index bb29b2b5327..6ecb88997d3 100644 --- a/packages/twenty-server/src/engine/api/rest/core/handlers/rest-api-group-by.handler.ts +++ b/packages/twenty-server/src/engine/api/rest/core/handlers/rest-api-group-by.handler.ts @@ -6,6 +6,7 @@ import { CommonGroupByQueryRunnerService } from 'src/engine/api/common/common-qu import { parseAggregateFieldsRestRequest } from 'src/engine/api/rest/input-request-parsers/aggregate-fields-parser-utils/parse-aggregate-fields-rest-request.util'; import { parseFilterRestRequest } from 'src/engine/api/rest/input-request-parsers/filter-parser-utils/parse-filter-rest-request.util'; import { parseGroupByRestRequest } from 'src/engine/api/rest/input-request-parsers/group-by-parser-utils/parse-group-by-rest-request.util'; +import { parseIncludeRecordsSampleRestRequest } from 'src/engine/api/rest/input-request-parsers/group-by-with-records/parse-include-records-sample-rest-request.util'; import { parseOrderByWithGroupByRestRequest } from 'src/engine/api/rest/input-request-parsers/order-by-with-group-by-parser-utils/parse-order-by-with-group-by-rest-request.util'; import { parseViewIdRestRequest } from 'src/engine/api/rest/input-request-parsers/view-id-parser-utils/parse-view-id-rest-request.util'; import { AuthenticatedRequest } from 'src/engine/api/rest/types/authenticated-request'; @@ -22,13 +23,16 @@ export class RestApiGroupByHandler extends RestApiBaseHandler { async handle(request: AuthenticatedRequest) { try { const { + filter, + orderBy, + viewId, + groupBy, + selectedFields, authContext, - objectMetadataItemWithFieldMaps, objectMetadataMaps, - } = await this.buildCommonOptions(request); - - const { filter, orderBy, viewId, groupBy, selectedFields } = - this.parseRequestArgs(request); + objectMetadataItemWithFieldMaps, + includeRecords, + } = await this.parseRequestArgs(request); return await this.commonGroupByQueryRunnerService.execute( { @@ -37,6 +41,7 @@ export class RestApiGroupByHandler extends RestApiBaseHandler { viewId, groupBy, selectedFields, + includeRecords, }, { authContext, @@ -49,20 +54,39 @@ export class RestApiGroupByHandler extends RestApiBaseHandler { } } - private parseRequestArgs(request: AuthenticatedRequest) { + private async parseRequestArgs(request: AuthenticatedRequest) { + const { authContext, objectMetadataItemWithFieldMaps, objectMetadataMaps } = + await this.buildCommonOptions(request); + const orderByWithGroupBy = parseOrderByWithGroupByRestRequest(request); const filter = parseFilterRestRequest(request); const viewId = parseViewIdRestRequest(request); const groupBy = parseGroupByRestRequest(request); + const includeRecords = parseIncludeRecordsSampleRestRequest(request); const aggregateFields = parseAggregateFieldsRestRequest(request); - const selectedFields = { ...aggregateFields, groupByDimensionValues: true }; + let selectedFields = { ...aggregateFields, groupByDimensionValues: true }; + + if (includeRecords) { + const selectableFields = await this.computeSelectedFields({ + depth: 0, + objectMetadataMapItem: objectMetadataItemWithFieldMaps, + objectMetadataMaps, + authContext, + }); + + selectedFields = { ...selectedFields, ...selectableFields }; + } return { + authContext, + objectMetadataItemWithFieldMaps, + objectMetadataMaps, filter, orderBy: orderByWithGroupBy, viewId, groupBy, selectedFields, + includeRecords, }; } } diff --git a/packages/twenty-server/src/engine/api/rest/core/interfaces/rest-api-base.handler.ts b/packages/twenty-server/src/engine/api/rest/core/interfaces/rest-api-base.handler.ts index 2ffd9547d52..b0d4bb47aa6 100644 --- a/packages/twenty-server/src/engine/api/rest/core/interfaces/rest-api-base.handler.ts +++ b/packages/twenty-server/src/engine/api/rest/core/interfaces/rest-api-base.handler.ts @@ -14,6 +14,7 @@ import { WorkspaceAuthContext } from 'src/engine/api/common/interfaces/workspace import { type ObjectRecordFilter } from 'src/engine/api/graphql/workspace-query-builder/interfaces/object-record.interface'; import { CommonGroupByOutputItem } from 'src/engine/api/common/types/common-group-by-output-item.type'; +import { CommonSelectedFields } from 'src/engine/api/common/types/common-selected-fields-result.type'; import { GraphqlQueryParser } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query.parser'; import { encodeCursor } from 'src/engine/api/graphql/graphql-query-runner/utils/cursors.util'; import { CoreQueryBuilderFactory } from 'src/engine/api/rest/core/query-builder/core-query-builder.factory'; @@ -596,7 +597,7 @@ export abstract class RestApiBaseHandler { depth?: Depth | undefined; objectMetadataMapItem: ObjectMetadataItemWithFieldMaps; objectMetadataMaps: ObjectMetadataMaps; - }) { + }): Promise { const { objectsPermissions } = await this.getObjectsPermissions(authContext); diff --git a/packages/twenty-server/src/engine/api/rest/core/query-builder/utils/path-parsers/parse-core-path.utils.ts b/packages/twenty-server/src/engine/api/rest/core/query-builder/utils/path-parsers/parse-core-path.utils.ts index f4bb56fe063..da69d296dec 100644 --- a/packages/twenty-server/src/engine/api/rest/core/query-builder/utils/path-parsers/parse-core-path.utils.ts +++ b/packages/twenty-server/src/engine/api/rest/core/query-builder/utils/path-parsers/parse-core-path.utils.ts @@ -37,7 +37,7 @@ export const parseCorePath = ( if ( queryAction[1] === 'duplicates' || - queryAction[1] === 'group' || + queryAction[1] === 'groupBy' || queryAction[1] === 'merge' ) { return { object: queryAction[0] }; diff --git a/packages/twenty-server/src/engine/api/rest/core/services/rest-api-core.service.ts b/packages/twenty-server/src/engine/api/rest/core/services/rest-api-core.service.ts index e8a617dc542..3fa8056d2cb 100644 --- a/packages/twenty-server/src/engine/api/rest/core/services/rest-api-core.service.ts +++ b/packages/twenty-server/src/engine/api/rest/core/services/rest-api-core.service.ts @@ -122,15 +122,7 @@ export class RestApiCoreService { } async groupBy(request: AuthenticatedRequest) { - const isCommonApiEnabled = await this.isCommonApiEnabled(request); - - if (isCommonApiEnabled) { - return await this.restApiGroupByHandler.handle(request); - } else { - throw new BadRequestException( - 'Activate feature flag to use GroupBy in the REST API', - ); - } + return await this.restApiGroupByHandler.handle(request); } async delete(request: AuthenticatedRequest) { diff --git a/packages/twenty-server/src/engine/api/rest/input-request-parsers/group-by-with-records/parse-include-records-sample-rest-request.util.ts b/packages/twenty-server/src/engine/api/rest/input-request-parsers/group-by-with-records/parse-include-records-sample-rest-request.util.ts new file mode 100644 index 00000000000..9323e790957 --- /dev/null +++ b/packages/twenty-server/src/engine/api/rest/input-request-parsers/group-by-with-records/parse-include-records-sample-rest-request.util.ts @@ -0,0 +1,13 @@ +import { isDefined } from 'twenty-shared/utils'; + +import { type AuthenticatedRequest } from 'src/engine/api/rest/types/authenticated-request'; + +export const parseIncludeRecordsSampleRestRequest = ( + request: AuthenticatedRequest, +): boolean => { + if (!isDefined(request.query.include_records_sample)) { + return false; + } + + return request.query.include_records_sample === 'true'; +}; diff --git a/packages/twenty-server/src/engine/twenty-orm/repository/permissions.utils.ts b/packages/twenty-server/src/engine/twenty-orm/repository/permissions.utils.ts index 357d303a8e9..0503fc3874a 100644 --- a/packages/twenty-server/src/engine/twenty-orm/repository/permissions.utils.ts +++ b/packages/twenty-server/src/engine/twenty-orm/repository/permissions.utils.ts @@ -17,7 +17,34 @@ import { import { type ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-metadata-maps'; import { getColumnNameToFieldMetadataIdMap } from 'src/engine/twenty-orm/utils/get-column-name-to-field-metadata-id.util'; -const getTargetEntityAndOperationType = (expressionMap: QueryExpressionMap) => { +const getTargetEntityAndOperationType = ( + expressionMap: QueryExpressionMap, +): + | { + isSubQuery: true; + mainEntity?: undefined; + operationType?: undefined; + } + | { + isSubQuery?: undefined; + mainEntity: string; + operationType: + | 'select' + | 'insert' + | 'update' + | 'delete' + | 'restore' + | 'soft-delete' + | 'relation'; + } => { + const isSubQuery = expressionMap.aliases[0].subQuery; + + if (isSubQuery) { + return { + isSubQuery: true, // will bypass permission checks because subQuery permissions will be evaluated when it is executed. This is valid for groupBy with records usecase. If your usecase is different, make sure permission checks are run. + }; + } + const mainEntity = expressionMap.aliases[0].metadata.name; const operationType = expressionMap.queryType; @@ -181,9 +208,13 @@ export const validateQueryIsPermittedOrThrow = ({ return; } - const { mainEntity, operationType } = + const { mainEntity, operationType, isSubQuery } = getTargetEntityAndOperationType(expressionMap); + if (isSubQuery) { + return; + } + const allFieldsSelected = expressionMap.selects.some( (select) => select.selection === mainEntity, ); diff --git a/packages/twenty-server/test/integration/graphql/suites/group-by-resolvers.integration-spec.ts b/packages/twenty-server/test/integration/graphql/suites/group-by-resolver.integration-spec.ts similarity index 99% rename from packages/twenty-server/test/integration/graphql/suites/group-by-resolvers.integration-spec.ts rename to packages/twenty-server/test/integration/graphql/suites/group-by-resolver.integration-spec.ts index a49a284e98a..5f8497ff7a2 100644 --- a/packages/twenty-server/test/integration/graphql/suites/group-by-resolvers.integration-spec.ts +++ b/packages/twenty-server/test/integration/graphql/suites/group-by-resolver.integration-spec.ts @@ -16,7 +16,7 @@ import { type FieldMetadataDTO } from 'src/engine/metadata-modules/field-metadat import { type ObjectMetadataDTO } from 'src/engine/metadata-modules/object-metadata/dtos/object-metadata.dto'; import { ViewFilterGroupLogicalOperator } from 'src/modules/view/standard-objects/view-filter-group.workspace-entity'; -describe('group-by resolvers (integration)', () => { +describe('group-by resolver (integration)', () => { describe('standard case', () => { const testPersonId = randomUUID(); const testPerson2Id = randomUUID(); diff --git a/packages/twenty-server/test/integration/graphql/suites/group-by-with-records-resolver.integration-spec.ts b/packages/twenty-server/test/integration/graphql/suites/group-by-with-records-resolver.integration-spec.ts new file mode 100644 index 00000000000..0afb3c08e6d --- /dev/null +++ b/packages/twenty-server/test/integration/graphql/suites/group-by-with-records-resolver.integration-spec.ts @@ -0,0 +1,352 @@ +import { randomUUID } from 'crypto'; + +import gql from 'graphql-tag'; +import { COMPANY_GQL_FIELDS } from 'test/integration/constants/company-gql-fields.constants'; +import { createOneOperationFactory } from 'test/integration/graphql/utils/create-one-operation-factory.util'; +import { destroyOneOperationFactory } from 'test/integration/graphql/utils/destroy-one-operation-factory.util'; +import { makeGraphqlAPIRequest } from 'test/integration/graphql/utils/make-graphql-api-request.util'; + +const OPPORTUNITY_GQL_FIELDS = ` + id + stage + amount { + amountMicros + } + companyId + createdAt + closeDate +`; + +// used not to mix records with the seeded ones +const FILTER_2020 = { + and: [ + { + createdAt: { + gte: '2020-01-01T00:00:00.000Z', + }, + }, + { + createdAt: { + lte: '2020-03-03T23:59:59.999Z', + }, + }, + ], +}; + +describe('basic group-by with records', () => { + const testOpportunityId1 = randomUUID(); + const testOpportunityId2 = randomUUID(); + const testOpportunityId3 = randomUUID(); + const testOpportunityId4 = randomUUID(); + const testCompanyId1 = randomUUID(); + const testCompanyId2 = randomUUID(); + + beforeAll(async () => { + // Create test companies + await makeGraphqlAPIRequest( + createOneOperationFactory({ + objectMetadataSingularName: 'company', + gqlFields: COMPANY_GQL_FIELDS, + data: { + id: testCompanyId1, + name: 'Company 1', + employees: 10, + }, + }), + ); + + await makeGraphqlAPIRequest( + createOneOperationFactory({ + objectMetadataSingularName: 'company', + gqlFields: COMPANY_GQL_FIELDS, + data: { + id: testCompanyId2, + name: 'Company 2', + employees: 20, + }, + }), + ); + + // Create test opportunities with different stages and dates + await makeGraphqlAPIRequest( + createOneOperationFactory({ + objectMetadataSingularName: 'opportunity', + gqlFields: OPPORTUNITY_GQL_FIELDS, + data: { + id: testOpportunityId1, + stage: 'NEW', + name: 'Opportunity 1', + amount: { amountMicros: 1000000000000 }, // 1000 + companyId: testCompanyId1, + closeDate: '2025-02-05T08:00:00.000Z', // Wednesday + createdAt: '2020-02-05T08:00:00.000Z', + }, + }), + ); + + await makeGraphqlAPIRequest( + createOneOperationFactory({ + objectMetadataSingularName: 'opportunity', + gqlFields: OPPORTUNITY_GQL_FIELDS, + data: { + id: testOpportunityId2, + stage: 'NEW', + name: 'Opportunity 2', + amount: { amountMicros: 2000000000000 }, // 2000 + companyId: testCompanyId1, + closeDate: '2025-02-06T08:00:00.000Z', // Thursday + createdAt: '2020-02-05T08:00:00.000Z', + }, + }), + ); + + await makeGraphqlAPIRequest( + createOneOperationFactory({ + objectMetadataSingularName: 'opportunity', + gqlFields: OPPORTUNITY_GQL_FIELDS, + data: { + id: testOpportunityId3, + stage: 'NEW', + name: 'Opportunity 3', + amount: { amountMicros: 3000000000000 }, // 3000 + companyId: testCompanyId2, + closeDate: '2025-02-06T08:00:00.000Z', // Thursday + createdAt: '2020-02-05T08:00:00.000Z', + }, + }), + ); + + await makeGraphqlAPIRequest( + createOneOperationFactory({ + objectMetadataSingularName: 'opportunity', + gqlFields: OPPORTUNITY_GQL_FIELDS, + data: { + id: testOpportunityId4, + stage: 'SCREENING', + name: 'Opportunity 4', + amount: { amountMicros: 4000000000000 }, // 4000 + companyId: testCompanyId2, + closeDate: '2025-02-06T08:00:00.000Z', // Thursday + createdAt: '2020-02-05T08:00:00.000Z', + }, + }), + ); + }); + + afterAll(async () => { + // Cleanup created opportunities + for (const id of [ + testOpportunityId1, + testOpportunityId2, + testOpportunityId3, + testOpportunityId4, + ]) { + await makeGraphqlAPIRequest( + destroyOneOperationFactory({ + objectMetadataSingularName: 'opportunity', + gqlFields: 'id', + recordId: id, + }), + ); + } + + // Cleanup created companies + for (const id of [testCompanyId1, testCompanyId2]) { + await makeGraphqlAPIRequest( + destroyOneOperationFactory({ + objectMetadataSingularName: 'company', + gqlFields: 'id', + recordId: id, + }), + ); + } + }); + + it('groups by stage and createdAt with records', async () => { + const response = await makeGraphqlAPIRequest({ + query: gql` + query OpportunitiesGroupBy( + $groupBy: [OpportunityGroupByInput!] + $filter: OpportunityFilterInput + ) { + opportunitiesGroupBy(groupBy: $groupBy, filter: $filter) { + minCloseDate + groupByDimensionValues + sumAmountAmountMicros + __typename + edges { + node { + stage + closeDate + name + amount { + amountMicros + } + } + } + } + } + `, + variables: { + groupBy: [ + { + closeDate: { + granularity: 'DAY_OF_THE_WEEK', + }, + }, + { + stage: true, + }, + ], + filter: FILTER_2020, + }, + }); + + expect(response.body.errors).toBeUndefined(); + expect(response.body.data).toBeDefined(); + + const groups = response.body.data.opportunitiesGroupBy; + + expect(groups).toBeDefined(); + expect(groups).toHaveLength(3); + + // Check that each group has the expected structure + groups.forEach((group: any) => { + expect(group.groupByDimensionValues).toBeDefined(); + expect(Array.isArray(group.groupByDimensionValues)).toBe(true); + expect(group.edges).toBeDefined(); + expect(Array.isArray(group.edges)).toBe(true); + }); + + // Find specific groups and verify their content + const wednesdayNewGroup = groups.find( + (group: any) => + group.groupByDimensionValues.includes('Wednesday') && + group.groupByDimensionValues.includes('NEW'), + ); + + expect(wednesdayNewGroup).toBeDefined(); + expect(wednesdayNewGroup.edges).toHaveLength(1); + expect(wednesdayNewGroup.edges[0].node.name).toBe('Opportunity 1'); + expect(wednesdayNewGroup.edges[0].node.stage).toBe('NEW'); + expect(wednesdayNewGroup.edges[0].node.amount.amountMicros).toBe( + 1000000000000, + ); + expect(wednesdayNewGroup.sumAmountAmountMicros).toBe(1000000000000); + + const thursdayNewGroup = groups.find( + (group: any) => + group.groupByDimensionValues.includes('Thursday') && + group.groupByDimensionValues.includes('NEW'), + ); + + expect(thursdayNewGroup).toBeDefined(); + expect(thursdayNewGroup.edges).toHaveLength(2); + expect(thursdayNewGroup.sumAmountAmountMicros).toBe(5000000000000); + for (const edge of thursdayNewGroup.edges) { + expect(edge.node.stage).toBe('NEW'); + expect(edge.node.name).toBeDefined(); + expect( + edge.node.name === 'Opportunity 2' || + edge.node.name === 'Opportunity 3', + ).toBe(true); + } + + const thursdayScreeningGroup = groups.find( + (group: any) => + group.groupByDimensionValues.includes('Thursday') && + group.groupByDimensionValues.includes('SCREENING'), + ); + + expect(thursdayScreeningGroup).toBeDefined(); + expect(thursdayScreeningGroup.edges).toHaveLength(1); + expect(thursdayScreeningGroup.edges[0].node.amount.amountMicros).toBe( + 4000000000000, + ); + expect(thursdayScreeningGroup.edges[0].node.stage).toBe('SCREENING'); + expect(thursdayScreeningGroup.edges[0].node.name).toBe('Opportunity 4'); + expect(thursdayScreeningGroup.sumAmountAmountMicros).toBe(4000000000000); + }); + + it('groups by stage and createdAt with records and filters', async () => { + // Test with filter to only include NEW stage opportunities + const response = await makeGraphqlAPIRequest({ + query: gql` + query OpportunitiesGroupBy( + $groupBy: [OpportunityGroupByInput!] + $filter: OpportunityFilterInput + ) { + opportunitiesGroupBy(groupBy: $groupBy, filter: $filter) { + minCloseDate + groupByDimensionValues + sumAmountAmountMicros + __typename + edges { + node { + stage + createdAt + amount { + amountMicros + } + } + } + } + } + `, + variables: { + groupBy: [ + { + closeDate: { + granularity: 'DAY_OF_THE_WEEK', + }, + }, + { + stage: true, + }, + ], + filter: { + and: [ + ...FILTER_2020.and, + { + stage: { + eq: 'NEW', + }, + }, + ], + }, + }, + }); + + expect(response.body.errors).toBeUndefined(); + expect(response.body.data).toBeDefined(); + + const groups = response.body.data.opportunitiesGroupBy; + + expect(groups).toHaveLength(2); + const wednesdayNewGroup = groups.find( + (group: any) => + group.groupByDimensionValues.includes('Wednesday') && + group.groupByDimensionValues.includes('NEW'), + ); + + expect(wednesdayNewGroup.groupByDimensionValues).toHaveLength(2); + expect(wednesdayNewGroup.groupByDimensionValues).toContain('NEW'); + expect(wednesdayNewGroup.groupByDimensionValues).toContain('Wednesday'); + expect(wednesdayNewGroup.edges).toHaveLength(1); + expect(wednesdayNewGroup.edges[0].node.stage).toBe('NEW'); + expect(wednesdayNewGroup.edges[0].node.amount.amountMicros).toBe( + 1000000000000, + ); + + const thursdayNewGroup = groups.find( + (group: any) => + group.groupByDimensionValues.includes('Thursday') && + group.groupByDimensionValues.includes('NEW'), + ); + + expect(thursdayNewGroup.groupByDimensionValues).toHaveLength(2); + expect(thursdayNewGroup.groupByDimensionValues).toContain('NEW'); + expect(thursdayNewGroup.groupByDimensionValues).toContain('Thursday'); + expect(thursdayNewGroup.edges).toHaveLength(2); + }); +});