From 5bb4abc23df2f8c9b59ff9e95983675e6bb862ce Mon Sep 17 00:00:00 2001 From: Marie <51697796+ijreilly@users.noreply.github.com> Date: Tue, 18 Nov 2025 17:25:33 +0100 Subject: [PATCH] Add optional limit variable to groupBy queries (#15885) Closes https://github.com/twentyhq/core-team-issues/issues/1600. Two remarks - This `limit` variable does not reduce postgre's work at it still needs to scan the whole table. It did not seem possible to me to optimize this as we cannot foresee which dimensions will be used by the user, and an optimization could only result from an index on the dimension(s) (e.g.: group companies by addressCity limit 50 can be optimized if we have an index on companies.addressCity + we had a default orderBy on adressCity). But this will still optimize the FE which at the moment receives all groups and truncates the result. - I have not done the work on the FE as the addition of limit is a breaking change, and will break until the workspaces' schema is rebuilt, so we need to flush the cache. I think this could be acceptable as the feature is in the lab but I preferred not doing it yet as it would have no impact since in the BE I added a default limit to 50 groups, and I expect more FE work will be done to allow the user to choose their own limit --- .../graph/hooks/useGraphWidgetGroupByQuery.ts | 2 + .../common-group-by-query-runner.service.ts | 9 ++++ .../common/types/common-query-args.type.ts | 1 + .../services/group-by-with-records.service.ts | 8 ++- .../group-by/utils/get-group-limit.util.ts | 14 +++++ .../utils/get-resolver-args.util.ts | 4 ++ .../group-by-resolver.integration-spec.ts | 51 ++++++++++++++++++- .../utils/group-by-operation-factory.util.ts | 7 ++- .../constants/DefaultNumberOfGroupsLimit.ts | 1 + packages/twenty-shared/src/constants/index.ts | 1 + 10 files changed, 93 insertions(+), 5 deletions(-) create mode 100644 packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/utils/get-group-limit.util.ts create mode 100644 packages/twenty-shared/src/constants/DefaultNumberOfGroupsLimit.ts diff --git a/packages/twenty-front/src/modules/page-layout/widgets/graph/hooks/useGraphWidgetGroupByQuery.ts b/packages/twenty-front/src/modules/page-layout/widgets/graph/hooks/useGraphWidgetGroupByQuery.ts index 8b2bcdfb82b..ce13a282685 100644 --- a/packages/twenty-front/src/modules/page-layout/widgets/graph/hooks/useGraphWidgetGroupByQuery.ts +++ b/packages/twenty-front/src/modules/page-layout/widgets/graph/hooks/useGraphWidgetGroupByQuery.ts @@ -6,6 +6,7 @@ import { generateGroupByQuery } from '@/page-layout/widgets/graph/utils/generate import { generateGroupByQueryVariablesFromChartConfiguration } from '@/page-layout/widgets/graph/utils/generateGroupByQueryVariablesFromChartConfiguration'; import { useQuery } from '@apollo/client'; import { useMemo } from 'react'; +import { DEFAULT_NUMBER_OF_GROUPS_LIMIT } from 'twenty-shared/constants'; import { isDefined } from 'twenty-shared/utils'; export const useGraphWidgetGroupByQuery = ({ @@ -52,6 +53,7 @@ export const useGraphWidgetGroupByQuery = ({ const variables = { ...groupByQueryVariables, filter: gqlOperationFilter, + limit: DEFAULT_NUMBER_OF_GROUPS_LIMIT, }; const query = generateGroupByQuery({ diff --git a/packages/twenty-server/src/engine/api/common/common-query-runners/common-group-by-query-runner.service.ts b/packages/twenty-server/src/engine/api/common/common-query-runners/common-group-by-query-runner.service.ts index 19f43423e6d..85e3d18816d 100644 --- a/packages/twenty-server/src/engine/api/common/common-query-runners/common-group-by-query-runner.service.ts +++ b/packages/twenty-server/src/engine/api/common/common-query-runners/common-group-by-query-runner.service.ts @@ -37,6 +37,7 @@ import { isGroupByDateField } from 'src/engine/api/graphql/graphql-query-runner/ import { parseGroupByArgs } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/parse-group-by-args.util'; import { removeQuotes } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/remove-quote.util'; import { GroupByWithRecordsService } from 'src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.service'; +import { getGroupLimit } from 'src/engine/api/graphql/graphql-query-runner/group-by/utils/get-group-limit.util'; import { ProcessAggregateHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-aggregate.helper'; import { isFieldMetadataRelationOrMorphRelation } from 'src/engine/api/graphql/workspace-schema-builder/utils/is-field-metadata-relation-or-morph-relation.utils'; import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps'; @@ -175,6 +176,7 @@ export class CommonGroupByQueryRunnerService extends CommonBaseQueryRunnerServic selectedFieldsResult: args.selectedFieldsResult, queryRunnerContext, orderByForRecords: args.orderByForRecords ?? [], + groupLimit: args.limit, }); } @@ -182,6 +184,7 @@ export class CommonGroupByQueryRunnerService extends CommonBaseQueryRunnerServic queryBuilder, groupByDefinitions, selectedFieldsResult: args.selectedFieldsResult, + groupLimit: args.limit, }); } @@ -320,11 +323,17 @@ export class CommonGroupByQueryRunnerService extends CommonBaseQueryRunnerServic queryBuilder, groupByDefinitions, selectedFieldsResult, + groupLimit, }: { queryBuilder: WorkspaceSelectQueryBuilder; groupByDefinitions: GroupByDefinition[]; selectedFieldsResult: GraphqlQuerySelectedFieldsResult; + groupLimit?: number; }): Promise { + const effectiveGroupLimit = getGroupLimit(groupLimit); + + queryBuilder.limit(effectiveGroupLimit); + const result = await queryBuilder.getRawMany(); return formatResultWithGroupByDimensionValues({ diff --git a/packages/twenty-server/src/engine/api/common/types/common-query-args.type.ts b/packages/twenty-server/src/engine/api/common/types/common-query-args.type.ts index 373bfcb2ecf..10c26de59ef 100644 --- a/packages/twenty-server/src/engine/api/common/types/common-query-args.type.ts +++ b/packages/twenty-server/src/engine/api/common/types/common-query-args.type.ts @@ -70,6 +70,7 @@ export interface GroupByQueryArgs { viewId?: string; includeRecords?: boolean; selectedFields: CommonSelectedFields; + limit?: number; } export interface DestroyOneQueryArgs { id: string; diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.service.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.service.ts index efad027c27b..990c562fda9 100644 --- a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.service.ts +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/services/group-by-with-records.service.ts @@ -14,6 +14,7 @@ import { type GraphqlQuerySelectedFieldsResult } from 'src/engine/api/graphql/gr import { GraphqlQueryParser } from 'src/engine/api/graphql/graphql-query-runner/graphql-query-parsers/graphql-query.parser'; import { type GroupByDefinition } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/types/group-by-definition.types'; import { formatResultWithGroupByDimensionValues } from 'src/engine/api/graphql/graphql-query-runner/group-by/resolvers/utils/format-result-with-group-by-dimension-values.util'; +import { getGroupLimit } from 'src/engine/api/graphql/graphql-query-runner/group-by/utils/get-group-limit.util'; import { ProcessNestedRelationsHelper } from 'src/engine/api/graphql/graphql-query-runner/helpers/process-nested-relations.helper'; import { buildColumnsToSelect } from 'src/engine/api/graphql/graphql-query-runner/utils/build-columns-to-select'; import { ObjectMetadataItemWithFieldMaps } from 'src/engine/metadata-modules/types/object-metadata-item-with-field-maps'; @@ -21,7 +22,6 @@ import { ObjectMetadataMaps } from 'src/engine/metadata-modules/types/object-met import { type WorkspaceSelectQueryBuilder } from 'src/engine/twenty-orm/repository/workspace-select-query-builder'; import { type WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository'; -const GROUPS_LIMIT = 50; const RECORDS_PER_GROUP_LIMIT = 10; const RELATIONS_PER_RECORD_LIMIT = 5; const SUB_QUERY_PREFIX = 'sub_query_'; @@ -41,6 +41,7 @@ export class GroupByWithRecordsService { selectedFieldsResult, queryRunnerContext, orderByForRecords, + groupLimit, }: { queryBuilderWithGroupBy: WorkspaceSelectQueryBuilder; queryBuilderWithFiltersAndWithoutGroupBy: WorkspaceSelectQueryBuilder; @@ -48,9 +49,12 @@ export class GroupByWithRecordsService { selectedFieldsResult: GraphqlQuerySelectedFieldsResult; queryRunnerContext: CommonExtendedQueryRunnerContext; orderByForRecords: ObjectRecordOrderBy; + groupLimit?: number; }): Promise { + const effectiveGroupLimit = getGroupLimit(groupLimit); + const groupsResult = await queryBuilderWithGroupBy - .limit(GROUPS_LIMIT) + .limit(effectiveGroupLimit) .getRawMany(); if (groupsResult.length === 0) { diff --git a/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/utils/get-group-limit.util.ts b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/utils/get-group-limit.util.ts new file mode 100644 index 00000000000..65d4a0ac05d --- /dev/null +++ b/packages/twenty-server/src/engine/api/graphql/graphql-query-runner/group-by/utils/get-group-limit.util.ts @@ -0,0 +1,14 @@ +import { DEFAULT_NUMBER_OF_GROUPS_LIMIT } from 'twenty-shared/constants'; + +export const getGroupLimit = (limit?: number): number => { + if ( + typeof limit === 'number' && + Number.isFinite(limit) && + limit > 0 && + Number.isInteger(limit) + ) { + return limit; + } + + return DEFAULT_NUMBER_OF_GROUPS_LIMIT; +}; diff --git a/packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/utils/get-resolver-args.util.ts b/packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/utils/get-resolver-args.util.ts index 1a887145df2..a8b5018c059 100644 --- a/packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/utils/get-resolver-args.util.ts +++ b/packages/twenty-server/src/engine/api/graphql/workspace-schema-builder/utils/get-resolver-args.util.ts @@ -186,6 +186,10 @@ export const getResolverArgs = ( type: UUIDScalarType, isNullable: true, }, + limit: { + type: GraphQLInt, + isNullable: true, + }, }; default: throw new Error(`Unknown resolver type: ${type}`); diff --git a/packages/twenty-server/test/integration/graphql/suites/group-by-resolver.integration-spec.ts b/packages/twenty-server/test/integration/graphql/suites/group-by-resolver.integration-spec.ts index 68502867560..4c4bde8c3f1 100644 --- a/packages/twenty-server/test/integration/graphql/suites/group-by-resolver.integration-spec.ts +++ b/packages/twenty-server/test/integration/graphql/suites/group-by-resolver.integration-spec.ts @@ -10,7 +10,7 @@ import { findManyObjectMetadata } from 'test/integration/metadata/suites/object- import { createOneCoreViewFilter } from 'test/integration/metadata/suites/view-filter/utils/create-one-core-view-filter.util'; import { createOneCoreView } from 'test/integration/metadata/suites/view/utils/create-one-core-view.util'; import { jestExpectToBeDefined } from 'test/utils/jest-expect-to-be-defined.util.test'; -import { ViewFilterOperand } from 'twenty-shared/types'; +import { OrderByDirection, ViewFilterOperand } from 'twenty-shared/types'; import { type FieldMetadataDTO } from 'src/engine/metadata-modules/field-metadata/dtos/field-metadata.dto'; import { type ObjectMetadataDTO } from 'src/engine/metadata-modules/object-metadata/dtos/object-metadata.dto'; @@ -77,6 +77,8 @@ describe('group-by resolver (integration)', () => { objectMetadataSingularName: 'person', objectMetadataPluralName: 'people', groupBy: [{ city: true }], + orderBy: [{ city: OrderByDirection.AscNullsFirst }], // needed for City groups to be in 300 first groups + limit: 300, }), ); @@ -105,6 +107,49 @@ describe('group-by resolver (integration)', () => { expect(groupWithCityB.totalCount).toEqual(2); }); + it('limits the number of groups when limit argument is provided', async () => { + const cityA = 'City A'; + const cityB = 'City B'; + const cityC = 'City C'; + + await makeGraphqlAPIRequest( + createOneOperationFactory({ + objectMetadataSingularName: 'person', + gqlFields: PERSON_GQL_FIELDS, + data: { id: testPersonId, city: cityA }, + }), + ); + await makeGraphqlAPIRequest( + createOneOperationFactory({ + objectMetadataSingularName: 'person', + gqlFields: PERSON_GQL_FIELDS, + data: { id: testPerson2Id, city: cityB }, + }), + ); + await makeGraphqlAPIRequest( + createOneOperationFactory({ + objectMetadataSingularName: 'person', + gqlFields: PERSON_GQL_FIELDS, + data: { id: testPerson3Id, city: cityC }, + }), + ); + + const response = await makeGraphqlAPIRequest( + groupByOperationFactory({ + objectMetadataSingularName: 'person', + objectMetadataPluralName: 'people', + groupBy: [{ city: true }], + limit: 2, + }), + ); + + const groups = response.body.data.peopleGroupBy; + + expect(groups).toBeDefined(); + expect(Array.isArray(groups)).toBe(true); + expect(groups).toHaveLength(2); + }); + it('computes aggregated metrics on date time field', async () => { const cityA = 'City A'; const cityB = 'City B'; @@ -142,7 +187,9 @@ describe('group-by resolver (integration)', () => { objectMetadataSingularName: 'person', objectMetadataPluralName: 'people', groupBy: [{ city: true }], + orderBy: [{ city: OrderByDirection.AscNullsFirst }], // needed for City groups to be in 300 first groups gqlFields: 'minCreatedAt', + limit: 300, }), ); @@ -592,6 +639,8 @@ describe('group-by resolver (integration)', () => { objectMetadataPluralName: 'people', groupBy: [{ city: true }], viewId, + orderBy: [{ city: OrderByDirection.AscNullsFirst }], // needed for City groups to be in 300 first groups + limit: 300, }), ); diff --git a/packages/twenty-server/test/integration/graphql/utils/group-by-operation-factory.util.ts b/packages/twenty-server/test/integration/graphql/utils/group-by-operation-factory.util.ts index a4b7367f1ef..6a6060e7b64 100644 --- a/packages/twenty-server/test/integration/graphql/utils/group-by-operation-factory.util.ts +++ b/packages/twenty-server/test/integration/graphql/utils/group-by-operation-factory.util.ts @@ -10,6 +10,7 @@ type GroupByOperationFactoryParams = { orderBy?: object[]; viewId?: string; gqlFields?: string; + limit?: number; }; export const groupByOperationFactory = ({ @@ -21,10 +22,11 @@ export const groupByOperationFactory = ({ orderByForRecords = [], viewId, gqlFields, + limit, }: GroupByOperationFactoryParams) => ({ query: gql` - query ${capitalize(objectMetadataPluralName)}GroupBy($groupBy: [${capitalize(objectMetadataSingularName)}GroupByInput!]!, $filter: ${capitalize(objectMetadataSingularName)}FilterInput, $orderBy: [${capitalize(objectMetadataSingularName)}OrderByWithGroupByInput!], $viewId: UUID) { - ${objectMetadataPluralName}GroupBy(groupBy: $groupBy, filter: $filter, orderBy: $orderBy, viewId: $viewId) { + query ${capitalize(objectMetadataPluralName)}GroupBy($groupBy: [${capitalize(objectMetadataSingularName)}GroupByInput!]!, $filter: ${capitalize(objectMetadataSingularName)}FilterInput, $orderBy: [${capitalize(objectMetadataSingularName)}OrderByWithGroupByInput!], $viewId: UUID, $limit: Int) { + ${objectMetadataPluralName}GroupBy(groupBy: $groupBy, filter: $filter, orderBy: $orderBy, viewId: $viewId, limit: $limit) { ${gqlFields ? gqlFields : ''} groupByDimensionValues totalCount @@ -36,6 +38,7 @@ export const groupByOperationFactory = ({ filter, orderBy, orderByForRecords, + limit, ...(viewId && { viewId }), }, }); diff --git a/packages/twenty-shared/src/constants/DefaultNumberOfGroupsLimit.ts b/packages/twenty-shared/src/constants/DefaultNumberOfGroupsLimit.ts new file mode 100644 index 00000000000..40132953dcc --- /dev/null +++ b/packages/twenty-shared/src/constants/DefaultNumberOfGroupsLimit.ts @@ -0,0 +1 @@ +export const DEFAULT_NUMBER_OF_GROUPS_LIMIT = 50; diff --git a/packages/twenty-shared/src/constants/index.ts b/packages/twenty-shared/src/constants/index.ts index e287f5c6b80..b6304bb8a69 100644 --- a/packages/twenty-shared/src/constants/index.ts +++ b/packages/twenty-shared/src/constants/index.ts @@ -12,6 +12,7 @@ export { COMPOSITE_FIELD_TYPE_SUB_FIELDS_NAMES } from './CompositeFieldTypeSubFi export { CurrencyCode } from './CurrencyCode'; export { CURRENCY_CODE_LABELS } from './CurrencyCodeLabels'; export { DATE_TYPE_FORMAT } from './DateTypeFormat'; +export { DEFAULT_NUMBER_OF_GROUPS_LIMIT } from './DefaultNumberOfGroupsLimit'; export { DEFAULT_RELATIVE_DATE_FILTER_VALUE } from './DefaultRelativeDateFilterValue'; export { FIELD_FOR_TOTAL_COUNT_AGGREGATE_OPERATION } from './FieldForTotalCountAggregateOperation'; export { MAX_OPTIONS_TO_DISPLAY } from './FieldMetadataMaxOptionsToDisplay';