Compare commits

...
Author SHA1 Message Date
neo773 3b521289fa revert 2026-05-18 21:28:09 +05:30
neo773 4b873a2b93 perf(server): cut redundant work in messaging/calendar sync pipeline
Seed lastCredentialsRefreshedAt on OAuth create/reconnect so token cache
is hit on first sync; thread fresh timestamp through return path to avoid
duplicate refreshes inside a single job. Fetch email aliases once at
OAuth callback via Gmail sendAs API instead of per-job via People API.
Propagate {lite: true} workspace context and transactionManager into
find() calls to cut pg-pool.connect overhead.
2026-05-18 21:02:39 +05:30
45 changed files with 2670 additions and 2416 deletions
@@ -69,6 +69,7 @@ import { ObjectMetadataEntity } from 'src/engine/metadata-modules/object-metadat
import { PermissionsModule } from 'src/engine/metadata-modules/permissions/permissions.module';
import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import { EmailAliasManagerModule } from 'src/modules/connected-account/email-alias-manager/email-alias-manager.module';
import { ConnectedAccountModule } from 'src/modules/connected-account/connected-account.module';
import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-common.module';
import { MessagingFolderSyncManagerModule } from 'src/modules/messaging/message-folder-manager/messaging-folder-sync-manager.module';
@@ -127,6 +128,7 @@ import { JwtAuthStrategy } from './strategies/jwt.auth.strategy';
EnterpriseModule,
FileModule,
ConnectedAccountTokenEncryptionModule,
EmailAliasManagerModule,
],
controllers: [
GoogleAuthController,
@@ -105,6 +105,7 @@ export class CreateConnectedAccountService {
userWorkspaceId,
scopes,
workspaceId,
lastCredentialsRefreshedAt: new Date(),
} as ConnectedAccountEntity);
}, authContext);
}
@@ -38,6 +38,7 @@ import {
type CalendarEventListFetchJobData,
} from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import { EmailAliasManagerService } from 'src/modules/connected-account/email-alias-manager/services/email-alias-manager.service';
import { AccountsToReconnectService } from 'src/modules/connected-account/services/accounts-to-reconnect.service';
import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service';
@@ -66,6 +67,7 @@ export class GoogleAPIsService {
private readonly googleAPIScopesService: GoogleAPIScopesService,
private readonly googleApisServiceAvailabilityService: GoogleApisServiceAvailabilityService,
private readonly syncMessageFoldersService: SyncMessageFoldersService,
private readonly emailAliasManagerService: EmailAliasManagerService,
@InjectRepository(ConnectedAccountEntity)
private readonly connectedAccountRepository: Repository<ConnectedAccountEntity>,
@InjectRepository(UserWorkspaceEntity)
@@ -238,6 +240,18 @@ export class GoogleAPIsService {
},
);
const connectedAccountForAliases =
await this.connectedAccountRepository.findOne({
where: { id: newOrExistingConnectedAccountId, workspaceId },
});
if (isDefined(connectedAccountForAliases)) {
await this.emailAliasManagerService.refreshHandleAliases(
connectedAccountForAliases,
workspaceId,
);
}
if (
isMessagingEnabled &&
isMessagingAvailable &&
@@ -36,6 +36,7 @@ import {
type CalendarEventListFetchJobData,
} from 'src/modules/calendar/calendar-event-import-manager/jobs/calendar-event-list-fetch.job';
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import { EmailAliasManagerService } from 'src/modules/connected-account/email-alias-manager/services/email-alias-manager.service';
import { AccountsToReconnectService } from 'src/modules/connected-account/services/accounts-to-reconnect.service';
import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service';
@@ -62,6 +63,7 @@ export class MicrosoftAPIsService {
private readonly updateConnectedAccountOnReconnectService: UpdateConnectedAccountOnReconnectService,
private readonly twentyConfigService: TwentyConfigService,
private readonly syncMessageFoldersService: SyncMessageFoldersService,
private readonly emailAliasManagerService: EmailAliasManagerService,
@InjectRepository(ConnectedAccountEntity)
private readonly connectedAccountRepository: Repository<ConnectedAccountEntity>,
@InjectRepository(UserWorkspaceEntity)
@@ -216,6 +218,18 @@ export class MicrosoftAPIsService {
},
);
const connectedAccountForAliases =
await this.connectedAccountRepository.findOne({
where: { id: newOrExistingConnectedAccountId, workspaceId },
});
if (isDefined(connectedAccountForAliases)) {
await this.emailAliasManagerService.refreshHandleAliases(
connectedAccountForAliases,
workspaceId,
);
}
if (
this.twentyConfigService.get(
'MESSAGING_PROVIDER_MICROSOFT_ENABLED',
@@ -56,6 +56,7 @@ export class UpdateConnectedAccountOnReconnectService {
refreshToken: encryptedRefreshToken,
scopes,
authFailedAt: null,
lastCredentialsRefreshedAt: new Date(),
},
);
}, authContext);
@@ -41,145 +41,149 @@ export class BlocklistItemDeleteCalendarEventsJob {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const blocklistItemIds = data.events.map(
(eventPayload) => eventPayload.recordId,
);
const blocklistRepository =
await this.globalWorkspaceOrmManager.getRepository<BlocklistWorkspaceEntity>(
workspaceId,
'blocklist',
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const blocklistItemIds = data.events.map(
(eventPayload) => eventPayload.recordId,
);
const blocklist = await blocklistRepository.find({
where: {
id: Any(blocklistItemIds),
},
});
const handlesToDeleteByWorkspaceMemberIdMap = blocklist.reduce(
(acc, blocklistItem) => {
const { handle, workspaceMemberId } = blocklistItem;
if (!acc.has(workspaceMemberId)) {
acc.set(workspaceMemberId, []);
}
if (!isDefined(handle)) {
return acc;
}
acc.get(workspaceMemberId)?.push(handle);
return acc;
},
new Map<string, string[]>(),
);
const calendarChannelEventAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelEventAssociationWorkspaceEntity>(
workspaceId,
'calendarChannelEventAssociation',
);
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
'workspaceMember',
{ shouldBypassPermissionChecks: true },
);
for (const workspaceMemberId of handlesToDeleteByWorkspaceMemberIdMap.keys()) {
const handles =
handlesToDeleteByWorkspaceMemberIdMap.get(workspaceMemberId);
if (!handles) {
continue;
}
const workspaceMember = await workspaceMemberRepository.findOne({
where: { id: workspaceMemberId },
});
if (!workspaceMember) {
continue;
}
const userWorkspace = await this.userWorkspaceRepository.findOne({
where: { userId: workspaceMember.userId, workspaceId },
select: ['id'],
});
if (!userWorkspace) {
continue;
}
const calendarChannels = await this.calendarChannelRepository.find({
select: {
id: true,
handle: true,
connectedAccount: {
handleAliases: true,
},
},
where: {
connectedAccount: { userWorkspaceId: userWorkspace.id },
const blocklistRepository =
await this.globalWorkspaceOrmManager.getRepository<BlocklistWorkspaceEntity>(
workspaceId,
'blocklist',
);
const blocklist = await blocklistRepository.find({
where: {
id: Any(blocklistItemIds),
},
relations: ['connectedAccount'],
});
for (const calendarChannel of calendarChannels) {
const calendarChannelHandles = [calendarChannel.handle];
const handlesToDeleteByWorkspaceMemberIdMap = blocklist.reduce(
(acc, blocklistItem) => {
const { handle, workspaceMemberId } = blocklistItem;
if (calendarChannel.connectedAccount.handleAliases) {
const rawAliases = calendarChannel.connectedAccount
.handleAliases as string | string[];
if (!acc.has(workspaceMemberId)) {
acc.set(workspaceMemberId, []);
}
const aliasList = Array.isArray(rawAliases)
? rawAliases
: rawAliases.split(',').map((alias: string) => alias.trim());
if (!isDefined(handle)) {
return acc;
}
calendarChannelHandles.push(...aliasList);
}
acc.get(workspaceMemberId)?.push(handle);
const handleConditions = handles.map((handle) => {
const isHandleDomain = handle.startsWith('@');
return acc;
},
new Map<string, string[]>(),
);
return isHandleDomain
? {
handle: And(
Or(ILike(`%${handle}`), ILike(`%.${handle.slice(1)}`)),
Not(In(calendarChannelHandles)),
),
}
: { handle };
});
const calendarChannelEventAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelEventAssociationWorkspaceEntity>(
workspaceId,
'calendarChannelEventAssociation',
);
const calendarEventsAssociationsToDelete =
await calendarChannelEventAssociationRepository.find({
where: {
calendarChannelId: calendarChannel.id,
calendarEvent: {
calendarEventParticipants: handleConditions,
},
},
});
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
'workspaceMember',
{ shouldBypassPermissionChecks: true },
);
if (calendarEventsAssociationsToDelete.length === 0) {
for (const workspaceMemberId of handlesToDeleteByWorkspaceMemberIdMap.keys()) {
const handles =
handlesToDeleteByWorkspaceMemberIdMap.get(workspaceMemberId);
if (!handles) {
continue;
}
await calendarChannelEventAssociationRepository.delete(
calendarEventsAssociationsToDelete.map(({ id }) => id),
);
}
}
const workspaceMember = await workspaceMemberRepository.findOne({
where: { id: workspaceMemberId },
});
await this.calendarEventCleanerService.cleanWorkspaceCalendarEvents(
workspaceId,
);
}, authContext);
if (!workspaceMember) {
continue;
}
const userWorkspace = await this.userWorkspaceRepository.findOne({
where: { userId: workspaceMember.userId, workspaceId },
select: ['id'],
});
if (!userWorkspace) {
continue;
}
const calendarChannels = await this.calendarChannelRepository.find({
select: {
id: true,
handle: true,
connectedAccount: {
handleAliases: true,
},
},
where: {
connectedAccount: { userWorkspaceId: userWorkspace.id },
workspaceId,
},
relations: ['connectedAccount'],
});
for (const calendarChannel of calendarChannels) {
const calendarChannelHandles = [calendarChannel.handle];
if (calendarChannel.connectedAccount.handleAliases) {
const rawAliases = calendarChannel.connectedAccount
.handleAliases as string | string[];
const aliasList = Array.isArray(rawAliases)
? rawAliases
: rawAliases.split(',').map((alias: string) => alias.trim());
calendarChannelHandles.push(...aliasList);
}
const handleConditions = handles.map((handle) => {
const isHandleDomain = handle.startsWith('@');
return isHandleDomain
? {
handle: And(
Or(ILike(`%${handle}`), ILike(`%.${handle.slice(1)}`)),
Not(In(calendarChannelHandles)),
),
}
: { handle };
});
const calendarEventsAssociationsToDelete =
await calendarChannelEventAssociationRepository.find({
where: {
calendarChannelId: calendarChannel.id,
calendarEvent: {
calendarEventParticipants: handleConditions,
},
},
});
if (calendarEventsAssociationsToDelete.length === 0) {
continue;
}
await calendarChannelEventAssociationRepository.delete(
calendarEventsAssociationsToDelete.map(({ id }) => id),
);
}
}
await this.calendarEventCleanerService.cleanWorkspaceCalendarEvents(
workspaceId,
);
},
authContext,
{ lite: true },
);
}
}
@@ -41,51 +41,55 @@ export class BlocklistReimportCalendarEventsJob {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
'workspaceMember',
{ shouldBypassPermissionChecks: true },
);
for (const eventPayload of data.events) {
const workspaceMemberId =
eventPayload.properties.before.workspaceMemberId;
const workspaceMember = await workspaceMemberRepository.findOne({
where: { id: workspaceMemberId },
});
if (!isDefined(workspaceMember)) {
continue;
}
const userWorkspace = await this.userWorkspaceRepository.findOne({
where: { userId: workspaceMember.userId, workspaceId },
select: ['id'],
});
if (!isDefined(userWorkspace)) {
continue;
}
const calendarChannels = await this.calendarChannelRepository.find({
select: ['id'],
where: {
connectedAccount: { userWorkspaceId: userWorkspace.id },
syncStage: Not(
CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING,
),
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
},
});
'workspaceMember',
{ shouldBypassPermissionChecks: true },
);
await this.calendarChannelSyncStatusService.resetAndMarkAsCalendarEventListFetchPending(
calendarChannels.map((calendarChannel) => calendarChannel.id),
workspaceId,
);
}
}, authContext);
for (const eventPayload of data.events) {
const workspaceMemberId =
eventPayload.properties.before.workspaceMemberId;
const workspaceMember = await workspaceMemberRepository.findOne({
where: { id: workspaceMemberId },
});
if (!isDefined(workspaceMember)) {
continue;
}
const userWorkspace = await this.userWorkspaceRepository.findOne({
where: { userId: workspaceMember.userId, workspaceId },
select: ['id'],
});
if (!isDefined(userWorkspace)) {
continue;
}
const calendarChannels = await this.calendarChannelRepository.find({
select: ['id'],
where: {
connectedAccount: { userWorkspaceId: userWorkspace.id },
syncStage: Not(
CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING,
),
workspaceId,
},
});
await this.calendarChannelSyncStatusService.resetAndMarkAsCalendarEventListFetchPending(
calendarChannels.map((calendarChannel) => calendarChannel.id),
workspaceId,
);
}
},
authContext,
{ lite: true },
);
}
}
@@ -24,90 +24,98 @@ export class CalendarEventCleanerService {
}) {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const calendarChannelEventAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
'calendarChannelEventAssociation',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const calendarChannelEventAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
'calendarChannelEventAssociation',
);
const workspaceDataSource =
await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource();
const workspaceDataSource =
await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource();
await workspaceDataSource.transaction(async (manager) => {
const transactionManager = manager as WorkspaceEntityManager;
await workspaceDataSource.transaction(async (manager) => {
const transactionManager = manager as WorkspaceEntityManager;
await deleteUsingPagination(
workspaceId,
500,
async (
limit: number,
offset: number,
_workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
const associations =
await calendarChannelEventAssociationRepository.find(
{
where: { calendarChannelId },
take: limit,
skip: offset,
},
await deleteUsingPagination(
workspaceId,
500,
async (
limit: number,
offset: number,
_workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
const associations =
await calendarChannelEventAssociationRepository.find(
{
where: { calendarChannelId },
take: limit,
skip: offset,
},
transactionManager,
);
return associations.map(({ id }) => id);
},
async (
ids: string[],
workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
this.logger.log(
`WorkspaceId: ${workspaceId} Deleting ${ids.length} calendar channel event associations for channel ${calendarChannelId}`,
);
await calendarChannelEventAssociationRepository.delete(
ids,
transactionManager,
);
return associations.map(({ id }) => id);
},
async (
ids: string[],
workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
this.logger.log(
`WorkspaceId: ${workspaceId} Deleting ${ids.length} calendar channel event associations for channel ${calendarChannelId}`,
);
await calendarChannelEventAssociationRepository.delete(
ids,
transactionManager,
);
},
transactionManager,
);
});
}, authContext);
},
transactionManager,
);
});
},
authContext,
{ lite: true },
);
}
public async cleanWorkspaceCalendarEvents(workspaceId: string) {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const calendarEventRepository =
await this.globalWorkspaceOrmManager.getRepository(
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const calendarEventRepository =
await this.globalWorkspaceOrmManager.getRepository(
workspaceId,
'calendarEvent',
);
await deleteUsingPagination(
workspaceId,
'calendarEvent',
);
await deleteUsingPagination(
workspaceId,
500,
async (limit, offset) => {
const nonAssociatedCalendarEvents =
await calendarEventRepository.find({
where: {
calendarChannelEventAssociations: {
id: IsNull(),
500,
async (limit, offset) => {
const nonAssociatedCalendarEvents =
await calendarEventRepository.find({
where: {
calendarChannelEventAssociations: {
id: IsNull(),
},
},
},
take: limit,
skip: offset,
});
take: limit,
skip: offset,
});
return nonAssociatedCalendarEvents.map(({ id }) => id);
},
async (ids) => {
await calendarEventRepository.delete({ id: Any(ids) });
},
);
}, authContext);
return nonAssociatedCalendarEvents.map(({ id }) => id);
},
async (ids) => {
await calendarEventRepository.delete({ id: Any(ids) });
},
);
},
authContext,
{ lite: true },
);
}
}
@@ -53,55 +53,60 @@ export class CalendarTriggerEventListFetchCommand extends CommandRunner {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const calendarChannels = await this.calendarChannelRepository.find({
where: {
isSyncEnabled: true,
syncStage: CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING,
...(calendarChannelId ? { id: calendarChannelId } : {}),
workspaceId,
},
});
if (calendarChannels.length === 0) {
this.logger.warn(
'No calendar channels found with CALENDAR_EVENT_LIST_FETCH_PENDING status',
);
return;
}
this.logger.log(
`Found ${calendarChannels.length} calendar channel(s) to process`,
);
for (const calendarChannel of calendarChannels) {
await this.calendarChannelRepository.update(
{ id: calendarChannel.id, workspaceId },
{
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const calendarChannels = await this.calendarChannelRepository.find({
where: {
isSyncEnabled: true,
syncStage:
CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED,
syncStageStartedAt: new Date().toISOString(),
},
);
await this.messageQueueService.add<CalendarEventListFetchJobData>(
CalendarEventListFetchJob.name,
{
calendarChannelId: calendarChannel.id,
CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING,
...(calendarChannelId ? { id: calendarChannelId } : {}),
workspaceId,
},
);
});
if (calendarChannels.length === 0) {
this.logger.warn(
'No calendar channels found with CALENDAR_EVENT_LIST_FETCH_PENDING status',
);
return;
}
this.logger.log(
`Triggered fetch for calendar channel ${calendarChannel.id}`,
`Found ${calendarChannels.length} calendar channel(s) to process`,
);
}
this.logger.log(
`Successfully triggered ${calendarChannels.length} calendar event list fetch job(s)`,
);
}, authContext);
for (const calendarChannel of calendarChannels) {
await this.calendarChannelRepository.update(
{ id: calendarChannel.id, workspaceId },
{
syncStage:
CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED,
syncStageStartedAt: new Date().toISOString(),
},
);
await this.messageQueueService.add<CalendarEventListFetchJobData>(
CalendarEventListFetchJob.name,
{
calendarChannelId: calendarChannel.id,
workspaceId,
},
);
this.logger.log(
`Triggered fetch for calendar channel ${calendarChannel.id}`,
);
}
this.logger.log(
`Successfully triggered ${calendarChannels.length} calendar event list fetch job(s)`,
);
},
authContext,
{ lite: true },
);
}
@Option({
@@ -35,32 +35,36 @@ export class CalendarEventListFetchJob {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const calendarChannel = await this.calendarChannelRepository.findOne({
where: {
id: calendarChannelId,
isSyncEnabled: true,
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const calendarChannel = await this.calendarChannelRepository.findOne({
where: {
id: calendarChannelId,
isSyncEnabled: true,
workspaceId,
},
relations: ['connectedAccount'],
});
if (!calendarChannel) {
return;
}
if (
calendarChannel.syncStage !==
CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED
) {
return;
}
await this.calendarFetchEventsService.fetchCalendarEvents(
calendarChannel as unknown as CalendarChannelEntity,
calendarChannel.connectedAccount,
workspaceId,
},
relations: ['connectedAccount'],
});
if (!calendarChannel) {
return;
}
if (
calendarChannel.syncStage !==
CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED
) {
return;
}
await this.calendarFetchEventsService.fetchCalendarEvents(
calendarChannel as unknown as CalendarChannelEntity,
calendarChannel.connectedAccount,
workspaceId,
);
}, authContext);
);
},
authContext,
{ lite: true },
);
}
}
@@ -35,32 +35,36 @@ export class CalendarEventsImportJob {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const calendarChannel = await this.calendarChannelRepository.findOne({
where: {
id: calendarChannelId,
isSyncEnabled: true,
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const calendarChannel = await this.calendarChannelRepository.findOne({
where: {
id: calendarChannelId,
isSyncEnabled: true,
workspaceId,
},
relations: ['connectedAccount'],
});
if (!calendarChannel?.isSyncEnabled) {
return;
}
if (
calendarChannel.syncStage !==
CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_SCHEDULED
) {
return;
}
await this.calendarEventsImportService.processCalendarEventsImport(
calendarChannel as unknown as CalendarChannelEntity,
calendarChannel.connectedAccount,
workspaceId,
},
relations: ['connectedAccount'],
});
if (!calendarChannel?.isSyncEnabled) {
return;
}
if (
calendarChannel.syncStage !==
CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_SCHEDULED
) {
return;
}
await this.calendarEventsImportService.processCalendarEventsImport(
calendarChannel as unknown as CalendarChannelEntity,
calendarChannel.connectedAccount,
workspaceId,
);
}, authContext);
);
},
authContext,
{ lite: true },
);
}
}
@@ -36,54 +36,58 @@ export class CalendarOngoingStaleJob {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const calendarChannels = await this.calendarChannelRepository.find({
where: {
syncStage: In([
CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_ONGOING,
CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING,
CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_SCHEDULED,
CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED,
]),
workspaceId,
},
});
for (const calendarChannel of calendarChannels) {
const syncStageStartedAt = calendarChannel.syncStageStartedAt;
if (isSyncStale(syncStageStartedAt?.toISOString() ?? null)) {
await this.calendarChannelSyncStatusService.resetSyncStageStartedAt(
[calendarChannel.id],
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const calendarChannels = await this.calendarChannelRepository.find({
where: {
syncStage: In([
CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_ONGOING,
CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING,
CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_SCHEDULED,
CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED,
]),
workspaceId,
);
},
});
switch (calendarChannel.syncStage) {
case CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING:
case CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED:
this.logger.log(
`Sync for calendar channel ${calendarChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to CALENDAR_EVENT_LIST_FETCH_PENDING`,
);
await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchPending(
[calendarChannel.id],
workspaceId,
);
break;
case CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_ONGOING:
case CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_SCHEDULED:
this.logger.log(
`Sync for calendar channel ${calendarChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to CALENDAR_EVENTS_IMPORT_PENDING`,
);
await this.calendarChannelSyncStatusService.markAsCalendarEventsImportPending(
[calendarChannel.id],
workspaceId,
);
break;
default:
break;
for (const calendarChannel of calendarChannels) {
const syncStageStartedAt = calendarChannel.syncStageStartedAt;
if (isSyncStale(syncStageStartedAt?.toISOString() ?? null)) {
await this.calendarChannelSyncStatusService.resetSyncStageStartedAt(
[calendarChannel.id],
workspaceId,
);
switch (calendarChannel.syncStage) {
case CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING:
case CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_SCHEDULED:
this.logger.log(
`Sync for calendar channel ${calendarChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to CALENDAR_EVENT_LIST_FETCH_PENDING`,
);
await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchPending(
[calendarChannel.id],
workspaceId,
);
break;
case CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_ONGOING:
case CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_SCHEDULED:
this.logger.log(
`Sync for calendar channel ${calendarChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to CALENDAR_EVENTS_IMPORT_PENDING`,
);
await this.calendarChannelSyncStatusService.markAsCalendarEventsImportPending(
[calendarChannel.id],
workspaceId,
);
break;
default:
break;
}
}
}
}
}, authContext);
},
authContext,
{ lite: true },
);
}
}
@@ -36,31 +36,37 @@ export class CalendarRelaunchFailedCalendarChannelJob {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const calendarChannel = await this.calendarChannelRepository.findOne({
where: {
id: calendarChannelId,
workspaceId,
},
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const calendarChannel = await this.calendarChannelRepository.findOne({
where: {
id: calendarChannelId,
workspaceId,
},
});
if (
!calendarChannel ||
calendarChannel.syncStage !== CalendarChannelSyncStage.FAILED ||
calendarChannel.syncStatus !== CalendarChannelSyncStatus.FAILED_UNKNOWN
) {
return;
}
if (
!calendarChannel ||
calendarChannel.syncStage !== CalendarChannelSyncStage.FAILED ||
calendarChannel.syncStatus !==
CalendarChannelSyncStatus.FAILED_UNKNOWN
) {
return;
}
await this.calendarChannelRepository.update(
{ id: calendarChannelId, workspaceId },
{
syncStage: CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING,
syncStatus: CalendarChannelSyncStatus.ACTIVE,
throttleFailureCount: 0,
syncStageStartedAt: null,
},
);
}, authContext);
await this.calendarChannelRepository.update(
{ id: calendarChannelId, workspaceId },
{
syncStage:
CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING,
syncStatus: CalendarChannelSyncStatus.ACTIVE,
throttleFailureCount: 0,
syncStageStartedAt: null,
},
);
},
authContext,
{ lite: true },
);
}
}
@@ -31,7 +31,6 @@ import { filterEventsAndReturnCancelledEvents } from 'src/modules/calendar/calen
import { CalendarChannelSyncStatusService } from 'src/modules/calendar/common/services/calendar-channel-sync-status.service';
import { type CalendarChannelEventAssociationWorkspaceEntity } from 'src/modules/calendar/common/standard-objects/calendar-channel-event-association.workspace-entity';
import { type FetchedCalendarEvent } from 'src/modules/calendar/common/types/fetched-calendar-event';
import { EmailAliasManagerService } from 'src/modules/connected-account/email-alias-manager/services/email-alias-manager.service';
import { type WorkspaceMemberWorkspaceEntity } from 'src/modules/workspace-member/standard-objects/workspace-member.workspace-entity';
@Injectable()
@@ -47,7 +46,6 @@ export class CalendarEventsImportService {
private readonly calendarSaveEventsService: CalendarSaveEventsService,
private readonly calendarEventImportErrorHandlerService: CalendarEventImportErrorHandlerService,
private readonly microsoftCalendarImportEventService: MicrosoftCalendarImportEventsService,
private readonly emailAliasManagerService: EmailAliasManagerService,
@InjectRepository(UserWorkspaceEntity)
private readonly userWorkspaceRepository: Repository<UserWorkspaceEntity>,
) {}
@@ -65,142 +63,138 @@ export class CalendarEventsImportService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
let calendarEvents: FetchedCalendarEvent[] = [];
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
let calendarEvents: FetchedCalendarEvent[] = [];
try {
if (fetchedCalendarEvents) {
calendarEvents = fetchedCalendarEvents;
} else {
const eventIdsToFetch: string[] = await this.cacheStorage.setPop(
`calendar-events-to-import:${workspaceId}:${calendarChannel.id}`,
CALENDAR_EVENT_IMPORT_BATCH_SIZE,
);
try {
if (fetchedCalendarEvents) {
calendarEvents = fetchedCalendarEvents;
} else {
const eventIdsToFetch: string[] = await this.cacheStorage.setPop(
`calendar-events-to-import:${workspaceId}:${calendarChannel.id}`,
CALENDAR_EVENT_IMPORT_BATCH_SIZE,
);
if (!eventIdsToFetch || eventIdsToFetch.length === 0) {
await this.calendarChannelSyncStatusService.markAsCompletedAndMarkAsCalendarEventListFetchPending(
if (!eventIdsToFetch || eventIdsToFetch.length === 0) {
await this.calendarChannelSyncStatusService.markAsCompletedAndMarkAsCalendarEventListFetchPending(
[calendarChannel.id],
workspaceId,
);
return;
}
switch (connectedAccount.provider) {
case 'microsoft':
calendarEvents =
await this.microsoftCalendarImportEventService.getCalendarEvents(
connectedAccount,
eventIdsToFetch,
);
break;
default:
break;
}
}
if (!calendarEvents || calendarEvents?.length === 0) {
await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchPending(
[calendarChannel.id],
workspaceId,
);
return;
}
switch (connectedAccount.provider) {
case 'microsoft':
calendarEvents =
await this.microsoftCalendarImportEventService.getCalendarEvents(
connectedAccount,
eventIdsToFetch,
);
break;
default:
break;
}
}
const userWorkspace = await this.userWorkspaceRepository.findOne({
where: {
id: (connectedAccount as unknown as { userWorkspaceId: string })
.userWorkspaceId,
},
});
if (!calendarEvents || calendarEvents?.length === 0) {
await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchPending(
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkspaceMemberWorkspaceEntity>(
workspaceId,
'workspaceMember',
{ shouldBypassPermissionChecks: true },
);
const workspaceMember = userWorkspace
? await workspaceMemberRepository.findOne({
where: { userId: userWorkspace.userId },
})
: null;
const blocklist = workspaceMember
? await this.blocklistRepository.getByWorkspaceMemberId(
workspaceMember.id,
workspaceId,
)
: [];
if (
!isDefined(connectedAccount.handleAliases) ||
!isDefined(calendarChannel.handle)
) {
throw new CalendarEventImportDriverException(
'Calendar channel handle or Handle aliases are required',
CalendarEventImportDriverExceptionCode.CHANNEL_MISCONFIGURED,
);
}
const { filteredEvents, cancelledEvents } =
filterEventsAndReturnCancelledEvents(
[calendarChannel.handle, ...connectedAccount.handleAliases],
calendarEvents,
blocklist.map((blocklist) => blocklist.handle ?? ''),
);
const cancelledEventExternalIds = cancelledEvents.map(
(event) => event.id,
);
const BATCH_SIZE = 1000;
for (let i = 0; i < filteredEvents.length; i = i + BATCH_SIZE) {
const eventsBatch = filteredEvents.slice(i, i + BATCH_SIZE);
await this.calendarSaveEventsService.saveCalendarEventsAndEnqueueContactCreationJob(
eventsBatch,
calendarChannel,
connectedAccount,
workspaceId,
);
}
const calendarChannelEventAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelEventAssociationWorkspaceEntity>(
workspaceId,
'calendarChannelEventAssociation',
);
await calendarChannelEventAssociationRepository.delete({
eventExternalId: Any(cancelledEventExternalIds),
calendarChannelId: calendarChannel.id,
});
await this.calendarEventCleanerService.cleanWorkspaceCalendarEvents(
workspaceId,
);
await this.calendarChannelSyncStatusService.markAsCompletedAndMarkAsCalendarEventListFetchPending(
[calendarChannel.id],
workspaceId,
);
}
const userWorkspace = await this.userWorkspaceRepository.findOne({
where: {
id: (connectedAccount as unknown as { userWorkspaceId: string })
.userWorkspaceId,
},
});
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkspaceMemberWorkspaceEntity>(
workspaceId,
'workspaceMember',
{ shouldBypassPermissionChecks: true },
);
const workspaceMember = userWorkspace
? await workspaceMemberRepository.findOne({
where: { userId: userWorkspace.userId },
})
: null;
const blocklist = workspaceMember
? await this.blocklistRepository.getByWorkspaceMemberId(
workspaceMember.id,
workspaceId,
)
: [];
const refreshedHandleAliases =
await this.emailAliasManagerService.refreshHandleAliases(
connectedAccount,
workspaceId,
);
connectedAccount.handleAliases = refreshedHandleAliases;
if (
!isDefined(connectedAccount.handleAliases) ||
!isDefined(calendarChannel.handle)
) {
throw new CalendarEventImportDriverException(
'Calendar channel handle or Handle aliases are required',
CalendarEventImportDriverExceptionCode.CHANNEL_MISCONFIGURED,
);
}
const { filteredEvents, cancelledEvents } =
filterEventsAndReturnCancelledEvents(
[calendarChannel.handle, ...connectedAccount.handleAliases],
calendarEvents,
blocklist.map((blocklist) => blocklist.handle ?? ''),
);
const cancelledEventExternalIds = cancelledEvents.map(
(event) => event.id,
);
const BATCH_SIZE = 1000;
for (let i = 0; i < filteredEvents.length; i = i + BATCH_SIZE) {
const eventsBatch = filteredEvents.slice(i, i + BATCH_SIZE);
await this.calendarSaveEventsService.saveCalendarEventsAndEnqueueContactCreationJob(
eventsBatch,
} catch (error) {
await this.calendarEventImportErrorHandlerService.handleDriverException(
error,
CalendarEventImportSyncStep.CALENDAR_EVENTS_IMPORT,
calendarChannel,
connectedAccount,
workspaceId,
);
}
const calendarChannelEventAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelEventAssociationWorkspaceEntity>(
workspaceId,
'calendarChannelEventAssociation',
);
await calendarChannelEventAssociationRepository.delete({
eventExternalId: Any(cancelledEventExternalIds),
calendarChannelId: calendarChannel.id,
});
await this.calendarEventCleanerService.cleanWorkspaceCalendarEvents(
workspaceId,
);
await this.calendarChannelSyncStatusService.markAsCompletedAndMarkAsCalendarEventListFetchPending(
[calendarChannel.id],
workspaceId,
);
} catch (error) {
await this.calendarEventImportErrorHandlerService.handleDriverException(
error,
CalendarEventImportSyncStep.CALENDAR_EVENTS_IMPORT,
calendarChannel,
workspaceId,
);
}
}, authContext);
},
authContext,
{ lite: true },
);
}
}
@@ -55,38 +55,52 @@ export class CalendarFetchEventsService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
try {
const { accessToken, refreshToken } =
await this.calendarAccountAuthenticationService.validateAndRefreshConnectedAccountAuthentication(
{
connectedAccount,
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
try {
const { accessToken, refreshToken } =
await this.calendarAccountAuthenticationService.validateAndRefreshConnectedAccountAuthentication(
{
connectedAccount,
workspaceId,
calendarChannelId: calendarChannel.id,
},
);
const connectedAccountWithFreshTokens = {
...connectedAccount,
accessToken,
refreshToken,
};
const getCalendarEventsResponse =
await this.getCalendarEventsService.getCalendarEvents(
connectedAccountWithFreshTokens,
calendarChannel.syncCursor || undefined,
);
const hasFullEvents = getCalendarEventsResponse.fullEvents;
const calendarEvents = hasFullEvents
? getCalendarEventsResponse.calendarEvents
: null;
const calendarEventIds = getCalendarEventsResponse.calendarEventIds;
const nextSyncCursor = getCalendarEventsResponse.nextSyncCursor;
if (!calendarEvents || calendarEvents?.length === 0) {
await this.calendarChannelRepository.update(
{ id: calendarChannel.id, workspaceId },
{
syncCursor: nextSyncCursor,
},
);
await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchPending(
[calendarChannel.id],
workspaceId,
calendarChannelId: calendarChannel.id,
},
);
);
}
const connectedAccountWithFreshTokens = {
...connectedAccount,
accessToken,
refreshToken,
};
const getCalendarEventsResponse =
await this.getCalendarEventsService.getCalendarEvents(
connectedAccountWithFreshTokens,
calendarChannel.syncCursor || undefined,
);
const hasFullEvents = getCalendarEventsResponse.fullEvents;
const calendarEvents = hasFullEvents
? getCalendarEventsResponse.calendarEvents
: null;
const calendarEventIds = getCalendarEventsResponse.calendarEventIds;
const nextSyncCursor = getCalendarEventsResponse.nextSyncCursor;
if (!calendarEvents || calendarEvents?.length === 0) {
await this.calendarChannelRepository.update(
{ id: calendarChannel.id, workspaceId },
{
@@ -94,53 +108,43 @@ export class CalendarFetchEventsService {
},
);
await this.calendarChannelSyncStatusService.markAsCalendarEventListFetchPending(
[calendarChannel.id],
workspaceId,
if (hasFullEvents && calendarEvents) {
await this.calendarEventsImportService.processCalendarEventsImport(
calendarChannel,
connectedAccount,
workspaceId,
calendarEvents,
);
} else if (!hasFullEvents && calendarEventIds) {
await this.cacheStorage.setAdd(
`calendar-events-to-import:${workspaceId}:${calendarChannel.id}`,
calendarEventIds,
);
await this.calendarChannelSyncStatusService.markAsCalendarEventsImportPending(
[calendarChannel.id],
workspaceId,
);
} else {
throw new CalendarEventImportDriverException(
"Expected 'calendarEvents' or 'calendarEventIds' to be present",
CalendarEventImportDriverExceptionCode.UNKNOWN,
);
}
} catch (error) {
this.logger.error(
`WorkspaceId: ${workspaceId}, CalendarChannelId: ${calendarChannel.id} - Calendar event fetch error: ${error.message}`,
);
}
await this.calendarChannelRepository.update(
{ id: calendarChannel.id, workspaceId },
{
syncCursor: nextSyncCursor,
},
);
if (hasFullEvents && calendarEvents) {
await this.calendarEventsImportService.processCalendarEventsImport(
await this.calendarEventImportErrorHandlerService.handleDriverException(
error,
CalendarEventImportSyncStep.CALENDAR_EVENT_LIST_FETCH,
calendarChannel,
connectedAccount,
workspaceId,
calendarEvents,
);
} else if (!hasFullEvents && calendarEventIds) {
await this.cacheStorage.setAdd(
`calendar-events-to-import:${workspaceId}:${calendarChannel.id}`,
calendarEventIds,
);
await this.calendarChannelSyncStatusService.markAsCalendarEventsImportPending(
[calendarChannel.id],
workspaceId,
);
} else {
throw new CalendarEventImportDriverException(
"Expected 'calendarEvents' or 'calendarEventIds' to be present",
CalendarEventImportDriverExceptionCode.UNKNOWN,
);
}
} catch (error) {
this.logger.error(
`WorkspaceId: ${workspaceId}, CalendarChannelId: ${calendarChannel.id} - Calendar event fetch error: ${error.message}`,
);
await this.calendarEventImportErrorHandlerService.handleDriverException(
error,
CalendarEventImportSyncStep.CALENDAR_EVENT_LIST_FETCH,
calendarChannel,
workspaceId,
);
}
}, authContext);
},
authContext,
{ lite: true },
);
}
}
@@ -34,284 +34,290 @@ export class CalendarSaveEventsService {
): Promise<void> {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const calendarEventRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarEventWorkspaceEntity>(
workspaceId,
'calendarEvent',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const calendarEventRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarEventWorkspaceEntity>(
workspaceId,
'calendarEvent',
);
const calendarChannelEventAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelEventAssociationWorkspaceEntity>(
workspaceId,
'calendarChannelEventAssociation',
);
const calendarChannelEventAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarChannelEventAssociationWorkspaceEntity>(
workspaceId,
'calendarChannelEventAssociation',
);
const workspaceDataSource =
await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource();
const workspaceDataSource =
await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource();
await workspaceDataSource.transaction(
async (transactionManager: WorkspaceEntityManager) => {
const existingAssociations =
await calendarChannelEventAssociationRepository.find(
{
where: {
eventExternalId: Any(
fetchedCalendarEvents.map((event) => event.id),
),
calendarChannelId: calendarChannel.id,
await workspaceDataSource.transaction(
async (transactionManager: WorkspaceEntityManager) => {
const existingAssociations =
await calendarChannelEventAssociationRepository.find(
{
where: {
eventExternalId: Any(
fetchedCalendarEvents.map((event) => event.id),
),
calendarChannelId: calendarChannel.id,
},
},
},
transactionManager,
);
const existingCalendarEventIdByExternalId = new Map(
existingAssociations.map((association) => [
association.eventExternalId,
association.calendarEventId,
]),
);
const existingAssociationIdByExternalId = new Map(
existingAssociations.map((association) => [
association.eventExternalId,
association.id,
]),
);
const fetchedCalendarEventsWithDBEvents: FetchedCalendarEventWithDBEvent[] =
fetchedCalendarEvents.map(
(event): FetchedCalendarEventWithDBEvent => {
const existingCalendarEventId =
existingCalendarEventIdByExternalId.get(event.id);
return {
fetchedCalendarEvent: event,
existingCalendarEvent: existingCalendarEventId
? ({
id: existingCalendarEventId,
} as CalendarEventWorkspaceEntity)
: null,
newlyCreatedCalendarEvent: null,
};
},
);
const newCalendarEventIdByExternalId = new Map<string, string>();
const newCalendarEventsToInsert = fetchedCalendarEventsWithDBEvents
.filter(
({ existingCalendarEvent }) => existingCalendarEvent === null,
)
.map(({ fetchedCalendarEvent }) => {
const calendarEventId = uuid();
newCalendarEventIdByExternalId.set(
fetchedCalendarEvent.id,
calendarEventId,
transactionManager,
);
return {
id: calendarEventId,
iCalUid: fetchedCalendarEvent.iCalUid,
title: fetchedCalendarEvent.title,
description: fetchedCalendarEvent.description,
startsAt: fetchedCalendarEvent.startsAt,
endsAt: fetchedCalendarEvent.endsAt,
location: fetchedCalendarEvent.location,
isFullDay: fetchedCalendarEvent.isFullDay,
isCanceled: fetchedCalendarEvent.isCanceled,
conferenceSolution: fetchedCalendarEvent.conferenceSolution,
conferenceLink: {
primaryLinkLabel: fetchedCalendarEvent.conferenceLinkLabel,
primaryLinkUrl: fetchedCalendarEvent.conferenceLinkUrl,
secondaryLinks: [],
},
externalCreatedAt: fetchedCalendarEvent.externalCreatedAt,
externalUpdatedAt: fetchedCalendarEvent.externalUpdatedAt,
};
});
if (newCalendarEventsToInsert.length > 0) {
await calendarEventRepository.insert(
newCalendarEventsToInsert,
transactionManager,
const existingCalendarEventIdByExternalId = new Map(
existingAssociations.map((association) => [
association.eventExternalId,
association.calendarEventId,
]),
);
}
const fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents: FetchedCalendarEventWithDBEvent[] =
fetchedCalendarEventsWithDBEvents.map(
({ fetchedCalendarEvent, existingCalendarEvent }) => {
const savedCalendarEventId = newCalendarEventIdByExternalId.get(
const existingAssociationIdByExternalId = new Map(
existingAssociations.map((association) => [
association.eventExternalId,
association.id,
]),
);
const fetchedCalendarEventsWithDBEvents: FetchedCalendarEventWithDBEvent[] =
fetchedCalendarEvents.map(
(event): FetchedCalendarEventWithDBEvent => {
const existingCalendarEventId =
existingCalendarEventIdByExternalId.get(event.id);
return {
fetchedCalendarEvent: event,
existingCalendarEvent: existingCalendarEventId
? ({
id: existingCalendarEventId,
} as CalendarEventWorkspaceEntity)
: null,
newlyCreatedCalendarEvent: null,
};
},
);
const newCalendarEventIdByExternalId = new Map<string, string>();
const newCalendarEventsToInsert = fetchedCalendarEventsWithDBEvents
.filter(
({ existingCalendarEvent }) => existingCalendarEvent === null,
)
.map(({ fetchedCalendarEvent }) => {
const calendarEventId = uuid();
newCalendarEventIdByExternalId.set(
fetchedCalendarEvent.id,
calendarEventId,
);
return {
fetchedCalendarEvent,
existingCalendarEvent,
newlyCreatedCalendarEvent: savedCalendarEventId
? ({
id: savedCalendarEventId,
} as CalendarEventWorkspaceEntity)
: null,
};
},
);
const existingEventsToUpdate =
fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents
.filter(
({ existingCalendarEvent }) => existingCalendarEvent !== null,
)
.map(({ fetchedCalendarEvent, existingCalendarEvent }) => {
if (!existingCalendarEvent) {
throw new Error(
`Existing calendar event with iCalUid ${fetchedCalendarEvent.iCalUid} not found - should never happen`,
);
}
return {
criteria: existingCalendarEvent.id,
partialEntity: {
iCalUid: fetchedCalendarEvent.iCalUid,
title: fetchedCalendarEvent.title,
description: fetchedCalendarEvent.description,
startsAt: fetchedCalendarEvent.startsAt,
endsAt: fetchedCalendarEvent.endsAt,
location: fetchedCalendarEvent.location,
isFullDay: fetchedCalendarEvent.isFullDay,
isCanceled: fetchedCalendarEvent.isCanceled,
conferenceSolution: fetchedCalendarEvent.conferenceSolution,
conferenceLink: {
primaryLinkLabel:
fetchedCalendarEvent.conferenceLinkLabel,
primaryLinkUrl: fetchedCalendarEvent.conferenceLinkUrl,
secondaryLinks: [],
},
externalCreatedAt: fetchedCalendarEvent.externalCreatedAt,
externalUpdatedAt: fetchedCalendarEvent.externalUpdatedAt,
id: calendarEventId,
iCalUid: fetchedCalendarEvent.iCalUid,
title: fetchedCalendarEvent.title,
description: fetchedCalendarEvent.description,
startsAt: fetchedCalendarEvent.startsAt,
endsAt: fetchedCalendarEvent.endsAt,
location: fetchedCalendarEvent.location,
isFullDay: fetchedCalendarEvent.isFullDay,
isCanceled: fetchedCalendarEvent.isCanceled,
conferenceSolution: fetchedCalendarEvent.conferenceSolution,
conferenceLink: {
primaryLinkLabel: fetchedCalendarEvent.conferenceLinkLabel,
primaryLinkUrl: fetchedCalendarEvent.conferenceLinkUrl,
secondaryLinks: [],
},
externalCreatedAt: fetchedCalendarEvent.externalCreatedAt,
externalUpdatedAt: fetchedCalendarEvent.externalUpdatedAt,
};
});
if (existingEventsToUpdate.length > 0) {
await calendarEventRepository.updateMany(
existingEventsToUpdate,
transactionManager,
);
}
if (newCalendarEventsToInsert.length > 0) {
await calendarEventRepository.insert(
newCalendarEventsToInsert,
transactionManager,
);
}
const calendarChannelEventAssociationsToSave: Pick<
CalendarChannelEventAssociationWorkspaceEntity,
| 'calendarEventId'
| 'eventExternalId'
| 'calendarChannelId'
| 'recurringEventExternalId'
>[] = fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents
.filter(
({ newlyCreatedCalendarEvent }) =>
newlyCreatedCalendarEvent !== null,
)
.map(({ fetchedCalendarEvent, newlyCreatedCalendarEvent }) => {
if (!newlyCreatedCalendarEvent?.id) {
throw new Error(
`Calendar event id not found for event with iCalUid ${fetchedCalendarEvent.iCalUid} - should never happen`,
);
}
const fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents: FetchedCalendarEventWithDBEvent[] =
fetchedCalendarEventsWithDBEvents.map(
({ fetchedCalendarEvent, existingCalendarEvent }) => {
const savedCalendarEventId =
newCalendarEventIdByExternalId.get(fetchedCalendarEvent.id);
return {
calendarEventId: newlyCreatedCalendarEvent.id,
eventExternalId: fetchedCalendarEvent.id,
calendarChannelId: calendarChannel.id,
recurringEventExternalId:
fetchedCalendarEvent.recurringEventExternalId ?? '',
};
});
if (calendarChannelEventAssociationsToSave.length > 0) {
await calendarChannelEventAssociationRepository.insert(
calendarChannelEventAssociationsToSave,
transactionManager,
);
}
const existingAssociationsToUpdate =
fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents
.filter(
({ existingCalendarEvent }) => existingCalendarEvent !== null,
)
.map(({ fetchedCalendarEvent }) => ({
criteria: existingAssociationIdByExternalId.get(
fetchedCalendarEvent.id,
)!,
partialEntity: {
recurringEventExternalId:
fetchedCalendarEvent.recurringEventExternalId ?? '',
return {
fetchedCalendarEvent,
existingCalendarEvent,
newlyCreatedCalendarEvent: savedCalendarEventId
? ({
id: savedCalendarEventId,
} as CalendarEventWorkspaceEntity)
: null,
};
},
}));
);
if (existingAssociationsToUpdate.length > 0) {
await calendarChannelEventAssociationRepository.updateMany(
existingAssociationsToUpdate,
transactionManager,
);
}
const existingEventsToUpdate =
fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents
.filter(
({ existingCalendarEvent }) => existingCalendarEvent !== null,
)
.map(({ fetchedCalendarEvent, existingCalendarEvent }) => {
if (!existingCalendarEvent) {
throw new Error(
`Existing calendar event with iCalUid ${fetchedCalendarEvent.iCalUid} not found - should never happen`,
);
}
const participantsToCreate =
fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents
return {
criteria: existingCalendarEvent.id,
partialEntity: {
iCalUid: fetchedCalendarEvent.iCalUid,
title: fetchedCalendarEvent.title,
description: fetchedCalendarEvent.description,
startsAt: fetchedCalendarEvent.startsAt,
endsAt: fetchedCalendarEvent.endsAt,
location: fetchedCalendarEvent.location,
isFullDay: fetchedCalendarEvent.isFullDay,
isCanceled: fetchedCalendarEvent.isCanceled,
conferenceSolution:
fetchedCalendarEvent.conferenceSolution,
conferenceLink: {
primaryLinkLabel:
fetchedCalendarEvent.conferenceLinkLabel,
primaryLinkUrl: fetchedCalendarEvent.conferenceLinkUrl,
secondaryLinks: [],
},
externalCreatedAt: fetchedCalendarEvent.externalCreatedAt,
externalUpdatedAt: fetchedCalendarEvent.externalUpdatedAt,
},
};
});
if (existingEventsToUpdate.length > 0) {
await calendarEventRepository.updateMany(
existingEventsToUpdate,
transactionManager,
);
}
const calendarChannelEventAssociationsToSave: Pick<
CalendarChannelEventAssociationWorkspaceEntity,
| 'calendarEventId'
| 'eventExternalId'
| 'calendarChannelId'
| 'recurringEventExternalId'
>[] = fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents
.filter(
({ newlyCreatedCalendarEvent }) =>
newlyCreatedCalendarEvent !== null,
)
.flatMap(
({ newlyCreatedCalendarEvent, fetchedCalendarEvent }) => {
if (!newlyCreatedCalendarEvent?.id) {
.map(({ fetchedCalendarEvent, newlyCreatedCalendarEvent }) => {
if (!newlyCreatedCalendarEvent?.id) {
throw new Error(
`Calendar event id not found for event with iCalUid ${fetchedCalendarEvent.iCalUid} - should never happen`,
);
}
return {
calendarEventId: newlyCreatedCalendarEvent.id,
eventExternalId: fetchedCalendarEvent.id,
calendarChannelId: calendarChannel.id,
recurringEventExternalId:
fetchedCalendarEvent.recurringEventExternalId ?? '',
};
});
if (calendarChannelEventAssociationsToSave.length > 0) {
await calendarChannelEventAssociationRepository.insert(
calendarChannelEventAssociationsToSave,
transactionManager,
);
}
const existingAssociationsToUpdate =
fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents
.filter(
({ existingCalendarEvent }) => existingCalendarEvent !== null,
)
.map(({ fetchedCalendarEvent }) => ({
criteria: existingAssociationIdByExternalId.get(
fetchedCalendarEvent.id,
)!,
partialEntity: {
recurringEventExternalId:
fetchedCalendarEvent.recurringEventExternalId ?? '',
},
}));
if (existingAssociationsToUpdate.length > 0) {
await calendarChannelEventAssociationRepository.updateMany(
existingAssociationsToUpdate,
transactionManager,
);
}
const participantsToCreate =
fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents
.filter(
({ newlyCreatedCalendarEvent }) =>
newlyCreatedCalendarEvent !== null,
)
.flatMap(
({ newlyCreatedCalendarEvent, fetchedCalendarEvent }) => {
if (!newlyCreatedCalendarEvent?.id) {
throw new Error(
`Newly created calendar event with iCalUid ${fetchedCalendarEvent.iCalUid} not found - should never happen`,
);
}
return fetchedCalendarEvent.participants.map(
(participant) => ({
...participant,
calendarEventId: newlyCreatedCalendarEvent.id,
}),
);
},
);
// todo: we should prevent duplicate rows on calendarEventAssociation by creating
// an index on calendarChannelId and calendarEventId
const participantsToUpdate =
fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents
.filter(
({ existingCalendarEvent }) => existingCalendarEvent !== null,
)
.flatMap(({ fetchedCalendarEvent, existingCalendarEvent }) => {
if (!existingCalendarEvent?.id) {
throw new Error(
`Newly created calendar event with iCalUid ${fetchedCalendarEvent.iCalUid} not found - should never happen`,
`Existing calendar event with iCalUid ${fetchedCalendarEvent.iCalUid} not found - should never happen`,
);
}
return fetchedCalendarEvent.participants.map(
(participant) => ({
...participant,
calendarEventId: newlyCreatedCalendarEvent.id,
calendarEventId: existingCalendarEvent.id,
}),
);
},
);
});
// todo: we should prevent duplicate rows on calendarEventAssociation by creating
// an index on calendarChannelId and calendarEventId
const participantsToUpdate =
fetchedCalendarEventsWithDBEventsEnrichedWithSavedEvents
.filter(
({ existingCalendarEvent }) => existingCalendarEvent !== null,
)
.flatMap(({ fetchedCalendarEvent, existingCalendarEvent }) => {
if (!existingCalendarEvent?.id) {
throw new Error(
`Existing calendar event with iCalUid ${fetchedCalendarEvent.iCalUid} not found - should never happen`,
);
}
return fetchedCalendarEvent.participants.map((participant) => ({
...participant,
calendarEventId: existingCalendarEvent.id,
}));
});
await this.calendarEventParticipantService.upsertAndDeleteCalendarEventParticipants(
{
participantsToCreate,
participantsToUpdate,
transactionManager,
calendarChannel,
connectedAccount,
workspaceId,
},
);
},
);
}, authContext);
await this.calendarEventParticipantService.upsertAndDeleteCalendarEventParticipants(
{
participantsToCreate,
participantsToUpdate,
transactionManager,
calendarChannel,
connectedAccount,
workspaceId,
},
);
},
);
},
authContext,
{ lite: true },
);
}
}
@@ -58,127 +58,132 @@ export class CalendarEventParticipantService {
}): Promise<void> {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const chunkedParticipantsToUpdate = chunk(participantsToUpdate, 200);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const chunkedParticipantsToUpdate = chunk(participantsToUpdate, 200);
const calendarEventParticipantRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarEventParticipantWorkspaceEntity>(
workspaceId,
'calendarEventParticipant',
);
const calendarEventParticipantRepository =
await this.globalWorkspaceOrmManager.getRepository<CalendarEventParticipantWorkspaceEntity>(
workspaceId,
'calendarEventParticipant',
);
for (const participantsToUpdateChunk of chunkedParticipantsToUpdate) {
const existingCalendarEventParticipants =
await calendarEventParticipantRepository.find({
where: {
calendarEventId: Any(
participantsToUpdateChunk
.map((participant) => participant.calendarEventId)
.filter(isDefined),
for (const participantsToUpdateChunk of chunkedParticipantsToUpdate) {
const existingCalendarEventParticipants =
await calendarEventParticipantRepository.find({
where: {
calendarEventId: Any(
participantsToUpdateChunk
.map((participant) => participant.calendarEventId)
.filter(isDefined),
),
},
});
const {
calendarEventParticipantsToUpdate,
newCalendarEventParticipants,
} = participantsToUpdateChunk.reduce<{
calendarEventParticipantsToUpdate: FetchedCalendarEventParticipantWithCalendarEventIdAndExistingId[];
newCalendarEventParticipants: FetchedCalendarEventParticipantWithCalendarEventId[];
}>(
(acc, calendarEventParticipant) => {
const existingCalendarEventParticipant =
existingCalendarEventParticipants.find(
(existingCalendarEventParticipant) =>
existingCalendarEventParticipant.handle ===
calendarEventParticipant.handle &&
existingCalendarEventParticipant.calendarEventId ===
calendarEventParticipant.calendarEventId,
);
if (existingCalendarEventParticipant) {
acc.calendarEventParticipantsToUpdate.push({
...calendarEventParticipant,
id: existingCalendarEventParticipant.id,
});
} else {
acc.newCalendarEventParticipants.push(calendarEventParticipant);
}
return acc;
},
{
calendarEventParticipantsToUpdate: [],
newCalendarEventParticipants: [],
},
);
const calendarEventParticipantsToDelete = differenceWith(
existingCalendarEventParticipants,
participantsToUpdateChunk,
(existingCalendarEventParticipant, participantToUpdate) =>
existingCalendarEventParticipant.handle ===
participantToUpdate.handle &&
existingCalendarEventParticipant.calendarEventId ===
participantToUpdate.calendarEventId,
);
await calendarEventParticipantRepository.delete(
{
id: Any(
calendarEventParticipantsToDelete.map(
(calendarEventParticipant) => calendarEventParticipant.id,
),
),
},
});
const {
calendarEventParticipantsToUpdate,
newCalendarEventParticipants,
} = participantsToUpdateChunk.reduce<{
calendarEventParticipantsToUpdate: FetchedCalendarEventParticipantWithCalendarEventIdAndExistingId[];
newCalendarEventParticipants: FetchedCalendarEventParticipantWithCalendarEventId[];
}>(
(acc, calendarEventParticipant) => {
const existingCalendarEventParticipant =
existingCalendarEventParticipants.find(
(existingCalendarEventParticipant) =>
existingCalendarEventParticipant.handle ===
calendarEventParticipant.handle &&
existingCalendarEventParticipant.calendarEventId ===
calendarEventParticipant.calendarEventId,
);
if (existingCalendarEventParticipant) {
acc.calendarEventParticipantsToUpdate.push({
...calendarEventParticipant,
id: existingCalendarEventParticipant.id,
});
} else {
acc.newCalendarEventParticipants.push(calendarEventParticipant);
}
return acc;
},
{
calendarEventParticipantsToUpdate: [],
newCalendarEventParticipants: [],
},
);
const calendarEventParticipantsToDelete = differenceWith(
existingCalendarEventParticipants,
participantsToUpdateChunk,
(existingCalendarEventParticipant, participantToUpdate) =>
existingCalendarEventParticipant.handle ===
participantToUpdate.handle &&
existingCalendarEventParticipant.calendarEventId ===
participantToUpdate.calendarEventId,
);
await calendarEventParticipantRepository.delete(
{
id: Any(
calendarEventParticipantsToDelete.map(
(calendarEventParticipant) => calendarEventParticipant.id,
),
),
},
transactionManager,
);
await calendarEventParticipantRepository.updateMany(
calendarEventParticipantsToUpdate.map((participant) => ({
criteria: participant.id,
partialEntity: participant,
})),
transactionManager,
);
participantsToCreate.push(...newCalendarEventParticipants);
}
const chunkedParticipantsToCreate = chunk(participantsToCreate, 200);
const savedParticipants: CalendarEventParticipantWorkspaceEntity[] = [];
for (const participantsToCreateChunk of chunkedParticipantsToCreate) {
const savedParticipantsChunk =
await calendarEventParticipantRepository.insert(
participantsToCreateChunk,
transactionManager,
);
savedParticipants.push(...savedParticipantsChunk.raw);
}
if (calendarChannel.isContactAutoCreationEnabled) {
await this.messageQueueService.add<CreateCompanyAndContactJobData>(
CreateCompanyAndContactJob.name,
{
workspaceId,
connectedAccount,
contactsToCreate: savedParticipants.map((participant) => ({
handle: participant.handle ?? '',
displayName: participant.displayName ?? participant.handle ?? '',
await calendarEventParticipantRepository.updateMany(
calendarEventParticipantsToUpdate.map((participant) => ({
criteria: participant.id,
partialEntity: participant,
})),
source: FieldActorSource.CALENDAR,
},
);
}
transactionManager,
);
participantsToCreate.push(...newCalendarEventParticipants);
}
await this.matchParticipantService.matchParticipants({
participants: savedParticipants,
objectMetadataName: 'calendarEventParticipant',
transactionManager,
matchWith: 'workspaceMemberAndPerson',
workspaceId,
});
}, authContext);
const chunkedParticipantsToCreate = chunk(participantsToCreate, 200);
const savedParticipants: CalendarEventParticipantWorkspaceEntity[] = [];
for (const participantsToCreateChunk of chunkedParticipantsToCreate) {
const savedParticipantsChunk =
await calendarEventParticipantRepository.insert(
participantsToCreateChunk,
transactionManager,
);
savedParticipants.push(...savedParticipantsChunk.raw);
}
if (calendarChannel.isContactAutoCreationEnabled) {
await this.messageQueueService.add<CreateCompanyAndContactJobData>(
CreateCompanyAndContactJob.name,
{
workspaceId,
connectedAccount,
contactsToCreate: savedParticipants.map((participant) => ({
handle: participant.handle ?? '',
displayName:
participant.displayName ?? participant.handle ?? '',
})),
source: FieldActorSource.CALENDAR,
},
);
}
await this.matchParticipantService.matchParticipants({
participants: savedParticipants,
objectMetadataName: 'calendarEventParticipant',
transactionManager,
matchWith: 'workspaceMemberAndPerson',
workspaceId,
});
},
authContext,
{ lite: true },
);
}
}
@@ -138,6 +138,7 @@ export class ApplyCalendarEventsVisibilityRestrictionsService {
return calendarEvents;
},
authContext,
{ lite: true },
);
}
}
@@ -46,15 +46,22 @@ export class CalendarChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.calendarChannelRepository.update(
{ id: In(calendarChannelIds), workspaceId },
{
syncStage: CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING,
...(!preserveSyncStageStartedAt ? { syncStageStartedAt: null } : {}),
},
);
}, authContext);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.calendarChannelRepository.update(
{ id: In(calendarChannelIds), workspaceId },
{
syncStage:
CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING,
...(!preserveSyncStageStartedAt
? { syncStageStartedAt: null }
: {}),
},
);
},
authContext,
{ lite: true },
);
}
public async markAsCalendarEventListFetchOngoing(
@@ -67,16 +74,21 @@ export class CalendarChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.calendarChannelRepository.update(
{ id: In(calendarChannelIds), workspaceId },
{
syncStage: CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING,
syncStatus: CalendarChannelSyncStatus.ONGOING,
syncStageStartedAt: new Date().toISOString(),
},
);
}, authContext);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.calendarChannelRepository.update(
{ id: In(calendarChannelIds), workspaceId },
{
syncStage:
CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_ONGOING,
syncStatus: CalendarChannelSyncStatus.ONGOING,
syncStageStartedAt: new Date().toISOString(),
},
);
},
authContext,
{ lite: true },
);
}
public async resetAndMarkAsCalendarEventListFetchPending(
@@ -95,16 +107,20 @@ export class CalendarChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.calendarChannelRepository.update(
{ id: In(calendarChannelIds), workspaceId },
{
syncCursor: '',
syncStageStartedAt: null,
throttleFailureCount: 0,
},
);
}, authContext);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.calendarChannelRepository.update(
{ id: In(calendarChannelIds), workspaceId },
{
syncCursor: '',
syncStageStartedAt: null,
throttleFailureCount: 0,
},
);
},
authContext,
{ lite: true },
);
await this.markAsCalendarEventListFetchPending(
calendarChannelIds,
@@ -122,14 +138,18 @@ export class CalendarChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.calendarChannelRepository.update(
{ id: In(calendarChannelIds), workspaceId },
{
syncStageStartedAt: null,
},
);
}, authContext);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.calendarChannelRepository.update(
{ id: In(calendarChannelIds), workspaceId },
{
syncStageStartedAt: null,
},
);
},
authContext,
{ lite: true },
);
}
public async markAsCalendarEventsImportPending(
@@ -143,15 +163,21 @@ export class CalendarChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.calendarChannelRepository.update(
{ id: In(calendarChannelIds), workspaceId },
{
syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_PENDING,
...(!preserveSyncStageStartedAt ? { syncStageStartedAt: null } : {}),
},
);
}, authContext);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.calendarChannelRepository.update(
{ id: In(calendarChannelIds), workspaceId },
{
syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_PENDING,
...(!preserveSyncStageStartedAt
? { syncStageStartedAt: null }
: {}),
},
);
},
authContext,
{ lite: true },
);
}
public async markAsCalendarEventsImportOngoing(
@@ -164,16 +190,20 @@ export class CalendarChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.calendarChannelRepository.update(
{ id: In(calendarChannelIds), workspaceId },
{
syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_ONGOING,
syncStatus: CalendarChannelSyncStatus.ONGOING,
syncStageStartedAt: new Date().toISOString(),
},
);
}, authContext);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.calendarChannelRepository.update(
{ id: In(calendarChannelIds), workspaceId },
{
syncStage: CalendarChannelSyncStage.CALENDAR_EVENTS_IMPORT_ONGOING,
syncStatus: CalendarChannelSyncStatus.ONGOING,
syncStageStartedAt: new Date().toISOString(),
},
);
},
authContext,
{ lite: true },
);
}
public async markAsCompletedAndMarkAsCalendarEventListFetchPending(
@@ -186,18 +216,23 @@ export class CalendarChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.calendarChannelRepository.update(
{ id: In(calendarChannelIds), workspaceId },
{
syncStage: CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING,
syncStatus: CalendarChannelSyncStatus.ACTIVE,
throttleFailureCount: 0,
syncStageStartedAt: null,
syncedAt: new Date().toISOString(),
},
);
}, authContext);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.calendarChannelRepository.update(
{ id: In(calendarChannelIds), workspaceId },
{
syncStage:
CalendarChannelSyncStage.CALENDAR_EVENT_LIST_FETCH_PENDING,
syncStatus: CalendarChannelSyncStatus.ACTIVE,
throttleFailureCount: 0,
syncStageStartedAt: null,
syncedAt: new Date().toISOString(),
},
);
},
authContext,
{ lite: true },
);
await this.markAsCalendarEventListFetchPending(
calendarChannelIds,
@@ -226,15 +261,19 @@ export class CalendarChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.calendarChannelRepository.update(
{ id: In(calendarChannelIds), workspaceId },
{
syncStatus: CalendarChannelSyncStatus.FAILED_UNKNOWN,
syncStage: CalendarChannelSyncStage.FAILED,
},
);
}, authContext);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.calendarChannelRepository.update(
{ id: In(calendarChannelIds), workspaceId },
{
syncStatus: CalendarChannelSyncStatus.FAILED_UNKNOWN,
syncStage: CalendarChannelSyncStage.FAILED,
},
);
},
authContext,
{ lite: true },
);
await this.metricsService.batchIncrementCounter({
key: MetricsKeys.CalendarEventSyncJobFailedUnknown,
@@ -258,36 +297,41 @@ export class CalendarChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.calendarChannelRepository.update(
{ id: In(calendarChannelIds), workspaceId },
{
syncStatus: CalendarChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS,
syncStage: CalendarChannelSyncStage.FAILED,
},
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.calendarChannelRepository.update(
{ id: In(calendarChannelIds), workspaceId },
{
syncStatus:
CalendarChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS,
syncStage: CalendarChannelSyncStage.FAILED,
},
);
const calendarChannels = await this.calendarChannelRepository.find({
select: ['id', 'connectedAccountId'],
where: { id: Any(calendarChannelIds), workspaceId },
});
const calendarChannels = await this.calendarChannelRepository.find({
select: ['id', 'connectedAccountId'],
where: { id: Any(calendarChannelIds), workspaceId },
});
const connectedAccountIds = calendarChannels.map(
(calendarChannel) => calendarChannel.connectedAccountId,
);
const connectedAccountIds = calendarChannels.map(
(calendarChannel) => calendarChannel.connectedAccountId,
);
await this.connectedAccountRepository.update(
{ id: Any(connectedAccountIds), workspaceId },
{
authFailedAt: new Date(),
},
);
await this.connectedAccountRepository.update(
{ id: Any(connectedAccountIds), workspaceId },
{
authFailedAt: new Date(),
},
);
await this.addToAccountsToReconnect(
calendarChannels.map((calendarChannel) => calendarChannel.id),
workspaceId,
);
}, authContext);
await this.addToAccountsToReconnect(
calendarChannels.map((calendarChannel) => calendarChannel.id),
workspaceId,
);
},
authContext,
{ lite: true },
);
await this.metricsService.batchIncrementCounter({
key: MetricsKeys.CalendarEventSyncJobFailedInsufficientPermissions,
@@ -19,31 +19,22 @@ export class GoogleEmailAliasManagerService {
connectedAccount,
);
const peopleClient = google.people({
const gmailClient = google.gmail({
version: 'v1',
auth: oAuth2Client,
});
const emailsResponse = await peopleClient.people
.get({
resourceName: 'people/me',
personFields: 'emailAddresses',
})
const sendAsResponse = await gmailClient.users.settings.sendAs
.list({ userId: 'me' })
.catch((error) => {
throw this.gmailEmailAliasErrorHandlerService.handleError(error);
});
const emailAddresses = emailsResponse.data.emailAddresses;
const handleAliases =
emailAddresses
?.filter((emailAddress) => {
return emailAddress.metadata?.primary !== true;
})
.map((emailAddress) => {
return emailAddress.value || '';
}) || [];
return handleAliases;
return (
sendAsResponse.data.sendAs
?.filter((alias) => alias.isPrimary !== true)
.map((alias) => alias.sendAsEmail || '')
.filter((email) => email.length > 0) ?? []
);
}
}
@@ -46,159 +46,164 @@ export class BlocklistItemDeleteMessagesJob {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const blocklistItemIds = data.events.map(
(eventPayload) => eventPayload.recordId,
);
const blocklistRepository =
await this.globalWorkspaceOrmManager.getRepository<BlocklistWorkspaceEntity>(
workspaceId,
'blocklist',
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const blocklistItemIds = data.events.map(
(eventPayload) => eventPayload.recordId,
);
const blocklist = await blocklistRepository.find({
where: {
id: Any(blocklistItemIds),
},
});
const handlesToDeleteByWorkspaceMemberIdMap = blocklist.reduce(
(acc, blocklistItem) => {
const { handle, workspaceMemberId } = blocklistItem;
if (!acc.has(workspaceMemberId)) {
acc.set(workspaceMemberId, []);
}
if (!isDefined(handle)) {
return acc;
}
acc.get(workspaceMemberId)?.push(handle);
return acc;
},
new Map<string, string[]>(),
);
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkspaceMemberWorkspaceEntity>(
workspaceId,
'workspaceMember',
{ shouldBypassPermissionChecks: true },
);
for (const workspaceMemberId of handlesToDeleteByWorkspaceMemberIdMap.keys()) {
const handles =
handlesToDeleteByWorkspaceMemberIdMap.get(workspaceMemberId);
if (!handles) {
continue;
}
const rolesToDelete = [
MessageParticipantRole.FROM,
MessageParticipantRole.TO,
] as const;
const workspaceMember = await workspaceMemberRepository.findOne({
where: { id: workspaceMemberId },
});
if (!workspaceMember) {
continue;
}
const userWorkspace = await this.userWorkspaceRepository.findOne({
where: { userId: workspaceMember.userId, workspaceId },
});
if (!userWorkspace) {
continue;
}
const connectedAccounts = await this.connectedAccountRepository.find({
where: { userWorkspaceId: userWorkspace.id, workspaceId },
});
const connectedAccountIds = connectedAccounts.map((ca) => ca.id);
if (connectedAccountIds.length === 0) {
continue;
}
const messageChannels = await this.messageChannelRepository.find({
select: {
id: true,
handle: true,
connectedAccount: {
handleAliases: true,
},
},
where: {
connectedAccountId: In(connectedAccountIds),
const blocklistRepository =
await this.globalWorkspaceOrmManager.getRepository<BlocklistWorkspaceEntity>(
workspaceId,
'blocklist',
);
const blocklist = await blocklistRepository.find({
where: {
id: Any(blocklistItemIds),
},
relations: { connectedAccount: true },
});
for (const messageChannel of messageChannels) {
const messageChannelHandles = [messageChannel.handle];
const handlesToDeleteByWorkspaceMemberIdMap = blocklist.reduce(
(acc, blocklistItem) => {
const { handle, workspaceMemberId } = blocklistItem;
const handleAliases = messageChannel.connectedAccount?.handleAliases;
if (!acc.has(workspaceMemberId)) {
acc.set(workspaceMemberId, []);
}
if (isDefined(handleAliases)) {
const aliasList: string[] = Array.isArray(handleAliases)
? handleAliases
: (handleAliases as string).split(',');
if (!isDefined(handle)) {
return acc;
}
messageChannelHandles.push(...aliasList);
}
acc.get(workspaceMemberId)?.push(handle);
const handleConditions = handles.map((handle) => {
const isHandleDomain = handle.startsWith('@');
return acc;
},
new Map<string, string[]>(),
);
return isHandleDomain
? {
handle: And(
Or(ILike(`%${handle}`), ILike(`%.${handle.slice(1)}`)),
Not(In(messageChannelHandles)),
),
role: In(rolesToDelete),
}
: { handle, role: In(rolesToDelete) };
});
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
const messageChannelMessageAssociationsToDelete =
await messageChannelMessageAssociationRepository.find({
where: {
messageChannelId: messageChannel.id,
message: {
messageParticipants: handleConditions,
},
},
});
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkspaceMemberWorkspaceEntity>(
workspaceId,
'workspaceMember',
{ shouldBypassPermissionChecks: true },
);
if (messageChannelMessageAssociationsToDelete.length === 0) {
for (const workspaceMemberId of handlesToDeleteByWorkspaceMemberIdMap.keys()) {
const handles =
handlesToDeleteByWorkspaceMemberIdMap.get(workspaceMemberId);
if (!handles) {
continue;
}
await messageChannelMessageAssociationRepository.delete(
messageChannelMessageAssociationsToDelete.map(({ id }) => id),
);
}
}
const rolesToDelete = [
MessageParticipantRole.FROM,
MessageParticipantRole.TO,
] as const;
await this.threadCleanerService.cleanOrphanMessagesAndThreads(
workspaceId,
);
}, authContext);
const workspaceMember = await workspaceMemberRepository.findOne({
where: { id: workspaceMemberId },
});
if (!workspaceMember) {
continue;
}
const userWorkspace = await this.userWorkspaceRepository.findOne({
where: { userId: workspaceMember.userId, workspaceId },
});
if (!userWorkspace) {
continue;
}
const connectedAccounts = await this.connectedAccountRepository.find({
where: { userWorkspaceId: userWorkspace.id, workspaceId },
});
const connectedAccountIds = connectedAccounts.map((ca) => ca.id);
if (connectedAccountIds.length === 0) {
continue;
}
const messageChannels = await this.messageChannelRepository.find({
select: {
id: true,
handle: true,
connectedAccount: {
handleAliases: true,
},
},
where: {
connectedAccountId: In(connectedAccountIds),
workspaceId,
},
relations: { connectedAccount: true },
});
for (const messageChannel of messageChannels) {
const messageChannelHandles = [messageChannel.handle];
const handleAliases =
messageChannel.connectedAccount?.handleAliases;
if (isDefined(handleAliases)) {
const aliasList: string[] = Array.isArray(handleAliases)
? handleAliases
: (handleAliases as string).split(',');
messageChannelHandles.push(...aliasList);
}
const handleConditions = handles.map((handle) => {
const isHandleDomain = handle.startsWith('@');
return isHandleDomain
? {
handle: And(
Or(ILike(`%${handle}`), ILike(`%.${handle.slice(1)}`)),
Not(In(messageChannelHandles)),
),
role: In(rolesToDelete),
}
: { handle, role: In(rolesToDelete) };
});
const messageChannelMessageAssociationsToDelete =
await messageChannelMessageAssociationRepository.find({
where: {
messageChannelId: messageChannel.id,
message: {
messageParticipants: handleConditions,
},
},
});
if (messageChannelMessageAssociationsToDelete.length === 0) {
continue;
}
await messageChannelMessageAssociationRepository.delete(
messageChannelMessageAssociationsToDelete.map(({ id }) => id),
);
}
}
await this.threadCleanerService.cleanOrphanMessagesAndThreads(
workspaceId,
);
},
authContext,
{ lite: true },
);
}
}
@@ -44,57 +44,63 @@ export class BlocklistReimportMessagesJob {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkspaceMemberWorkspaceEntity>(
workspaceId,
'workspaceMember',
{ shouldBypassPermissionChecks: true },
);
for (const eventPayload of data.events) {
const workspaceMemberId =
eventPayload.properties.before.workspaceMemberId;
const workspaceMember = await workspaceMemberRepository.findOne({
where: { id: workspaceMemberId },
});
if (!workspaceMember) {
continue;
}
const userWorkspace = await this.userWorkspaceRepository.findOne({
where: { userId: workspaceMember.userId, workspaceId },
});
if (!userWorkspace) {
continue;
}
const connectedAccounts = await this.connectedAccountRepository.find({
where: { userWorkspaceId: userWorkspace.id, workspaceId },
});
const connectedAccountIds = connectedAccounts.map((ca) => ca.id);
if (connectedAccountIds.length === 0) {
continue;
}
const messageChannels = await this.messageChannelRepository.find({
where: {
connectedAccountId: In(connectedAccountIds),
syncStage: Not(MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING),
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkspaceMemberWorkspaceEntity>(
workspaceId,
},
});
'workspaceMember',
{ shouldBypassPermissionChecks: true },
);
await this.messagingChannelSyncStatusService.resetAndMarkAsMessagesListFetchPending(
messageChannels.map((messageChannel) => messageChannel.id),
workspaceId,
);
}
}, authContext);
for (const eventPayload of data.events) {
const workspaceMemberId =
eventPayload.properties.before.workspaceMemberId;
const workspaceMember = await workspaceMemberRepository.findOne({
where: { id: workspaceMemberId },
});
if (!workspaceMember) {
continue;
}
const userWorkspace = await this.userWorkspaceRepository.findOne({
where: { userId: workspaceMember.userId, workspaceId },
});
if (!userWorkspace) {
continue;
}
const connectedAccounts = await this.connectedAccountRepository.find({
where: { userWorkspaceId: userWorkspace.id, workspaceId },
});
const connectedAccountIds = connectedAccounts.map((ca) => ca.id);
if (connectedAccountIds.length === 0) {
continue;
}
const messageChannels = await this.messageChannelRepository.find({
where: {
connectedAccountId: In(connectedAccountIds),
syncStage: Not(
MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
),
workspaceId,
},
});
await this.messagingChannelSyncStatusService.resetAndMarkAsMessagesListFetchPending(
messageChannels.map((messageChannel) => messageChannel.id),
workspaceId,
);
}
},
authContext,
{ lite: true },
);
}
}
@@ -157,6 +157,7 @@ export class ApplyMessagesVisibilityRestrictionsService {
return messages;
},
authContext,
{ lite: true },
);
}
}
@@ -53,15 +53,21 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
...(!preserveSyncStageStartedAt ? { syncStageStartedAt: null } : {}),
},
);
}, authContext);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
...(!preserveSyncStageStartedAt
? { syncStageStartedAt: null }
: {}),
},
);
},
authContext,
{ lite: true },
);
}
public async markAsMessagesImportPending(
@@ -75,15 +81,21 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_PENDING,
...(!preserveSyncStageStartedAt ? { syncStageStartedAt: null } : {}),
},
);
}, authContext);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_PENDING,
...(!preserveSyncStageStartedAt
? { syncStageStartedAt: null }
: {}),
},
);
},
authContext,
{ lite: true },
);
}
public async resetAndMarkAsMessagesListFetchPending(
@@ -102,26 +114,31 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncCursor: '',
syncStageStartedAt: null,
throttleFailureCount: 0,
throttleRetryAfter: null,
pendingGroupEmailsAction: MessageChannelPendingGroupEmailsAction.NONE,
},
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncCursor: '',
syncStageStartedAt: null,
throttleFailureCount: 0,
throttleRetryAfter: null,
pendingGroupEmailsAction:
MessageChannelPendingGroupEmailsAction.NONE,
},
);
await this.messageFolderRepository.update(
{ messageChannelId: In(messageChannelIds), workspaceId },
{
syncCursor: '',
pendingSyncAction: MessageFolderPendingSyncAction.NONE,
},
);
}, authContext);
await this.messageFolderRepository.update(
{ messageChannelId: In(messageChannelIds), workspaceId },
{
syncCursor: '',
pendingSyncAction: MessageFolderPendingSyncAction.NONE,
},
);
},
authContext,
{ lite: true },
);
await this.markAsMessagesListFetchPending(messageChannelIds, workspaceId);
}
@@ -136,12 +153,16 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{ syncStageStartedAt: null },
);
}, authContext);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{ syncStageStartedAt: null },
);
},
authContext,
{ lite: true },
);
}
public async markAsMessagesListFetchScheduled(
@@ -154,16 +175,20 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED,
syncStatus: MessageChannelSyncStatus.ONGOING,
syncStageStartedAt: new Date().toISOString(),
},
);
}, authContext);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED,
syncStatus: MessageChannelSyncStatus.ONGOING,
syncStageStartedAt: new Date().toISOString(),
},
);
},
authContext,
{ lite: true },
);
}
public async markAsMessagesListFetchOngoing(
@@ -176,16 +201,20 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING,
syncStatus: MessageChannelSyncStatus.ONGOING,
syncStageStartedAt: new Date().toISOString(),
},
);
}, authContext);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING,
syncStatus: MessageChannelSyncStatus.ONGOING,
syncStageStartedAt: new Date().toISOString(),
},
);
},
authContext,
{ lite: true },
);
}
public async markAsCompletedAndMarkAsMessagesListFetchPending(
@@ -198,19 +227,23 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStatus: MessageChannelSyncStatus.ACTIVE,
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
throttleFailureCount: 0,
throttleRetryAfter: null,
syncStageStartedAt: null,
syncedAt: new Date().toISOString(),
},
);
}, authContext);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStatus: MessageChannelSyncStatus.ACTIVE,
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
throttleFailureCount: 0,
throttleRetryAfter: null,
syncStageStartedAt: null,
syncedAt: new Date().toISOString(),
},
);
},
authContext,
{ lite: true },
);
await this.metricsService.batchIncrementCounter({
key: MetricsKeys.MessageChannelSyncJobActive,
@@ -228,14 +261,18 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED,
},
);
}, authContext);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED,
},
);
},
authContext,
{ lite: true },
);
}
public async markAsMessagesImportOngoing(
@@ -248,16 +285,20 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING,
syncStatus: MessageChannelSyncStatus.ONGOING,
syncStageStartedAt: new Date().toISOString(),
},
);
}, authContext);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING,
syncStatus: MessageChannelSyncStatus.ONGOING,
syncStageStartedAt: new Date().toISOString(),
},
);
},
authContext,
{ lite: true },
);
}
public async markAsFailed(
@@ -273,50 +314,56 @@ export class MessageChannelSyncStatusService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
syncStage: MessageChannelSyncStage.FAILED,
syncStatus: syncStatus,
throttleRetryAfter: null,
},
);
const metricsKey =
syncStatus === MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS
? MetricsKeys.MessageChannelSyncJobFailedInsufficientPermissions
: MetricsKeys.MessageChannelSyncJobFailedUnknown;
await this.metricsService.batchIncrementCounter({
key: metricsKey,
eventIds: messageChannelIds,
});
if (
syncStatus === MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS
) {
const messageChannels = await this.messageChannelRepository.find({
where: { id: In(messageChannelIds), workspaceId },
});
const connectedAccountIds = messageChannels.map(
(messageChannel) => messageChannel.connectedAccountId,
);
await this.connectedAccountRepository.update(
{ id: Any(connectedAccountIds), workspaceId },
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.messageChannelRepository.update(
{ id: In(messageChannelIds), workspaceId },
{
authFailedAt: new Date(),
syncStage: MessageChannelSyncStage.FAILED,
syncStatus: syncStatus,
throttleRetryAfter: null,
},
);
await this.addToAccountsToReconnect(
messageChannels.map((messageChannel) => messageChannel.id),
workspaceId,
);
}
}, authContext);
const metricsKey =
syncStatus ===
MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS
? MetricsKeys.MessageChannelSyncJobFailedInsufficientPermissions
: MetricsKeys.MessageChannelSyncJobFailedUnknown;
await this.metricsService.batchIncrementCounter({
key: metricsKey,
eventIds: messageChannelIds,
});
if (
syncStatus ===
MessageChannelSyncStatus.FAILED_INSUFFICIENT_PERMISSIONS
) {
const messageChannels = await this.messageChannelRepository.find({
where: { id: In(messageChannelIds), workspaceId },
});
const connectedAccountIds = messageChannels.map(
(messageChannel) => messageChannel.connectedAccountId,
);
await this.connectedAccountRepository.update(
{ id: Any(connectedAccountIds), workspaceId },
{
authFailedAt: new Date(),
},
);
await this.addToAccountsToReconnect(
messageChannels.map((messageChannel) => messageChannel.id),
workspaceId,
);
}
},
authContext,
{ lite: true },
);
}
private async addToAccountsToReconnect(
@@ -42,44 +42,48 @@ export class MessagingResetChannelCommand extends CommandRunner {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
this.logger.log(
`No message channel ID provided, resetting all message channels in workspace ${workspaceId}`,
);
const messageChannels = await this.messageChannelRepository.find({
where: {
...(isDefined(messageChannelId) ? { id: messageChannelId } : {}),
workspaceId,
},
});
if (messageChannels.length === 0) {
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
this.logger.log(
`No message channels found in workspace ${workspaceId}`,
`No message channel ID provided, resetting all message channels in workspace ${workspaceId}`,
);
return;
}
const messageChannels = await this.messageChannelRepository.find({
where: {
...(isDefined(messageChannelId) ? { id: messageChannelId } : {}),
workspaceId,
},
});
this.logger.log(
`Found ${messageChannels.length} message channels to reset`,
);
if (messageChannels.length === 0) {
this.logger.log(
`No message channels found in workspace ${workspaceId}`,
);
for (const messageChannel of messageChannels) {
await this.messagingChannelSyncStatusService.resetAndMarkAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
return;
}
this.logger.log(
`Found ${messageChannels.length} message channels to reset`,
);
await this.messagingMessageCleanerService.cleanOrphanMessagesAndThreads(
workspaceId,
);
}
this.logger.log(
`Successfully reset all ${messageChannels.length} message channels in workspace ${workspaceId}`,
);
}, authContext);
for (const messageChannel of messageChannels) {
await this.messagingChannelSyncStatusService.resetAndMarkAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
await this.messagingMessageCleanerService.cleanOrphanMessagesAndThreads(
workspaceId,
);
}
this.logger.log(
`Successfully reset all ${messageChannels.length} message channels in workspace ${workspaceId}`,
);
},
authContext,
{ lite: true },
);
}
@Option({
@@ -29,95 +29,99 @@ export class MessagingMessageCleanerService {
}) {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageWorkspaceEntity>(
workspaceId,
'message',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const messageRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageWorkspaceEntity>(
workspaceId,
'message',
);
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
const messageThreadRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageThreadWorkspaceEntity>(
workspaceId,
'messageThread',
);
const messageThreadRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageThreadWorkspaceEntity>(
workspaceId,
'messageThread',
);
const messageExternalIdsChunks = chunk(messageExternalIds, 500);
const messageExternalIdsChunks = chunk(messageExternalIds, 500);
for (const messageExternalIdsChunk of messageExternalIdsChunks) {
const messageChannelMessageAssociationsToDelete =
await messageChannelMessageAssociationRepository.find({
for (const messageExternalIdsChunk of messageExternalIdsChunks) {
const messageChannelMessageAssociationsToDelete =
await messageChannelMessageAssociationRepository.find({
where: {
messageExternalId: In(messageExternalIdsChunk),
messageChannelId,
},
});
if (messageChannelMessageAssociationsToDelete.length <= 0) {
continue;
}
await messageChannelMessageAssociationRepository.delete(
messageChannelMessageAssociationsToDelete.map(({ id }) => id),
);
this.logger.log(
`WorkspaceId: ${workspaceId} Deleting ${messageChannelMessageAssociationsToDelete.length} message channel message associations`,
);
const orphanMessages = await messageRepository.find({
where: {
messageExternalId: In(messageExternalIdsChunk),
messageChannelId,
id: In(
messageChannelMessageAssociationsToDelete.map(
({ messageId }) => messageId,
),
),
messageChannelMessageAssociations: {
id: IsNull(),
},
},
});
if (messageChannelMessageAssociationsToDelete.length <= 0) {
continue;
}
if (orphanMessages.length <= 0) {
continue;
}
await messageChannelMessageAssociationRepository.delete(
messageChannelMessageAssociationsToDelete.map(({ id }) => id),
);
this.logger.debug(
`WorkspaceId: ${workspaceId} Deleting ${orphanMessages.length} orphan messages`,
);
this.logger.log(
`WorkspaceId: ${workspaceId} Deleting ${messageChannelMessageAssociationsToDelete.length} message channel message associations`,
);
await messageRepository.delete(orphanMessages.map(({ id }) => id));
const orphanMessages = await messageRepository.find({
where: {
id: In(
messageChannelMessageAssociationsToDelete.map(
({ messageId }) => messageId,
const orphanMessageThreads = await messageThreadRepository.find({
where: {
id: In(
orphanMessages.map(({ messageThreadId }) => messageThreadId),
),
),
messageChannelMessageAssociations: {
id: IsNull(),
messages: {
id: IsNull(),
},
},
},
});
});
if (orphanMessages.length <= 0) {
continue;
if (orphanMessageThreads.length <= 0) {
continue;
}
this.logger.debug(
`WorkspaceId: ${workspaceId} Deleting ${orphanMessageThreads.length} orphan message threads`,
);
await messageThreadRepository.delete(
orphanMessageThreads.map(({ id }) => id),
);
}
this.logger.debug(
`WorkspaceId: ${workspaceId} Deleting ${orphanMessages.length} orphan messages`,
);
await messageRepository.delete(orphanMessages.map(({ id }) => id));
const orphanMessageThreads = await messageThreadRepository.find({
where: {
id: In(
orphanMessages.map(({ messageThreadId }) => messageThreadId),
),
messages: {
id: IsNull(),
},
},
});
if (orphanMessageThreads.length <= 0) {
continue;
}
this.logger.debug(
`WorkspaceId: ${workspaceId} Deleting ${orphanMessageThreads.length} orphan message threads`,
);
await messageThreadRepository.delete(
orphanMessageThreads.map(({ id }) => id),
);
}
}, authContext);
},
authContext,
{ lite: true },
);
}
async deleteMessageChannelMessageAssociationsByChannelId({
@@ -129,80 +133,20 @@ export class MessagingMessageCleanerService {
}) {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
const workspaceDataSource =
await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource();
const workspaceDataSource =
await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource();
await workspaceDataSource.transaction(async (manager) => {
const transactionManager = manager as WorkspaceEntityManager;
await workspaceDataSource.transaction(async (manager) => {
const transactionManager = manager as WorkspaceEntityManager;
await deleteUsingPagination(
workspaceId,
500,
async (
limit: number,
offset: number,
_workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
const associations =
await messageChannelMessageAssociationRepository.find(
{
where: { messageChannelId },
take: limit,
skip: offset,
},
transactionManager,
);
return associations.map(({ id }) => id);
},
async (
ids: string[],
workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
this.logger.log(
`WorkspaceId: ${workspaceId} Deleting ${ids.length} message channel message associations for channel ${messageChannelId}`,
);
await messageChannelMessageAssociationRepository.delete(
ids,
transactionManager,
);
},
transactionManager,
);
});
}, authContext);
}
public async cleanOrphanMessagesAndThreads(workspaceId: string) {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageThreadRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageThreadWorkspaceEntity>(
workspaceId,
'messageThread',
);
const messageRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageWorkspaceEntity>(
workspaceId,
'message',
);
const workspaceDataSource =
await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource();
await workspaceDataSource.transaction(
async (transactionManager: WorkspaceEntityManager) => {
await deleteUsingPagination(
workspaceId,
500,
@@ -210,72 +154,140 @@ export class MessagingMessageCleanerService {
limit: number,
offset: number,
_workspaceId: string,
transactionManager: WorkspaceEntityManager,
transactionManager?: WorkspaceEntityManager,
) => {
const nonAssociatedMessages = await messageRepository.find(
{
where: {
messageChannelMessageAssociations: {
id: IsNull(),
},
const associations =
await messageChannelMessageAssociationRepository.find(
{
where: { messageChannelId },
take: limit,
skip: offset,
},
take: limit,
skip: offset,
relations: ['messageChannelMessageAssociations'],
},
transactionManager,
);
transactionManager,
);
return nonAssociatedMessages.map(({ id }) => id);
return associations.map(({ id }) => id);
},
async (
ids: string[],
workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
this.logger.debug(
`WorkspaceId: ${workspaceId} Deleting ${ids.length} messages from message cleaner`,
this.logger.log(
`WorkspaceId: ${workspaceId} Deleting ${ids.length} message channel message associations for channel ${messageChannelId}`,
);
await messageRepository.delete(ids, transactionManager);
},
transactionManager,
);
await deleteUsingPagination(
workspaceId,
500,
async (
limit: number,
offset: number,
_workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
const orphanThreads = await messageThreadRepository.find(
{
where: {
messages: {
id: IsNull(),
},
},
take: limit,
skip: offset,
},
await messageChannelMessageAssociationRepository.delete(
ids,
transactionManager,
);
return orphanThreads.map(({ id }) => id);
},
async (
ids: string[],
_workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
await messageThreadRepository.delete(ids, transactionManager);
},
transactionManager,
);
},
);
}, authContext);
});
},
authContext,
{ lite: true },
);
}
public async cleanOrphanMessagesAndThreads(workspaceId: string) {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const messageThreadRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageThreadWorkspaceEntity>(
workspaceId,
'messageThread',
);
const messageRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageWorkspaceEntity>(
workspaceId,
'message',
);
const workspaceDataSource =
await this.globalWorkspaceOrmManager.getGlobalWorkspaceDataSource();
await workspaceDataSource.transaction(
async (transactionManager: WorkspaceEntityManager) => {
await deleteUsingPagination(
workspaceId,
500,
async (
limit: number,
offset: number,
_workspaceId: string,
transactionManager: WorkspaceEntityManager,
) => {
const nonAssociatedMessages = await messageRepository.find(
{
where: {
messageChannelMessageAssociations: {
id: IsNull(),
},
},
take: limit,
skip: offset,
relations: ['messageChannelMessageAssociations'],
},
transactionManager,
);
return nonAssociatedMessages.map(({ id }) => id);
},
async (
ids: string[],
workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
this.logger.debug(
`WorkspaceId: ${workspaceId} Deleting ${ids.length} messages from message cleaner`,
);
await messageRepository.delete(ids, transactionManager);
},
transactionManager,
);
await deleteUsingPagination(
workspaceId,
500,
async (
limit: number,
offset: number,
_workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
const orphanThreads = await messageThreadRepository.find(
{
where: {
messages: {
id: IsNull(),
},
},
take: limit,
skip: offset,
},
transactionManager,
);
return orphanThreads.map(({ id }) => id);
},
async (
ids: string[],
_workspaceId: string,
transactionManager?: WorkspaceEntityManager,
) => {
await messageThreadRepository.delete(ids, transactionManager);
},
transactionManager,
);
},
);
},
authContext,
{ lite: true },
);
}
}
@@ -188,6 +188,7 @@ export class SyncMessageFoldersService {
return [...updatedExistingFolders, ...createdFolders];
},
authContext,
{ lite: true },
);
}
}
@@ -53,54 +53,58 @@ export class MessagingTriggerMessageListFetchCommand extends CommandRunner {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannels = await this.messageChannelRepository.find({
where: {
isSyncEnabled: true,
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
...(messageChannelId ? { id: messageChannelId } : {}),
workspaceId,
},
});
if (messageChannels.length === 0) {
this.logger.warn(
'No message channels found with MESSAGE_LIST_FETCH_PENDING status',
);
return;
}
this.logger.log(
`Found ${messageChannels.length} message channel(s) to process`,
);
for (const messageChannel of messageChannels) {
await this.messageChannelRepository.update(
{ id: messageChannel.id, workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED,
syncStageStartedAt: new Date().toISOString(),
},
);
await this.messageQueueService.add<MessagingMessageListFetchJobData>(
MessagingMessageListFetchJob.name,
{
messageChannelId: messageChannel.id,
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const messageChannels = await this.messageChannelRepository.find({
where: {
isSyncEnabled: true,
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
...(messageChannelId ? { id: messageChannelId } : {}),
workspaceId,
},
);
});
if (messageChannels.length === 0) {
this.logger.warn(
'No message channels found with MESSAGE_LIST_FETCH_PENDING status',
);
return;
}
this.logger.log(
`Triggered fetch for message channel ${messageChannel.id}`,
`Found ${messageChannels.length} message channel(s) to process`,
);
}
this.logger.log(
`Successfully triggered ${messageChannels.length} message list fetch job(s)`,
);
}, authContext);
for (const messageChannel of messageChannels) {
await this.messageChannelRepository.update(
{ id: messageChannel.id, workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED,
syncStageStartedAt: new Date().toISOString(),
},
);
await this.messageQueueService.add<MessagingMessageListFetchJobData>(
MessagingMessageListFetchJob.name,
{
messageChannelId: messageChannel.id,
workspaceId,
},
);
this.logger.log(
`Triggered fetch for message channel ${messageChannel.id}`,
);
}
this.logger.log(
`Successfully triggered ${messageChannels.length} message list fetch job(s)`,
);
},
authContext,
{ lite: true },
);
}
@Option({
@@ -94,14 +94,18 @@ export class InboundEmailImportService {
);
}
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
await this.messagingSaveMessagesAndEnqueueContactCreationService.saveMessagesAndEnqueueContactCreation(
[parsedInboundMessage.message],
messageChannel,
connectedAccount,
workspaceId,
);
}, buildSystemAuthContext(workspaceId));
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
await this.messagingSaveMessagesAndEnqueueContactCreationService.saveMessagesAndEnqueueContactCreation(
[parsedInboundMessage.message],
messageChannel,
connectedAccount,
workspaceId,
);
},
buildSystemAuthContext(workspaceId),
{ lite: true },
);
await this.inboundEmailStorageService.deleteRawMessage(s3Key);
@@ -48,59 +48,63 @@ export class MessagingMessageListFetchJob {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannel = await this.messageChannelRepository.findOne({
where: {
id: messageChannelId,
workspaceId,
},
relations: { connectedAccount: true, messageFolders: true },
});
if (!messageChannel) {
await this.messagingMonitoringService.track({
eventName: 'message_list_fetch_job.error.message_channel_not_found',
messageChannelId,
workspaceId,
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const messageChannel = await this.messageChannelRepository.findOne({
where: {
id: messageChannelId,
workspaceId,
},
relations: { connectedAccount: true, messageFolders: true },
});
return;
}
if (!messageChannel) {
await this.messagingMonitoringService.track({
eventName: 'message_list_fetch_job.error.message_channel_not_found',
messageChannelId,
workspaceId,
});
if (
messageChannel.syncStage !==
MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED
) {
return;
}
return;
}
try {
await this.messagingMonitoringService.track({
eventName: 'message_list_fetch.started',
workspaceId,
connectedAccountId: messageChannel.connectedAccount.id,
messageChannelId: messageChannel.id,
});
if (
messageChannel.syncStage !==
MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED
) {
return;
}
await this.messagingMessageListFetchService.processMessageListFetch(
messageChannel,
workspaceId,
);
try {
await this.messagingMonitoringService.track({
eventName: 'message_list_fetch.started',
workspaceId,
connectedAccountId: messageChannel.connectedAccount.id,
messageChannelId: messageChannel.id,
});
await this.messagingMonitoringService.track({
eventName: 'message_list_fetch.completed',
workspaceId,
connectedAccountId: messageChannel.connectedAccount.id,
messageChannelId: messageChannel.id,
});
} catch (error) {
await this.messageImportErrorHandlerService.handleDriverException(
error,
MessageImportSyncStep.MESSAGE_LIST_FETCH,
messageChannel,
workspaceId,
);
}
}, authContext);
await this.messagingMessageListFetchService.processMessageListFetch(
messageChannel,
workspaceId,
);
await this.messagingMonitoringService.track({
eventName: 'message_list_fetch.completed',
workspaceId,
connectedAccountId: messageChannel.connectedAccount.id,
messageChannelId: messageChannel.id,
});
} catch (error) {
await this.messageImportErrorHandlerService.handleDriverException(
error,
MessageImportSyncStep.MESSAGE_LIST_FETCH,
messageChannel,
workspaceId,
);
}
},
authContext,
{ lite: true },
);
}
}
@@ -43,41 +43,45 @@ export class MessagingMessagesImportJob {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannel = await this.messageChannelRepository.findOne({
where: {
id: messageChannelId,
workspaceId,
},
relations: { connectedAccount: true, messageFolders: true },
});
if (!messageChannel) {
await this.messagingMonitoringService.track({
eventName: 'messages_import.error.message_channel_not_found',
messageChannelId,
workspaceId,
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const messageChannel = await this.messageChannelRepository.findOne({
where: {
id: messageChannelId,
workspaceId,
},
relations: { connectedAccount: true, messageFolders: true },
});
return;
}
if (!messageChannel) {
await this.messagingMonitoringService.track({
eventName: 'messages_import.error.message_channel_not_found',
messageChannelId,
workspaceId,
});
if (!messageChannel?.isSyncEnabled) {
return;
}
return;
}
if (
messageChannel.syncStage !==
MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED
) {
return;
}
if (!messageChannel?.isSyncEnabled) {
return;
}
await this.messagingMessagesImportService.processMessageBatchImport(
messageChannel,
messageChannel.connectedAccount,
workspaceId,
);
}, authContext);
if (
messageChannel.syncStage !==
MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED
) {
return;
}
await this.messagingMessagesImportService.processMessageBatchImport(
messageChannel,
messageChannel.connectedAccount,
workspaceId,
);
},
authContext,
{ lite: true },
);
}
}
@@ -37,52 +37,58 @@ export class MessagingOngoingStaleJob {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannels = await this.messageChannelRepository.find({
where: {
syncStage: In([
MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING,
MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING,
MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED,
MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED,
]),
workspaceId,
},
});
for (const messageChannel of messageChannels) {
if (isSyncStale(toIsoStringOrNull(messageChannel.syncStageStartedAt))) {
await this.messageChannelSyncStatusService.resetSyncStageStartedAt(
[messageChannel.id],
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const messageChannels = await this.messageChannelRepository.find({
where: {
syncStage: In([
MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING,
MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING,
MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED,
MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED,
]),
workspaceId,
);
},
});
switch (messageChannel.syncStage) {
case MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING:
case MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED:
this.logger.log(
`Sync for message channel ${messageChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to MESSAGE_LIST_FETCH_PENDING`,
);
await this.messageChannelSyncStatusService.markAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
break;
case MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING:
case MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED:
this.logger.log(
`Sync for message channel ${messageChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to MESSAGES_IMPORT_PENDING`,
);
await this.messageChannelSyncStatusService.markAsMessagesImportPending(
[messageChannel.id],
workspaceId,
);
break;
default:
break;
for (const messageChannel of messageChannels) {
if (
isSyncStale(toIsoStringOrNull(messageChannel.syncStageStartedAt))
) {
await this.messageChannelSyncStatusService.resetSyncStageStartedAt(
[messageChannel.id],
workspaceId,
);
switch (messageChannel.syncStage) {
case MessageChannelSyncStage.MESSAGE_LIST_FETCH_ONGOING:
case MessageChannelSyncStage.MESSAGE_LIST_FETCH_SCHEDULED:
this.logger.log(
`Sync for message channel ${messageChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to MESSAGE_LIST_FETCH_PENDING`,
);
await this.messageChannelSyncStatusService.markAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
break;
case MessageChannelSyncStage.MESSAGES_IMPORT_ONGOING:
case MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED:
this.logger.log(
`Sync for message channel ${messageChannel.id} and workspace ${workspaceId} is stale. Setting sync stage to MESSAGES_IMPORT_PENDING`,
);
await this.messageChannelSyncStatusService.markAsMessagesImportPending(
[messageChannel.id],
workspaceId,
);
break;
default:
break;
}
}
}
}
}, authContext);
},
authContext,
{ lite: true },
);
}
}
@@ -36,32 +36,36 @@ export class MessagingRelaunchFailedMessageChannelJob {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageChannel = await this.messageChannelRepository.findOne({
where: {
id: messageChannelId,
workspaceId,
},
});
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const messageChannel = await this.messageChannelRepository.findOne({
where: {
id: messageChannelId,
workspaceId,
},
});
if (
!messageChannel ||
messageChannel.syncStage !== MessageChannelSyncStage.FAILED ||
messageChannel.syncStatus !== MessageChannelSyncStatus.FAILED_UNKNOWN
) {
return;
}
if (
!messageChannel ||
messageChannel.syncStage !== MessageChannelSyncStage.FAILED ||
messageChannel.syncStatus !== MessageChannelSyncStatus.FAILED_UNKNOWN
) {
return;
}
await this.messageChannelRepository.update(
{ id: messageChannelId, workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
syncStatus: MessageChannelSyncStatus.ACTIVE,
throttleFailureCount: 0,
throttleRetryAfter: null,
syncStageStartedAt: null,
},
);
}, authContext);
await this.messageChannelRepository.update(
{ id: messageChannelId, workspaceId },
{
syncStage: MessageChannelSyncStage.MESSAGE_LIST_FETCH_PENDING,
syncStatus: MessageChannelSyncStatus.ACTIVE,
throttleFailureCount: 0,
throttleRetryAfter: null,
syncStageStartedAt: null,
},
);
},
authContext,
{ lite: true },
);
}
}
@@ -26,37 +26,41 @@ export class MessagingCursorService {
) {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
if (!folderId) {
await this.messageChannelRepository.update(
{ id: messageChannel.id, workspaceId },
{
throttleFailureCount: 0,
throttleRetryAfter: null,
syncStageStartedAt: null,
syncCursor:
!messageChannel.syncCursor ||
nextSyncCursor > messageChannel.syncCursor
? nextSyncCursor
: messageChannel.syncCursor,
},
);
} else {
await this.messageFolderRepository.update(
{ id: folderId, workspaceId },
{
syncCursor: nextSyncCursor,
},
);
await this.messageChannelRepository.update(
{ id: messageChannel.id, workspaceId },
{
throttleFailureCount: 0,
throttleRetryAfter: null,
syncStageStartedAt: null,
},
);
}
}, authContext);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
if (!folderId) {
await this.messageChannelRepository.update(
{ id: messageChannel.id, workspaceId },
{
throttleFailureCount: 0,
throttleRetryAfter: null,
syncStageStartedAt: null,
syncCursor:
!messageChannel.syncCursor ||
nextSyncCursor > messageChannel.syncCursor
? nextSyncCursor
: messageChannel.syncCursor,
},
);
} else {
await this.messageFolderRepository.update(
{ id: folderId, workspaceId },
{
syncCursor: nextSyncCursor,
},
);
await this.messageChannelRepository.update(
{ id: messageChannel.id, workspaceId },
{
throttleFailureCount: 0,
throttleRetryAfter: null,
syncStageStartedAt: null,
},
);
}
},
authContext,
{ lite: true },
);
}
}
@@ -37,77 +37,81 @@ export class MessagingDeleteFolderMessagesService {
let totalDeletedCount = 0;
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageFolderAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationMessageFolderWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociationMessageFolder',
);
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
let hasMoreData = true;
while (hasMoreData) {
const folderAssociations =
await messageFolderAssociationRepository.find({
where: {
messageFolderId: messageFolder.id,
},
take: BATCH_SIZE,
});
if (folderAssociations.length === 0) {
hasMoreData = false;
continue;
}
const folderAssociationIds = folderAssociations.map(
(folderAssociation) => folderAssociation.id,
);
const messageChannelMessageAssociationIds = folderAssociations.map(
(folderAssociation) =>
folderAssociation.messageChannelMessageAssociationId,
);
const associations =
await messageChannelMessageAssociationRepository.find({
where: {
id: In(messageChannelMessageAssociationIds),
messageChannelId: messageChannel.id,
},
});
const messageExternalIds = associations
.map((association) => association.messageExternalId)
.filter(isDefined);
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id}, FolderId: ${messageFolder.id} - Deleting ${messageExternalIds.length} messages`,
);
if (messageExternalIds.length > 0) {
await this.messagingMessageCleanerService.deleteMessagesChannelMessageAssociationsAndRelatedOrphans(
{
workspaceId,
messageExternalIds,
messageChannelId: messageChannel.id,
},
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const messageFolderAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationMessageFolderWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociationMessageFolder',
);
totalDeletedCount += messageExternalIds.length;
}
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
await messageFolderAssociationRepository.delete({
id: In(folderAssociationIds),
});
}
}, authContext);
let hasMoreData = true;
while (hasMoreData) {
const folderAssociations =
await messageFolderAssociationRepository.find({
where: {
messageFolderId: messageFolder.id,
},
take: BATCH_SIZE,
});
if (folderAssociations.length === 0) {
hasMoreData = false;
continue;
}
const folderAssociationIds = folderAssociations.map(
(folderAssociation) => folderAssociation.id,
);
const messageChannelMessageAssociationIds = folderAssociations.map(
(folderAssociation) =>
folderAssociation.messageChannelMessageAssociationId,
);
const associations =
await messageChannelMessageAssociationRepository.find({
where: {
id: In(messageChannelMessageAssociationIds),
messageChannelId: messageChannel.id,
},
});
const messageExternalIds = associations
.map((association) => association.messageExternalId)
.filter(isDefined);
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id}, FolderId: ${messageFolder.id} - Deleting ${messageExternalIds.length} messages`,
);
if (messageExternalIds.length > 0) {
await this.messagingMessageCleanerService.deleteMessagesChannelMessageAssociationsAndRelatedOrphans(
{
workspaceId,
messageExternalIds,
messageChannelId: messageChannel.id,
},
);
totalDeletedCount += messageExternalIds.length;
}
await messageFolderAssociationRepository.delete({
id: In(folderAssociationIds),
});
}
},
authContext,
{ lite: true },
);
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id}, FolderId: ${messageFolder.id} - Completed deleting ${totalDeletedCount} messages from folder: ${messageFolder.name}`,
@@ -146,6 +146,7 @@ export class MessagingDeleteGroupEmailMessagesService {
return totalDeletedCount;
},
authContext,
{ lite: true },
);
}
}
@@ -29,57 +29,61 @@ export class MessagingMessageFolderAssociationService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const repository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationMessageFolderWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociationMessageFolder',
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const repository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationMessageFolderWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociationMessageFolder',
);
const records = associations.flatMap((association) =>
association.messageFolderIds.map((folderId) => ({
messageChannelMessageAssociationId:
association.messageChannelMessageAssociationId,
messageFolderId: folderId,
})),
);
const records = associations.flatMap((association) =>
association.messageFolderIds.map((folderId) => ({
messageChannelMessageAssociationId:
association.messageChannelMessageAssociationId,
messageFolderId: folderId,
})),
);
if (records.length === 0) {
return;
}
if (records.length === 0) {
return;
}
const associationIds = [
...new Set(
records.map((record) => record.messageChannelMessageAssociationId),
),
];
const existingRecords = await repository.find(
{
where: {
messageChannelMessageAssociationId: In(associationIds),
},
},
transactionManager,
);
const existingKeys = new Set(
existingRecords.map(
(record) =>
`${record.messageChannelMessageAssociationId}:${record.messageFolderId}`,
),
);
const recordsToInsert = records.filter(
(record) =>
!existingKeys.has(
`${record.messageChannelMessageAssociationId}:${record.messageFolderId}`,
const associationIds = [
...new Set(
records.map((record) => record.messageChannelMessageAssociationId),
),
);
];
if (recordsToInsert.length > 0) {
await repository.insert(recordsToInsert, transactionManager);
}
}, authContext);
const existingRecords = await repository.find(
{
where: {
messageChannelMessageAssociationId: In(associationIds),
},
},
transactionManager,
);
const existingKeys = new Set(
existingRecords.map(
(record) =>
`${record.messageChannelMessageAssociationId}:${record.messageFolderId}`,
),
);
const recordsToInsert = records.filter(
(record) =>
!existingKeys.has(
`${record.messageChannelMessageAssociationId}:${record.messageFolderId}`,
),
);
if (recordsToInsert.length > 0) {
await repository.insert(recordsToInsert, transactionManager);
}
},
authContext,
{ lite: true },
);
}
}
@@ -61,241 +61,246 @@ export class MessagingMessageListFetchService {
) {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
try {
const pendingGroupEmailActionsProcessed =
await this.processPendingGroupEmailActions(
messageChannel,
workspaceId,
);
const pendingFolderActionsProcessed =
await this.processPendingFolderActions(messageChannel, workspaceId);
await this.messageChannelSyncStatusService.markAsMessagesListFetchOngoing(
[messageChannel.id],
workspaceId,
);
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Processing message list fetch`,
);
const freshMessageChannel =
pendingGroupEmailActionsProcessed || pendingFolderActionsProcessed
? await this.messageChannelRepository.findOne({
where: {
id: messageChannel.id,
workspaceId,
},
relations: { connectedAccount: true, messageFolders: true },
})
: messageChannel;
if (!isDefined(freshMessageChannel)) {
this.logger.error(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Message channel not found`,
);
return;
}
const { accessToken, refreshToken } =
await this.messagingAccountAuthenticationService.validateAndRefreshConnectedAccountAuthentication(
{
connectedAccount: freshMessageChannel.connectedAccount,
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
try {
const pendingGroupEmailActionsProcessed =
await this.processPendingGroupEmailActions(
messageChannel,
workspaceId,
messageChannelId: freshMessageChannel.id,
},
);
);
const messageChannelWithFreshTokens = {
...freshMessageChannel,
connectedAccount: {
...freshMessageChannel.connectedAccount,
accessToken,
refreshToken,
},
};
const pendingFolderActionsProcessed =
await this.processPendingFolderActions(messageChannel, workspaceId);
const messageFolders =
await this.syncMessageFoldersService.syncMessageFolders({
messageChannel: messageChannelWithFreshTokens,
await this.messageChannelSyncStatusService.markAsMessagesListFetchOngoing(
[messageChannel.id],
workspaceId,
});
const messageFoldersToSync = messageFolders.filter(
(folder) =>
folder.pendingSyncAction === MessageFolderPendingSyncAction.NONE,
);
const messageLists =
await this.messagingGetMessageListService.getMessageLists(
messageChannelWithFreshTokens,
messageFoldersToSync,
);
await this.cacheStorage.del(
`messages-to-import:${workspaceId}:${freshMessageChannel.id}`,
);
const messageExternalIds = messageLists.flatMap(
(messageList) => messageList.messageExternalIds,
);
const messageExternalIdsToDelete = messageLists.flatMap(
(messageList) => messageList.messageExternalIdsToDelete,
);
const isFullSync =
messageLists.every(
(messageList) => !isNonEmptyString(messageList.previousSyncCursor),
) && !isNonEmptyString(freshMessageChannel.syncCursor);
let totalMessagesToImportCount = 0;
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${freshMessageChannel.id} - Is full sync: ${isFullSync}, toImportCount: ${messageExternalIds.length}, toDeleteCount: ${messageExternalIdsToDelete.length}`,
);
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Processing message list fetch`,
);
const messageExternalIdsChunks = chunk(messageExternalIds, 200);
const freshMessageChannel =
pendingGroupEmailActionsProcessed || pendingFolderActionsProcessed
? await this.messageChannelRepository.findOne({
where: {
id: messageChannel.id,
workspaceId,
},
relations: { connectedAccount: true, messageFolders: true },
})
: messageChannel;
for (const [
index,
messageExternalIdsChunk,
] of messageExternalIdsChunks.entries()) {
const existingMessageChannelMessageAssociations =
await messageChannelMessageAssociationRepository.find({
where: {
if (!isDefined(freshMessageChannel)) {
this.logger.error(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Message channel not found`,
);
return;
}
const { accessToken, refreshToken } =
await this.messagingAccountAuthenticationService.validateAndRefreshConnectedAccountAuthentication(
{
connectedAccount: freshMessageChannel.connectedAccount,
workspaceId,
messageChannelId: freshMessageChannel.id,
messageExternalId: In(messageExternalIdsChunk),
},
);
const messageChannelWithFreshTokens = {
...freshMessageChannel,
connectedAccount: {
...freshMessageChannel.connectedAccount,
accessToken,
refreshToken,
},
};
const messageFolders =
await this.syncMessageFoldersService.syncMessageFolders({
messageChannel: messageChannelWithFreshTokens,
workspaceId,
});
const existingMessageChannelMessageAssociationsExternalIds =
existingMessageChannelMessageAssociations.map(
const messageFoldersToSync = messageFolders.filter(
(folder) =>
folder.pendingSyncAction === MessageFolderPendingSyncAction.NONE,
);
const messageLists =
await this.messagingGetMessageListService.getMessageLists(
messageChannelWithFreshTokens,
messageFoldersToSync,
);
await this.cacheStorage.del(
`messages-to-import:${workspaceId}:${freshMessageChannel.id}`,
);
const messageExternalIds = messageLists.flatMap(
(messageList) => messageList.messageExternalIds,
);
const messageExternalIdsToDelete = messageLists.flatMap(
(messageList) => messageList.messageExternalIdsToDelete,
);
const isFullSync =
messageLists.every(
(messageList) =>
!isNonEmptyString(messageList.previousSyncCursor),
) && !isNonEmptyString(freshMessageChannel.syncCursor);
let totalMessagesToImportCount = 0;
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${freshMessageChannel.id} - Is full sync: ${isFullSync}, toImportCount: ${messageExternalIds.length}, toDeleteCount: ${messageExternalIdsToDelete.length}`,
);
const messageChannelMessageAssociationRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageChannelMessageAssociationWorkspaceEntity>(
workspaceId,
'messageChannelMessageAssociation',
);
const messageExternalIdsChunks = chunk(messageExternalIds, 200);
for (const [
index,
messageExternalIdsChunk,
] of messageExternalIdsChunks.entries()) {
const existingMessageChannelMessageAssociations =
await messageChannelMessageAssociationRepository.find({
where: {
messageChannelId: freshMessageChannel.id,
messageExternalId: In(messageExternalIdsChunk),
},
});
const existingMessageChannelMessageAssociationsExternalIds =
existingMessageChannelMessageAssociations.map(
(messageChannelMessageAssociation) =>
messageChannelMessageAssociation.messageExternalId,
);
const messageExternalIdsToImport = messageExternalIdsChunk.filter(
(messageExternalId) =>
!existingMessageChannelMessageAssociationsExternalIds.includes(
messageExternalId,
),
);
if (messageExternalIdsToImport.length) {
this.logger.debug(
`messageChannelId: ${freshMessageChannel.id} Adding ${messageExternalIdsToImport.length} message external ids to import in batch ${index + 1}`,
);
totalMessagesToImportCount += messageExternalIdsToImport.length;
await this.cacheStorage.setAdd(
`messages-to-import:${workspaceId}:${messageChannelWithFreshTokens.id}`,
messageExternalIdsToImport,
ONE_WEEK_IN_MILLISECONDS,
);
}
}
for (const messageList of messageLists) {
const { nextSyncCursor, folderId } = messageList;
await this.messagingCursorService.updateCursor(
messageChannelWithFreshTokens,
nextSyncCursor,
workspaceId,
folderId,
);
}
const fullSyncMessageChannelMessageAssociationsToDelete = isFullSync
? await this.computeFullSyncMessageChannelMessageAssociationsToDelete(
freshMessageChannel,
messageExternalIds,
workspaceId,
)
: [];
const allMessageExternalIdsToDelete = [
...messageExternalIdsToDelete,
...fullSyncMessageChannelMessageAssociationsToDelete.map(
(messageChannelMessageAssociation) =>
messageChannelMessageAssociation.messageExternalId,
),
];
if (allMessageExternalIdsToDelete.length) {
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${freshMessageChannel.id} - Deleting ${allMessageExternalIdsToDelete.length} message channel message associations`,
);
const messageExternalIdsToImport = messageExternalIdsChunk.filter(
(messageExternalId) =>
!existingMessageChannelMessageAssociationsExternalIds.includes(
messageExternalId,
),
);
const toDeleteChunks = chunk(allMessageExternalIdsToDelete, 200);
if (messageExternalIdsToImport.length) {
this.logger.debug(
`messageChannelId: ${freshMessageChannel.id} Adding ${messageExternalIdsToImport.length} message external ids to import in batch ${index + 1}`,
);
for (const [index, toDeleteChunk] of toDeleteChunks.entries()) {
this.logger.debug(
`messageChannelId: ${freshMessageChannel.id} Deleting ${toDeleteChunk.length} message channel message associations in batch ${index + 1}`,
);
totalMessagesToImportCount += messageExternalIdsToImport.length;
await this.cacheStorage.setAdd(
`messages-to-import:${workspaceId}:${messageChannelWithFreshTokens.id}`,
messageExternalIdsToImport,
ONE_WEEK_IN_MILLISECONDS,
);
await this.messagingMessageCleanerService.deleteMessagesChannelMessageAssociationsAndRelatedOrphans(
{
workspaceId,
messageExternalIds: toDeleteChunk.filter(
(messageExternalId) => isNonEmptyString(messageExternalId),
),
messageChannelId: messageChannelWithFreshTokens.id,
},
);
}
}
}
for (const messageList of messageLists) {
const { nextSyncCursor, folderId } = messageList;
await this.messagingCursorService.updateCursor(
messageChannelWithFreshTokens,
nextSyncCursor,
workspaceId,
folderId,
);
}
const fullSyncMessageChannelMessageAssociationsToDelete = isFullSync
? await this.computeFullSyncMessageChannelMessageAssociationsToDelete(
freshMessageChannel,
messageExternalIds,
workspaceId,
)
: [];
const allMessageExternalIdsToDelete = [
...messageExternalIdsToDelete,
...fullSyncMessageChannelMessageAssociationsToDelete.map(
(messageChannelMessageAssociation) =>
messageChannelMessageAssociation.messageExternalId,
),
];
if (allMessageExternalIdsToDelete.length) {
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${freshMessageChannel.id} - Deleting ${allMessageExternalIdsToDelete.length} message channel message associations`,
`WorkspaceId: ${workspaceId}, MessageChannelId: ${freshMessageChannel.id} - Total messages to import count: ${totalMessagesToImportCount}`,
);
const toDeleteChunks = chunk(allMessageExternalIdsToDelete, 200);
for (const [index, toDeleteChunk] of toDeleteChunks.entries()) {
this.logger.debug(
`messageChannelId: ${freshMessageChannel.id} Deleting ${toDeleteChunk.length} message channel message associations in batch ${index + 1}`,
if (totalMessagesToImportCount === 0) {
await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending(
[messageChannelWithFreshTokens.id],
workspaceId,
);
await this.messagingMessageCleanerService.deleteMessagesChannelMessageAssociationsAndRelatedOrphans(
{
workspaceId,
messageExternalIds: toDeleteChunk.filter((messageExternalId) =>
isNonEmptyString(messageExternalId),
),
messageChannelId: messageChannelWithFreshTokens.id,
},
);
return;
}
}
this.logger.log(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${freshMessageChannel.id} - Total messages to import count: ${totalMessagesToImportCount}`,
);
this.logger.debug(
`messageChannelId: ${freshMessageChannel.id} Scheduling direct messages import`,
);
if (totalMessagesToImportCount === 0) {
await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending(
await this.messageChannelSyncStatusService.markAsMessagesImportScheduled(
[messageChannelWithFreshTokens.id],
workspaceId,
);
return;
await this.messagingMessagesImportService.processMessageBatchImport(
{
...messageChannelWithFreshTokens,
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED,
},
messageChannelWithFreshTokens.connectedAccount,
workspaceId,
);
} catch (error) {
await this.messageImportErrorHandlerService.handleDriverException(
error,
MessageImportSyncStep.MESSAGE_LIST_FETCH,
messageChannel,
workspaceId,
);
}
this.logger.debug(
`messageChannelId: ${freshMessageChannel.id} Scheduling direct messages import`,
);
await this.messageChannelSyncStatusService.markAsMessagesImportScheduled(
[messageChannelWithFreshTokens.id],
workspaceId,
);
await this.messagingMessagesImportService.processMessageBatchImport(
{
...messageChannelWithFreshTokens,
syncStage: MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED,
},
messageChannelWithFreshTokens.connectedAccount,
workspaceId,
);
} catch (error) {
await this.messageImportErrorHandlerService.handleDriverException(
error,
MessageImportSyncStep.MESSAGE_LIST_FETCH,
messageChannel,
workspaceId,
);
}
}, authContext);
},
authContext,
{ lite: true },
);
}
private async processPendingGroupEmailActions(
@@ -81,13 +81,16 @@ export class MessagingMessageService {
const messageAccumulatorMap = new Map<string, MessageAccumulator>();
const existingMessagesInDB = await messageRepository.find({
where: {
headerMessageId: In(
messages.map((message) => message.headerMessageId),
),
const existingMessagesInDB = await messageRepository.find(
{
where: {
headerMessageId: In(
messages.map((message) => message.headerMessageId),
),
},
},
});
transactionManager,
);
const messageChannelMessageAssociationsReferencingMessageThread =
await messageChannelMessageAssociationRepository.find(
@@ -104,12 +107,17 @@ export class MessagingMessageService {
);
const existingMessageChannelMessageAssociations =
await messageChannelMessageAssociationRepository.find({
where: {
messageId: In(existingMessagesInDB.map((message) => message.id)),
messageChannelId,
await messageChannelMessageAssociationRepository.find(
{
where: {
messageId: In(
existingMessagesInDB.map((message) => message.id),
),
messageChannelId,
},
},
});
transactionManager,
);
await this.enrichMessageAccumulatorWithExistingMessages(
messages,
@@ -311,6 +319,7 @@ export class MessagingMessageService {
};
},
authContext,
{ lite: true },
);
}
@@ -17,7 +17,6 @@ import { GlobalWorkspaceOrmManager } from 'src/engine/twenty-orm/global-workspac
import { buildSystemAuthContext } from 'src/engine/twenty-orm/utils/build-system-auth-context.util';
import { BlocklistRepository } from 'src/modules/blocklist/repositories/blocklist.repository';
import { BlocklistWorkspaceEntity } from 'src/modules/blocklist/standard-objects/blocklist.workspace-entity';
import { EmailAliasManagerService } from 'src/modules/connected-account/email-alias-manager/services/email-alias-manager.service';
import { MessageChannelSyncStatusService } from 'src/modules/messaging/common/services/message-channel-sync-status.service';
import {
MessageImportDriverException,
@@ -47,7 +46,6 @@ export class MessagingMessagesImportService {
private readonly messagingMonitoringService: MessagingMonitoringService,
@InjectObjectMetadataRepository(BlocklistWorkspaceEntity)
private readonly blocklistRepository: BlocklistRepository,
private readonly emailAliasManagerService: EmailAliasManagerService,
private readonly globalWorkspaceOrmManager: GlobalWorkspaceOrmManager,
@InjectRepository(MessageChannelEntity)
private readonly messageChannelRepository: Repository<MessageChannelEntity>,
@@ -74,58 +72,192 @@ export class MessagingMessagesImportService {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
try {
if (
messageChannel.syncStage !==
MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED
) {
return;
}
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
try {
if (
messageChannel.syncStage !==
MessageChannelSyncStage.MESSAGES_IMPORT_SCHEDULED
) {
return;
}
await this.messagingMonitoringService.track({
eventName: 'messages_import.started',
workspaceId,
connectedAccountId: messageChannel.connectedAccountId,
messageChannelId: messageChannel.id,
});
await this.messagingMonitoringService.track({
eventName: 'messages_import.started',
workspaceId,
connectedAccountId: messageChannel.connectedAccountId,
messageChannelId: messageChannel.id,
});
await this.messageChannelSyncStatusService.markAsMessagesImportOngoing(
[messageChannel.id],
workspaceId,
);
const { accessToken, refreshToken } =
await this.messagingAccountAuthenticationService.validateAndRefreshConnectedAccountAuthentication(
{
connectedAccount,
workspaceId,
messageChannelId: messageChannel.id,
},
);
const connectedAccountWithFreshTokens = {
...connectedAccount,
accessToken,
refreshToken,
};
const refreshedHandleAliases =
await this.emailAliasManagerService.refreshHandleAliases(
connectedAccountWithFreshTokens,
await this.messageChannelSyncStatusService.markAsMessagesImportOngoing(
[messageChannel.id],
workspaceId,
);
connectedAccountWithFreshTokens.handleAliases = refreshedHandleAliases;
const { accessToken, refreshToken } =
await this.messagingAccountAuthenticationService.validateAndRefreshConnectedAccountAuthentication(
{
connectedAccount,
workspaceId,
messageChannelId: messageChannel.id,
},
);
messageIdsToFetch = await this.cacheStorage.setPop(
`messages-to-import:${workspaceId}:${messageChannel.id}`,
messagesGetBatchSize,
);
const connectedAccountWithFreshTokens = {
...connectedAccount,
accessToken,
refreshToken,
};
if (!messageIdsToFetch?.length) {
await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending(
[messageChannel.id],
messageIdsToFetch = await this.cacheStorage.setPop(
`messages-to-import:${workspaceId}:${messageChannel.id}`,
messagesGetBatchSize,
);
if (!messageIdsToFetch?.length) {
await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
return await this.trackMessageImportCompleted(
messageChannel,
workspaceId,
);
}
const allMessages =
await this.messagingGetMessagesService.getMessages(
messageIdsToFetch,
connectedAccountWithFreshTokens,
messageChannel,
);
// Map external folder IDs to internal folder IDs
const messageFolders = messageChannel.messageFolders ?? [];
const foldersWithExternalId = messageFolders.filter(
(folder): folder is typeof folder & { externalId: string } =>
isDefined(folder.externalId),
);
const folderExternalToInternalMap = new Map<string, string>(
foldersWithExternalId.map((folder) => [
folder.externalId,
folder.id,
]),
);
for (const message of allMessages) {
const externalFolderIds = message.messageFolderExternalIds ?? [];
message.messageFolderIds = externalFolderIds
.map((externalId) => folderExternalToInternalMap.get(externalId))
.filter(isDefined);
}
const userWorkspace = await this.userWorkspaceRepository.findOne({
where: {
id: connectedAccountWithFreshTokens.userWorkspaceId,
},
});
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkspaceMemberWorkspaceEntity>(
workspaceId,
'workspaceMember',
{ shouldBypassPermissionChecks: true },
);
const workspaceMember = userWorkspace
? await workspaceMemberRepository.findOne({
where: { userId: userWorkspace.userId },
})
: null;
const blocklist = workspaceMember
? await this.blocklistRepository.getByWorkspaceMemberId(
workspaceMember.id,
workspaceId,
)
: [];
if (!isDefined(messageChannel.handle)) {
throw new MessageImportDriverException(
'Message channel handle is required',
MessageImportDriverExceptionCode.CHANNEL_MISCONFIGURED,
);
}
if (!isDefined(connectedAccountWithFreshTokens.handleAliases)) {
throw new MessageImportDriverException(
'Message channel handle is required',
MessageImportDriverExceptionCode.CHANNEL_MISCONFIGURED,
);
}
const workspace = await this.workspaceRepository.findOne({
where: { id: workspaceId },
select: ['id', 'isInternalMessagesImportEnabled'],
});
const messagesToSave = filterEmails(
messageChannel.handle,
[...connectedAccountWithFreshTokens.handleAliases],
allMessages,
blocklist
.map((blocklistItem) => blocklistItem.handle)
.filter(isDefined),
messageChannel.excludeGroupEmails,
workspace?.isInternalMessagesImportEnabled ?? false,
);
if (messagesToSave.length > 0) {
await this.saveMessagesAndEnqueueContactCreationService.saveMessagesAndEnqueueContactCreation(
messagesToSave,
messageChannel,
connectedAccountWithFreshTokens,
workspaceId,
);
}
if (messageIdsToFetch.length < messagesGetBatchSize) {
await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
} else {
await this.messageChannelSyncStatusService.markAsMessagesImportPending(
[messageChannel.id],
workspaceId,
);
}
await this.messageChannelRepository.update(
{ id: messageChannel.id, workspaceId },
{
throttleFailureCount: 0,
throttleRetryAfter: null,
syncStageStartedAt: null,
},
);
return await this.trackMessageImportCompleted(
messageChannel,
workspaceId,
);
} catch (error) {
this.logger.error(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Error (${error.code}) importing messages: ${error.message}`,
);
await this.cacheStorage.setAdd(
`messages-to-import:${workspaceId}:${messageChannel.id}`,
messageIdsToFetch,
);
await this.messageImportErrorHandlerService.handleDriverException(
error,
MessageImportSyncStep.MESSAGES_IMPORT_ONGOING,
messageChannel,
workspaceId,
);
@@ -134,144 +266,10 @@ export class MessagingMessagesImportService {
workspaceId,
);
}
const allMessages = await this.messagingGetMessagesService.getMessages(
messageIdsToFetch,
connectedAccountWithFreshTokens,
messageChannel,
);
// Map external folder IDs to internal folder IDs
const messageFolders = messageChannel.messageFolders ?? [];
const foldersWithExternalId = messageFolders.filter(
(folder): folder is typeof folder & { externalId: string } =>
isDefined(folder.externalId),
);
const folderExternalToInternalMap = new Map<string, string>(
foldersWithExternalId.map((folder) => [folder.externalId, folder.id]),
);
for (const message of allMessages) {
const externalFolderIds = message.messageFolderExternalIds ?? [];
message.messageFolderIds = externalFolderIds
.map((externalId) => folderExternalToInternalMap.get(externalId))
.filter(isDefined);
}
const userWorkspace = await this.userWorkspaceRepository.findOne({
where: {
id: connectedAccountWithFreshTokens.userWorkspaceId,
},
});
const workspaceMemberRepository =
await this.globalWorkspaceOrmManager.getRepository<WorkspaceMemberWorkspaceEntity>(
workspaceId,
'workspaceMember',
{ shouldBypassPermissionChecks: true },
);
const workspaceMember = userWorkspace
? await workspaceMemberRepository.findOne({
where: { userId: userWorkspace.userId },
})
: null;
const blocklist = workspaceMember
? await this.blocklistRepository.getByWorkspaceMemberId(
workspaceMember.id,
workspaceId,
)
: [];
if (!isDefined(messageChannel.handle)) {
throw new MessageImportDriverException(
'Message channel handle is required',
MessageImportDriverExceptionCode.CHANNEL_MISCONFIGURED,
);
}
if (!isDefined(connectedAccountWithFreshTokens.handleAliases)) {
throw new MessageImportDriverException(
'Message channel handle is required',
MessageImportDriverExceptionCode.CHANNEL_MISCONFIGURED,
);
}
const workspace = await this.workspaceRepository.findOne({
where: { id: workspaceId },
select: ['id', 'isInternalMessagesImportEnabled'],
});
const messagesToSave = filterEmails(
messageChannel.handle,
[...connectedAccountWithFreshTokens.handleAliases],
allMessages,
blocklist
.map((blocklistItem) => blocklistItem.handle)
.filter(isDefined),
messageChannel.excludeGroupEmails,
workspace?.isInternalMessagesImportEnabled ?? false,
);
if (messagesToSave.length > 0) {
await this.saveMessagesAndEnqueueContactCreationService.saveMessagesAndEnqueueContactCreation(
messagesToSave,
messageChannel,
connectedAccountWithFreshTokens,
workspaceId,
);
}
if (messageIdsToFetch.length < messagesGetBatchSize) {
await this.messageChannelSyncStatusService.markAsCompletedAndMarkAsMessagesListFetchPending(
[messageChannel.id],
workspaceId,
);
} else {
await this.messageChannelSyncStatusService.markAsMessagesImportPending(
[messageChannel.id],
workspaceId,
);
}
await this.messageChannelRepository.update(
{ id: messageChannel.id, workspaceId },
{
throttleFailureCount: 0,
throttleRetryAfter: null,
syncStageStartedAt: null,
},
);
return await this.trackMessageImportCompleted(
messageChannel,
workspaceId,
);
} catch (error) {
this.logger.error(
`WorkspaceId: ${workspaceId}, MessageChannelId: ${messageChannel.id} - Error (${error.code}) importing messages: ${error.message}`,
);
await this.cacheStorage.setAdd(
`messages-to-import:${workspaceId}:${messageChannel.id}`,
messageIdsToFetch,
);
await this.messageImportErrorHandlerService.handleDriverException(
error,
MessageImportSyncStep.MESSAGES_IMPORT_ONGOING,
messageChannel,
workspaceId,
);
return await this.trackMessageImportCompleted(
messageChannel,
workspaceId,
);
}
}, authContext);
},
authContext,
{ lite: true },
);
}
private async trackMessageImportCompleted(
@@ -114,6 +114,7 @@ export class MessagingProcessFolderActionsService {
}
},
authContext,
{ lite: true },
);
}
}
@@ -79,6 +79,7 @@ export class MessagingProcessGroupEmailActionsService {
}
},
authContext,
{ lite: true },
);
await this.messageChannelRepository.update(
@@ -157,6 +157,7 @@ export class MessagingSaveMessagesAndEnqueueContactCreationService {
);
},
authContext,
{ lite: true },
);
if (
@@ -23,57 +23,64 @@ export class MessagingMessageParticipantService {
): Promise<void> {
const authContext = buildSystemAuthContext(workspaceId);
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(async () => {
const messageParticipantRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageParticipantWorkspaceEntity>(
workspaceId,
'messageParticipant',
await this.globalWorkspaceOrmManager.executeInWorkspaceContext(
async () => {
const messageParticipantRepository =
await this.globalWorkspaceOrmManager.getRepository<MessageParticipantWorkspaceEntity>(
workspaceId,
'messageParticipant',
);
const existingParticipantsBasedOnMessageIds =
await messageParticipantRepository.find(
{
where: {
messageId: In(
participants.map((participant) => participant.messageId),
),
},
},
transactionManager,
);
const participantsToCreate: Pick<
MessageParticipantWorkspaceEntity,
'messageId' | 'handle' | 'displayName' | 'role'
>[] = participants
.filter(
(participant) =>
!existingParticipantsBasedOnMessageIds.find(
(existingParticipant) =>
existingParticipant.messageId === participant.messageId &&
existingParticipant.handle === participant.handle &&
existingParticipant.displayName === participant.displayName &&
existingParticipant.role === participant.role,
),
)
.map((participant) => {
return {
messageId: participant.messageId,
handle: participant.handle,
displayName: participant.displayName,
role: participant.role,
};
});
const createdParticipants = await messageParticipantRepository.insert(
participantsToCreate,
transactionManager,
);
const existingParticipantsBasedOnMessageIds =
await messageParticipantRepository.find({
where: {
messageId: In(
participants.map((participant) => participant.messageId),
),
},
await this.matchParticipantService.matchParticipants({
participants: createdParticipants.raw ?? [],
objectMetadataName: 'messageParticipant',
transactionManager,
matchWith: 'workspaceMemberAndPerson',
workspaceId,
});
const participantsToCreate: Pick<
MessageParticipantWorkspaceEntity,
'messageId' | 'handle' | 'displayName' | 'role'
>[] = participants
.filter(
(participant) =>
!existingParticipantsBasedOnMessageIds.find(
(existingParticipant) =>
existingParticipant.messageId === participant.messageId &&
existingParticipant.handle === participant.handle &&
existingParticipant.displayName === participant.displayName &&
existingParticipant.role === participant.role,
),
)
.map((participant) => {
return {
messageId: participant.messageId,
handle: participant.handle,
displayName: participant.displayName,
role: participant.role,
};
});
const createdParticipants = await messageParticipantRepository.insert(
participantsToCreate,
transactionManager,
);
await this.matchParticipantService.matchParticipants({
participants: createdParticipants.raw ?? [],
objectMetadataName: 'messageParticipant',
transactionManager,
matchWith: 'workspaceMemberAndPerson',
workspaceId,
});
}, authContext);
},
authContext,
{ lite: true },
);
}
}
@@ -74,6 +74,7 @@ export class MessagingMessageChannelSyncStatusMonitoringCronJob {
}
},
authContext,
{ lite: true },
);
} catch (error) {
this.exceptionHandlerService.captureExceptions([error], {