Compare commits

...
Author SHA1 Message Date
Sonarly Claude Code 5359915232 chore: improve monitoring for fix: prevent DB connection timeout during long-run
**Monitoring improvement: Treat connection termination as temporary error**

Added `CONNECTION_TERMINATED` exception code to TwentyORM and configured the messaging exception handler to treat it as a temporary error (like QUERY_READ_TIMEOUT). This means:

1. When "Connection terminated unexpectedly" errors occur, they are now wrapped in a `TwentyORMException` with code `CONNECTION_TERMINATED`
2. The messaging import service handles this as a temporary error, incrementing `throttleFailureCount` and scheduling a retry
3. Sentry is only notified after max retry attempts (default 5), reducing noise from transient infrastructure issues

Files changed:
1. `twenty-orm.exception.ts` - Added `CONNECTION_TERMINATED` enum value and user-friendly message
2. `compute-twenty-orm-exception.ts` - Added handler to convert "Connection terminated unexpectedly" QueryFailedError to TwentyORMException
3. `messaging-import-exception-handler.service.ts` - Added `CONNECTION_TERMINATED` to the temporary error cases
2026-05-08 02:15:45 +00:00
Sonarly Claude Code 964249b7b0 fix: prevent DB connection timeout during long-running external API calls
https://sonarly.com/issue/36017?type=bug

Background worker's PostgreSQL connection becomes stale during 30+ second Gmail API batch fetch operations, causing "Connection terminated unexpectedly" when subsequent database queries execute.

Fix: **Fix: Enable TCP keepalives on PostgreSQL connections**

Added TCP keepalive settings to all database connection configurations:
- `keepAlive: true` - enables TCP keepalive probes
- `keepAliveInitialDelayMillis: 10000` - sends first probe after 10 seconds of idle time

This prevents AWS load balancers/RDS proxy from terminating idle connections during long-running external API operations (like Gmail batch fetches that take 30+ seconds).

Files changed for the fix:
1. `global-workspace-datasource.service.ts` - primary and replica datasource configurations
2. `core.datasource.ts` - core datasource configuration

Both locations now have consistent keepalive settings that will maintain TCP connections through infrastructure idle timeouts.
2026-05-08 02:15:45 +00:00
5 changed files with 26 additions and 0 deletions
@@ -75,6 +75,10 @@ export const typeORMCoreModuleOptions: TypeOrmModuleOptions = {
query_timeout: Number(process.env.PG_DATABASE_PRIMARY_TIMEOUT_MS ?? 10000),
idleTimeoutMillis: Number(process.env.PG_POOL_IDLE_TIMEOUT_MS ?? 600000),
allowExitOnIdle: process.env.PG_POOL_ALLOW_EXIT_ON_IDLE === 'true',
// TCP keepalive prevents connections from being terminated by load
// balancers/proxies during long-running external operations
keepAlive: true,
keepAliveInitialDelayMillis: 10000,
},
};
@@ -35,6 +35,16 @@ export const computeTwentyORMException = async (
);
}
if (error.message.includes('Connection terminated unexpectedly')) {
return new TwentyORMException(
'Connection terminated unexpectedly',
TwentyORMExceptionCode.CONNECTION_TERMINATED,
{
userFriendlyMessage: msg`Database connection was interrupted. Please try again.`,
},
);
}
const errorCode = (error as QueryFailedErrorWithCode).code;
if (
@@ -21,6 +21,7 @@ export enum TwentyORMExceptionCode {
METHOD_NOT_ALLOWED = 'METHOD_NOT_ALLOWED',
ENUM_TYPE_NAME_NOT_FOUND = 'ENUM_TYPE_NAME_NOT_FOUND',
QUERY_READ_TIMEOUT = 'QUERY_READ_TIMEOUT',
CONNECTION_TERMINATED = 'CONNECTION_TERMINATED',
DUPLICATE_ENTRY_DETECTED = 'DUPLICATE_ENTRY_DETECTED',
TOO_MANY_RECORDS_TO_UPDATE = 'TOO_MANY_RECORDS_TO_UPDATE',
INVALID_INPUT = 'INVALID_INPUT',
@@ -61,6 +62,8 @@ const getTwentyORMExceptionUserFriendlyMessage = (
return msg`This operation is not allowed.`;
case TwentyORMExceptionCode.QUERY_READ_TIMEOUT:
return msg`Query timed out. Please try again.`;
case TwentyORMExceptionCode.CONNECTION_TERMINATED:
return msg`Database connection was interrupted. Please try again.`;
case TwentyORMExceptionCode.DUPLICATE_ENTRY_DETECTED:
return msg`A duplicate entry was detected.`;
case TwentyORMExceptionCode.TOO_MANY_RECORDS_TO_UPDATE:
@@ -50,6 +50,10 @@ export class GlobalWorkspaceDataSourceService
allowExitOnIdle: this.twentyConfigService.get(
'PG_POOL_ALLOW_EXIT_ON_IDLE',
),
// TCP keepalive prevents connections from being terminated by load
// balancers/proxies during long-running external operations
keepAlive: true,
keepAliveInitialDelayMillis: 10000,
},
},
this.workspaceEventEmitter,
@@ -85,6 +89,10 @@ export class GlobalWorkspaceDataSourceService
allowExitOnIdle: this.twentyConfigService.get(
'PG_POOL_ALLOW_EXIT_ON_IDLE',
),
// TCP keepalive prevents connections from being terminated by load
// balancers/proxies during long-running external operations
keepAlive: true,
keepAliveInitialDelayMillis: 10000,
},
},
this.workspaceEventEmitter,
@@ -59,6 +59,7 @@ export class MessageImportExceptionHandlerService {
);
break;
case TwentyORMExceptionCode.QUERY_READ_TIMEOUT:
case TwentyORMExceptionCode.CONNECTION_TERMINATED:
case MessageImportDriverExceptionCode.TEMPORARY_ERROR:
case MessageNetworkExceptionCode.ECONNABORTED:
case MessageNetworkExceptionCode.ENOTFOUND: