Compare commits

...
Author SHA1 Message Date
Sonarly Claude Code 3c62416512 chore: improve monitoring for FK violation on billingSubscriptionItem due to rac
**Monitoring: Add Stripe event context to unhandled billing error messages**

In `BillingWebhookController.handleWebhooks()`:
- Hoisted the `event` variable outside the try block so it's available in the catch block
- Appended Stripe event type and ID to the `BILLING_UNHANDLED_ERROR` exception message (e.g., `"error message (event: customer.subscription.deleted, id: evt_xxx)"`)
- This makes Sentry error grouping more meaningful and immediately tells engineers which Stripe event type triggered the unhandled error, without needing to dig through breadcrumbs or request body
2026-03-14 10:58:54 +00:00
Sonarly Claude Code d3f5299860 FK violation on billingSubscriptionItem due to race condition in concurrent Stripe webhook processing
https://sonarly.com/issue/4349?type=bug

Concurrent processing of Stripe subscription cancellation webhooks causes a race condition where one handler deletes the workspace (and cascade-deletes subscriptions) while another handler is still inserting subscription items for the same workspace.

Fix: **Fix: Handle FK violation from concurrent webhook processing gracefully**

In `BillingWebhookSubscriptionService.updateBillingSubscriptionItems()`:
- Added a try-catch around the `billingSubscriptionItemRepository.upsert()` call that specifically catches `QueryFailedError` with PostgreSQL error code `23503` (foreign key violation)
- When caught, returns `false` to signal to the caller that the subscription was deleted concurrently
- All other errors are re-thrown — this is NOT a broad catch

In `BillingWebhookSubscriptionService.processStripeEvent()`:
- Checks the return value of `updateBillingSubscriptionItems`
- If items could not be updated (subscription deleted by concurrent handler), logs a warning and returns early with a 200 response
- This prevents the remaining logic (workspace suspension, Stripe customer metadata update, billing threshold setup) from running on a now-deleted subscription

**Why this approach:**
- The race condition occurs when Stripe sends near-simultaneous events (`customer.subscription.updated` + `customer.subscription.deleted`) for the same cancellation, and one handler's `deleteWorkspace()` cascade-deletes the subscription while the other handler is still inserting items
- A distributed lock would be the ideal prevention, but this graceful handling is the appropriate surgical fix: the workspace is already being cleaned up by the concurrent handler, so skipping the remaining processing is correct behavior
2026-03-14 10:58:54 +00:00
2 changed files with 54 additions and 18 deletions
@@ -68,8 +68,10 @@ export class BillingWebhookController {
);
}
let event: Stripe.Event | undefined;
try {
const event = this.stripeWebhookService.constructEventFromPayload(
event = this.stripeWebhookService.constructEventFromPayload(
signature,
req.rawBody,
);
@@ -86,8 +88,12 @@ export class BillingWebhookController {
const errorMessage =
error instanceof Error ? error.message : JSON.stringify(error);
const eventContext = event
? ` (event: ${event.type}, id: ${event.id})`
: '';
throw new BillingException(
errorMessage,
`${errorMessage}${eventContext}`,
BillingExceptionCode.BILLING_UNHANDLED_ERROR,
);
}
@@ -6,10 +6,11 @@ import { InjectRepository } from '@nestjs/typeorm';
import { msg } from '@lingui/core/macro';
import { isDefined } from 'twenty-shared/utils';
import { WorkspaceActivationStatus } from 'twenty-shared/workspace';
import { In, Repository } from 'typeorm';
import { In, QueryFailedError, Repository } from 'typeorm';
import type Stripe from 'stripe';
import { POSTGRESQL_ERROR_CODES } from 'src/engine/api/graphql/workspace-query-runner/constants/postgres-error-codes.constants';
import { getDeletedStripeSubscriptionItemIdsFromStripeSubscriptionEvent } from 'src/engine/core-modules/billing-webhook/utils/get-deleted-stripe-subscription-item-ids-from-stripe-subscription-event.util';
import { transformStripeSubscriptionEventToDatabaseCustomer } from 'src/engine/core-modules/billing-webhook/utils/transform-stripe-subscription-event-to-database-customer.util';
import { transformStripeSubscriptionEventToDatabaseSubscriptionItem } from 'src/engine/core-modules/billing-webhook/utils/transform-stripe-subscription-event-to-database-subscription-item.util';
@@ -134,10 +135,22 @@ export class BillingWebhookSubscriptionService {
);
}
await this.updateBillingSubscriptionItems(
updatedBillingSubscription.id,
event,
);
const subscriptionItemsUpdated =
await this.updateBillingSubscriptionItems(
updatedBillingSubscription.id,
event,
);
if (!subscriptionItemsUpdated) {
this.logger.warn(
`Subscription ${data.object.id} was deleted by a concurrent webhook handler, skipping remaining processing for workspace ${workspaceId}`,
);
return {
stripeSubscriptionId: data.object.id,
stripeCustomerId: data.object.customer,
};
}
const shouldSuspend = this.shouldSuspendWorkspace(data);
@@ -224,7 +237,7 @@ export class BillingWebhookSubscriptionService {
| Stripe.CustomerSubscriptionUpdatedEvent
| Stripe.CustomerSubscriptionCreatedEvent
| Stripe.CustomerSubscriptionDeletedEvent,
) {
): Promise<boolean> {
const deletedSubscriptionItemIds =
getDeletedStripeSubscriptionItemIdsFromStripeSubscriptionEvent(event);
@@ -235,15 +248,32 @@ export class BillingWebhookSubscriptionService {
});
}
await this.billingSubscriptionItemRepository.upsert(
transformStripeSubscriptionEventToDatabaseSubscriptionItem(
subscriptionId,
event.data,
),
{
conflictPaths: ['stripeSubscriptionItemId'],
skipUpdateIfNoValuesChanged: true,
},
);
try {
await this.billingSubscriptionItemRepository.upsert(
transformStripeSubscriptionEventToDatabaseSubscriptionItem(
subscriptionId,
event.data,
),
{
conflictPaths: ['stripeSubscriptionItemId'],
skipUpdateIfNoValuesChanged: true,
},
);
} catch (error) {
// When concurrent webhook handlers process the same workspace,
// one handler may delete the workspace (cascade-deleting subscriptions)
// while the other is still inserting subscription items.
if (
error instanceof QueryFailedError &&
(error as QueryFailedError & { code?: string }).code ===
POSTGRESQL_ERROR_CODES.FOREIGN_KEY_VIOLATION
) {
return false;
}
throw error;
}
return true;
}
}