fix(file-storage): add adaptive retry and backoff for S3 rate-limit errors

https://sonarly.com/issue/32646?type=bug The workflow worker fails to upload built logic function code to S3 when AWS returns a 503 SlowDown rate-limit error, causing workflow runs to fail. The S3 client has no custom retry configuration and the default 3-attempt retry with standard backoff is insufficient during request bursts. Fix: packages/twenty-server/src/engine/core-modules/file-storage/drivers/s3.driver.ts Add `maxAttempts: 5` and `retryMode: 'adaptive'` to both S3 client instantiations in the constructor (main client and presign client). The adaptive retry mode uses a client-side token bucket to dynamically reduce request rate when S3 returns 503 SlowDown responses, preventing cascading failures during high-concurrency worker bursts. The increased maxAttempts (from default 3 to 5) gives the adaptive backoff strategy sufficient headroom to succeed on transient rate limits. Constructor: S3 client now created with `{ ...s3Options, region, endpoint, maxAttempts: 5, retryMode: 'adaptive' }` instead of `{ ...s3Options, region, endpoint }`. Same change applied to the presign client when presignEndpoint is provided.
2026-04-29 19:46:51 +00:00
1 changed files with 22 additions and 3 deletions
@@ -57,12 +57,24 @@ export class S3Driver implements StorageDriver {
      return;
    }

-    this.s3Client = new S3({ ...s3Options, region, endpoint });
+    this.s3Client = new S3({
+      ...s3Options,
+      region,
+      endpoint,
+      maxAttempts: 5,
+      retryMode: 'adaptive',
+    });
    this.bucketName = bucketName;

    if (presignEnabled) {
      this.presignClient = presignEndpoint
-        ? new S3({ ...s3Options, region, endpoint: presignEndpoint })
+        ? new S3({
+            ...s3Options,
+            region,
+            endpoint: presignEndpoint,
+            maxAttempts: 5,
+            retryMode: 'adaptive',
+          })
        : this.s3Client;
    }
  }
@@ -109,7 +121,14 @@ export class S3Driver implements StorageDriver {
      Bucket: this.bucketName,
    });

-    await this.s3Client.send(command);
+    try {
+      await this.s3Client.send(command);
+    } catch (error) {
+      this.logger.error(
+        `S3 writeFile failed for key "${params.filePath}": ${error.name ?? 'UnknownError'} - ${error.message}`,
+      );
+      throw error;
+    }
  }

  private async createFolder(path: string) {