change limit

packages/twenty-server/src/database/commands/workspace-export/utils/__tests__/format-copy-value.util.spec.ts

@@ -0,0 +1,71 @@
+import { formatPgCopyField } from 'src/database/commands/workspace-export/utils/format-pg-copy-value.util';
+
+describe('formatPgCopyField', () => {
+  it('should return \\N for null and undefined', () => {
+    expect(formatPgCopyField(null)).toBe('\\N');
+    expect(formatPgCopyField(undefined)).toBe('\\N');
+  });
+
+  it('should return t/f for booleans (not true/false)', () => {
+    expect(formatPgCopyField(true)).toBe('t');
+    expect(formatPgCopyField(false)).toBe('f');
+  });
+
+  it('should return \\N for non-finite numbers', () => {
+    expect(formatPgCopyField(NaN)).toBe('\\N');
+    expect(formatPgCopyField(Infinity)).toBe('\\N');
+  });
+
+  it('should escape tabs, newlines, and backslashes in strings', () => {
+    expect(formatPgCopyField('col1\tcol2')).toBe('col1\\tcol2');
+    expect(formatPgCopyField('line1\nline2')).toBe('line1\\nline2');
+    expect(formatPgCopyField('path\\to\\file')).toBe('path\\\\to\\\\file');
+    expect(formatPgCopyField('a\r\nb')).toBe('a\\r\\nb');
+  });
+
+  it('should not quote strings (COPY format is unquoted)', () => {
+    const result = formatPgCopyField('hello world');
+
+    expect(result).toBe('hello world');
+    expect(result).not.toContain("'");
+  });
+
+  it('should format dates as ISO strings without quotes', () => {
+    const date = new Date('2024-01-15T10:30:00.000Z');
+
+    expect(formatPgCopyField(date)).toBe('2024-01-15T10:30:00.000Z');
+  });
+
+  it('should escape special chars in JSON column values', () => {
+    const value = { key: 'value\twith\ttabs' };
+
+    expect(formatPgCopyField(value, true)).toBe(
+      '{"key":"value\\\\twith\\\\ttabs"}',
+    );
+  });
+
+  it('should format PostgreSQL array literals', () => {
+    expect(formatPgCopyField([])).toBe('{}');
+    expect(formatPgCopyField(['a', 'b'])).toBe('{"a","b"}');
+  });
+
+  it('should JSON-serialize arrays of objects with escaping', () => {
+    const value = [{ id: 1, name: 'test\ttab' }];
+
+    expect(formatPgCopyField(value)).toBe('[{"id":1,"name":"test\\\\ttab"}]');
+  });
+
+  it('should handle a row with mixed types matching COPY tab-delimited format', () => {
+    const fields = [
+      formatPgCopyField('abc-123'),
+      formatPgCopyField(null),
+      formatPgCopyField(true),
+      formatPgCopyField(42),
+      formatPgCopyField(new Date('2025-03-28T00:00:00.000Z')),
+    ];
+
+    expect(fields.join('\t')).toBe(
+      'abc-123\t\\N\tt\t42\t2025-03-28T00:00:00.000Z',
+    );
+  });
+});

packages/twenty-server/src/database/commands/workspace-export/utils/format-pg-copy-value.util.ts

@@ -0,0 +1,55 @@
+import { isDefined } from 'twenty-shared/utils';
+
+const escapeCopyText = (text: string): string => {
+  return text
+    .replace(/\\/g, '\\\\')
+    .replace(/\t/g, '\\t')
+    .replace(/\n/g, '\\n')
+    .replace(/\r/g, '\\r');
+};
+
+export const formatPgCopyField = (
+  value: unknown,
+  isJsonColumn = false,
+): string => {
+  if (!isDefined(value)) return '\\N';
+
+  if (isJsonColumn) return escapeCopyText(JSON.stringify(value));
+
+  if (typeof value === 'boolean') return value ? 't' : 'f';
+
+  if (typeof value === 'number') {
+    if (!Number.isFinite(value)) return '\\N';
+
+    return String(value);
+  }
+
+  if (typeof value === 'bigint') return String(value);
+
+  if (value instanceof Date) return value.toISOString();
+
+  if (Array.isArray(value)) {
+    if (value.length === 0) return '{}';
+
+    if (isDefined(value[0]) && typeof value[0] === 'object') {
+      return escapeCopyText(JSON.stringify(value));
+    }
+
+    const formattedElements = value.map((element) => {
+      if (!isDefined(element)) return 'NULL';
+
+      const stringElement = String(element);
+      const escapedElement = stringElement
+        .replace(/\\/g, '\\\\')
+        .replace(/"/g, '\\"');
+
+      return `"${escapedElement}"`;
+    });
+
+    return `{${formattedElements.join(',')}}`;
+  }
+
+  if (typeof value === 'object') return escapeCopyText(JSON.stringify(value));
+
+  return escapeCopyText(String(value));
+};

packages/twenty-server/src/database/commands/workspace-export/utils/format-sql-value.util.ts

@@ -2,6 +2,7 @@ import { isDefined } from 'twenty-shared/utils';
 
 import { escapeLiteral } from 'src/engine/workspace-manager/workspace-migration/utils/remove-sql-injection.util';
 
+
 export const formatSqlValue = (
   value: unknown,
   isJsonColumn = false,

packages/twenty-server/src/database/commands/workspace-export/workspace-export.service.ts

@@ -21,10 +21,12 @@ import { getCoreEntityMetadatasWithWorkspaceId } from 'src/database/commands/wor
 import { generateWorkspaceSchemaDdl } from 'src/database/commands/workspace-export/utils/generate-workspace-schema-ddl.util';
 import { buildInsertPrefix } from 'src/database/commands/workspace-export/utils/build-insert-prefix.util';
 import { buildWorkspaceTableColumnSets } from 'src/database/commands/workspace-export/utils/build-workspace-table-column-sets.util';
-import { formatSqlValue } from 'src/database/commands/workspace-export/utils/format-sql-value.util';
-import { generateInsertStatement } from 'src/database/commands/workspace-export/utils/generate-insert-statement.util';
+import {
+  formatSqlValue,
+} from 'src/database/commands/workspace-export/utils/format-sql-value.util';
+import { formatPgCopyField } from './utils/format-pg-copy-value.util';
 
-const BATCH_SIZE = 5000;
+const BATCH_SIZE = 10_000;
 
 type WorkspaceExportParams = {
   workspaceId: string;
@@ -174,7 +176,7 @@ export class WorkspaceExportService {
           whereClause: '"workspaceId" = $1',
           queryParameters: [workspaceId],
           jsonColumns: this.buildJsonColumnSet(entityMetadata),
-        });
+          });
       } catch (error) {
         this.logger.warn(`${entityMetadata.tableName}: skipped`, error);
       }
@@ -219,47 +221,103 @@ export class WorkspaceExportService {
     excludedColumns,
   }: WriteRowsOptions): Promise<void> {
     const whereFragment = whereClause ? ` WHERE ${whereClause}` : '';
-
-    const [{ count: totalCount }] = await queryRunner.query(
-      `SELECT COUNT(*)::int as count FROM "${schemaName}"."${tableName}"${whereFragment}`,
-      queryParameters,
-    );
-
-    if (totalCount === 0) return;
-
-    this.logger.log(`  ${displayName}: ${totalCount} rows`);
-
+    let columnNames: string[] | undefined;
     let insertPrefix: string | undefined;
+    let totalRows = 0;
 
-    for (let offset = 0; offset < totalCount; offset += BATCH_SIZE) {
+    for (let offset = 0; ; offset += BATCH_SIZE) {
       const rows: Record<string, unknown>[] = await queryRunner.query(
         `SELECT * FROM "${schemaName}"."${tableName}"${whereFragment} ORDER BY "id" LIMIT ${BATCH_SIZE} OFFSET ${offset}`,
         queryParameters,
       );
 
-      const batchStatements: string[] = [];
+      if (rows.length === 0) break;
 
-      for (const row of rows) {
-        const columnNames = Object.keys(row).filter(
+      if (!columnNames) {
+        columnNames = Object.keys(rows[0]).filter(
           (columnName) => !excludedColumns?.has(columnName),
         );
+        insertPrefix = buildInsertPrefix(schemaName, tableName, columnNames);
+      }
 
-        if (!insertPrefix) {
-          insertPrefix = buildInsertPrefix(schemaName, tableName, columnNames);
-        }
+      totalRows += rows.length;
 
+      const valueTuples: string[] = [];
+
+      for (const row of rows) {
         const formattedValues = columnNames.map((columnName) =>
           formatSqlValue(row[columnName], jsonColumns?.has(columnName)),
         );
 
-        batchStatements.push(
-          generateInsertStatement(insertPrefix, formattedValues),
+        valueTuples.push(`(${formattedValues.join(', ')})`);
+      }
+
+      const statement = `${insertPrefix}${valueTuples.join(', ')};\n`;
+
+      if (!stream.write(statement)) {
+        await once(stream, 'drain');
+      }
+
+      if (rows.length < BATCH_SIZE) break;
+    }
+
+    if (totalRows > 0) {
+      this.logger.log(`  ${displayName}: ${totalRows} rows`);
+    }
+  }
+
+  private async writeCopyRows({
+    schemaName,
+    tableName,
+    displayName,
+    queryRunner,
+    stream,
+    jsonColumns,
+    excludedColumns,
+  }: Omit<WriteRowsOptions, 'onConflictDoNothing'>): Promise<void> {
+    let columnNames: string[] | undefined;
+    let totalRows = 0;
+
+    for (let offset = 0; ; offset += BATCH_SIZE) {
+      const rows: Record<string, unknown>[] = await queryRunner.query(
+        `SELECT * FROM "${schemaName}"."${tableName}" ORDER BY "id" LIMIT ${BATCH_SIZE} OFFSET ${offset}`,
+      );
+
+      if (rows.length === 0) break;
+
+      if (!columnNames) {
+        columnNames = Object.keys(rows[0]).filter(
+          (columnName) => !excludedColumns?.has(columnName),
+        );
+
+        const escapedColumns = columnNames.map(escapeIdentifier).join(', ');
+
+        stream.write(
+          `COPY ${escapeIdentifier(schemaName)}.${escapeIdentifier(tableName)} (${escapedColumns}) FROM stdin;\n`,
         );
       }
 
-      if (!stream.write(batchStatements.join(''))) {
-        await once(stream, 'drain');
+      totalRows += rows.length;
+
+      for (const row of rows) {
+        const values = columnNames.map((columnName) =>
+          formatPgCopyField(row[columnName], jsonColumns?.has(columnName)),
+        );
+
+        if (!stream.write(values.join('\t') + '\n')) {
+          await once(stream, 'drain');
+        }
       }
+
+      if (rows.length < BATCH_SIZE) break;
+    }
+
+    if (columnNames) {
+      stream.write('\\.\n\n');
+    }
+
+    if (totalRows > 0) {
+      this.logger.log(`  ${displayName}: ${totalRows} rows`);
     }
   }
 
@@ -319,7 +377,7 @@ export class WorkspaceExportService {
       );
 
       try {
-        await this.writeRows({
+        await this.writeCopyRows({
           schemaName,
           tableName,
           displayName: objectMetadata.nameSingular,