Support header in Routing Form CSV (#5133)

* Dont actually delete fields and now add headers to CSV which is possible

* Fix TS issues

* Handle cases due to soft deletion

* Add submission time as a column

* Fix bug with fields getting deleted

Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>
Co-authored-by: Peer Richelsen <peeroke@gmail.com>

packages/app-store/ee/routing-forms/api/responses/[formId].ts

@@ -2,14 +2,21 @@ import type { NextApiRequest, NextApiResponse } from "next";
 
 import prisma from "@calcom/prisma";
 
-import { Response } from "../../types/types";
+import { getSerializableForm } from "../../lib/getSerializableForm";
+import { Response, SerializableForm } from "../../types/types";
+import { App_RoutingForms_Form } from ".prisma/client";
 
 function escapeCsvText(str: string) {
   return str.replace(/,/, "%2C");
 }
-async function* getResponses(formId: string) {
+async function* getResponses(
+  formId: string,
+  headerFields: NonNullable<SerializableForm<App_RoutingForms_Form>["fields"]>
+) {
   let responses;
   let skip = 0;
+  // Keep it small enough to be in Vercel limits of Serverless Function in terms of memory.
+  // To avoid limit in terms of execution time there is an RFC https://linear.app/calcom/issue/CAL-204/rfc-routing-form-improved-csv-exports
   const take = 100;
   while (
     (responses = await prisma.app_RoutingForms_FormResponse.findMany({
@@ -22,25 +29,22 @@ async function* getResponses(formId: string) {
     responses.length
   ) {
     const csv: string[] = [];
-    // Because fields can be added or removed at any time we can't have fixed columns.
-    // Because there can be huge amount of data we can't keep all that in memory to identify columns from all the data at once.
-    // TODO: So, for now add the field label in front of it. It certainly needs improvement.
-    // TODO: Email CSV when we need to scale it.
     responses.forEach((response) => {
       const fieldResponses = response.response as Response;
-      const csvLineColumns = [];
-      for (const [, fieldResponse] of Object.entries(fieldResponses)) {
-        const label = escapeCsvText(fieldResponse.label);
-        const value = fieldResponse.value;
+      const csvCells: string[] = [];
+      headerFields.forEach((headerField) => {
+        const fieldResponse = fieldResponses[headerField.id];
+        const value = fieldResponse?.value || "";
         let serializedValue = "";
         if (value instanceof Array) {
           serializedValue = value.map((val) => escapeCsvText(val)).join(" | ");
         } else {
           serializedValue = escapeCsvText(value);
         }
-        csvLineColumns.push(`"${label} :=> ${serializedValue}"`);
-      }
-      csv.push(csvLineColumns.join(","));
+        csvCells.push(serializedValue);
+      });
+      csvCells.push(response.createdAt.toISOString());
+      csv.push(csvCells.join(","));
     });
     skip += take;
     yield csv.join("\n");
@@ -63,13 +67,28 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       id: formId,
     },
   });
+
   if (!form) {
     throw new Error("Form not found");
   }
+  const serializableForm = getSerializableForm(form, true);
   res.setHeader("Content-Type", "text/csv; charset=UTF-8");
-  res.setHeader("Content-Disposition", `attachment; filename="${form.name}-${form.id}.csv"`);
+  res.setHeader(
+    "Content-Disposition",
+    `attachment; filename="${serializableForm.name}-${serializableForm.id}.csv"`
+  );
   res.setHeader("Transfer-Encoding", "chunked");
-  const csvIterator = getResponses(formId);
+  const headerFields = serializableForm.fields || [];
+  const csvIterator = getResponses(formId, headerFields);
+
+  // Make Header
+  res.write(
+    headerFields
+      .map((field) => `${field.label}${field.deleted ? "(Deleted)" : ""}`)
+      .concat(["Submission Time"])
+      .join(",") + "\n"
+  );
+
   for await (const partialCsv of csvIterator) {
     res.write(partialCsv);
     res.write("\n");

packages/app-store/ee/routing-forms/lib/getSerializableForm.ts

@@ -5,7 +5,13 @@ import { RoutingFormSettings } from "@calcom/prisma/zod-utils";
 import { SerializableForm } from "../types/types";
 import { zodFields, zodRoutes } from "../zod";
 
-export function getSerializableForm<TForm extends App_RoutingForms_Form>(form: TForm) {
+/**
+ * Doesn't have deleted fields by default
+ */
+export function getSerializableForm<TForm extends App_RoutingForms_Form>(
+  form: TForm,
+  withDeletedFields = false
+) {
   const routesParsed = zodRoutes.safeParse(form.routes);
   if (!routesParsed.success) {
     throw new Error("Error parsing routes");
@@ -26,7 +32,11 @@ export function getSerializableForm<TForm extends App_RoutingForms_Form>(form: T
   const serializableForm: SerializableForm<TForm> = {
     ...form,
     settings: settings,
-    fields: fieldsParsed.data,
+    fields: fieldsParsed.data
+      ? withDeletedFields
+        ? fieldsParsed.data
+        : fieldsParsed.data.filter((f) => !f.deleted)
+      : [],
     routes: routesParsed.data,
     createdAt: form.createdAt.toString(),
     updatedAt: form.updatedAt.toString(),

packages/app-store/ee/routing-forms/playwright/tests/basic.e2e.ts

@@ -215,13 +215,22 @@ test.describe("Routing Forms", () => {
           resolve(body);
         });
       });
+      const csvRows = csv.trim().split("\n");
+      const csvHeaderRow = csvRows[0];
+      expect(csvHeaderRow).toEqual("Test field,Multi Select,Submission Time");
 
-      expect(csv.trim()).toEqual(
-        `
-"Test field :=> event-routing"
-"Test field :=> external-redirect"
-"Test field :=> custom-page"`.trim()
-      );
+      const firstResponseCells = csvRows[1].split(",");
+      const secondResponseCells = csvRows[2].split(",");
+      const thirdResponseCells = csvRows[3].split(",");
+
+      expect(firstResponseCells.slice(0, -1).join(",")).toEqual("event-routing,");
+      expect(new Date(firstResponseCells.at(-1)).getDay()).toEqual(new Date().getDay());
+
+      expect(secondResponseCells.slice(0, -1).join(",")).toEqual("external-redirect,");
+      expect(new Date(secondResponseCells.at(-1)).getDay()).toEqual(new Date().getDay());
+
+      expect(thirdResponseCells.slice(0, -1).join(",")).toEqual("custom-page,");
+      expect(new Date(thirdResponseCells.at(-1)).getDay()).toEqual(new Date().getDay());
     });
 
     test("Router URL should work", async ({ page, users }) => {

packages/app-store/ee/routing-forms/trpc-router.ts

@@ -277,6 +277,40 @@ const app_RoutingForms = createRouter()
 
           fields = fields || [];
 
+          const form = await prisma.app_RoutingForms_Form.findUnique({
+            where: {
+              id: id,
+            },
+            select: {
+              id: true,
+              user: true,
+              name: true,
+              description: true,
+              userId: true,
+              disabled: true,
+              createdAt: true,
+              updatedAt: true,
+              routes: true,
+              fields: true,
+              settings: true,
+            },
+          });
+
+          // Add back deleted fields in the end. Fields can't be deleted, to make sure columns never decrease which hugely simplifies CSV generation
+          if (form) {
+            const serializedForm = getSerializableForm(form, true);
+            // Find all fields that are in DB(including deleted) but not in the mutation
+            const deletedFields =
+              serializedForm.fields?.filter((f) => !fields!.find((field) => field.id === f.id)) || [];
+
+            fields = fields.concat(
+              deletedFields.map((f) => {
+                f.deleted = true;
+                return f;
+              })
+            );
+          }
+
           if (addFallback) {
             const uuid = uuidv4();
             routes = routes || [];

packages/app-store/ee/routing-forms/types/types.d.ts

@@ -6,6 +6,7 @@ import { RoutingFormSettings } from "@calcom/prisma/zod-utils";
 import { zodFields, zodRoutes } from "../zod";
 
 export type Response = Record<
+  // Field ID
   string,
   {
     value: string | string[];

packages/app-store/ee/routing-forms/zod.ts

@@ -9,6 +9,7 @@ export const zodFields = z
       type: z.string(),
       selectText: z.string().optional(),
       required: z.boolean().optional(),
+      deleted: z.boolean().optional(),
     })
   )
   .optional();