From deeb8f38e5aa28b4afc0b007ec1d9dd14c300c8d Mon Sep 17 00:00:00 2001 From: Hariom Balhara Date: Tue, 8 Nov 2022 19:51:53 +0530 Subject: [PATCH] Support header in Routing Form CSV (#5133) * Dont actually delete fields and now add headers to CSV which is possible * Fix TS issues * Handle cases due to soft deletion * Add submission time as a column * Fix bug with fields getting deleted Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> Co-authored-by: Peer Richelsen --- .../routing-forms/api/responses/[formId].ts | 49 +++++++++++++------ .../routing-forms/lib/getSerializableForm.ts | 14 +++++- .../playwright/tests/basic.e2e.ts | 21 +++++--- .../app-store/ee/routing-forms/trpc-router.ts | 34 +++++++++++++ .../ee/routing-forms/types/types.d.ts | 1 + packages/app-store/ee/routing-forms/zod.ts | 1 + 6 files changed, 97 insertions(+), 23 deletions(-) diff --git a/packages/app-store/ee/routing-forms/api/responses/[formId].ts b/packages/app-store/ee/routing-forms/api/responses/[formId].ts index 30f9388ead..8688dda8ae 100644 --- a/packages/app-store/ee/routing-forms/api/responses/[formId].ts +++ b/packages/app-store/ee/routing-forms/api/responses/[formId].ts @@ -2,14 +2,21 @@ import type { NextApiRequest, NextApiResponse } from "next"; import prisma from "@calcom/prisma"; -import { Response } from "../../types/types"; +import { getSerializableForm } from "../../lib/getSerializableForm"; +import { Response, SerializableForm } from "../../types/types"; +import { App_RoutingForms_Form } from ".prisma/client"; function escapeCsvText(str: string) { return str.replace(/,/, "%2C"); } -async function* getResponses(formId: string) { +async function* getResponses( + formId: string, + headerFields: NonNullable["fields"]> +) { let responses; let skip = 0; + // Keep it small enough to be in Vercel limits of Serverless Function in terms of memory. + // To avoid limit in terms of execution time there is an RFC https://linear.app/calcom/issue/CAL-204/rfc-routing-form-improved-csv-exports const take = 100; while ( (responses = await prisma.app_RoutingForms_FormResponse.findMany({ @@ -22,25 +29,22 @@ async function* getResponses(formId: string) { responses.length ) { const csv: string[] = []; - // Because fields can be added or removed at any time we can't have fixed columns. - // Because there can be huge amount of data we can't keep all that in memory to identify columns from all the data at once. - // TODO: So, for now add the field label in front of it. It certainly needs improvement. - // TODO: Email CSV when we need to scale it. responses.forEach((response) => { const fieldResponses = response.response as Response; - const csvLineColumns = []; - for (const [, fieldResponse] of Object.entries(fieldResponses)) { - const label = escapeCsvText(fieldResponse.label); - const value = fieldResponse.value; + const csvCells: string[] = []; + headerFields.forEach((headerField) => { + const fieldResponse = fieldResponses[headerField.id]; + const value = fieldResponse?.value || ""; let serializedValue = ""; if (value instanceof Array) { serializedValue = value.map((val) => escapeCsvText(val)).join(" | "); } else { serializedValue = escapeCsvText(value); } - csvLineColumns.push(`"${label} :=> ${serializedValue}"`); - } - csv.push(csvLineColumns.join(",")); + csvCells.push(serializedValue); + }); + csvCells.push(response.createdAt.toISOString()); + csv.push(csvCells.join(",")); }); skip += take; yield csv.join("\n"); @@ -63,13 +67,28 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) id: formId, }, }); + if (!form) { throw new Error("Form not found"); } + const serializableForm = getSerializableForm(form, true); res.setHeader("Content-Type", "text/csv; charset=UTF-8"); - res.setHeader("Content-Disposition", `attachment; filename="${form.name}-${form.id}.csv"`); + res.setHeader( + "Content-Disposition", + `attachment; filename="${serializableForm.name}-${serializableForm.id}.csv"` + ); res.setHeader("Transfer-Encoding", "chunked"); - const csvIterator = getResponses(formId); + const headerFields = serializableForm.fields || []; + const csvIterator = getResponses(formId, headerFields); + + // Make Header + res.write( + headerFields + .map((field) => `${field.label}${field.deleted ? "(Deleted)" : ""}`) + .concat(["Submission Time"]) + .join(",") + "\n" + ); + for await (const partialCsv of csvIterator) { res.write(partialCsv); res.write("\n"); diff --git a/packages/app-store/ee/routing-forms/lib/getSerializableForm.ts b/packages/app-store/ee/routing-forms/lib/getSerializableForm.ts index 6edc814ef8..953148d565 100644 --- a/packages/app-store/ee/routing-forms/lib/getSerializableForm.ts +++ b/packages/app-store/ee/routing-forms/lib/getSerializableForm.ts @@ -5,7 +5,13 @@ import { RoutingFormSettings } from "@calcom/prisma/zod-utils"; import { SerializableForm } from "../types/types"; import { zodFields, zodRoutes } from "../zod"; -export function getSerializableForm(form: TForm) { +/** + * Doesn't have deleted fields by default + */ +export function getSerializableForm( + form: TForm, + withDeletedFields = false +) { const routesParsed = zodRoutes.safeParse(form.routes); if (!routesParsed.success) { throw new Error("Error parsing routes"); @@ -26,7 +32,11 @@ export function getSerializableForm(form: T const serializableForm: SerializableForm = { ...form, settings: settings, - fields: fieldsParsed.data, + fields: fieldsParsed.data + ? withDeletedFields + ? fieldsParsed.data + : fieldsParsed.data.filter((f) => !f.deleted) + : [], routes: routesParsed.data, createdAt: form.createdAt.toString(), updatedAt: form.updatedAt.toString(), diff --git a/packages/app-store/ee/routing-forms/playwright/tests/basic.e2e.ts b/packages/app-store/ee/routing-forms/playwright/tests/basic.e2e.ts index 8190cbbcd5..5d9046927c 100644 --- a/packages/app-store/ee/routing-forms/playwright/tests/basic.e2e.ts +++ b/packages/app-store/ee/routing-forms/playwright/tests/basic.e2e.ts @@ -215,13 +215,22 @@ test.describe("Routing Forms", () => { resolve(body); }); }); + const csvRows = csv.trim().split("\n"); + const csvHeaderRow = csvRows[0]; + expect(csvHeaderRow).toEqual("Test field,Multi Select,Submission Time"); - expect(csv.trim()).toEqual( - ` -"Test field :=> event-routing" -"Test field :=> external-redirect" -"Test field :=> custom-page"`.trim() - ); + const firstResponseCells = csvRows[1].split(","); + const secondResponseCells = csvRows[2].split(","); + const thirdResponseCells = csvRows[3].split(","); + + expect(firstResponseCells.slice(0, -1).join(",")).toEqual("event-routing,"); + expect(new Date(firstResponseCells.at(-1)).getDay()).toEqual(new Date().getDay()); + + expect(secondResponseCells.slice(0, -1).join(",")).toEqual("external-redirect,"); + expect(new Date(secondResponseCells.at(-1)).getDay()).toEqual(new Date().getDay()); + + expect(thirdResponseCells.slice(0, -1).join(",")).toEqual("custom-page,"); + expect(new Date(thirdResponseCells.at(-1)).getDay()).toEqual(new Date().getDay()); }); test("Router URL should work", async ({ page, users }) => { diff --git a/packages/app-store/ee/routing-forms/trpc-router.ts b/packages/app-store/ee/routing-forms/trpc-router.ts index af454ab9a3..eca9f68d10 100644 --- a/packages/app-store/ee/routing-forms/trpc-router.ts +++ b/packages/app-store/ee/routing-forms/trpc-router.ts @@ -277,6 +277,40 @@ const app_RoutingForms = createRouter() fields = fields || []; + const form = await prisma.app_RoutingForms_Form.findUnique({ + where: { + id: id, + }, + select: { + id: true, + user: true, + name: true, + description: true, + userId: true, + disabled: true, + createdAt: true, + updatedAt: true, + routes: true, + fields: true, + settings: true, + }, + }); + + // Add back deleted fields in the end. Fields can't be deleted, to make sure columns never decrease which hugely simplifies CSV generation + if (form) { + const serializedForm = getSerializableForm(form, true); + // Find all fields that are in DB(including deleted) but not in the mutation + const deletedFields = + serializedForm.fields?.filter((f) => !fields!.find((field) => field.id === f.id)) || []; + + fields = fields.concat( + deletedFields.map((f) => { + f.deleted = true; + return f; + }) + ); + } + if (addFallback) { const uuid = uuidv4(); routes = routes || []; diff --git a/packages/app-store/ee/routing-forms/types/types.d.ts b/packages/app-store/ee/routing-forms/types/types.d.ts index 33b97c8cb7..e547788f49 100644 --- a/packages/app-store/ee/routing-forms/types/types.d.ts +++ b/packages/app-store/ee/routing-forms/types/types.d.ts @@ -6,6 +6,7 @@ import { RoutingFormSettings } from "@calcom/prisma/zod-utils"; import { zodFields, zodRoutes } from "../zod"; export type Response = Record< + // Field ID string, { value: string | string[]; diff --git a/packages/app-store/ee/routing-forms/zod.ts b/packages/app-store/ee/routing-forms/zod.ts index 4a84519773..7f54dc0aa2 100644 --- a/packages/app-store/ee/routing-forms/zod.ts +++ b/packages/app-store/ee/routing-forms/zod.ts @@ -9,6 +9,7 @@ export const zodFields = z type: z.string(), selectText: z.string().optional(), required: z.boolean().optional(), + deleted: z.boolean().optional(), }) ) .optional();