fix: Flakiness in tests (#12929)

apps/web/playwright/booking-pages.e2e.ts

@@ -29,7 +29,10 @@ test.describe("free user", () => {
 
   test("cannot book same slot multiple times", async ({ page, users, emails }) => {
     const [user] = users.get();
-    const bookerObj = { email: `testEmail-${randomString(4)}@example.com`, name: "testBooker" };
+    const bookerObj = {
+      email: users.trackEmail({ username: "testEmail", domain: "example.com" }),
+      name: "testBooker",
+    };
     // Click first event type
     await page.click('[data-testid="event-type-link"]');
 

apps/web/playwright/fixtures/emails.ts

@@ -0,0 +1,34 @@
+import mailhog from "mailhog";
+
+import { IS_MAILHOG_ENABLED } from "@calcom/lib/constants";
+
+const unimplemented = () => {
+  throw new Error("Mailhog is not enabled");
+};
+
+const hasUUID = (query: string) => {
+  return /[a-zA-Z0-9]{22}/.test(query) || /[0-9a-f]{8}/.test(query);
+};
+export const createEmailsFixture = () => {
+  if (IS_MAILHOG_ENABLED) {
+    const mailhogAPI = mailhog();
+    return {
+      search: (query: string, kind?: string, start?: number, limit?: number) => {
+        if (kind === "from" || kind === "to") {
+          if (!hasUUID(query)) {
+            throw new Error(
+              `You should not use "from" or "to" queries without UUID in emails. Because mailhog maintains all the emails sent through tests, you should be able to uniquely identify the email among those. Found query: ${query}`
+            );
+          }
+        }
+        return mailhogAPI.search.bind(mailhogAPI)(query, kind, start, limit);
+      },
+      deleteMessage: mailhogAPI.deleteMessage.bind(mailhogAPI),
+    };
+  } else {
+    return {
+      search: unimplemented,
+      deleteMessage: unimplemented,
+    };
+  }
+};

apps/web/playwright/fixtures/users.ts

@@ -1,8 +1,9 @@
 import type { Page, WorkerInfo } from "@playwright/test";
 import type Prisma from "@prisma/client";
+import type { Team } from "@prisma/client";
 import { Prisma as PrismaType } from "@prisma/client";
 import { hashSync as hash } from "bcryptjs";
-import type { API } from "mailhog";
+import { uuid } from "short-uuid";
 
 import stripe from "@calcom/features/ee/payments/server/stripe";
 import { DEFAULT_SCHEDULE, getAvailabilityFromSchedule } from "@calcom/lib/availability";
@@ -13,6 +14,7 @@ import { teamMetadataSchema } from "@calcom/prisma/zod-utils";
 import type { Schedule } from "@calcom/types/schedule";
 
 import { selectFirstAvailableTimeSlotNextMonth, teamEventSlug, teamEventTitle } from "../lib/testUtils";
+import type { createEmailsFixture } from "./emails";
 import { TimeZoneEnum } from "./types";
 
 // Don't import hashPassword from app as that ends up importing next-auth and initializing it before NEXTAUTH_URL can be updated during tests.
@@ -101,7 +103,7 @@ const createTeamAndAddUser = async (
 ) => {
   const slug = `${isOrg ? "org" : "team"}-${workerInfo.workerIndex}-${Date.now()}`;
   const data: PrismaType.TeamCreateInput = {
-    name: `user-id-${user.id}'s Team ${isOrg ? "Org" : "Team"}`,
+    name: `user-id-${user.id}'s ${isOrg ? "Org" : "Team"}`,
   };
   data.metadata = {
     ...(isUnpublished ? { requestedSlug: slug } : {}),
@@ -140,8 +142,17 @@ const createTeamAndAddUser = async (
 };
 
 // creates a user fixture instance and stores the collection
-export const createUsersFixture = (page: Page, emails: API | undefined, workerInfo: WorkerInfo) => {
-  const store = { users: [], page } as { users: UserFixture[]; page: typeof page };
+export const createUsersFixture = (
+  page: Page,
+  emails: ReturnType<typeof createEmailsFixture>,
+  workerInfo: WorkerInfo
+) => {
+  const store = { users: [], trackedEmails: [], page, teams: [] } as {
+    users: UserFixture[];
+    trackedEmails: { email: string }[];
+    page: typeof page;
+    teams: Team[];
+  };
   return {
     buildForSignup: (opts?: Pick<CustomUserOpts, "email" | "username" | "useExactUsername" | "password">) => {
       const uname =
@@ -322,6 +333,7 @@ export const createUsersFixture = (page: Page, emails: API | undefined, workerIn
           },
           workerInfo
         );
+        store.teams.push(team);
         const teamEvent = await createTeamEventType(user, team, scenario);
         if (scenario.teammates) {
           // Create Teammate users
@@ -379,6 +391,16 @@ export const createUsersFixture = (page: Page, emails: API | undefined, workerIn
       store.users.push(userFixture);
       return userFixture;
     },
+    /**
+     * Use this method to get an email that can be automatically cleaned up from all the places in DB
+     */
+    trackEmail: ({ username, domain }: { username: string; domain: string }) => {
+      const email = `${username}-${uuid().substring(0, 8)}@${domain}`;
+      store.trackedEmails.push({
+        email,
+      });
+      return email;
+    },
     get: () => store.users,
     logout: async () => {
       await page.goto("/auth/logout");
@@ -387,7 +409,7 @@ export const createUsersFixture = (page: Page, emails: API | undefined, workerIn
       const ids = store.users.map((u) => u.id);
       if (emails) {
         const emailMessageIds: string[] = [];
-        for (const user of store.users) {
+        for (const user of store.trackedEmails.concat(store.users.map((u) => ({ email: u.email })))) {
           const emailMessages = await emails.search(user.email);
           if (emailMessages && emailMessages.count > 0) {
             emailMessages.items.forEach((item) => {
@@ -401,7 +423,12 @@ export const createUsersFixture = (page: Page, emails: API | undefined, workerIn
       }
 
       await prisma.user.deleteMany({ where: { id: { in: ids } } });
+      // Delete all users that were tracked by email(if they were created)
+      await prisma.user.deleteMany({ where: { email: { in: store.trackedEmails.map((e) => e.email) } } });
+      await prisma.team.deleteMany({ where: { id: { in: store.teams.map((org) => org.id) } } });
       store.users = [];
+      store.teams = [];
+      store.trackedEmails = [];
     },
     delete: async (id: number) => {
       await prisma.user.delete({ where: { id } });

apps/web/playwright/lib/fixtures.ts

@@ -1,14 +1,11 @@
 import type { Page } from "@playwright/test";
 import { test as base } from "@playwright/test";
-import type { API } from "mailhog";
-import mailhog from "mailhog";
 
-import { IS_MAILHOG_ENABLED } from "@calcom/lib/constants";
-import logger from "@calcom/lib/logger";
 import prisma from "@calcom/prisma";
 
 import type { ExpectedUrlDetails } from "../../../../playwright.config";
 import { createBookingsFixture } from "../fixtures/bookings";
+import { createEmailsFixture } from "../fixtures/emails";
 import { createEmbedsFixture } from "../fixtures/embeds";
 import { createFeatureFixture } from "../fixtures/features";
 import { createOrgsFixture } from "../fixtures/orgs";
@@ -27,7 +24,7 @@ export interface Fixtures {
   embeds: ReturnType<typeof createEmbedsFixture>;
   servers: ReturnType<typeof createServersFixture>;
   prisma: typeof prisma;
-  emails?: API;
+  emails: ReturnType<typeof createEmailsFixture>;
   routingForms: ReturnType<typeof createRoutingFormsFixture>;
   bookingPage: ReturnType<typeof createBookingPageFixture>;
   features: ReturnType<typeof createFeatureFixture>;
@@ -84,14 +81,7 @@ export const test = base.extend<Fixtures>({
     await use(createRoutingFormsFixture());
   },
   emails: async ({}, use) => {
-    if (IS_MAILHOG_ENABLED) {
-      const mailhogAPI = mailhog();
-      await use(mailhogAPI);
-    } else {
-      //FIXME: Ideally we should error out here. If someone is running tests with mailhog disabled, they should be aware of it
-      logger.warn("Mailhog is not enabled - Skipping Emails verification");
-      await use(undefined);
-    }
+    await use(createEmailsFixture());
   },
   bookingPage: async ({ page }, use) => {
     const bookingPage = createBookingPageFixture(page);

apps/web/playwright/lib/testUtils.ts

@@ -6,13 +6,14 @@ import type { IncomingMessage, ServerResponse } from "http";
 import { createServer } from "http";
 // eslint-disable-next-line no-restricted-imports
 import { noop } from "lodash";
-import type { API, Messages } from "mailhog";
+import type { Messages } from "mailhog";
 import { totp } from "otplib";
 
 import type { Prisma } from "@calcom/prisma/client";
 import { BookingStatus } from "@calcom/prisma/enums";
 import type { IntervalLimit } from "@calcom/types/Calendar";
 
+import type { createEmailsFixture } from "../fixtures/emails";
 import type { Fixtures } from "./fixtures";
 import { test } from "./fixtures";
 
@@ -218,11 +219,15 @@ export async function getEmailsReceivedByUser({
   emails,
   userEmail,
 }: {
-  emails?: API;
+  emails?: ReturnType<typeof createEmailsFixture>;
   userEmail: string;
 }): Promise<Messages | null> {
   if (!emails) return null;
-  return emails.search(userEmail, "to");
+  const matchingEmails = await emails.search(userEmail, "to");
+  if (!matchingEmails?.total) {
+    console.log(`No emails received by ${userEmail}`);
+  }
+  return matchingEmails;
 }
 
 export async function expectEmailsToHaveSubject({
@@ -231,7 +236,7 @@ export async function expectEmailsToHaveSubject({
   booker,
   eventTitle,
 }: {
-  emails?: API;
+  emails?: ReturnType<typeof createEmailsFixture>;
   organizer: { name?: string | null; email: string };
   booker: { name: string; email: string };
   eventTitle: string;

apps/web/playwright/oauth-provider.e2e.ts

@@ -104,7 +104,7 @@ test.describe("OAuth Provider", () => {
     expect(meData.username.startsWith("test user")).toBe(true);
   });
 
-  test("should create valid access toke & refresh token for team", async ({ page, users }) => {
+  test("should create valid access token & refresh token for team", async ({ page, users }) => {
     const user = await users.create({ username: "test user", name: "test user" }, { hasTeam: true });
     await user.apiLogin();
 
@@ -157,8 +157,8 @@ test.describe("OAuth Provider", () => {
 
     const meData = await meResponse.json();
 
-    // check if team access token is valid
-    expect(meData.username.endsWith("Team Team")).toBe(true);
+    // Check if team access token is valid
+    expect(meData.username).toEqual(`user-id-${user.id}'s Team`);
 
     // request new token with refresh token
     const refreshTokenResponse = await fetch(`${WEBAPP_URL}/api/auth/oauth/refreshToken`, {
@@ -186,7 +186,7 @@ test.describe("OAuth Provider", () => {
       },
     });
 
-    expect(meData.username.endsWith("Team Team")).toBe(true);
+    expect(meData.username).toEqual(`user-id-${user.id}'s Team`);
   });
 
   test("redirect not logged-in users to login page and after forward to authorization page", async ({

apps/web/playwright/organization/across-org/across-org.e2e.ts

@@ -5,9 +5,8 @@ import prisma from "@calcom/prisma";
 
 import { test } from "../../lib/fixtures";
 
-test.afterAll(({ users, emails }) => {
+test.afterAll(({ users }) => {
   users.deleteAll();
-  emails?.deleteAll();
 });
 
 test.describe("user1NotMemberOfOrg1 is part of team1MemberOfOrg1", () => {

apps/web/playwright/organization/expects.ts

@@ -2,13 +2,14 @@ import type { Page } from "@playwright/test";
 import { expect } from "@playwright/test";
 import { JSDOM } from "jsdom";
 // eslint-disable-next-line no-restricted-imports
-import type { API, Messages } from "mailhog";
+import type { Messages } from "mailhog";
+import type { createEmailsFixture } from "playwright/fixtures/emails";
 
 import { getEmailsReceivedByUser } from "../lib/testUtils";
 
 export async function expectInvitationEmailToBeReceived(
   page: Page,
-  emails: API | undefined,
+  emails: ReturnType<typeof createEmailsFixture>,
   userEmail: string,
   subject: string,
   returnLink?: string
@@ -16,7 +17,7 @@ export async function expectInvitationEmailToBeReceived(
   if (!emails) return null;
   // We need to wait for the email to go through, otherwise it will fail
   // eslint-disable-next-line playwright/no-wait-for-timeout
-  await page.waitForTimeout(5000);
+  await page.waitForTimeout(2000);
   const receivedEmails = await getEmailsReceivedByUser({ emails, userEmail });
   expect(receivedEmails?.total).toBe(1);
   const [firstReceivedEmail] = (receivedEmails as Messages).items;

apps/web/playwright/organization/organization-creation.e2e.ts

@@ -1,13 +1,14 @@
 import { expect } from "@playwright/test";
 import path from "path";
+import { uuid } from "short-uuid";
 
 import { test } from "../lib/fixtures";
 import { generateTotpCode } from "../lib/testUtils";
 import { expectInvitationEmailToBeReceived } from "./expects";
 
-test.afterAll(({ users, emails }) => {
+test.afterAll(({ users, orgs }) => {
   users.deleteAll();
-  emails?.deleteAll();
+  orgs.deleteAll();
 });
 
 function capitalize(text: string) {
@@ -26,6 +27,11 @@ test.describe("Organization", () => {
     const orgOwner = await users.create({
       role: "ADMIN",
     });
+    const instanceAdmin = await users.create({
+      username: `admin-${uuid()}`,
+      email: users.trackEmail({ username: "admin", domain: "example.com" }),
+      role: "ADMIN",
+    });
     const orgDomain = `${orgOwner.username}-org`;
     const orgName = capitalize(`${orgOwner.username}-org`);
     await orgOwner.apiLogin();
@@ -38,7 +44,8 @@ test.describe("Organization", () => {
       await expect(page.locator(".text-red-700")).toHaveCount(3);
 
       // Happy path
-      await page.locator("input[name=adminEmail]").fill(`john@${orgDomain}.com`);
+      const adminEmail = users.trackEmail({ username: "john", domain: `${orgDomain}.com` });
+      await page.locator("input[name=adminEmail]").fill(adminEmail);
       expect(await page.locator("input[name=name]").inputValue()).toEqual(orgName);
       expect(await page.locator("input[name=slug]").inputValue()).toEqual(orgDomain);
       await page.locator("button[type=submit]").click();
@@ -48,7 +55,7 @@ test.describe("Organization", () => {
       await expectInvitationEmailToBeReceived(
         page,
         emails,
-        `john@${orgOwner.username}-org.com`,
+        adminEmail,
         "Verify your email to create an organization"
       );
 
@@ -56,12 +63,11 @@ test.describe("Organization", () => {
         // Code verification
         await expect(page.locator("#modal-title")).toBeVisible();
         await page.locator("input[name='2fa1']").fill(generateTotpCode(`john@${orgDomain}.com`));
-
         // Check admin email about DNS pending action
         await expectInvitationEmailToBeReceived(
           page,
           emails,
-          "admin@example.com",
+          instanceAdmin.email,
           "New organization created: pending action"
         );
 
@@ -105,14 +111,15 @@ test.describe("Organization", () => {
       await page.locator("button[type=submit]").click();
 
       // Happy path
-      await page.locator('textarea[name="emails"]').fill(`rick@${orgDomain}.com`);
+      const adminEmail = users.trackEmail({ username: "rick", domain: `${orgDomain}.com` });
+      await page.locator('textarea[name="emails"]').fill(adminEmail);
       await page.locator("button[type=submit]").click();
 
       // Check if invited admin received the invitation email
       await expectInvitationEmailToBeReceived(
         page,
         emails,
-        `rick@${orgDomain}.com`,
+        adminEmail,
         `${orgName}'s admin invited you to join the organization ${orgName} on Cal.com`
       );
 

apps/web/playwright/organization/organization-invitation.e2e.ts

@@ -9,12 +9,12 @@ import { expectInvitationEmailToBeReceived } from "./expects";
 
 test.describe.configure({ mode: "parallel" });
 
-test.afterEach(async ({ users, emails }) => {
+test.afterEach(async ({ users, orgs }) => {
   await users.deleteAll();
-  emails?.deleteAll();
+  await orgs.deleteAll();
 });
 
-test.describe.serial("Organization", () => {
+test.describe("Organization", () => {
   test.describe("Email not matching orgAutoAcceptEmail", () => {
     test("Org Invitation", async ({ browser, page, users, emails }) => {
       const orgOwner = await users.create(undefined, { hasTeam: true, isOrg: true });
@@ -24,9 +24,10 @@ test.describe.serial("Organization", () => {
       await page.waitForLoadState("networkidle");
 
       await test.step("By email", async () => {
-        const invitedUserEmail = `rick-${Date.now()}@domain.com`;
+        const invitedUserEmail = users.trackEmail({ username: "rick", domain: "domain.com" });
         // '-domain' because the email doesn't match orgAutoAcceptEmail
         const usernameDerivedFromEmail = `${invitedUserEmail.split("@")[0]}-domain`;
+
         await inviteAnEmail(page, invitedUserEmail);
         const inviteLink = await expectInvitationEmailToBeReceived(
           page,
@@ -66,7 +67,7 @@ test.describe.serial("Organization", () => {
 
       await test.step("By invite link", async () => {
         const inviteLink = await copyInviteLink(page);
-        const email = `rick-${Date.now()}@domain.com`;
+        const email = users.trackEmail({ username: "rick", domain: "domain.com" });
         // '-domain' because the email doesn't match orgAutoAcceptEmail
         const usernameDerivedFromEmail = `${email.split("@")[0]}-domain`;
         await signupFromInviteLink({ browser, inviteLink, email });
@@ -91,7 +92,7 @@ test.describe.serial("Organization", () => {
       await test.step("By email", async () => {
         await page.goto(`/settings/teams/${team.id}/members`);
         await page.waitForLoadState("networkidle");
-        const invitedUserEmail = `rick-${Date.now()}@domain.com`;
+        const invitedUserEmail = users.trackEmail({ username: "rick", domain: "domain.com" });
         // '-domain' because the email doesn't match orgAutoAcceptEmail
         const usernameDerivedFromEmail = `${invitedUserEmail.split("@")[0]}-domain`;
         await inviteAnEmail(page, invitedUserEmail);
@@ -154,7 +155,7 @@ test.describe.serial("Organization", () => {
       await test.step("By invite link", async () => {
         await page.goto(`/settings/teams/${team.id}/members`);
         const inviteLink = await copyInviteLink(page);
-        const email = `rick-${Date.now()}@domain.com`;
+        const email = users.trackEmail({ username: "rick", domain: "domain.com" });
         // '-domain' because the email doesn't match orgAutoAcceptEmail
         const usernameDerivedFromEmail = `${email.split("@")[0]}-domain`;
         await signupFromInviteLink({ browser, inviteLink, email });
@@ -190,7 +191,7 @@ test.describe.serial("Organization", () => {
       await page.waitForLoadState("networkidle");
 
       await test.step("By email", async () => {
-        const invitedUserEmail = `rick-${Date.now()}@example.com`;
+        const invitedUserEmail = users.trackEmail({ username: "rick", domain: "example.com" });
         const usernameDerivedFromEmail = invitedUserEmail.split("@")[0];
         await inviteAnEmail(page, invitedUserEmail);
         const inviteLink = await expectInvitationEmailToBeReceived(
@@ -231,7 +232,7 @@ test.describe.serial("Organization", () => {
 
       await test.step("By invite link", async () => {
         const inviteLink = await copyInviteLink(page);
-        const email = `rick-${Date.now()}@example.com`;
+        const email = users.trackEmail({ username: "rick", domain: "example.com" });
         const usernameDerivedFromEmail = email.split("@")[0];
         await signupFromInviteLink({ browser, inviteLink, email });
 
@@ -262,7 +263,7 @@ test.describe.serial("Organization", () => {
       await test.step("By email", async () => {
         await page.goto(`/settings/teams/${team.id}/members`);
         await page.waitForLoadState("networkidle");
-        const invitedUserEmail = `rick-${Date.now()}@example.com`;
+        const invitedUserEmail = users.trackEmail({ username: "rick", domain: "example.com" });
         const usernameDerivedFromEmail = invitedUserEmail.split("@")[0];
         await inviteAnEmail(page, invitedUserEmail);
         await expectUserToBeAMemberOfTeam({
@@ -323,7 +324,7 @@ test.describe.serial("Organization", () => {
         await page.goto(`/settings/teams/${team.id}/members`);
 
         const inviteLink = await copyInviteLink(page);
-        const email = `rick-${Date.now()}@example.com`;
+        const email = users.trackEmail({ username: "rick", domain: "example.com" });
         // '-domain' because the email doesn't match orgAutoAcceptEmail
         const usernameDerivedFromEmail = `${email.split("@")[0]}`;
 
@@ -460,6 +461,7 @@ async function expectUserToBeAMemberOfTeam({
 }) {
   // Check newly invited member is not pending anymore
   await page.goto(`/settings/teams/${teamId}/members`);
+  await page.reload();
   expect(
     (
       await page.locator(`[data-testid="member-${username}"] [data-testid=member-role]`).textContent()

apps/web/playwright/signup.e2e.ts

@@ -14,9 +14,8 @@ test.describe("Signup Flow Test", async () => {
   test.beforeEach(async ({ features }) => {
     features.reset(); // This resets to the inital state not an empt yarray
   });
-  test.afterAll(async ({ users, emails }) => {
+  test.afterAll(async ({ users }) => {
     await users.deleteAll();
-    emails?.deleteAll();
   });
   test("Username is taken", async ({ page, users }) => {
     // log in trail user
@@ -204,6 +203,7 @@ test.describe("Signup Flow Test", async () => {
       data: { enabled: true },
     });
     const userToCreate = users.buildForSignup({
+      email: users.trackEmail({ username: "email-verify", domain: "example.com" }),
       username: "email-verify",
       password: "Password99!",
     });

apps/web/playwright/team/expects.ts

@@ -1,13 +1,14 @@
 import type { Page } from "@playwright/test";
 import { expect } from "@playwright/test";
 import { JSDOM } from "jsdom";
-import type { API, Messages } from "mailhog";
+import type { Messages } from "mailhog";
+import type { createEmailsFixture } from "playwright/fixtures/emails";
 
 import { getEmailsReceivedByUser } from "../lib/testUtils";
 
 export async function expectInvitationEmailToBeReceived(
   page: Page,
-  emails: API | undefined,
+  emails: ReturnType<typeof createEmailsFixture>,
   userEmail: string,
   subject: string,
   returnLink?: string
@@ -15,7 +16,7 @@ export async function expectInvitationEmailToBeReceived(
   if (!emails) return null;
 
   // eslint-disable-next-line playwright/no-wait-for-timeout
-  await page.waitForTimeout(10000);
+  await page.waitForTimeout(2000);
   const receivedEmails = await getEmailsReceivedByUser({ emails, userEmail });
   expect(receivedEmails?.total).toBe(1);
 

apps/web/playwright/team/team-invitation.e2e.ts

@@ -8,9 +8,8 @@ import { expectInvitationEmailToBeReceived } from "./expects";
 
 test.describe.configure({ mode: "parallel" });
 
-test.afterEach(async ({ users, emails }) => {
+test.afterEach(async ({ users }) => {
   await users.deleteAll();
-  emails?.deleteAll();
 });
 
 test.describe("Team", () => {
@@ -23,7 +22,10 @@ test.describe("Team", () => {
     await page.waitForLoadState("networkidle");
 
     await test.step("To the team by email (external user)", async () => {
-      const invitedUserEmail = `rick_${Date.now()}@domain-${Date.now()}.com`;
+      const invitedUserEmail = users.trackEmail({
+        username: "rick",
+        domain: `domain-${Date.now()}.com`,
+      });
       await page.locator(`button:text("${t("add")}")`).click();
       await page.locator('input[name="inviteUser"]').fill(invitedUserEmail);
       await page.locator(`button:text("${t("send_invite")}")`).click();
@@ -104,7 +106,10 @@ test.describe("Team", () => {
     await page.waitForLoadState("networkidle");
 
     await test.step("To the organization by email (internal user)", async () => {
-      const invitedUserEmail = `rick@example.com`;
+      const invitedUserEmail = users.trackEmail({
+        username: "rick",
+        domain: `example.com`,
+      });
       await page.locator(`button:text("${t("add")}")`).click();
       await page.locator('input[name="inviteUser"]').fill(invitedUserEmail);
       await page.locator(`button:text("${t("send_invite")}")`).click();