From 054a0fa7dbb151354abe975e8e0dadf5cf589ebd Mon Sep 17 00:00:00 2001 From: Keith Williams Date: Tue, 23 Dec 2025 22:39:26 -0300 Subject: [PATCH] refactor: Add dedicated setup-db job to eliminate cache race condition (#26171) * refactor: Add dedicated setup-db job to eliminate cache race condition - Create new setup-db.yml workflow that runs cache-db action - Update pr.yml to add setup-db job that runs before all E2E jobs - Update integration-test, e2e, e2e-api-v2, e2e-app-store, e2e-embed, and e2e-embed-react jobs to depend on setup-db instead of build-api-v1 - Add setup-db to required job needs list and result check This eliminates the race condition where multiple jobs could try to write to the same database cache simultaneously. Co-Authored-By: keith@cal.com * refactor: Remove DB parts from api-v1-production-build.yml The database setup is now handled by the dedicated setup-db job, so the postgres service and cache-db action are no longer needed in the API v1 build workflow. Co-Authored-By: keith@cal.com * Apply suggestion from @keithwillcode * Made integration tests dependency consistent * refactor: Remove unnecessary env vars from setup-db.yml Only keep the env vars needed for database setup: - CALENDSO_ENCRYPTION_KEY (for db-seed) - DATABASE_URL / DATABASE_DIRECT_URL (for database connection) - TURBO_TOKEN / TURBO_TEAM (for turbo caching) Co-Authored-By: keith@cal.com * fix: Add back E2E_TEST_CALCOM_QA_* env vars needed by db-seed These env vars are used by scripts/seed.ts to create the QA user with Google Calendar credentials during database seeding. Co-Authored-By: keith@cal.com * fix: Restore postgres service and cache-db to api-v1-production-build.yml The API v1 build still needs a database to run properly. Restored the postgres service and cache-db action, and added dependency on setup-db so it waits for the database cache to be created first. Co-Authored-By: keith@cal.com * E2E for API v2 no longer waiting on API v2 build * Reordering the jobs by dependencies * add GOOGLE_API_CREDENTIALS env var to setup db * Added back all env vars to setup-db * fix: Include commit SHA in cache-db key to invalidate cache on new commits Co-Authored-By: keith@cal.com * refactor: Strip back setup-db.yml env vars to minimal set needed for db-seed Co-Authored-By: keith@cal.com --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/actions/cache-db/action.yml | 6 +++- .github/workflows/e2e.yml | 4 +-- .github/workflows/pr.yml | 27 ++++++++++----- .github/workflows/setup-db.yml | 54 +++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/setup-db.yml diff --git a/.github/actions/cache-db/action.yml b/.github/actions/cache-db/action.yml index 32b9b0c083..dff9ba52e8 100644 --- a/.github/actions/cache-db/action.yml +++ b/.github/actions/cache-db/action.yml @@ -7,6 +7,10 @@ inputs: path: required: false default: "backups/backup.sql" + COMMIT_SHA: + required: false + default: "" + description: "Commit SHA to include in cache key (passed from workflow)" runs: using: "composite" steps: @@ -17,7 +21,7 @@ runs: cache-name: cache-db key-1: ${{ hashFiles('packages/prisma/schema.prisma', 'packages/prisma/migrations/**/**.sql', 'packages/prisma/*.ts') }} key-2: ${{ github.event.pull_request.number || github.ref }} - key-3: ${{ github.event.pull_request.head.sha }} + key-3: ${{ inputs.COMMIT_SHA || github.event.pull_request.head.sha || github.sha }} DATABASE_URL: ${{ inputs.DATABASE_URL }} DATABASE_DIRECT_URL: ${{ inputs.DATABASE_URL }} E2E_TEST_CALCOM_QA_EMAIL: ${{ inputs.E2E_TEST_CALCOM_QA_EMAIL }} diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index d85f6d99fe..65d098c438 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -5,7 +5,6 @@ permissions: actions: write contents: read env: - NODE_OPTIONS: --max-old-space-size=4096 ALLOWED_HOSTNAMES: ${{ vars.CI_ALLOWED_HOSTNAMES }} CALENDSO_ENCRYPTION_KEY: ${{ secrets.CI_CALENDSO_ENCRYPTION_KEY }} DAILY_API_KEY: ${{ secrets.CI_DAILY_API_KEY }} @@ -15,11 +14,11 @@ env: E2E_TEST_APPLE_CALENDAR_EMAIL: ${{ secrets.E2E_TEST_APPLE_CALENDAR_EMAIL }} E2E_TEST_APPLE_CALENDAR_PASSWORD: ${{ secrets.E2E_TEST_APPLE_CALENDAR_PASSWORD }} E2E_TEST_MAILHOG_ENABLED: ${{ vars.E2E_TEST_MAILHOG_ENABLED }} - GOOGLE_API_CREDENTIALS: ${{ secrets.CI_GOOGLE_API_CREDENTIALS }} EMAIL_SERVER_HOST: ${{ secrets.CI_EMAIL_SERVER_HOST }} EMAIL_SERVER_PORT: ${{ secrets.CI_EMAIL_SERVER_PORT }} EMAIL_SERVER_USER: ${{ secrets.CI_EMAIL_SERVER_USER }} EMAIL_SERVER_PASSWORD: ${{ secrets.CI_EMAIL_SERVER_PASSWORD}} + GOOGLE_API_CREDENTIALS: ${{ secrets.CI_GOOGLE_API_CREDENTIALS }} GOOGLE_LOGIN_ENABLED: ${{ vars.CI_GOOGLE_LOGIN_ENABLED }} NEXTAUTH_SECRET: ${{ secrets.CI_NEXTAUTH_SECRET }} NEXTAUTH_URL: ${{ secrets.CI_NEXTAUTH_URL }} @@ -30,6 +29,7 @@ env: NEXT_PUBLIC_STRIPE_PUBLIC_KEY: ${{ secrets.CI_NEXT_PUBLIC_STRIPE_PUBLIC_KEY }} NEXT_PUBLIC_WEBAPP_URL: ${{ vars.CI_NEXT_PUBLIC_WEBAPP_URL }} NEXT_PUBLIC_WEBSITE_URL: ${{ vars.CI_NEXT_PUBLIC_WEBSITE_URL }} + NODE_OPTIONS: --max-old-space-size=4096 PAYMENT_FEE_FIXED: ${{ vars.CI_PAYMENT_FEE_FIXED }} PAYMENT_FEE_PERCENTAGE: ${{ vars.CI_PAYMENT_FEE_PERCENTAGE }} SAML_ADMINS: ${{ secrets.CI_SAML_ADMINS }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index ddfdec1381..968070e169 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -139,9 +139,18 @@ jobs: uses: ./.github/workflows/unit-tests.yml secrets: inherit + setup-db: + name: Setup Database + needs: [prepare, deps] + if: ${{ needs.prepare.outputs.run-e2e == 'true' && needs.prepare.outputs.has-files-requiring-all-checks == 'true' }} + uses: ./.github/workflows/setup-db.yml + with: + COMMIT_SHA: ${{ needs.prepare.outputs.commit-sha }} + secrets: inherit + build-api-v1: name: Production builds - needs: [prepare, deps] + needs: [prepare, deps, setup-db] if: ${{ needs.prepare.outputs.run-e2e == 'true' && needs.prepare.outputs.has-files-requiring-all-checks == 'true' }} uses: ./.github/workflows/api-v1-production-build.yml secrets: inherit @@ -183,42 +192,42 @@ jobs: integration-test: name: Tests - needs: [prepare, build, build-api-v1] - if: ${{ needs.prepare.outputs.has-files-requiring-all-checks == 'true' }} + needs: [prepare, build, setup-db] + if: ${{ needs.prepare.outputs.run-e2e == 'true' && needs.prepare.outputs.has-files-requiring-all-checks == 'true' }} uses: ./.github/workflows/integration-tests.yml secrets: inherit e2e: name: Tests - needs: [prepare, build, build-api-v1] + needs: [prepare, build, setup-db] if: ${{ needs.prepare.outputs.run-e2e == 'true' && needs.prepare.outputs.has-files-requiring-all-checks == 'true' }} uses: ./.github/workflows/e2e.yml secrets: inherit e2e-api-v2: name: Tests - needs: [prepare, build-api-v2] + needs: [prepare, setup-db] if: ${{ needs.prepare.outputs.run-e2e == 'true' && needs.prepare.outputs.has-files-requiring-all-checks == 'true' }} uses: ./.github/workflows/e2e-api-v2.yml secrets: inherit e2e-app-store: name: Tests - needs: [prepare, build] + needs: [prepare, build, setup-db] if: ${{ needs.prepare.outputs.run-e2e == 'true' && needs.prepare.outputs.has-files-requiring-all-checks == 'true' }} uses: ./.github/workflows/e2e-app-store.yml secrets: inherit e2e-embed: name: Tests - needs: [prepare, build] + needs: [prepare, build, setup-db] if: ${{ needs.prepare.outputs.run-e2e == 'true' && needs.prepare.outputs.has-files-requiring-all-checks == 'true' }} uses: ./.github/workflows/e2e-embed.yml secrets: inherit e2e-embed-react: name: Tests - needs: [prepare, build] + needs: [prepare, build, setup-db] if: ${{ needs.prepare.outputs.run-e2e == 'true' && needs.prepare.outputs.has-files-requiring-all-checks == 'true' }} uses: ./.github/workflows/e2e-embed-react.yml secrets: inherit @@ -243,6 +252,7 @@ jobs: build-atoms, build-docs, build-companion, + setup-db, e2e, e2e-api-v2, e2e-embed, @@ -266,6 +276,7 @@ jobs: needs.build-api-v2.result != 'success' || needs.build-atoms.result != 'success' || needs.build-docs.result != 'success' || + needs.setup-db.result != 'success' || needs.integration-test.result != 'success' || needs.e2e.result != 'success' || needs.e2e-api-v2.result != 'success' || diff --git a/.github/workflows/setup-db.yml b/.github/workflows/setup-db.yml new file mode 100644 index 0000000000..53a29936ee --- /dev/null +++ b/.github/workflows/setup-db.yml @@ -0,0 +1,54 @@ +name: Setup Database + +on: + workflow_call: + inputs: + COMMIT_SHA: + required: false + type: string + default: "" + description: "Commit SHA to include in cache key" + +permissions: + contents: read + +env: + CALENDSO_ENCRYPTION_KEY: ${{ secrets.CI_CALENDSO_ENCRYPTION_KEY }} + DATABASE_URL: ${{ secrets.CI_DATABASE_URL }} + DATABASE_DIRECT_URL: ${{ secrets.CI_DATABASE_URL }} + E2E_TEST_CALCOM_QA_EMAIL: ${{ secrets.E2E_TEST_CALCOM_QA_EMAIL }} + E2E_TEST_CALCOM_QA_PASSWORD: ${{ secrets.E2E_TEST_CALCOM_QA_PASSWORD }} + E2E_TEST_CALCOM_QA_GCAL_CREDENTIALS: ${{ secrets.E2E_TEST_CALCOM_QA_GCAL_CREDENTIALS }} + GOOGLE_API_CREDENTIALS: ${{ secrets.CI_GOOGLE_API_CREDENTIALS }} + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_TEAM: ${{ secrets.TURBO_TEAM }} + +jobs: + setup-db: + name: Setup Database + runs-on: buildjet-4vcpu-ubuntu-2204 + timeout-minutes: 15 + services: + postgres: + image: postgres:13 + credentials: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: calendso + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/dangerous-git-checkout + - uses: ./.github/actions/yarn-install + - uses: ./.github/actions/cache-db + with: + COMMIT_SHA: ${{ inputs.COMMIT_SHA || github.sha }}