dev: exception handling

apiserver/.env.example

@@ -8,11 +8,11 @@ SENTRY_DSN=""
 SENTRY_ENVIRONMENT="development"
 
 # Database Settings
-PGUSER="plane"
-PGPASSWORD="plane"
-PGHOST="plane-db"
-PGDATABASE="plane"
-DATABASE_URL=postgresql://${PGUSER}:${PGPASSWORD}@${PGHOST}/${PGDATABASE}
+POSTGRES_USER="plane"
+POSTGRES_PASSWORD="plane"
+POSTGRES_HOST="plane-db"
+POSTGRES_DB="plane"
+DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}
 
 # Oauth variables
 GOOGLE_CLIENT_ID=""

apiserver/bin/beat

@@ -2,4 +2,5 @@
 set -e
 
 python manage.py wait_for_db
+python manage.py wait_for_migrations
 celery -A plane beat -l info

apiserver/bin/takeoff

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e
 python manage.py wait_for_db
-python manage.py migrate
+python manage.py safe_migrate
 
 # Create the default bucket
 #!/bin/bash

apiserver/bin/takeoff.local

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e
 python manage.py wait_for_db
-python manage.py migrate
+python manage.py safe_migrate
 
 # Create the default bucket
 #!/bin/bash

apiserver/bin/worker

@@ -2,4 +2,5 @@
 set -e
 
 python manage.py wait_for_db
+python manage.py wait_for_migrations
 celery -A plane worker -l info

apiserver/plane/db/management/commands/safe_migrate.py

@@ -0,0 +1,55 @@
+# Python imports
+import time
+import uuid
+import atexit
+
+# Django imports
+from django.core.management.base import BaseCommand
+from django.core.management import call_command
+from django.db.migrations.executor import MigrationExecutor
+from django.db import connections, DEFAULT_DB_ALIAS
+
+
+class Command(BaseCommand):
+    help = 'Wait for migrations to be completed and acquire lock before starting migrations'
+
+    def handle(self, *args, **kwargs):
+        self.lock_key = 'django_migration_lock'
+        self.lock_value = str(uuid.uuid4())  # Unique value for the lock
+        self.client = redis_instance()
+
+        # Register the cleanup function
+        atexit.register(self.cleanup)
+
+        while self._pending_migrations():
+            # Try acquiring the lock
+            if self.client.set(self.lock_key, self.lock_value, nx=True, ex=300):  # 5 minutes expiry
+                try:
+                    self.stdout.write("Acquired migration lock, running migrations...")
+                    call_command('migrate')
+                except Exception as e:
+                    self.stdout.write(f"An error occurred during migrations: {e}")
+                finally:
+                    # Release the lock if it belongs to this process
+                    self.cleanup()
+                return  # Exit after attempting migration
+            else:
+                self.stdout.write("Migration lock is held by another instance. Waiting 10 seconds to retry...")
+                time.sleep(10)  # Wait for 10 seconds before retrying
+
+        self.stdout.write("No pending migrations.")
+
+    def _pending_migrations(self):
+        connection = connections[DEFAULT_DB_ALIAS]
+        executor = MigrationExecutor(connection)
+        targets = executor.loader.graph.leaf_nodes()
+        return bool(executor.migration_plan(targets))
+
+    def cleanup(self):
+        """
+        Clean up function to release the lock.
+        """
+        stored_value = self.client.get(self.lock_key)
+        if stored_value and stored_value.decode() == self.lock_value:
+            self.client.delete(self.lock_key)
+            self.stdout.write("Released migration lock.")

apiserver/plane/db/management/commands/wait_for_migrations.py

@@ -0,0 +1,21 @@
+# wait_for_migrations.py
+import time
+from django.core.management.base import BaseCommand
+from django.db.migrations.executor import MigrationExecutor
+from django.db import connections, DEFAULT_DB_ALIAS
+
+class Command(BaseCommand):
+    help = 'Wait for database migrations to complete before starting Celery worker/beat'
+
+    def handle(self, *args, **kwargs):
+        while self._pending_migrations():
+            self.stdout.write("Waiting for database migrations to complete...")
+            time.sleep(10)  # wait for 10 seconds before checking again
+
+        self.stdout.write("All migrations are complete. Safe to start Celery worker/beat.")
+
+    def _pending_migrations(self):
+        connection = connections[DEFAULT_DB_ALIAS]
+        executor = MigrationExecutor(connection)
+        targets = executor.loader.graph.leaf_nodes()
+        return bool(executor.migration_plan(targets))

docker-compose-local.yml

@@ -76,6 +76,8 @@ services:
       - web
 
   api:
+    deploy:
+      replicas: 3
     build:
       context: ./apiserver
       dockerfile: Dockerfile.dev
@@ -86,7 +88,7 @@ services:
       - dev_env
     volumes:
       - ./apiserver:/code
-    # command: /bin/sh -c "python manage.py migrate && python manage.py runserver 0.0.0.0:8000 --settings=plane.settings.local"
+    command: ./bin/takeoff.local
     env_file:
       - ./apiserver/.env
     depends_on:
@@ -104,7 +106,7 @@ services:
       - dev_env
     volumes:
       - ./apiserver:/code
-    command: /bin/sh -c "celery -A plane worker -l info"
+    command: ./bin/worker
     env_file:
       - ./apiserver/.env
     depends_on:
@@ -123,7 +125,7 @@ services:
       - dev_env
     volumes:
       - ./apiserver:/code
-    command: /bin/sh -c "celery -A plane beat -l info"
+    command: ./bin/beat
     env_file:
       - ./apiserver/.env
     depends_on: