[SILO-707] fix: ratelimit issues and fails with jira cloud (#4842)

apps/api/plane/bgtasks/data_import_task.py

@@ -1,6 +1,7 @@
 # Python imports
 import logging
 import random
+import uuid
 
 # Third party imports
 from celery import shared_task
@@ -138,6 +139,23 @@ def update_job_batch_completion(
         log_exception(e)
 
 
+def is_valid_uuid(value):
+    """
+    Check if a value is a valid UUID string.
+
+    Args:
+        value: The value to check
+
+    Returns:
+        bool: True if the value is a valid UUID, False otherwise
+    """
+    try:
+        uuid.UUID(str(value))
+        return True
+    except (ValueError, AttributeError):
+        return False
+
+
 def sanitize_issue_data(issue_data):
     """
     Sanitize the issue data
@@ -156,8 +174,12 @@ def process_single_issue(slug, project, user_id, issue_data):
             # Process the main issue
             issue_data = sanitize_issue_data(issue_data)
 
-            # Extract labels before serialization (they're label names, not UUIDs)
-            labels = issue_data.pop("labels", [])
+            # Handle labels based on whether they are UUIDs or names
+            labels = []
+            if "labels" in issue_data:
+                label_values = issue_data.get("labels", [])
+                if label_values and not is_valid_uuid(label_values[0]):
+                    labels = issue_data.pop("labels")
 
             serializer = IssueSerializer(
                 data=issue_data,

packages/etl/src/jira-server/services/api.service.ts

@@ -1,5 +1,5 @@
 // services
-import axios from "axios";
+import axios, { AxiosError } from "axios";
 import type { Paginated } from "jira.js";
 import { Board as BoardClient } from "jira.js/out/agile";
 import { Version2Client } from "jira.js/out/version2";
@@ -30,6 +30,24 @@ export class JiraV2Service {
         personalAccessToken: props.patToken,
       },
     });
+
+    this.jiraClient.handleFailedResponse = async (request) => {
+      const error = request as AxiosError;
+      if (error.response?.status === 429) {
+        const retryAfter = 60; // 60 seconds default
+        console.log("Rate limit exceeded ====== in jira client, waiting for", retryAfter, "seconds");
+        await new Promise((resolve) => setTimeout(resolve, retryAfter * 1000));
+
+        // Actually retry the request
+        const originalConfig = error.config;
+        if (originalConfig) {
+          console.log("Retrying request after rate limit...");
+          const response = await axios.request(originalConfig);
+          return response.data;
+        }
+      }
+      throw error;
+    };
   }
 
   async getServerInfo() {

packages/etl/src/jira/etl/pull.ts

@@ -34,11 +34,16 @@ export function pullUsers(users: string): ImportedJiraUser[] {
 
 export async function pullLabels(client: JiraService): Promise<string[]> {
   const labels: string[] = [];
-  await fetchPaginatedData(
-    (startAt) => client.getResourceLabels(startAt),
-    (values) => labels.push(...(values as string[])),
-    "values"
-  );
+  try {
+    await fetchPaginatedData(
+      (startAt) => client.getResourceLabels(startAt),
+      (values) => labels.push(...(values as string[])),
+      "values"
+    );
+  } catch (error) {
+    console.log("Error while pulling labels", error);
+  }
+
   return labels;
 }
 
@@ -60,7 +65,19 @@ export async function pullIssues(client: JiraService, projectKey: string, from?:
 }
 
 export async function pullComments(issues: IJiraIssue[], client: JiraService): Promise<any[]> {
-  return await pullCommentsInBatches(issues, 20, client);
+  const comments: JiraComment[] = [];
+
+  try {
+    // Pull comments for each issue
+    for (const issue of issues) {
+      const issueComments = await pullCommentsForIssue(issue, client);
+      comments.push(...issueComments);
+    }
+  } catch (error) {
+    console.log("Error while pulling comments for issues", error);
+  }
+
+  return comments;
 }
 
 export async function pullSprints(client: JiraService, projectId: string): Promise<JiraSprint[]> {
@@ -133,23 +150,15 @@ export const pullCommentsForIssue = async (issue: IJiraIssue, client: JiraServic
   return comments;
 };
 
-export const pullCommentsInBatches = async (
-  issues: IJiraIssue[],
-  batchSize: number,
-  client: JiraService
-): Promise<JiraComment[]> => {
-  const comments: JiraComment[] = [];
-  for (let i = 0; i < issues.length; i += batchSize) {
-    const batch = issues.slice(i, i + batchSize);
-    const batchComments = await Promise.all(batch.map((issue) => pullCommentsForIssue(issue, client)));
-    comments.push(...batchComments.flat());
+export const pullIssueTypes = async (client: JiraService, projectId: string): Promise<JiraIssueTypeDetails[]> => {
+  try {
+    return await client.getProjectIssueTypes(projectId);
+  } catch (error) {
+    console.log("Error while pulling issue types", error);
+    return [];
   }
-  return comments;
 };
 
-export const pullIssueTypes = async (client: JiraService, projectId: string): Promise<JiraIssueTypeDetails[]> =>
-  await client.getProjectIssueTypes(projectId);
-
 export const pullIssueFields = async (
   client: JiraService,
   issueTypes: JiraIssueTypeDetails[],

packages/etl/src/jira/etl/transform.ts

@@ -112,7 +112,7 @@ export const transformComment = (
   created_by: comment.author?.displayName,
   comment_html: comment.renderedBody ?? "<p></p>",
   actor: comment.author?.displayName,
-  issue: comment.issue_id,
+  issue: `${projectId}_${resourceId}_${comment.issue_id}`,
 });
 
 export const transformUser = (user: ImportedJiraUser): Partial<PlaneUser> => {

packages/etl/src/jira/services/api.service.ts

@@ -61,6 +61,24 @@ export class JiraService {
           },
         },
       });
+
+      this.jiraClient.handleFailedResponse = async (request) => {
+        const error = request as AxiosError;
+        if (error.response?.status === 429) {
+          const retryAfter = 60; // 60 seconds default
+          console.log("Rate limit exceeded ====== in jira client, waiting for", retryAfter, "seconds");
+          await new Promise((resolve) => setTimeout(resolve, retryAfter * 1000));
+
+          // Actually retry the request
+          const originalConfig = error.config;
+          if (originalConfig) {
+            console.log("Retrying request after rate limit...");
+            const response = await axios.request(originalConfig);
+            return response.data;
+          }
+        }
+        throw error;
+      };
     }
   }