diff --git a/apps/api/plane/bgtasks/data_import_task.py b/apps/api/plane/bgtasks/data_import_task.py index 877708f9cf..dcec91afda 100644 --- a/apps/api/plane/bgtasks/data_import_task.py +++ b/apps/api/plane/bgtasks/data_import_task.py @@ -1,6 +1,7 @@ # Python imports import logging import random +import uuid # Third party imports from celery import shared_task @@ -138,6 +139,23 @@ def update_job_batch_completion( log_exception(e) +def is_valid_uuid(value): + """ + Check if a value is a valid UUID string. + + Args: + value: The value to check + + Returns: + bool: True if the value is a valid UUID, False otherwise + """ + try: + uuid.UUID(str(value)) + return True + except (ValueError, AttributeError): + return False + + def sanitize_issue_data(issue_data): """ Sanitize the issue data @@ -156,8 +174,12 @@ def process_single_issue(slug, project, user_id, issue_data): # Process the main issue issue_data = sanitize_issue_data(issue_data) - # Extract labels before serialization (they're label names, not UUIDs) - labels = issue_data.pop("labels", []) + # Handle labels based on whether they are UUIDs or names + labels = [] + if "labels" in issue_data: + label_values = issue_data.get("labels", []) + if label_values and not is_valid_uuid(label_values[0]): + labels = issue_data.pop("labels") serializer = IssueSerializer( data=issue_data, diff --git a/packages/etl/src/jira-server/services/api.service.ts b/packages/etl/src/jira-server/services/api.service.ts index bce0266756..68dc72f6a1 100644 --- a/packages/etl/src/jira-server/services/api.service.ts +++ b/packages/etl/src/jira-server/services/api.service.ts @@ -1,5 +1,5 @@ // services -import axios from "axios"; +import axios, { AxiosError } from "axios"; import type { Paginated } from "jira.js"; import { Board as BoardClient } from "jira.js/out/agile"; import { Version2Client } from "jira.js/out/version2"; @@ -30,6 +30,24 @@ export class JiraV2Service { personalAccessToken: props.patToken, }, }); + + this.jiraClient.handleFailedResponse = async (request) => { + const error = request as AxiosError; + if (error.response?.status === 429) { + const retryAfter = 60; // 60 seconds default + console.log("Rate limit exceeded ====== in jira client, waiting for", retryAfter, "seconds"); + await new Promise((resolve) => setTimeout(resolve, retryAfter * 1000)); + + // Actually retry the request + const originalConfig = error.config; + if (originalConfig) { + console.log("Retrying request after rate limit..."); + const response = await axios.request(originalConfig); + return response.data; + } + } + throw error; + }; } async getServerInfo() { diff --git a/packages/etl/src/jira/etl/pull.ts b/packages/etl/src/jira/etl/pull.ts index d36c86516d..3e9fde6aa5 100644 --- a/packages/etl/src/jira/etl/pull.ts +++ b/packages/etl/src/jira/etl/pull.ts @@ -34,11 +34,16 @@ export function pullUsers(users: string): ImportedJiraUser[] { export async function pullLabels(client: JiraService): Promise { const labels: string[] = []; - await fetchPaginatedData( - (startAt) => client.getResourceLabels(startAt), - (values) => labels.push(...(values as string[])), - "values" - ); + try { + await fetchPaginatedData( + (startAt) => client.getResourceLabels(startAt), + (values) => labels.push(...(values as string[])), + "values" + ); + } catch (error) { + console.log("Error while pulling labels", error); + } + return labels; } @@ -60,7 +65,19 @@ export async function pullIssues(client: JiraService, projectKey: string, from?: } export async function pullComments(issues: IJiraIssue[], client: JiraService): Promise { - return await pullCommentsInBatches(issues, 20, client); + const comments: JiraComment[] = []; + + try { + // Pull comments for each issue + for (const issue of issues) { + const issueComments = await pullCommentsForIssue(issue, client); + comments.push(...issueComments); + } + } catch (error) { + console.log("Error while pulling comments for issues", error); + } + + return comments; } export async function pullSprints(client: JiraService, projectId: string): Promise { @@ -133,23 +150,15 @@ export const pullCommentsForIssue = async (issue: IJiraIssue, client: JiraServic return comments; }; -export const pullCommentsInBatches = async ( - issues: IJiraIssue[], - batchSize: number, - client: JiraService -): Promise => { - const comments: JiraComment[] = []; - for (let i = 0; i < issues.length; i += batchSize) { - const batch = issues.slice(i, i + batchSize); - const batchComments = await Promise.all(batch.map((issue) => pullCommentsForIssue(issue, client))); - comments.push(...batchComments.flat()); +export const pullIssueTypes = async (client: JiraService, projectId: string): Promise => { + try { + return await client.getProjectIssueTypes(projectId); + } catch (error) { + console.log("Error while pulling issue types", error); + return []; } - return comments; }; -export const pullIssueTypes = async (client: JiraService, projectId: string): Promise => - await client.getProjectIssueTypes(projectId); - export const pullIssueFields = async ( client: JiraService, issueTypes: JiraIssueTypeDetails[], diff --git a/packages/etl/src/jira/etl/transform.ts b/packages/etl/src/jira/etl/transform.ts index b0beb5b4e8..e7d93710c9 100644 --- a/packages/etl/src/jira/etl/transform.ts +++ b/packages/etl/src/jira/etl/transform.ts @@ -112,7 +112,7 @@ export const transformComment = ( created_by: comment.author?.displayName, comment_html: comment.renderedBody ?? "

", actor: comment.author?.displayName, - issue: comment.issue_id, + issue: `${projectId}_${resourceId}_${comment.issue_id}`, }); export const transformUser = (user: ImportedJiraUser): Partial => { diff --git a/packages/etl/src/jira/services/api.service.ts b/packages/etl/src/jira/services/api.service.ts index c41a6edb13..1c0eefae71 100644 --- a/packages/etl/src/jira/services/api.service.ts +++ b/packages/etl/src/jira/services/api.service.ts @@ -61,6 +61,24 @@ export class JiraService { }, }, }); + + this.jiraClient.handleFailedResponse = async (request) => { + const error = request as AxiosError; + if (error.response?.status === 429) { + const retryAfter = 60; // 60 seconds default + console.log("Rate limit exceeded ====== in jira client, waiting for", retryAfter, "seconds"); + await new Promise((resolve) => setTimeout(resolve, retryAfter * 1000)); + + // Actually retry the request + const originalConfig = error.config; + if (originalConfig) { + console.log("Retrying request after rate limit..."); + const response = await axios.request(originalConfig); + return response.data; + } + } + throw error; + }; } }