diff --git a/apps/silo/src/apps/jira-server-importer/controllers/index.ts b/apps/silo/src/apps/jira-server-importer/controllers/index.ts index 88c9a62b8c..420b1ec3e4 100644 --- a/apps/silo/src/apps/jira-server-importer/controllers/index.ts +++ b/apps/silo/src/apps/jira-server-importer/controllers/index.ts @@ -2,9 +2,8 @@ import type { Request, Response } from "express"; // etl import { Controller, Get, Post } from "@plane/decorators"; import { E_IMPORTER_KEYS } from "@plane/etl/core"; -import type { JiraResource } from "@plane/etl/jira"; import type { JiraProject, JiraV2Service } from "@plane/etl/jira-server"; -import { createJiraService } from "@plane/etl/jira-server"; +import { createJiraService, EJiraAuthenticationType } from "@plane/etl/jira-server"; // db // helpers import { compareAndGetAdditionalUsers } from "@/helpers/additional-users"; @@ -24,14 +23,18 @@ class JiraDataCenterController { async upsertCredentials(req: Request, res: Response) { try { const { workspaceId, userId, apiToken, personalAccessToken, userEmail, hostname } = req.body; - if (!workspaceId || !userId || !apiToken || !personalAccessToken) { - res.status(400).json({ message: "Workspace ID, User ID, API Token and Personal Access Token are required" }); + if (!workspaceId || !userId || !apiToken || !personalAccessToken || !userEmail) { + return res + .status(400) + .json({ message: "Workspace ID, User ID, API Token, Personal Access Token and User Email are required" }); } try { const jiraService = createJiraService({ patToken: personalAccessToken, hostname: hostname, + email: (userEmail as string).trim(), + authenticationType: EJiraAuthenticationType.PERSONAL_ACCESS_TOKEN, }); await jiraService.getCurrentUser(); @@ -64,8 +67,7 @@ class JiraDataCenterController { try { const jiraClient = await createJiraServerClient(workspaceId, userId); const serverInfo = await jiraClient.getServerInfo(); - // @ts-expect-error - const resource: JiraResource = { + const resource = { id: serverInfo.scmInfo ?? "", url: serverInfo.baseUrl ?? "", name: serverInfo.serverTitle ?? "", @@ -185,6 +187,8 @@ export const createJiraServerClient = async (workspaceId: string, userId: string return createJiraService({ hostname: jiraCredentials.source_hostname, patToken: jiraCredentials.source_access_token, + email: jiraCredentials.source_auth_email, + authenticationType: EJiraAuthenticationType.PERSONAL_ACCESS_TOKEN, }); }; diff --git a/apps/silo/src/apps/jira-server-importer/helpers/migration-helpers.ts b/apps/silo/src/apps/jira-server-importer/helpers/migration-helpers.ts index 2f753300e0..4d4f4e7d91 100644 --- a/apps/silo/src/apps/jira-server-importer/helpers/migration-helpers.ts +++ b/apps/silo/src/apps/jira-server-importer/helpers/migration-helpers.ts @@ -6,7 +6,7 @@ import type { } from "jira.js/out/version2/models"; import { E_IMPORTER_KEYS } from "@plane/etl/core"; import type { IPriorityConfig, IStateConfig, JiraComponent, JiraConfig, JiraSprint } from "@plane/etl/jira-server"; -import { JiraV2Service } from "@plane/etl/jira-server"; +import { EJiraAuthenticationType, JiraV2Service } from "@plane/etl/jira-server"; import type { ExIssueAttachment, ExState } from "@plane/sdk"; import type { TImportJob, TWorkspaceCredential } from "@plane/types"; @@ -92,8 +92,14 @@ export const createJiraClient = ( if (!credentials.source_access_token || !credentials.source_hostname || !credentials.source_auth_email) { throw new Error(`Missing credentials in job config for job ${job.id}`); } + + const authenticationType = + job.source === E_IMPORTER_KEYS.JIRA ? EJiraAuthenticationType.BASIC : EJiraAuthenticationType.PERSONAL_ACCESS_TOKEN; + return new JiraV2Service({ + email: credentials.source_auth_email, patToken: credentials.source_access_token!, hostname: credentials.source_hostname, + authenticationType, }); }; diff --git a/apps/silo/src/apps/jira-server-importer/migrator/transformers/etl.ts b/apps/silo/src/apps/jira-server-importer/migrator/transformers/etl.ts index c559280498..25bace7447 100644 --- a/apps/silo/src/apps/jira-server-importer/migrator/transformers/etl.ts +++ b/apps/silo/src/apps/jira-server-importer/migrator/transformers/etl.ts @@ -1,5 +1,5 @@ import { v4 as uuid } from "uuid"; -import type { TIssuePropertyValuesPayload } from "@plane/etl/core"; +import type { E_IMPORTER_KEYS, TIssuePropertyValuesPayload } from "@plane/etl/core"; import type { JiraCustomFieldKeys, IJiraIssue, @@ -48,7 +48,17 @@ export const getTransformedIssues = ( const resourceId = job.config.resource ? job.config.resource.id : uuid(); return entities.issues.map((issue: IJiraIssue): Partial => { - const transformedIssue = transformIssue(resourceId, job.project_id, issue, resourceUrl, stateMap, priorityMap); + const transformedIssue = transformIssue( + { + resourceId, + projectId: job.project_id, + source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA, + }, + issue, + resourceUrl, + stateMap, + priorityMap + ); if (job.config?.issueType && issue.fields.issuetype?.name) { const issueTypeValue = job.config.issueType; @@ -71,7 +81,16 @@ export const getTransformedComments = ( entities: JiraEntity ): Partial[] => { const resourceId = job.config.resource ? job.config.resource.id : uuid(); - return entities.issue_comments.map((comment) => transformComment(resourceId, job.project_id, comment)); + return entities.issue_comments.map((comment) => + transformComment( + { + resourceId, + projectId: job.project_id, + source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA, + }, + comment + ) + ); }; export const getTransformedUsers = (_job: TImportJob, entities: JiraEntity): Partial[] => @@ -79,17 +98,44 @@ export const getTransformedUsers = (_job: TImportJob, entities: Jira export const getTransformedSprints = (job: TImportJob, entities: JiraEntity): Partial[] => { const resourceId = job.config.resource ? job.config.resource.id : uuid(); - return entities.sprints.map((sprint) => transformSprint(resourceId, job.project_id, sprint)); + return entities.sprints.map((sprint) => + transformSprint( + { + resourceId, + projectId: job.project_id, + source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA, + }, + sprint + ) + ); }; export const getTransformedComponents = (job: TImportJob, entities: JiraEntity): Partial[] => { const resourceId = job.config.resource ? job.config.resource.id : uuid(); - return entities.components.map((component) => transformComponent(resourceId, job.project_id, component)); + return entities.components.map((component) => + transformComponent( + { + resourceId, + projectId: job.project_id, + source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA, + }, + component + ) + ); }; export const getTransformedIssueTypes = (job: TImportJob, entities: JiraEntity): Partial[] => { const resourceId = job.config.resource ? job.config.resource.id : uuid(); - return entities.issueTypes.map((issueType) => transformIssueType(resourceId, job.project_id, issueType)); + return entities.issueTypes.map((issueType) => + transformIssueType( + { + resourceId, + projectId: job.project_id, + source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA, + }, + issueType + ) + ); }; export const getTransformedIssueFields = ( @@ -98,7 +144,16 @@ export const getTransformedIssueFields = ( ): Partial[] => { const resourceId = job.config.resource ? job.config.resource.id : uuid(); return entities.issueFields - .map((issueField) => transformIssueFields(resourceId, job.project_id, issueField)) + .map((issueField) => + transformIssueFields( + { + resourceId, + projectId: job.project_id, + source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA, + }, + issueField + ) + ) .filter((field) => field && field.property_type) as Partial[]; }; @@ -114,7 +169,16 @@ export const getTransformedIssueFieldOptions = ( OPTION_CUSTOM_FIELD_TYPES.includes(issueField.schema?.custom as JiraCustomFieldKeys) ) .flatMap((issueField) => - issueField?.options?.map((fieldOption) => transformIssueFieldOptions(resourceId, job.project_id, fieldOption)) + issueField?.options?.map((fieldOption) => + transformIssueFieldOptions( + { + resourceId, + projectId: job.project_id, + source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA, + }, + fieldOption + ) + ) ) .filter(Boolean) as Partial[]; }; @@ -146,8 +210,7 @@ export const getTransformedIssuePropertyValues = ( entities.issues.forEach((issue: IJiraIssue) => { if (issue.id && issue.fields) { transformedIssuePropertyValues[`${projectId}_${resourceId}_${issue.id}`] = transformIssuePropertyValues( - resourceId, - projectId, + { resourceId, projectId, source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA }, issue, planeIssuePropertiesMap, jiraCustomFieldMap @@ -194,8 +257,7 @@ export const getTransformedIssuePropertyValuesV2 = ( // Transform custom field property values const customPropertyValues = transformIssuePropertyValues( - resourceId, - projectId, + { resourceId, projectId, source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA }, issue, planeIssuePropertiesMap, jiraCustomFieldMap @@ -203,7 +265,11 @@ export const getTransformedIssuePropertyValuesV2 = ( // Transform default property values (fix versions, affected versions, reporter) const defaultPropertyValues = issueTypeId - ? transformDefaultPropertyValues(resourceId, projectId, issue, issueTypeId) + ? transformDefaultPropertyValues( + { resourceId, projectId, source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA }, + issue, + issueTypeId + ) : {}; // Merge both custom and default property values diff --git a/apps/silo/src/apps/jira-server-importer/v2/helpers/properties.ts b/apps/silo/src/apps/jira-server-importer/v2/helpers/properties.ts index 0d528a643f..8e1a26c91b 100644 --- a/apps/silo/src/apps/jira-server-importer/v2/helpers/properties.ts +++ b/apps/silo/src/apps/jira-server-importer/v2/helpers/properties.ts @@ -1,4 +1,4 @@ -import { E_IMPORTER_KEYS } from "@plane/etl/core"; +import type { E_IMPORTER_KEYS } from "@plane/etl/core"; import type { ExIssueProperty } from "@plane/sdk"; export enum E_DEFAULT_PROPERTY_TYPES { @@ -23,12 +23,13 @@ export const getSupportedDefaultProperties = ( resourceId: string, projectId: string, issueTypeId: string, - issueTypeExternalId: string + issueTypeExternalId: string, + source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA ): Partial[] => [ { type_id: issueTypeExternalId, - external_id: `${resourceId}-${projectId}-${issueTypeId}-${E_DEFAULT_PROPERTY_TYPES.FIX_VERSION}`, - external_source: E_IMPORTER_KEYS.JIRA_SERVER, + external_id: getDefaultPropertyExternalId(resourceId, projectId, issueTypeId, E_DEFAULT_PROPERTY_TYPES.FIX_VERSION), + external_source: source, display_name: "Fix Version", property_type: "TEXT", settings: { @@ -38,8 +39,13 @@ export const getSupportedDefaultProperties = ( }, { type_id: issueTypeExternalId, - external_id: `${resourceId}-${projectId}-${issueTypeId}-${E_DEFAULT_PROPERTY_TYPES.AFFECTED_VERSION}`, - external_source: E_IMPORTER_KEYS.JIRA_SERVER, + external_id: getDefaultPropertyExternalId( + resourceId, + projectId, + issueTypeId, + E_DEFAULT_PROPERTY_TYPES.AFFECTED_VERSION + ), + external_source: source, display_name: "Affected Version", property_type: "TEXT", settings: { @@ -49,8 +55,8 @@ export const getSupportedDefaultProperties = ( }, { type_id: issueTypeExternalId, - external_id: `${resourceId}-${projectId}-${issueTypeId}-${E_DEFAULT_PROPERTY_TYPES.REPORTER}`, - external_source: E_IMPORTER_KEYS.JIRA_SERVER, + external_id: getDefaultPropertyExternalId(resourceId, projectId, issueTypeId, E_DEFAULT_PROPERTY_TYPES.REPORTER), + external_source: source, display_name: "Reporter", property_type: "RELATION", relation_type: "USER", diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/index.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/index.ts index e69de29bb2..010746d4f7 100644 --- a/apps/silo/src/apps/jira-server-importer/v2/migrator/index.ts +++ b/apps/silo/src/apps/jira-server-importer/v2/migrator/index.ts @@ -0,0 +1,10 @@ +import type { MQ, Store } from "@/worker/base"; +import { JiraImportOrchestrator } from "./orchestrator"; +import JIRA_CLOUD_STEPS from "./steps/cloud"; +import JIRA_SERVER_STEPS from "./steps/server"; + +export const getJiraCloudImportOrchestrator = (mq: MQ, store: Store) => + new JiraImportOrchestrator(mq, store, JIRA_CLOUD_STEPS); + +export const getJiraServerImportOrchestrator = (mq: MQ, store: Store) => + new JiraImportOrchestrator(mq, store, JIRA_SERVER_STEPS); diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/orchestrator.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/orchestrator.ts index 5ba209096e..51323a7787 100644 --- a/apps/silo/src/apps/jira-server-importer/v2/migrator/orchestrator.ts +++ b/apps/silo/src/apps/jira-server-importer/v2/migrator/orchestrator.ts @@ -4,7 +4,6 @@ import { logger } from "@plane/logger"; import type { TImportJob } from "@plane/types"; import { createJiraClient } from "@/apps/jira-server-importer/helpers/migration-helpers"; import { flushJob } from "@/apps/jira-server-importer/v2/helpers/cache"; -import JIRA_SERVER_STEPS from "@/apps/jira-server-importer/v2/migrator/steps"; import type { IStep, IStorageService, @@ -15,7 +14,7 @@ import type { } from "@/apps/jira-server-importer/v2/types"; import { getJobCredentials, getJobData } from "@/helpers/job"; import { getPlaneAPIClient } from "@/helpers/plane-api-client"; -import { getAPIClient } from "@/services/client"; +import { getAPIClient, getAPIClientInternal } from "@/services/client"; import type { TaskHandler, TaskHeaders } from "@/types"; import type { MQ, Store } from "@/worker/base"; import { redisStorageService } from "../services/storage.service"; @@ -37,18 +36,16 @@ type OrchestratorTaskData = { stepContext?: TStepExecutionContext; }; -const client = getAPIClient(); +const client = getAPIClientInternal(); -export class JiraServerImportOrchestrator implements TaskHandler { - private steps: IStep[]; +export class JiraImportOrchestrator implements TaskHandler { private storage: IStorageService = redisStorageService; constructor( private readonly mq: MQ, - private readonly store: Store - ) { - this.steps = JIRA_SERVER_STEPS; - } + private readonly store: Store, + private readonly steps: IStep[] + ) { } /** * Main task handler - called by worker @@ -143,8 +140,74 @@ export class JiraServerImportOrchestrator implements TaskHandler { // Load execution context const { state, step, jobContext } = await this.loadStepExecution(jobId, data); - // Execute step and handle result - await this.executeAndHandleResult(headers, state, step, jobContext); + try { + // Execute step and handle result + await this.executeAndHandleResult(headers, state, step, jobContext); + } catch (error) { + // Log error and move to next step instead of failing the job + await this.handleStepError(headers, state, step, error); + } + } + + /** + * Handle step error: Log error and move to next step + */ + private async handleStepError( + headers: TaskHeaders, + state: TOrchestratorState, + step: IStep, + error: unknown + ): Promise { + const errorMessage = error instanceof Error ? error.message : String(error); + + logger.error(`[ORCHESTRATOR] Step failed, continuing to next step`, { + jobId: state.jobId, + step: step.name, + stepIndex: state.currentStepIndex + 1, + totalSteps: state.totalSteps, + error: errorMessage, + }); + + // Track failed step + if (!state.failedSteps) { + state.failedSteps = []; + } + state.failedSteps.push({ + name: step.name, + error: errorMessage, + failedAt: new Date().toISOString(), + }); + + // Move to next step + state.currentStepIndex++; + state.stepContext = undefined; + await this.saveState(state); + + if (state.currentStepIndex >= this.steps.length) { + // All steps processed (some may have failed) + await this.mq.sendMessage( + {}, + { + headers: { + ...headers, + type: EOrchestratorTaskType.COMPLETE, + }, + } + ); + } else { + // Move to next step + state.currentStepName = this.steps[state.currentStepIndex].name; + await this.saveState(state); + await this.mq.sendMessage( + { state }, + { + headers: { + ...headers, + type: EOrchestratorTaskType.EXECUTE_STEP, + }, + } + ); + } } /** @@ -178,6 +241,18 @@ export class JiraServerImportOrchestrator implements TaskHandler { return { state, step, jobContext }; } + /** + * Initialize job context + */ + private async initializeJobContext(jobId: string): Promise { + const job = await getJobData(jobId); + const credentials = await getJobCredentials(job as TImportJob); + const planeClient = await getPlaneAPIClient(credentials, E_IMPORTER_KEYS.IMPORTER); + const sourceClient = createJiraClient(job as TImportJob, credentials); + + return { job, credentials, planeClient, sourceClient }; + } + /** * Execute step and handle the result */ @@ -298,7 +373,7 @@ export class JiraServerImportOrchestrator implements TaskHandler { // Delete orchestrator state await this.store.del(stateKey); - logger.info(`[ORCHESTRATOR] Job completed successfully`, { jobId }); + logger.info(`[ORCHESTRATOR] Job completed`, { jobId }); } /** @@ -362,18 +437,6 @@ export class JiraServerImportOrchestrator implements TaskHandler { return dependencyData; } - /** - * Initialize job context - */ - private async initializeJobContext(jobId: string): Promise { - const job = await getJobData(jobId); - const credentials = await getJobCredentials(job as TImportJob); - const planeClient = await getPlaneAPIClient(credentials, E_IMPORTER_KEYS.IMPORTER); - const sourceClient = createJiraClient(job as TImportJob, credentials); - - return { job, credentials, planeClient, sourceClient }; - } - /** * Check if job is cancelled */ diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/cloud/entities/index.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/cloud/entities/index.ts new file mode 100644 index 0000000000..9d9708df57 --- /dev/null +++ b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/cloud/entities/index.ts @@ -0,0 +1,3 @@ +export * from "./issue-properties.step"; +export * from "./users.step"; +export * from "./issue-types.step"; diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/cloud/entities/issue-properties.step.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/cloud/entities/issue-properties.step.ts new file mode 100644 index 0000000000..5a21dbe766 --- /dev/null +++ b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/cloud/entities/issue-properties.step.ts @@ -0,0 +1,29 @@ +import { pullIssueFields } from "@plane/etl/jira"; +import type { JiraIssueField } from "@plane/etl/jira-server"; +import { createJiraClient } from "@/apps/jira-importer/helpers/migration-helpers"; +import { getJobCredentials } from "@/helpers/job"; +import type { TJobContext, TIssueTypesData } from "../../../../types"; +import { JiraIssuePropertiesStep } from "../../shared/entities"; + +export class JiraCloudIssuePropertiesStep extends JiraIssuePropertiesStep { + protected async pull( + jobCtx: TJobContext, + projectId: string, + issueTypesData: TIssueTypesData + ): Promise { + const { job } = jobCtx; + const credentials = await getJobCredentials(job); + const client = createJiraClient(job, credentials); + const issueTypes = issueTypesData + .map((type) => { + const id = type.external_id.split("_").pop(); + if (typeof id !== "string" || id.length === 0) { + return null; + } + return { id, name: type.name }; + }) + .filter((item): item is { id: string; name: string } => item !== null); + + return pullIssueFields(client, issueTypes, projectId); + } +} diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/cloud/entities/issue-types.step.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/cloud/entities/issue-types.step.ts new file mode 100644 index 0000000000..be66a65417 --- /dev/null +++ b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/cloud/entities/issue-types.step.ts @@ -0,0 +1,29 @@ +import { pullIssueTypes } from "@plane/etl/jira"; +import { logger } from "@plane/logger"; +import { createJiraClient } from "@/apps/jira-importer/helpers/migration-helpers"; +import type { TJobContext } from "@/apps/jira-server-importer/v2/types"; +import { getJobCredentials } from "@/helpers/job"; +import { JiraIssueTypesStep } from "../../shared"; + +export class JiraCloudIssueTypesStep extends JiraIssueTypesStep { + protected async pull(jobContext: TJobContext, projectId: string, startAt: number) { + const { job } = jobContext; + const credentials = await getJobCredentials(job); + const client = createJiraClient(job, credentials); + const result = await pullIssueTypes(client, projectId); + + logger.info(`[${jobContext.job.id}] [${this.name}] Pulled issue types from Jira Cloud`, { + jobId: jobContext.job.id, + count: result.length, + hasMore: false, + startAt, + }); + + return { + items: result, + hasMore: false, + startAt, + maxResults: result.length, + }; + } +} diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/cloud/entities/users.step.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/cloud/entities/users.step.ts new file mode 100644 index 0000000000..ea008e63fc --- /dev/null +++ b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/cloud/entities/users.step.ts @@ -0,0 +1,51 @@ +import { pullUsers } from "@plane/etl/jira"; +import type { JiraV2Service, PaginatedResult, ImportedJiraUser } from "@plane/etl/jira-server"; +import { logger } from "@plane/logger"; +import type { TJobContext } from "../../../../types"; +import { JiraUsersStep } from "../../shared/entities"; + +export class JiraCloudUserStep extends JiraUsersStep { + /* + * For jira cloud, we don't have access to user emails, hence + * we take a csv in order to import the users, and we use a different + * pull function, hence we need to overwrite this. + */ + protected async pull( + jobContext: TJobContext, + _client: JiraV2Service, + _startAt: number, + _jobId: string + ): Promise> { + const { job } = jobContext; + if (!job.config.users) { + logger.error(`[${job.id}] [${this.name}] No users found in config`, { jobId: job.id }); + return { + items: [], + hasMore: false, + total: 0, + startAt: 0, + maxResults: 0, + }; + } + + const users = pullUsers(job.config.users); + const items = users.map((user) => ({ + avatarUrl: "", + user_id: user.user_id, + email: user.email, + full_name: "", + user_name: user.user_name, + added_to_org: user.added_to_org, + org_role: user.org_role, + user_status: "Active", + })); + + return { + items, + hasMore: false, + total: items.length, + startAt: 0, + maxResults: items.length, + }; + } +} diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/cloud/index.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/cloud/index.ts new file mode 100644 index 0000000000..73134e0a04 --- /dev/null +++ b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/cloud/index.ts @@ -0,0 +1,37 @@ +import { E_IMPORTER_KEYS } from "@plane/etl/core"; +import { + // Pre-run steps + PlaneProjectConfigurationStep, + // Entity steps + JiraBoardsStep, + JiraCyclesStep, + JiraDefaultPropertiesStep, + JiraIssuePropertyOptionsStep, + JiraModulesStep, + WaitForCeleryStep, + // Association steps + JiraRelationsStep, +} from "../shared"; +import { JiraCloudUserStep, JiraCloudIssuePropertiesStep, JiraCloudIssueTypesStep } from "./entities"; +import { JiraCloudIssuesStep } from "./issues"; + +const JIRA_CLOUD_STEPS = [ + // Pre-run steps + new PlaneProjectConfigurationStep(), + // Entity steps + new JiraCloudUserStep(), + new JiraModulesStep(E_IMPORTER_KEYS.JIRA), + new JiraBoardsStep(), + new JiraCyclesStep(E_IMPORTER_KEYS.JIRA), + new JiraCloudIssueTypesStep(E_IMPORTER_KEYS.JIRA), + new JiraCloudIssuePropertiesStep(E_IMPORTER_KEYS.JIRA), + new JiraDefaultPropertiesStep(E_IMPORTER_KEYS.JIRA), + new JiraIssuePropertyOptionsStep(E_IMPORTER_KEYS.JIRA), + // Issue steps + new JiraCloudIssuesStep(E_IMPORTER_KEYS.JIRA), + new WaitForCeleryStep(), + // Association steps + new JiraRelationsStep(E_IMPORTER_KEYS.JIRA), +]; + +export default JIRA_CLOUD_STEPS; diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/cloud/issues/index.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/cloud/issues/index.ts new file mode 100644 index 0000000000..2ab03072ed --- /dev/null +++ b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/cloud/issues/index.ts @@ -0,0 +1 @@ +export * from "./issues.step"; diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/cloud/issues/issues.step.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/cloud/issues/issues.step.ts new file mode 100644 index 0000000000..0fbde5b73d --- /dev/null +++ b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/cloud/issues/issues.step.ts @@ -0,0 +1,155 @@ +import { pullIssuesV2 } from "@plane/etl/jira"; +import { logger } from "@plane/logger"; +import type { TImportJob } from "@plane/types"; +import { createJiraClient } from "@/apps/jira-importer/helpers/migration-helpers"; +import type { TStepExecutionContext } from "@/apps/jira-server-importer/v2/types"; +import { getJobCredentials } from "@/helpers/job"; +import { withCache } from "../../../../helpers/cache"; +import { buildExternalId } from "../../../../helpers/job"; +import type { TJobContext } from "../../../../types"; +import { JiraIssuesStep } from "../../shared/issues"; + +/** + * Jira Cloud Issues Step + * + * Extends JiraServerIssuesStep to use nextPageToken pagination + * instead of startAt pagination for Jira Cloud API + */ +export class JiraCloudIssuesStep extends JiraIssuesStep { + /** + * Override pagination context to use nextPageToken for Jira Cloud + */ + protected getPaginationContext(previousContext?: TStepExecutionContext) { + return { + startAt: 0, // Not used for Cloud pagination + totalProcessed: previousContext?.pageCtx.totalProcessed ?? 0, + nextPageToken: previousContext?.state?.nextPageToken as string | undefined, + }; + } + + /** + * Pull paginated issues from Jira Cloud using nextPageToken + */ + protected async pull(props: { + jobContext: TJobContext; + projectKey: string; + paginationCtx: { startAt: number; totalProcessed: number; nextPageToken?: string }; + }) { + const { job } = props.jobContext; + const credentials = await getJobCredentials(job); + const client = createJiraClient(job, credentials); + + // Get total number of issues first + const total = await withCache( + `jira-cloud-issues-total-${props.projectKey}`, + job, + async () => await client.getNumberOfIssues(props.projectKey) + ); + + const result = await pullIssuesV2( + { + client, + nextPageToken: props.paginationCtx.nextPageToken, + }, + props.projectKey, + total + ); + + logger.info(`[${job.id}] [${this.name}] Pulled issues from Jira Cloud`, { + jobId: job.id, + count: result.items.length, + hasMore: result.hasMore, + nextPageToken: result.nextPageToken ? "present" : "none", + }); + + return { + items: result.items as any[], + hasMore: result.hasMore, + total: total ?? 0, + nextPageToken: result.nextPageToken, + }; + } + + /** + * Build next context with nextPageToken for Jira Cloud pagination + */ + protected buildNextContext( + issuesResult: { items: any[]; hasMore: boolean; total: number; nextPageToken?: string }, + paginationCtx: { startAt: number; totalProcessed: number; nextPageToken?: string }, + pulled: number, + pushed: number + ) { + const newTotalProcessed = paginationCtx.totalProcessed + pulled; + + if (issuesResult.hasMore && issuesResult.nextPageToken) { + return { + pageCtx: { + startAt: 0, // Not used for Cloud + hasMore: true, + totalProcessed: newTotalProcessed, + }, + results: { + pulled, + pushed, + errors: [], + }, + state: { + nextPageToken: issuesResult.nextPageToken, + }, + }; + } + + return { + pageCtx: { + startAt: 0, + hasMore: false, + totalProcessed: newTotalProcessed, + }, + results: { + pulled, + pushed, + errors: [], + }, + }; + } + + /** + * Override shouldReturnEmpty to check first page correctly for Cloud + */ + protected shouldReturnEmpty( + issuesResult: { items: any[]; hasMore: boolean; total: number }, + paginationCtx: { startAt: number; totalProcessed: number; nextPageToken?: string } + ): boolean { + // First page in Cloud is when there's no nextPageToken in context + return issuesResult.items.length === 0 && !paginationCtx.nextPageToken; + } + + protected extractSprints(job: TImportJob, issue: any): string[] { + /* We go through the issue fields and find the custom field that matches + * the definition of a sprint, which should be an array of objects, and + * each object should contain boardId, name, startDate, endDate, and state + */ + + const isSprintField = (value: any): boolean => + Array.isArray(value) && + value.length > 0 && + value[0].boardId && + value[0].name && + value[0].startDate && + value[0].endDate && + value[0].state; + + const config = job.config; + const resourceId = config.resource?.id || ""; + const projectId = job.project_id || ""; + + for (const [fieldKey, fieldValue] of Object.entries(issue.fields)) { + if (!fieldKey.startsWith("customfield_")) continue; + if (isSprintField(fieldValue)) { + return (fieldValue as any[]).map((s: any) => buildExternalId(projectId, resourceId, s.id.toString())); + } + } + + return []; + } +} diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/index.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/index.ts deleted file mode 100644 index 3e2c259d84..0000000000 --- a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/index.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { - JiraServerUsersStep, - JiraServerModulesStep, - JiraServerBoardsStep, - JiraServerCyclesStep, - JiraServerIssueTypesStep, - JiraServerIssuePropertiesStep, - JiraServerDefaultPropertiesStep, - JiraServerIssuePropertyOptionsStep, -} from "./entities"; -import { JiraServerIssuesStep, WaitForCeleryStep } from "./issues"; -import { PlaneProjectConfigurationStep } from "./pre-run"; -import { JiraServerRelationsStep } from "./relations"; - -const JIRA_SERVER_STEPS = [ - // Pre-run steps - new PlaneProjectConfigurationStep(), - // Entity steps - new JiraServerUsersStep(), - new JiraServerModulesStep(), - new JiraServerBoardsStep(), - new JiraServerCyclesStep(), - new JiraServerIssueTypesStep(), - new JiraServerIssuePropertiesStep(), - new JiraServerDefaultPropertiesStep(), - new JiraServerIssuePropertyOptionsStep(), - // Issue steps - new JiraServerIssuesStep(), - new WaitForCeleryStep(), - // Association steps - new JiraServerRelationsStep(), -]; - -export default JIRA_SERVER_STEPS; diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/server/index.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/server/index.ts new file mode 100644 index 0000000000..cd5691d7c1 --- /dev/null +++ b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/server/index.ts @@ -0,0 +1,40 @@ +import { E_IMPORTER_KEYS } from "@plane/etl/core"; +import { + // Pre-run steps + PlaneProjectConfigurationStep, + // Entity steps + JiraBoardsStep, + JiraCyclesStep, + JiraDefaultPropertiesStep, + JiraIssuePropertiesStep, + JiraIssuePropertyOptionsStep, + JiraIssueTypesStep, + JiraModulesStep, + JiraUsersStep, + // Issue steps + JiraIssuesStep, + WaitForCeleryStep, + // Association steps + JiraRelationsStep, +} from "../shared"; + +const JIRA_SERVER_STEPS = [ + // Pre-run steps + new PlaneProjectConfigurationStep(), + // Entity steps + new JiraUsersStep(), + new JiraModulesStep(E_IMPORTER_KEYS.JIRA_SERVER), + new JiraBoardsStep(), + new JiraCyclesStep(E_IMPORTER_KEYS.JIRA_SERVER), + new JiraIssueTypesStep(E_IMPORTER_KEYS.JIRA_SERVER), + new JiraIssuePropertiesStep(E_IMPORTER_KEYS.JIRA_SERVER), + new JiraDefaultPropertiesStep(E_IMPORTER_KEYS.JIRA_SERVER), + new JiraIssuePropertyOptionsStep(E_IMPORTER_KEYS.JIRA_SERVER), + // Issue steps + new JiraIssuesStep(E_IMPORTER_KEYS.JIRA_SERVER), + new WaitForCeleryStep(), + // Association steps + new JiraRelationsStep(E_IMPORTER_KEYS.JIRA_SERVER), +]; + +export default JIRA_SERVER_STEPS; diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/entities/boards.step.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/entities/boards.step.ts similarity index 94% rename from apps/silo/src/apps/jira-server-importer/v2/migrator/steps/entities/boards.step.ts rename to apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/entities/boards.step.ts index 02f90ec36c..b0ff67c5eb 100644 --- a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/entities/boards.step.ts +++ b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/entities/boards.step.ts @@ -15,7 +15,7 @@ import type { TStepExecutionContext, TStepExecutionInput, } from "@/apps/jira-server-importer/v2/types"; -import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types"; +import { EJiraStep } from "@/apps/jira-server-importer/v2/types"; /** * Handles the extraction of boards from Jira Server. @@ -28,8 +28,8 @@ import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types"; * for mapping issues to modules/views. If future requirements need board entities * in Plane, add transform/push logic here. */ -export class JiraServerBoardsStep implements IStep { - name = EJiraServerStep.BOARDS; +export class JiraBoardsStep implements IStep { + name = EJiraStep.BOARDS; dependencies = []; private readonly PAGE_SIZE = 100; @@ -52,12 +52,13 @@ export class JiraServerBoardsStep implements IStep { logger.info(`[${job.id}] [${this.name}] Pulling boards`, { startAt, totalProcessed }); const result = await this.pullBoards(sourceClient, projectId, startAt, job.id); + const scrumBoards = result.items.filter((board) => board.type === "scrum"); if (this.shouldReturnEmpty(result, startAt)) { return createEmptyContext(); } - await this.storeBoards(storage, job.id, result.items); + await this.storeBoards(storage, job.id, scrumBoards); const newTotalProcessed = totalProcessed + result.items.length; diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/entities/cycles.step.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/entities/cycles.step.ts similarity index 93% rename from apps/silo/src/apps/jira-server-importer/v2/migrator/steps/entities/cycles.step.ts rename to apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/entities/cycles.step.ts index 6ef48ead62..7b59ced019 100644 --- a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/entities/cycles.step.ts +++ b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/entities/cycles.step.ts @@ -1,5 +1,6 @@ import type { Sprint } from "jira.js/out/agile/models"; import { v4 as uuid } from "uuid"; +import type { E_IMPORTER_KEYS } from "@plane/etl/core"; import type { JiraConfig, JiraSprint } from "@plane/etl/jira-server"; import { pullSprintsForBoardV2, transformSprint } from "@plane/etl/jira-server"; import { logger } from "@plane/logger"; @@ -14,7 +15,7 @@ import type { TStepExecutionContext, TStepExecutionInput, } from "@/apps/jira-server-importer/v2/types"; -import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types"; +import { EJiraStep } from "@/apps/jira-server-importer/v2/types"; import { createAllCyclesV2 } from "@/etl/migrator/cycles.migrator"; /** @@ -23,12 +24,14 @@ import { createAllCyclesV2 } from "@/etl/migrator/cycles.migrator"; * * Dependencies: boards - board IDs to iterate through for sprint collection */ -export class JiraServerCyclesStep implements IStep { - name = EJiraServerStep.CYCLES; - dependencies = [EJiraServerStep.BOARDS]; +export class JiraCyclesStep implements IStep { + name = EJiraStep.CYCLES; + dependencies = [EJiraStep.BOARDS]; private readonly PAGE_SIZE = 100; + constructor(private readonly source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA) { } + /** * Executes the sprint extraction process: * 1. Extracts current board ID from dependency data @@ -64,7 +67,7 @@ export class JiraServerCyclesStep implements IStep { } catch (error) { logger.error(`[${jobContext.job.id}] [${this.name}] Step failed`, { jobId: jobContext.job.id, - error: error instanceof Error ? error.message : String(error), + error: error }); throw error; } @@ -152,7 +155,16 @@ export class JiraServerCyclesStep implements IStep { issues: [], // Issues are handled in separate step })); - return formattedSprints.map((sprint) => transformSprint(resourceId, job.project_id, sprint)); + return formattedSprints.map((sprint) => + transformSprint( + { + resourceId, + projectId: job.project_id, + source: this.source, + }, + sprint + ) + ); } /** diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/entities/default-properties.step.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/entities/default-properties.step.ts similarity index 80% rename from apps/silo/src/apps/jira-server-importer/v2/migrator/steps/entities/default-properties.step.ts rename to apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/entities/default-properties.step.ts index a6a73bd4b1..a844c26a2e 100644 --- a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/entities/default-properties.step.ts +++ b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/entities/default-properties.step.ts @@ -2,8 +2,8 @@ import { createSuccessContext } from "@/apps/jira-server-importer/v2/helpers/ctx import { extractJobData } from "@/apps/jira-server-importer/v2/helpers/job"; import { getSupportedDefaultProperties } from "@/apps/jira-server-importer/v2/helpers/properties"; import type { TIssueTypesData, TStepExecutionContext, TStepExecutionInput } from "@/apps/jira-server-importer/v2/types"; -import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types"; -import { JiraServerIssuePropertiesStep } from "./issue-properties.step"; +import { EJiraStep } from "@/apps/jira-server-importer/v2/types"; +import { JiraIssuePropertiesStep } from "./issue-properties.step"; /* * @overview @@ -14,9 +14,9 @@ import { JiraServerIssuePropertiesStep } from "./issue-properties.step"; * to all the available issue types, which will allow us to map those properties * to the corresponding plane properties. */ -export class JiraServerDefaultPropertiesStep extends JiraServerIssuePropertiesStep { - name: EJiraServerStep = EJiraServerStep.DEFAULT_PROPERTIES; - dependencies: EJiraServerStep[] = [EJiraServerStep.ISSUE_TYPES]; +export class JiraDefaultPropertiesStep extends JiraIssuePropertiesStep { + name: EJiraStep = EJiraStep.DEFAULT_PROPERTIES; + dependencies: EJiraStep[] = [EJiraStep.ISSUE_TYPES]; async execute(input: TStepExecutionInput): Promise { // Get the issue types from the dependency data and for each issue type, populate @@ -37,13 +37,14 @@ export class JiraServerDefaultPropertiesStep extends JiraServerIssuePropertiesSt resourceId, projectId, issueType.id, - issueType.external_id + issueType.external_id, + this.source ); defaultPropertiesToCreate.push(...defaultProperties); } const pushed = await this.push(jobContext, defaultPropertiesToCreate, issueTypes); - await this.storePropertiesData(EJiraServerStep.ISSUE_PROPERTIES, job, pushed, storage); + await this.storePropertiesData(EJiraStep.ISSUE_PROPERTIES, job, pushed, storage); return createSuccessContext({ pulled: defaultPropertiesToCreate.length, diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/entities/index.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/entities/index.ts similarity index 100% rename from apps/silo/src/apps/jira-server-importer/v2/migrator/steps/entities/index.ts rename to apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/entities/index.ts diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/entities/issue-properties.step.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/entities/issue-properties.step.ts similarity index 94% rename from apps/silo/src/apps/jira-server-importer/v2/migrator/steps/entities/issue-properties.step.ts rename to apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/entities/issue-properties.step.ts index 8a45763a1b..a663e0b918 100644 --- a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/entities/issue-properties.step.ts +++ b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/entities/issue-properties.step.ts @@ -1,4 +1,5 @@ import { v4 as uuid } from "uuid"; +import type { E_IMPORTER_KEYS } from "@plane/etl/core"; import type { JiraConfig, JiraIssueField } from "@plane/etl/jira-server"; import { pullIssueFieldsV2, transformIssueFields } from "@plane/etl/jira-server"; import { logger } from "@plane/logger"; @@ -15,7 +16,7 @@ import type { TStepExecutionContext, TStepExecutionInput, } from "@/apps/jira-server-importer/v2/types"; -import { E_ADDITIONAL_STORAGE_KEYS, EJiraServerStep } from "@/apps/jira-server-importer/v2/types"; +import { E_ADDITIONAL_STORAGE_KEYS, EJiraStep } from "@/apps/jira-server-importer/v2/types"; import { createOrUpdateIssueProperties } from "@/etl/migrator/issue-types/issue-property.migrator"; /** @@ -24,9 +25,11 @@ import { createOrUpdateIssueProperties } from "@/etl/migrator/issue-types/issue- * * Note: No pagination needed - fields are a small dataset */ -export class JiraServerIssuePropertiesStep implements IStep { - name = EJiraServerStep.ISSUE_PROPERTIES; - dependencies = [EJiraServerStep.ISSUE_TYPES]; +export class JiraIssuePropertiesStep implements IStep { + name = EJiraStep.ISSUE_PROPERTIES; + dependencies = [EJiraStep.ISSUE_TYPES]; + + constructor(protected readonly source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA) {} async execute(input: TStepExecutionInput): Promise { const { jobContext, storage, dependencyData } = input; @@ -91,7 +94,7 @@ export class JiraServerIssuePropertiesStep implements IStep { /** * Pull custom fields from Jira Server */ - private async pull( + protected async pull( jobCtx: TJobContext, projectId: string, issueTypesData: TIssueTypesData @@ -119,7 +122,7 @@ export class JiraServerIssuePropertiesStep implements IStep { const resourceId = job.config.resource ? job.config.resource.id : uuid(); return jiraFields - .map((field) => transformIssueFields(resourceId, job.project_id, field)) + .map((field) => transformIssueFields({ resourceId, projectId: job.project_id, source: this.source }, field)) .filter((field) => field && field.property_type) as Partial[]; } @@ -261,7 +264,7 @@ export class JiraServerIssuePropertiesStep implements IStep { */ protected async storePropertiesData( // We need to pass the name of the step, as default properties step uses this function to store data - name: EJiraServerStep, + name: EJiraStep, job: TImportJob, properties: ExIssueProperty[], storage: IStorageService diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/entities/issue-property-options.step.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/entities/issue-property-options.step.ts similarity index 93% rename from apps/silo/src/apps/jira-server-importer/v2/migrator/steps/entities/issue-property-options.step.ts rename to apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/entities/issue-property-options.step.ts index 11894eeac7..6caecb9db5 100644 --- a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/entities/issue-property-options.step.ts +++ b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/entities/issue-property-options.step.ts @@ -1,4 +1,5 @@ import { v4 as uuid } from "uuid"; +import type { E_IMPORTER_KEYS } from "@plane/etl/core"; import type { JiraConfig, JiraCustomFieldKeys, JiraIssueField } from "@plane/etl/jira-server"; import { OPTION_CUSTOM_FIELD_TYPES, transformIssueFieldOptions } from "@plane/etl/jira-server"; import { logger } from "@plane/logger"; @@ -14,7 +15,7 @@ import type { TStepExecutionContext, TStepExecutionInput, } from "@/apps/jira-server-importer/v2/types"; -import { E_ADDITIONAL_STORAGE_KEYS, EJiraServerStep } from "@/apps/jira-server-importer/v2/types"; +import { E_ADDITIONAL_STORAGE_KEYS, EJiraStep } from "@/apps/jira-server-importer/v2/types"; import { createOrUpdateIssuePropertiesOptions } from "@/etl/migrator/issue-types/issue-property.migrator"; /** @@ -23,9 +24,11 @@ import { createOrUpdateIssuePropertiesOptions } from "@/etl/migrator/issue-types * * Note: Only processes select/multi-select fields */ -export class JiraServerIssuePropertyOptionsStep implements IStep { - name = EJiraServerStep.ISSUE_PROPERTY_OPTIONS; - dependencies = [EJiraServerStep.ISSUE_PROPERTIES]; +export class JiraIssuePropertyOptionsStep implements IStep { + name = EJiraStep.ISSUE_PROPERTY_OPTIONS; + dependencies = [EJiraStep.ISSUE_PROPERTIES]; + + constructor(private readonly source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA) {} async execute(input: TStepExecutionInput): Promise { const { jobContext, storage, dependencyData } = input; @@ -85,7 +88,10 @@ export class JiraServerIssuePropertyOptionsStep implements IStep { field.schema?.custom && OPTION_CUSTOM_FIELD_TYPES.includes(field.schema.custom as JiraCustomFieldKeys) ) .flatMap( - (field) => field.options?.map((option) => transformIssueFieldOptions(resourceId, job.project_id, option)) || [] + (field) => + field.options?.map((option) => + transformIssueFieldOptions({ resourceId, projectId: job.project_id, source: this.source }, option) + ) || [] ) .filter(Boolean) as Partial[]; diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/entities/issue-types.step.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/entities/issue-types.step.ts similarity index 95% rename from apps/silo/src/apps/jira-server-importer/v2/migrator/steps/entities/issue-types.step.ts rename to apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/entities/issue-types.step.ts index 38290539e9..4875ced6c3 100644 --- a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/entities/issue-types.step.ts +++ b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/entities/issue-types.step.ts @@ -1,6 +1,7 @@ import type { IssueTypeDetails } from "jira.js/out/version2/models"; import { v4 as uuid } from "uuid"; import { E_FEATURE_FLAGS } from "@plane/constants"; +import type { E_IMPORTER_KEYS } from "@plane/etl/core"; import type { JiraConfig } from "@plane/etl/jira-server"; import { pullIssueTypesV2, transformIssueType } from "@plane/etl/jira-server"; import { logger } from "@plane/logger"; @@ -16,7 +17,7 @@ import type { TStepExecutionContext, TStepExecutionInput, } from "@/apps/jira-server-importer/v2/types"; -import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types"; +import { EJiraStep } from "@/apps/jira-server-importer/v2/types"; import { createOrUpdateIssueTypes } from "@/etl/migrator/issue-types/issue-type.migrator"; import { getPlaneFeatureFlagService } from "@/helpers/plane-api-client"; import { protect } from "@/lib"; @@ -27,10 +28,12 @@ import { protect } from "@/lib"; * * Handles Epic types and conflict resolution automatically */ -export class JiraServerIssueTypesStep implements IStep { - name = EJiraServerStep.ISSUE_TYPES; +export class JiraIssueTypesStep implements IStep { + name = EJiraStep.ISSUE_TYPES; dependencies = []; + constructor(private readonly source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA) {} + private readonly PAGE_SIZE = 50; /** @@ -135,7 +138,7 @@ export class JiraServerIssueTypesStep implements IStep { /** * Pull one page of issue types from Jira Server */ - private async pull(jobContext: TJobContext, projectId: string, startAt: number) { + protected async pull(jobContext: TJobContext, projectId: string, startAt: number) { const result = await pullIssueTypesV2( { client: jobContext.sourceClient, @@ -161,7 +164,9 @@ export class JiraServerIssueTypesStep implements IStep { private transform(job: TImportJob, jiraIssueTypes: IssueTypeDetails[]): Partial[] { const resourceId = job.config.resource ? job.config.resource.id : uuid(); - return jiraIssueTypes.map((issueType) => transformIssueType(resourceId, job.project_id, issueType)); + return jiraIssueTypes.map((issueType) => + transformIssueType({ resourceId, projectId: job.project_id, source: this.source }, issueType) + ); } /** diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/entities/modules.step.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/entities/modules.step.ts similarity index 90% rename from apps/silo/src/apps/jira-server-importer/v2/migrator/steps/entities/modules.step.ts rename to apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/entities/modules.step.ts index 689d7356cc..5703ee9af9 100644 --- a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/entities/modules.step.ts +++ b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/entities/modules.step.ts @@ -1,6 +1,7 @@ /* ----------------- Modules Migrator Step ----------------- */ import type { ComponentWithIssueCount } from "jira.js/out/version2/models"; import { v4 as uuid } from "uuid"; +import type { E_IMPORTER_KEYS } from "@plane/etl/core"; import type { JiraConfig, JiraV2Service } from "@plane/etl/jira-server"; import { pullComponentsV2, transformComponentV2 } from "@plane/etl/jira-server"; import { logger } from "@plane/logger"; @@ -14,19 +15,21 @@ import type { TStepExecutionContext, TStepExecutionInput, } from "@/apps/jira-server-importer/v2/types"; -import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types"; +import { EJiraStep } from "@/apps/jira-server-importer/v2/types"; import { createAllModulesV2 } from "@/etl/migrator/modules.migrator"; /** * Jira Server Modules Step (Components in Jira) * Pulls components from Jira Server with pagination, transforms to modules, and pushes to Plane */ -export class JiraServerModulesStep implements IStep { - name = EJiraServerStep.MODULES; +export class JiraModulesStep implements IStep { + name = EJiraStep.MODULES; dependencies = []; private readonly PAGE_SIZE = 100; + constructor(private readonly source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA) {} + async execute(input: TStepExecutionInput): Promise { const { jobContext, storage, previousContext } = input; const { job, sourceClient } = jobContext; @@ -102,7 +105,9 @@ export class JiraServerModulesStep implements IStep { */ private transform(job: TImportJob, jiraComponents: ComponentWithIssueCount[]) { const resourceId = job.config.resource ? job.config.resource.id : uuid(); - return jiraComponents.map((component) => transformComponentV2(resourceId, job.project_id, component)); + return jiraComponents.map((component) => + transformComponentV2({ resourceId, projectId: job.project_id, source: this.source }, component) + ); } /** diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/entities/users.step.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/entities/users.step.ts similarity index 87% rename from apps/silo/src/apps/jira-server-importer/v2/migrator/steps/entities/users.step.ts rename to apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/entities/users.step.ts index da4846b425..9c63c557c6 100644 --- a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/entities/users.step.ts +++ b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/entities/users.step.ts @@ -12,7 +12,7 @@ import type { TStepExecutionContext, TStepExecutionInput, } from "@/apps/jira-server-importer/v2/types"; -import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types"; +import { EJiraStep } from "@/apps/jira-server-importer/v2/types"; import { createUsers } from "@/etl/migrator/users.migrator"; import { protect } from "@/lib"; @@ -25,8 +25,8 @@ import { protect } from "@/lib"; * * User avatars are also imported to Plane during this process. */ -export class JiraServerUsersStep implements IStep { - name = EJiraServerStep.USERS; +export class JiraUsersStep implements IStep { + name = EJiraStep.USERS; dependencies = []; private readonly PAGE_SIZE = 100; @@ -37,7 +37,7 @@ export class JiraServerUsersStep implements IStep { private shouldExecute(input: TStepExecutionInput): boolean { const { jobContext } = input; const job = jobContext.job as TImportJob; - return !job.config?.skipUserImport; + return !job.config.skipUserImport; } /** @@ -81,7 +81,7 @@ export class JiraServerUsersStep implements IStep { }); // Pull users from Jira Server (paginated) - const pulledUsers = await this.pull(sourceClient, startAt, job.id); + const pulledUsers = await this.pull(jobContext, sourceClient, startAt, job.id); if (pulledUsers.items.length === 0) { logger.info(`[${job.id}] [${this.name}] No users found`, { jobId: job.id }); @@ -119,14 +119,14 @@ export class JiraServerUsersStep implements IStep { * @param jobId - The import job ID for logging purposes * @returns Paginated result containing user items and hasMore flag */ - private async pull(client: JiraV2Service, startAt: number, jobId: string) { + protected async pull(_jobContext: TJobContext, client: JiraV2Service, startAt: number, jobId: string) { const result = await pullUsersV2({ client, startAt, maxResults: this.PAGE_SIZE, }); - logger.info(`[${jobId}] [${this.name}] Pulled users from Jira Server`, { + logger.info(`[${jobId}] [${this.name}] Pulled users`, { jobId, count: result.items.length, hasMore: result.hasMore, @@ -256,18 +256,36 @@ export class JiraServerUsersStep implements IStep { storage: IStorageService ): Promise { const allUsers = [...existingUsers, ...createdUsers]; + + /* + * In jira cloud, we don't have access to user emails, hence insead + * of poluting the space with multiple mappings and if else conditions + * we are using a merge of displayName and email, in case of transformation + * if the email is not present, we can fallback to displayName, which can be + * expected to be present in the mapping. + */ const mappings = allUsers .filter((user) => user.email && user.id) - .map((user) => ({ - externalId: user.email!, - planeId: user.id!, - })); + .flatMap((user) => { + const displayName = user.display_name; + const email = user.email; + + return [ + ...(email + ? [ + { + externalId: email, + planeId: user.id!, + }, + ] + : []), + { + externalId: displayName, + planeId: user.id!, + }, + ]; + }); await storage.storeMapping(jobId, this.name, mappings); - - logger.info(`[${jobId}] [${this.name}] Stored mappings`, { - jobId, - mappingCount: mappings.length, - }); } } diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/index.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/index.ts new file mode 100644 index 0000000000..85958f4893 --- /dev/null +++ b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/index.ts @@ -0,0 +1,4 @@ +export * from "./entities"; +export * from "./issues"; +export * from "./pre-run"; +export * from "./relations"; diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/issues/index.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/issues/index.ts similarity index 100% rename from apps/silo/src/apps/jira-server-importer/v2/migrator/steps/issues/index.ts rename to apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/issues/index.ts diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/issues/issues.step.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/issues/issues.step.ts similarity index 59% rename from apps/silo/src/apps/jira-server-importer/v2/migrator/steps/issues/issues.step.ts rename to apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/issues/issues.step.ts index 5c4747bdf1..17567bb939 100644 --- a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/issues/issues.step.ts +++ b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/issues/issues.step.ts @@ -1,7 +1,14 @@ +import type { Worklog } from "jira.js/out/version2/models"; import { v4 as uuidv4 } from "uuid"; -import type { TIssuePropertyValuesPayload } from "@plane/etl/core"; -import type { IJiraIssue, JiraConfig, JiraIssueField } from "@plane/etl/jira-server"; -import { pullIssuesV2, transformComment, transformIssueV2 } from "@plane/etl/jira-server"; +import type { E_IMPORTER_KEYS, TIssuePropertyValuesPayload } from "@plane/etl/core"; +import type { IJiraIssue, JiraConfig, JiraIssueField, JiraV2Service } from "@plane/etl/jira-server"; +import { + pullAllCommentsForIssue, + pullAllWorklogsForIssue, + pullIssuesV2, + transformComment, + transformIssueV2, +} from "@plane/etl/jira-server"; import { logger } from "@plane/logger"; import type { ExIssue, ExIssueComment, ExIssueProperty, ExIssuePropertyOption, TWorklog } from "@plane/sdk"; import type { TImportJob } from "@plane/types"; @@ -20,7 +27,7 @@ import type { TStepExecutionContext, TStepExecutionInput, } from "@/apps/jira-server-importer/v2/types"; -import { E_ADDITIONAL_STORAGE_KEYS, EJiraServerStep } from "@/apps/jira-server-importer/v2/types"; +import { E_ADDITIONAL_STORAGE_KEYS, EJiraStep } from "@/apps/jira-server-importer/v2/types"; import { generateIssuePayloadV2 } from "@/etl/migrator/issues.migrator"; import { getAPIClient } from "@/services/client"; import type { BulkIssuePayload } from "@/types"; @@ -41,90 +48,71 @@ import { celeryProducer } from "@/worker"; * NOT dependencies (accessed via storage.lookupMapping): * - users, labels, issue_types */ -export class JiraServerIssuesStep implements IStep { - name = EJiraServerStep.ISSUES; +export class JiraIssuesStep implements IStep { + name = EJiraStep.ISSUES; dependencies = [ - EJiraServerStep.ISSUE_TYPES, // Provides issue types - EJiraServerStep.ISSUE_PROPERTIES, // Provides properties + rawFields - EJiraServerStep.ISSUE_PROPERTY_OPTIONS, // Provides options + EJiraStep.ISSUE_TYPES, // Provides issue types + EJiraStep.ISSUE_PROPERTIES, // Provides properties + rawFields + EJiraStep.ISSUE_PROPERTY_OPTIONS, // Provides options ]; + constructor(private readonly source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA) { } + private readonly PAGE_SIZE = 50; - /** - * Initialize report batch count on first page - * Automatically detects if it's the first page and calculates total batches - */ - private async initializeReportBatchCount(input: TStepExecutionInput, totalIssues: number): Promise { - const { jobContext, previousContext } = input; - const startAt = previousContext?.pageCtx.startAt || 0; - - // Only initialize on first page - if (startAt !== 0) return; - - const totalBatches = Math.ceil(totalIssues / this.PAGE_SIZE); - const client = getAPIClient(); - - await client.importReport.updateImportReport(jobContext.job.report_id, { - total_batch_count: totalBatches, - }); - - logger.info(`[${jobContext.job.id}] [${this.name}] Initialized report batch count`, { - jobId: jobContext.job.id, - totalIssues, - pageSize: this.PAGE_SIZE, - totalBatches, - }); - } - async execute(input: TStepExecutionInput): Promise { const { jobContext, storage, previousContext, dependencyData } = input; const { job } = jobContext; try { - const projectKey = job.config?.project?.key; - if (!projectKey) { - throw new Error("Project key not found in job config"); - } + const projectKey = this.getProjectKey(job); + const paginationCtx = this.getPaginationContext(previousContext); - const startAt = previousContext?.pageCtx.startAt || 0; - const totalProcessed = previousContext?.pageCtx.totalProcessed || 0; - - logger.info(`[${jobContext.job.id}] [${this.name}] Starting execution`, { + logger.info(`[${job.id}] [${this.name}] Starting execution`, { jobId: job.id, - startAt, - totalProcessed, + ...paginationCtx, }); // Get property data from dependencies (needed for property value transformation) const propertyData = this.getPropertyData(dependencyData); const additionalData = await this.getAdditionalData(storage, job); - // Pull paginated issues, comments, and transform property values - const pulled = await this.pull(jobContext, projectKey, startAt, propertyData, additionalData); + // Pull paginated issues from Jira (can be overridden in subclasses) + const issuesResult = await this.pull({ + jobContext, + projectKey, + paginationCtx, + }); - if (pulled.total !== undefined) { - await this.initializeReportBatchCount(input, pulled.total); - } + await this.initializeReportBatchCount(paginationCtx, issuesResult.total, job); - const associations = this.extractAssociations(job, pulled.issues); - - if (pulled.issues.length === 0) { - logger.info(`[${jobContext.job.id}] [${this.name}] No issues found`, { jobId: job.id }); + if (this.shouldReturnEmpty(issuesResult, paginationCtx)) { + logger.info(`[${job.id}] [${this.name}] No issues found`, { jobId: job.id }); return createEmptyContext(); } + // Process issues: extract comments and transform property values + const processed = await this.processIssuesData({ + jobContext, + issuesResult, + propertyData, + additionalData, + }); + + const associations = await this.extractAssociations(job, jobContext.sourceClient, processed.issues); + // Load entity mappings from storage - const mappings = await this.loadMappings(job, pulled.issues, associations, storage); + const mappings = await this.loadMappings(job, processed.issues, associations, storage); // Transform issues (uses existing transformIssue function) - const transformed = await this.transform(job, pulled.issues); + const transformed = await this.transform(job, processed.issues); + // Generate payload and send to Celery const pushed = await this.push( transformed, - pulled.comments, - pulled.propertyValues, + processed.comments, + processed.propertyValues, mappings, associations, propertyData, @@ -132,26 +120,19 @@ export class JiraServerIssuesStep implements IStep { ); // Store relations for Relations Step - const relations = this.extractRelations(job, pulled.issues); + const relations = this.extractRelations(job, processed.issues); await this.storeRelations(relations, storage, job.id); - logger.info(`[${jobContext.job.id}] [${this.name}] Completed page`, { + logger.info(`[${job.id}] [${this.name}] Completed page`, { jobId: job.id, - issues: pulled.issues.length, - comments: pulled.comments.length, + issues: processed.issues.length, + comments: processed.comments.length, pushed: pushed.length, }); - return createPaginationContext({ - hasMore: pulled.hasMore, - startAt: startAt, - pageSize: this.PAGE_SIZE, - pulled: pulled.issues.length, - pushed: pushed.length, - totalProcessed: totalProcessed + pulled.issues.length, - }); + return this.buildNextContext(issuesResult, paginationCtx, processed.issues.length, pushed.length); } catch (error) { - logger.error(`[${jobContext.job.id}] [${this.name}] Step failed`, { + logger.error(`[${job.id}] [${this.name}] Step failed`, { jobId: job.id, error: error instanceof Error ? error.message : String(error), }); @@ -160,50 +141,152 @@ export class JiraServerIssuesStep implements IStep { } /** - * Pull issues and comments, then transform property values - * Uses existing transformIssuePropertyValues for proper conversion + * Extract pagination state from previous context + * Override this in subclasses to support different pagination strategies + * (e.g., nextPageToken for Jira Cloud) */ - private async pull( - jobContext: TJobContext, - projectKey: string, - startAt: number, - propertyData: { - issueTypes: TIssueTypesData; - planeIssueProperties: ExIssueProperty[]; - planeIssuePropertiesOptions: ExIssuePropertyOption[]; - }, - additionalData: { - rawFields: JiraIssueField[]; + protected getPaginationContext(previousContext?: TStepExecutionContext) { + return { + startAt: previousContext?.pageCtx.startAt ?? 0, + totalProcessed: previousContext?.pageCtx.totalProcessed ?? 0, + }; + } + + /** + * Extract and validate project key from job configuration + */ + protected getProjectKey(job: TImportJob): string { + const projectKey = job.config?.project?.key; + if (!projectKey) { + throw new Error("Project key not found in job config"); } - ): Promise<{ - issues: IJiraIssue[]; - comments: ExIssueComment[]; - propertyValues: TIssuePropertyValuesPayload; + return projectKey; + } + + /** + * Initialize report batch count on first page + */ + protected async initializeReportBatchCount( + paginationCtx: { startAt: number; totalProcessed: number }, + totalIssues: number | undefined, + job: TImportJob + ): Promise { + // Only initialize on first page + if (paginationCtx.startAt !== 0 || !totalIssues) return; + + const totalBatches = Math.ceil(totalIssues / this.PAGE_SIZE); + const client = getAPIClient(); + + await client.importReport.updateImportReport(job.report_id, { + total_batch_count: totalBatches, + }); + + logger.info(`[${job.id}] [${this.name}] Initialized report batch count`, { + jobId: job.id, + totalIssues, + pageSize: this.PAGE_SIZE, + totalBatches, + }); + } + + /** + * Check if step should return empty context + */ + protected shouldReturnEmpty( + issuesResult: { items: IJiraIssue[]; hasMore: boolean; total: number }, + paginationCtx: { startAt: number; totalProcessed: number } + ): boolean { + return issuesResult.items.length === 0 && paginationCtx.startAt === 0; + } + + /** + * Build the next context based on pagination state + * Override this in subclasses to support different pagination strategies + */ + protected buildNextContext( + issuesResult: { items: IJiraIssue[]; hasMore: boolean; total: number }, + paginationCtx: { startAt: number; totalProcessed: number }, + pulled: number, + pushed: number + ): TStepExecutionContext { + return createPaginationContext({ + hasMore: issuesResult.hasMore, + startAt: paginationCtx.startAt, + pageSize: this.PAGE_SIZE, + pulled, + pushed, + totalProcessed: paginationCtx.totalProcessed + pulled, + }); + } + + /** + * Pull paginated issues from Jira + * Override this in subclasses to customize issue fetching behavior + * (e.g., use nextPageToken for Jira Cloud) + */ + protected async pull(props: { + jobContext: TJobContext; + projectKey: string; + paginationCtx: { startAt: number; totalProcessed: number }; + }): Promise<{ + items: IJiraIssue[]; hasMore: boolean; total: number; }> { - const { projectId, resourceId } = extractJobData(jobContext.job); - // Pull paginated issues const issuesResult = await pullIssuesV2( { - client: jobContext.sourceClient, - startAt, + client: props.jobContext.sourceClient, + startAt: props.paginationCtx.startAt, maxResults: this.PAGE_SIZE, }, - projectKey + props.projectKey ); - // Pull comments for these issues - // Merge: timestamps from fields.comment, HTML body from renderedFields.comment - const comments: Partial[] = issuesResult.items - .map((issue) => { - const fieldsComments = issue.fields?.comment?.comments || []; - const renderedComments = issue.renderedFields?.comment?.comments || []; + logger.info(`[${props.jobContext.job.id}] [${this.name}] Pulled issues`, { + jobId: props.jobContext.job.id, + count: issuesResult.items.length, + hasMore: issuesResult.hasMore, + startAt: props.paginationCtx.startAt, + }); + return { + items: issuesResult.items, + hasMore: issuesResult.hasMore, + total: issuesResult.total ?? 0, + }; + } + + /** + * Extract and transform comments from issues + * Merges timestamps from fields.comment with HTML body from renderedFields.comment + */ + protected async extractCommentsFromIssues(props: { + _jobContext: TJobContext; + issues: IJiraIssue[]; + resourceId: string; + projectId: string; + }): Promise { + const comments = []; + for (const issue of props.issues) { + const fieldsComments = issue.fields?.comment?.comments || []; + const renderedComments = issue.renderedFields?.comment?.comments || []; + /* + * Jira provides us with the comments, but those comments are paginated, in + * case the number of comments are more than what we have in the comments property + * we will need to fetch the remaining comments. + */ + const shouldFetchMoreComments = issue.fields.comment?.total > issue.fields.comment?.comments?.length; + if (shouldFetchMoreComments) { + const allComments = await pullAllCommentsForIssue(issue, props._jobContext.sourceClient); + const transformedComments = allComments.map((comment) => + transformComment({ resourceId: props.resourceId, projectId: props.projectId, source: this.source }, comment) + ); + comments.push(transformedComments); + } else { // Create a map of rendered comments by ID for quick lookup const renderedCommentsMap = new Map(renderedComments.map((rendered) => [rendered.id, rendered])); - return fieldsComments.map((comment) => { + const transformedComments = fieldsComments.map((comment) => { // Get the rendered version for HTML body const renderedComment = renderedCommentsMap.get(comment.id); @@ -217,35 +300,72 @@ export class JiraServerIssuesStep implements IStep { issue_id: issue.id, }; - return transformComment(resourceId, projectId, mergedComment); + return transformComment( + { resourceId: props.resourceId, projectId: props.projectId, source: this.source }, + mergedComment + ); }); - }) - .flat(); + comments.push(transformedComments); + } + } + + return comments.flat() as ExIssueComment[]; + } + + /** + * Process pulled issues: extract comments and transform property values + * Uses existing transformIssuePropertyValues for proper conversion + */ + protected async processIssuesData(props: { + jobContext: TJobContext; + issuesResult: { + items: IJiraIssue[]; + hasMore: boolean; + total: number; + }; + propertyData: { + issueTypes: TIssueTypesData; + planeIssueProperties: ExIssueProperty[]; + planeIssuePropertiesOptions: ExIssuePropertyOption[]; + }; + additionalData: { + rawFields: JiraIssueField[]; + }; + }): Promise<{ + issues: IJiraIssue[]; + comments: ExIssueComment[]; + propertyValues: TIssuePropertyValuesPayload; + }> { + const { projectId, resourceId } = extractJobData(props.jobContext.job); + + // Extract comments from issues + const comments = await this.extractCommentsFromIssues({ + _jobContext: props.jobContext, + issues: props.issuesResult.items, + resourceId, + projectId, + }); // Transform property values for each issue using existing function const propertyValues = getTransformedIssuePropertyValuesV2( - jobContext.job, - issuesResult.items, - additionalData.rawFields, - propertyData.planeIssueProperties, - propertyData.issueTypes + props.jobContext.job, + props.issuesResult.items, + props.additionalData.rawFields, + props.propertyData.planeIssueProperties, + props.propertyData.issueTypes ); - logger.info(`[${jobContext.job.id}] [${this.name}] Pulled from Jira`, { - jobId: jobContext.job.id, - issues: issuesResult.items.length, + logger.info(`[${props.jobContext.job.id}] [${this.name}] Processed issues data`, { + jobId: props.jobContext.job.id, + issues: props.issuesResult.items.length, comments: comments.length, issuesWithPropertyValues: Object.keys(propertyValues).length, - hasMore: issuesResult.hasMore, - startAt, }); return { - issues: issuesResult.items, - comments: comments as ExIssueComment[], + issues: props.issuesResult.items, + comments, propertyValues, - hasMore: issuesResult.hasMore, - total: issuesResult.total ?? 0, }; } @@ -265,9 +385,9 @@ export class JiraServerIssuesStep implements IStep { }; } - const issueTypes = dependencyData[EJiraServerStep.ISSUE_TYPES] as TIssueTypesData; - const propertiesData = dependencyData[EJiraServerStep.ISSUE_PROPERTIES] as TIssuePropertiesData; - const optionsData = dependencyData[EJiraServerStep.ISSUE_PROPERTY_OPTIONS]; + const issueTypes = dependencyData[EJiraStep.ISSUE_TYPES] as TIssueTypesData; + const propertiesData = dependencyData[EJiraStep.ISSUE_PROPERTIES] as TIssuePropertiesData; + const optionsData = dependencyData[EJiraStep.ISSUE_PROPERTY_OPTIONS]; return { issueTypes, @@ -336,10 +456,10 @@ export class JiraServerIssuesStep implements IStep { // Load mappings in parallel from storage const [userMap, issueTypeMap, cycleMap, moduleMap] = await Promise.all([ // We are retrieving all users instead of associated users, as there can be custom fields that associates with users, and we can't select those - storage.retrieveMapping(jobId, EJiraServerStep.USERS), - storage.lookupMapping(jobId, EJiraServerStep.ISSUE_TYPES, Array.from(issueTypeIds)), - storage.lookupMapping(jobId, EJiraServerStep.CYCLES, Array.from(sprintExternalIds)), - storage.lookupMapping(jobId, EJiraServerStep.MODULES, Array.from(componentExternalIds)), + storage.retrieveMapping(jobId, EJiraStep.USERS), + storage.lookupMapping(jobId, EJiraStep.ISSUE_TYPES, Array.from(issueTypeIds)), + storage.lookupMapping(jobId, EJiraStep.CYCLES, Array.from(sprintExternalIds)), + storage.lookupMapping(jobId, EJiraStep.MODULES, Array.from(componentExternalIds)), ]); logger.info(`[${jobId}] [${this.name}] Loaded mappings`, { @@ -362,7 +482,13 @@ export class JiraServerIssuesStep implements IStep { const priorityMapping = job.config?.priority || {}; const transformed = issues.map((issue) => - transformIssueV2(resourceId, job.project_id, issue, resourceUrl, stateMapping, priorityMapping) + transformIssueV2( + { resourceId, projectId: job.project_id, source: this.source }, + issue, + resourceUrl, + stateMapping, + priorityMapping + ) ); return transformed; @@ -395,7 +521,11 @@ export class JiraServerIssuesStep implements IStep { ); } - private extractAssociations(job: TImportJob, issues: IJiraIssue[]): TIssuesAssociationsData { + private async extractAssociations( + job: TImportJob, + client: JiraV2Service, + issues: IJiraIssue[] + ): Promise { const { projectId, resourceId } = extractJobData(job); const cycles = new Map(); const modules = new Map(); @@ -404,7 +534,7 @@ export class JiraServerIssuesStep implements IStep { const issueExternalId = buildExternalId(projectId, resourceId, issue.id); const sprintExternalIds = this.extractSprints(job, issue); const componentExternalIds = this.extractComponents(job, issue); - const issueWorklogs = this.extractWorklogs(job, issue); + const issueWorklogs = await this.extractWorklogs(job, client, issue); cycles.set(issueExternalId, sprintExternalIds); modules.set(issueExternalId, componentExternalIds); worklogs.set(issueExternalId, issueWorklogs); @@ -412,7 +542,7 @@ export class JiraServerIssuesStep implements IStep { return { cycles, modules, worklogs }; } - private extractSprints(job: TImportJob, issue: IJiraIssue): string[] { + protected extractSprints(job: TImportJob, issue: IJiraIssue): string[] { const { projectId, resourceId } = extractJobData(job); const sprintFieldKey = detectSprintFieldId(issue); @@ -424,8 +554,8 @@ export class JiraServerIssuesStep implements IStep { : null; return sprintObjects ? sprintObjects - .map((s) => (s ? buildExternalId(projectId, resourceId, s.id.toString()) : null)) - .filter((s) => s !== null) + .map((s) => (s ? buildExternalId(projectId, resourceId, s.id.toString()) : null)) + .filter((s) => s !== null) : []; } @@ -467,15 +597,34 @@ export class JiraServerIssuesStep implements IStep { /** * Extract worklogs from issue */ - private extractWorklogs(_job: TImportJob, issue: IJiraIssue): Partial[] { + protected async extractWorklogs( + _job: TImportJob, + client: JiraV2Service, + issue: IJiraIssue + ): Promise[]> { + const transformWorklog = (worklog: Worklog) => ({ + description: worklog.comment ?? "", + duration: worklog.timeSpentSeconds ? worklog.timeSpentSeconds / 60 : 0, + logged_by: worklog.author?.emailAddress, + created_at: worklog.created, + updated_at: worklog.updated, + }); + + const shouldPullMoreWorklogs = issue.fields?.worklog?.total > issue.fields?.worklog?.worklogs?.length; + if (shouldPullMoreWorklogs) { + const worklogs = await pullAllWorklogsForIssue(issue, client); + return worklogs.map(transformWorklog); + } + return ( - issue.fields.worklog?.worklogs?.map((worklog) => ({ - description: worklog.comment ?? "", - duration: worklog.timeSpentSeconds ? worklog.timeSpentSeconds / 60 : 0, - logged_by: worklog.author?.emailAddress, - created_at: worklog.created, - updated_at: worklog.updated, - })) || [] + issue.fields?.worklog?.worklogs?.map((worklog, index) => { + const comment = + typeof worklog.comment === "string" + ? worklog.comment + : issue.renderedFields?.worklog?.worklogs?.[index]?.comment; + + return transformWorklog({ ...worklog, comment }); + }) || [] ); } diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/issues/wait-for-celery.step.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/issues/wait-for-celery.step.ts similarity index 92% rename from apps/silo/src/apps/jira-server-importer/v2/migrator/steps/issues/wait-for-celery.step.ts rename to apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/issues/wait-for-celery.step.ts index 067a855b90..682029e889 100644 --- a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/issues/wait-for-celery.step.ts +++ b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/issues/wait-for-celery.step.ts @@ -1,12 +1,12 @@ import { logger } from "@plane/logger"; import { createEmptyContext } from "@/apps/jira-server-importer/v2/helpers/ctx"; import type { IStep, TStepExecutionContext, TStepExecutionInput } from "@/apps/jira-server-importer/v2/types"; -import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types"; +import { EJiraStep } from "@/apps/jira-server-importer/v2/types"; import { wait } from "@/helpers/delay"; import { getAPIClient } from "@/services/client"; export class WaitForCeleryStep implements IStep { - name = EJiraServerStep.WAIT_FOR_CELERY; + name = EJiraStep.WAIT_FOR_CELERY; dependencies = []; async execute(input: TStepExecutionInput): Promise { diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/pre-run/index.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/pre-run/index.ts similarity index 100% rename from apps/silo/src/apps/jira-server-importer/v2/migrator/steps/pre-run/index.ts rename to apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/pre-run/index.ts diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/pre-run/project-config.step.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/pre-run/project-config.step.ts similarity index 93% rename from apps/silo/src/apps/jira-server-importer/v2/migrator/steps/pre-run/project-config.step.ts rename to apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/pre-run/project-config.step.ts index b3954b30eb..d81bb3cfe9 100644 --- a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/pre-run/project-config.step.ts +++ b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/pre-run/project-config.step.ts @@ -8,7 +8,7 @@ import { E_FEATURE_FLAGS } from "@plane/constants"; import { logger } from "@plane/logger"; import { createEmptyContext } from "@/apps/jira-server-importer/v2/helpers/ctx"; import type { IStep, TStepExecutionContext, TStepExecutionInput } from "@/apps/jira-server-importer/v2/types"; -import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types"; +import { EJiraStep } from "@/apps/jira-server-importer/v2/types"; import { getPlaneFeatureFlagService } from "@/helpers/plane-api-client"; export type TRequiredFlags = { @@ -18,8 +18,8 @@ export type TRequiredFlags = { }; export class PlaneProjectConfigurationStep implements IStep { - name = EJiraServerStep.PLANE_PROJECT_CONFIGURATION; - dependencies: EJiraServerStep[] = []; + name = EJiraStep.PLANE_PROJECT_CONFIGURATION; + dependencies: EJiraStep[] = []; async execute(input: TStepExecutionInput): Promise { const { jobContext } = input; diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/relations/index.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/relations/index.ts similarity index 100% rename from apps/silo/src/apps/jira-server-importer/v2/migrator/steps/relations/index.ts rename to apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/relations/index.ts diff --git a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/relations/relations.step.ts b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/relations/relations.step.ts similarity index 89% rename from apps/silo/src/apps/jira-server-importer/v2/migrator/steps/relations/relations.step.ts rename to apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/relations/relations.step.ts index f59bebff98..d9234ee29f 100644 --- a/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/relations/relations.step.ts +++ b/apps/silo/src/apps/jira-server-importer/v2/migrator/steps/shared/relations/relations.step.ts @@ -1,4 +1,5 @@ import { v4 as uuidv4 } from "uuid"; +import type { E_IMPORTER_KEYS } from "@plane/etl/core"; import { logger } from "@plane/logger"; import { createEmptyContext } from "@/apps/jira-server-importer/v2/helpers/ctx"; import type { @@ -7,7 +8,7 @@ import type { TStepExecutionContext, TStepExecutionInput, } from "@/apps/jira-server-importer/v2/types"; -import { E_ADDITIONAL_STORAGE_KEYS, EJiraServerStep } from "@/apps/jira-server-importer/v2/types"; +import { E_ADDITIONAL_STORAGE_KEYS, EJiraStep } from "@/apps/jira-server-importer/v2/types"; import { celeryProducer } from "@/worker"; /** @@ -16,9 +17,10 @@ import { celeryProducer } from "@/worker"; * Retrieves all accumulated relations from storage and sends to Celery in one batch * Only handles relationships (parent, blocking, etc) - cycles/modules already handled */ -export class JiraServerRelationsStep implements IStep { - name = EJiraServerStep.RELATIONS; +export class JiraRelationsStep implements IStep { + name = EJiraStep.RELATIONS; dependencies = []; + constructor(private readonly source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA) {} async execute(input: TStepExecutionInput): Promise { const { jobContext, storage } = input; @@ -68,7 +70,7 @@ export class JiraServerRelationsStep implements IStep { job_id: job.id, workspace_id: job.workspace_id, project_id: job.project_id, - source: job.source, + source: this.source, user_id: credentials.user_id, }, job.workspace_slug, diff --git a/apps/silo/src/apps/jira-server-importer/v2/types/index.ts b/apps/silo/src/apps/jira-server-importer/v2/types/index.ts index d5f39743b4..7081a85926 100644 --- a/apps/silo/src/apps/jira-server-importer/v2/types/index.ts +++ b/apps/silo/src/apps/jira-server-importer/v2/types/index.ts @@ -2,7 +2,7 @@ import type { JiraV2Service } from "@plane/etl/jira-server"; import type { Client as PlaneClient, TWorklog } from "@plane/sdk"; import type { TImportJob, TWorkspaceCredential } from "@plane/types"; -export enum EJiraServerStep { +export enum EJiraStep { // Pre-run PLANE_PROJECT_CONFIGURATION = "plane_project_configuration", @@ -45,6 +45,15 @@ export type TIssuesAssociationsData = { worklogs: Map[]>; }; +/** + * Failed step tracking structure + */ +export type TFailedStep = { + name: string; + error: string; + failedAt: string; +}; + export type TOrchestratorState = { jobId: string; @@ -54,6 +63,9 @@ export type TOrchestratorState = { totalSteps: number; completedSteps: string[]; + // Failed steps (logged and skipped) + failedSteps?: TFailedStep[]; + // Timestamps startedAt: string; lastUpdatedAt: string; @@ -103,10 +115,10 @@ export type TIssuePropertiesData = Array<{ */ export interface IStep { /** Step name (e.g., 'users', 'issues') */ - name: EJiraServerStep; + name: EJiraStep; /** data that should be loaded before handling this step */ - dependencies: EJiraServerStep[]; + dependencies: EJiraStep[]; shouldFail?: boolean; /** Execute pull, transform, and push for this step */ diff --git a/apps/silo/src/worker/manager.ts b/apps/silo/src/worker/manager.ts index 7978cf5018..172d712000 100644 --- a/apps/silo/src/worker/manager.ts +++ b/apps/silo/src/worker/manager.ts @@ -13,8 +13,10 @@ import { FlatfileMigrator } from "@/apps/flatfile/migrator/flatfile.migrator"; import { GithubWebhookWorker } from "@/apps/github/workers"; import { PlaneGithubWebhookWorker } from "@/apps/github/workers/plane"; import { GitlabWebhookWorker } from "@/apps/gitlab"; -import { JiraDataMigrator } from "@/apps/jira-importer/migrator/jira.migrator"; -import { JiraServerImportOrchestrator } from "@/apps/jira-server-importer/v2/migrator/orchestrator"; +import { + getJiraCloudImportOrchestrator, + getJiraServerImportOrchestrator, +} from "@/apps/jira-server-importer/v2/migrator"; import { LinearDocsMigrator } from "@/apps/linear-importer/migrator/linear-docs.migrator"; import { LinearDataMigrator } from "@/apps/linear-importer/migrator/linear.migrator"; import { NotionDataMigrator } from "@/apps/notion-importer/worker"; @@ -46,10 +48,9 @@ class WorkerFactory { static createWorker(type: string, mq: MQ, store: Store): TaskHandler { switch (type) { case "jira": - return new JiraDataMigrator(mq, store); + return getJiraCloudImportOrchestrator(mq, store); case "jira_server": - // return new JiraDataCenterMigrator(mq, store); - return new JiraServerImportOrchestrator(mq, store); + return getJiraServerImportOrchestrator(mq, store); case "linear": return new LinearDataMigrator(mq, store); case "linear_docs": diff --git a/apps/web/ee/components/importers/jira/steps/import-users-from-jira/root.tsx b/apps/web/ee/components/importers/jira/steps/import-users-from-jira/root.tsx index 4eceb8d4da..afa1553454 100644 --- a/apps/web/ee/components/importers/jira/steps/import-users-from-jira/root.tsx +++ b/apps/web/ee/components/importers/jira/steps/import-users-from-jira/root.tsx @@ -64,7 +64,7 @@ export const ImportUsersFromJira: FC = observer(() => { } if (key === "userData" && !formData.userSkipToggle && typeof value === "string") { handleSyncJobConfig("users", value); - handleSyncJobConfig("skipUserImport", true); + handleSyncJobConfig("skipUserImport", formData.userSkipToggle); } }; diff --git a/packages/etl/src/jira-server/etl/pull-v2.ts b/packages/etl/src/jira-server/etl/pull-v2.ts index a82f0bd1cb..5965e06924 100644 --- a/packages/etl/src/jira-server/etl/pull-v2.ts +++ b/packages/etl/src/jira-server/etl/pull-v2.ts @@ -5,6 +5,7 @@ import type { Comment as JComment, IssueTypeDetails as JiraIssueTypeDetails, FieldDetails, + Worklog } from "jira.js/out/version2/models"; import type { ImportedJiraUser, @@ -15,7 +16,7 @@ import type { JiraCustomFieldKeys, JiraV2Service, } from ".."; -import { formatDateStringForHHMM, OPTION_CUSTOM_FIELD_TYPES } from "../helpers"; +import { fetchPaginatedDataByKey, formatDateStringForHHMM, OPTION_CUSTOM_FIELD_TYPES } from "../helpers"; type BasePaginationContext = { client: JiraV2Service; @@ -191,6 +192,34 @@ export async function pullComponentIssuesV2( }; } +export const pullAllCommentsForIssue = async (issue: IJiraIssue, client: JiraV2Service): Promise => { + const values = await fetchPaginatedDataByKey( + (startAt) => client.getIssueComments(issue.id, startAt, 500), + "comments" + ); + + return values.map( + (comment): JiraComment => ({ + ...comment, + issue_id: issue.id, + }) + ); +}; + +export const pullAllWorklogsForIssue = async (issue: IJiraIssue, client: JiraV2Service): Promise => { + const values = await fetchPaginatedDataByKey( + (startAt) => client.getIssueWorklogs(issue.id, startAt, 500), + "worklogs" + ); + + return values.map( + (worklog) => ({ + ...worklog, + issue_id: issue.id, + }) + ); +}; + export async function pullCommentsForIssueV2( ctx: BasePaginationContext, issue: IJiraIssue diff --git a/packages/etl/src/jira-server/etl/transform.ts b/packages/etl/src/jira-server/etl/transform.ts index f9279a27e1..c763efe4a0 100644 --- a/packages/etl/src/jira-server/etl/transform.ts +++ b/packages/etl/src/jira-server/etl/transform.ts @@ -35,14 +35,20 @@ import type { JiraIssueFieldOptions, } from "../types"; +export type TTransformationContext = { + resourceId: string; + projectId: string; + source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA; +}; + export const transformIssue = ( - resourceId: string, - projectId: string, + ctx: TTransformationContext, issue: IJiraIssue, resourceUrl: string, stateMap: IStateConfig[], priorityMap: IPriorityConfig[] ): Partial => { + const { resourceId, projectId, source } = ctx; const targetState = getTargetState(stateMap, issue.fields.status); const targetPriority = getTargetPriority(priorityMap, issue.fields.priority); const attachments = getTargetAttachments(resourceId, projectId, issue.fields.attachment); @@ -66,7 +72,7 @@ export const transformIssue = ( assignees: issue.fields.assignee?.name ? [issue.fields.assignee.name] : [], links, external_id: `${projectId}_${resourceId}_${issue.id}`, - external_source: E_IMPORTER_KEYS.JIRA_SERVER, + external_source: source, created_by: issue.fields.creator?.name, name: issue.fields.summary ?? "Untitled", description_html: description, @@ -84,13 +90,13 @@ export const transformIssue = ( }; export const transformIssueV2 = ( - resourceId: string, - projectId: string, + ctx: TTransformationContext, issue: IJiraIssue, resourceUrl: string, stateMap: IStateConfig[], priorityMap: IPriorityConfig[] ): Partial => { + const { resourceId, projectId, source } = ctx; const targetState = getTargetState(stateMap, issue.fields.status); const targetPriority = getTargetPriority(priorityMap, issue.fields.priority); const attachments = getTargetAttachments(resourceId, projectId, issue.fields.attachment); @@ -111,11 +117,15 @@ export const transformIssueV2 = ( issue.fields.labels.push("JIRA IMPORTED"); return { - assignees: issue.fields.assignee?.emailAddress ? [issue.fields.assignee.emailAddress] : [], + assignees: issue.fields.assignee?.emailAddress + ? [issue.fields.assignee.emailAddress] + : issue.fields.assignee?.displayName + ? [issue.fields.assignee.displayName] + : [], links, external_id: `${projectId}_${resourceId}_${issue.id}`, - external_source: E_IMPORTER_KEYS.JIRA_SERVER, - created_by: issue.fields.creator?.emailAddress, + external_source: source, + created_by: issue.fields.creator?.emailAddress || issue.fields.creator?.displayName, name: issue.fields.summary ?? "Untitled", description_html: description, target_date: issue.fields.duedate, @@ -136,20 +146,19 @@ export const transformLabel = (label: string): Partial => ({ color: getRandomColor(), }); -export const transformComment = ( - resourceId: string, - projectId: string, - comment: JiraComment -): Partial => ({ - external_id: `${projectId}_${resourceId}_${comment.id}`, - external_source: E_IMPORTER_KEYS.JIRA_SERVER, - created_at: getFormattedDate(comment.created), - created_by: comment.author?.emailAddress, - // @ts-expect-error - comment_html: comment.body ?? "

", - actor: comment.author?.emailAddress, - issue: `${projectId}_${resourceId}_${comment.issue_id}`, -}); +export const transformComment = (ctx: TTransformationContext, comment: JiraComment): Partial => { + const { resourceId, projectId, source } = ctx; + return { + external_id: `${projectId}_${resourceId}_${comment.id}`, + external_source: source, + created_at: comment.created, + created_by: comment.author?.emailAddress || comment.author?.displayName, + // @ts-expect-error + comment_html: comment.renderedBody ? comment.renderedBody : (comment.body ?? "

"), + actor: comment.author?.emailAddress || comment.author?.displayName, + issue: `${projectId}_${resourceId}_${comment.issue_id}`, + }; +}; export const transformUser = (user: ImportedJiraUser): Partial => { const [first_name, last_name] = user.full_name.split(" "); @@ -165,42 +174,46 @@ export const transformUser = (user: ImportedJiraUser): Partial => { }; }; -export const transformSprint = (resourceId: string, projectId: string, sprint: JiraSprint): Partial => ({ - external_id: `${projectId}_${resourceId}_${sprint.sprint.id.toString()}`, - external_source: E_IMPORTER_KEYS.JIRA_SERVER, - name: sprint.sprint.name, - start_date: getFormattedDate(sprint.sprint.startDate), - end_date: getFormattedDate(sprint.sprint.endDate), - created_at: getFormattedDate(sprint.sprint.createdDate), - issues: sprint.issues.map((issue) => `${projectId}_${resourceId}_${issue.id}`), -}); +export const transformSprint = (ctx: TTransformationContext, sprint: JiraSprint): Partial => { + const { resourceId, projectId, source } = ctx; + return { + external_id: `${projectId}_${resourceId}_${sprint.sprint.id.toString()}`, + external_source: source, + name: sprint.sprint.name, + start_date: getFormattedDate(sprint.sprint.startDate), + end_date: getFormattedDate(sprint.sprint.endDate), + created_at: getFormattedDate(sprint.sprint.createdDate), + issues: sprint.issues.map((issue) => `${projectId}_${resourceId}_${issue.id}`), + }; +}; -export const transformComponent = ( - resourceId: string, - projectId: string, - component: JiraComponent -): Partial => ({ - external_id: `${projectId}_${resourceId}_${component.component.id}`, - external_source: E_IMPORTER_KEYS.JIRA_SERVER, - name: component.component.name, - issues: component.issues.map((issue) => `${projectId}_${resourceId}_${issue.id}`), -}); +export const transformComponent = (ctx: TTransformationContext, component: JiraComponent): Partial => { + const { resourceId, projectId, source } = ctx; + return { + external_id: `${projectId}_${resourceId}_${component.component.id}`, + external_source: source, + name: component.component.name, + issues: component.issues.map((issue) => `${projectId}_${resourceId}_${issue.id}`), + }; +}; export const transformComponentV2 = ( - resourceId: string, - projectId: string, + ctx: TTransformationContext, component: ComponentWithIssueCount -): Partial => ({ - external_id: `${projectId}_${resourceId}_${component.id}`, - external_source: E_IMPORTER_KEYS.JIRA_SERVER, - name: component.name, -}); +): Partial => { + const { resourceId, projectId, source } = ctx; + return { + external_id: `${projectId}_${resourceId}_${component.id}`, + external_source: source, + name: component.name, + }; +}; export const transformIssueType = ( - resourceId: string, - projectId: string, + ctx: TTransformationContext, issueType: JiraIssueTypeDetails ): Partial => { + const { resourceId, projectId, source } = ctx; const isEpic = issueType.name?.toLowerCase().includes("epic"); return { @@ -209,15 +222,15 @@ export const transformIssueType = ( is_active: true, is_epic: isEpic, external_id: `${projectId}_${resourceId}_${issueType.id}`, - external_source: E_IMPORTER_KEYS.JIRA_SERVER, + external_source: source, }; }; export const transformIssueFields = ( - resourceId: string, - projectId: string, + ctx: TTransformationContext, issueField: JiraIssueField ): Partial | undefined => { + const { resourceId, projectId, source } = ctx; if ( !issueField.schema || !issueField.scope?.type || @@ -231,7 +244,7 @@ export const transformIssueFields = ( return { external_id: `${projectId}_${resourceId}_${fieldId}`, - external_source: E_IMPORTER_KEYS.JIRA_SERVER, + external_source: source, display_name: issueField.name, type_id: issueField.scope?.type ? `${projectId}_${resourceId}_${issueField.scope?.type}` : undefined, is_required: false, @@ -241,26 +254,28 @@ export const transformIssueFields = ( }; export const transformIssueFieldOptions = ( - resourceId: string, - projectId: string, + ctx: TTransformationContext, issueFieldOption: JiraIssueFieldOptions -): Partial => ({ - external_id: `${projectId}_${resourceId}_${issueFieldOption.id}`, - external_source: E_IMPORTER_KEYS.JIRA_SERVER, - name: issueFieldOption.value, - is_active: issueFieldOption.disabled ? false : true, - property_id: `${projectId}_${resourceId}_${issueFieldOption.fieldId}`, -}); +): Partial => { + const { resourceId, projectId, source } = ctx; + return { + external_id: `${projectId}_${resourceId}_${issueFieldOption.id}`, + external_source: source, + name: issueFieldOption.value, + is_active: issueFieldOption.disabled ? false : true, + property_id: `${projectId}_${resourceId}_${issueFieldOption.fieldId}`, + }; +}; export const transformIssuePropertyValues = ( - resourceId: string, - projectId: string, + ctx: TTransformationContext, issue: IJiraIssue, // eslint-disable-next-line no-undef planeIssueProperties: Map>, // TODO: replace Map with Record> in the future // eslint-disable-next-line no-undef jiraCustomFieldMap: Map // TODO: replace Map with Record in the future ): TPropertyValuesPayload => { + const { resourceId, projectId } = ctx; // Get all custom fields that are present in the issue and are also present in the plane issue properties const customFieldKeysToTransform = Object.keys(issue.fields).filter( (key) => key.startsWith("customfield_") && planeIssueProperties.has(key) @@ -283,18 +298,18 @@ export const transformIssuePropertyValues = ( }; export const transformDefaultPropertyValues = ( - resourceId: string, - projectId: string, + ctx: TTransformationContext, issue: IJiraIssue, issueTypeId: string ): TPropertyValuesPayload => { + const { resourceId, projectId, source } = ctx; const propertyValuesPayload: TPropertyValuesPayload = {}; // Fix Versions if (issue.fields.fixVersions && Array.isArray(issue.fields.fixVersions)) { const fixVersionExternalId = `${resourceId}-${projectId}-${issueTypeId}-fix-version`; propertyValuesPayload[fixVersionExternalId] = issue.fields.fixVersions.map((version: any) => ({ - external_source: "JIRA_SERVER", + external_source: source, external_id: version.id ? String(version.id) : undefined, value: version.name || "", })); @@ -304,7 +319,7 @@ export const transformDefaultPropertyValues = ( if (issue.fields.versions && Array.isArray(issue.fields.versions)) { const affectedVersionExternalId = `${resourceId}-${projectId}-${issueTypeId}-affected-version`; propertyValuesPayload[affectedVersionExternalId] = issue.fields.versions.map((version: any) => ({ - external_source: "JIRA_SERVER", + external_source: source, external_id: version.id ? String(version.id) : undefined, value: version.name || "", })); @@ -315,8 +330,8 @@ export const transformDefaultPropertyValues = ( const reporterExternalId = `${resourceId}-${projectId}-${issueTypeId}-reporter`; propertyValuesPayload[reporterExternalId] = [ { - external_source: "JIRA_SERVER", - external_id: issue.fields.reporter.emailAddress || issue.fields.reporter.name, + external_source: source, + external_id: issue.fields.reporter.emailAddress || issue.fields.reporter.displayName || "", value: issue.fields.reporter.emailAddress || issue.fields.reporter.displayName || "", }, ]; diff --git a/packages/etl/src/jira-server/helpers/etl.ts b/packages/etl/src/jira-server/helpers/etl.ts index f1a817cbe5..a5a14d718f 100644 --- a/packages/etl/src/jira-server/helpers/etl.ts +++ b/packages/etl/src/jira-server/helpers/etl.ts @@ -191,11 +191,11 @@ export const getPropertyValues = ( break; case "com.atlassian.jira.plugin.system.customfieldtypes:userpicker": // Handle userpicker - if (value.emailAddress) { + if (value.emailAddress || value.displayName) { propertyValues.push({ ...commonPropertyProp, - external_id: value.emailAddress, - value: value.emailAddress, + external_id: value.emailAddress ?? value.displayName, + value: value.emailAddress ?? value.displayName, }); } break; @@ -277,11 +277,11 @@ export const getPropertyValues = ( // Handle multiuserpicker if (Array.isArray(value)) { value.forEach((val) => { - if (val.emailAddress) { + if (val.emailAddress || val.displayName) { propertyValues.push({ ...commonPropertyProp, - external_id: val.emailAddress, - value: val.emailAddress, + external_id: val.emailAddress ?? val.displayName, + value: val.emailAddress ?? val.displayName, }); } }); diff --git a/packages/etl/src/jira-server/services/api.service.ts b/packages/etl/src/jira-server/services/api.service.ts index 68dc72f6a1..15ec470251 100644 --- a/packages/etl/src/jira-server/services/api.service.ts +++ b/packages/etl/src/jira-server/services/api.service.ts @@ -1,5 +1,6 @@ // services -import axios, { AxiosError } from "axios"; +import type { AxiosError } from "axios"; +import axios from "axios"; import type { Paginated } from "jira.js"; import { Board as BoardClient } from "jira.js/out/agile"; import { Version2Client } from "jira.js/out/version2"; @@ -7,29 +8,45 @@ import type { CustomFieldContextOption, FieldDetails, Issue, - IssueTypeDetails, JiraStatus, Project, + IssueTypeDetails, } from "jira.js/out/version2/models"; import type { JiraCustomFieldWithCtx } from "@/jira-server/types/custom-fields"; import type { JiraApiUser, JiraProps } from ".."; -import { fetchPaginatedData } from ".."; +import { fetchPaginatedData, EJiraAuthenticationType } from ".."; export class JiraV2Service { private jiraClient: Version2Client; private hostname: string; private patToken: string; + private email: string; + private authenticationType: EJiraAuthenticationType = EJiraAuthenticationType.PERSONAL_ACCESS_TOKEN; constructor(props: JiraProps) { this.hostname = props.hostname; this.patToken = props.patToken; + this.email = props.email; + this.authenticationType = props.authenticationType; - this.jiraClient = new Version2Client({ - host: props.hostname, - authentication: { - personalAccessToken: props.patToken, - }, - }); + if (this.authenticationType === EJiraAuthenticationType.BASIC) { + this.jiraClient = new Version2Client({ + host: props.hostname, + authentication: { + basic: { + email: this.email, + apiToken: props.patToken, + }, + }, + }); + } else { + this.jiraClient = new Version2Client({ + host: props.hostname, + authentication: { + personalAccessToken: props.patToken, + }, + }); + } this.jiraClient.handleFailedResponse = async (request) => { const error = request as AxiosError; @@ -70,6 +87,14 @@ export class JiraV2Service { })) as JiraApiUser[]; } + async getIssueWorklogs(issueId: string, startAt: number, maxResults: number) { + return this.jiraClient.issueWorklogs.getIssueWorklog({ + issueIdOrKey: issueId, + startAt: startAt, + maxResults: maxResults, + }); + } + // Verified async getNumberOfIssues(projectKey: string) { const issues = await this.jiraClient.issueSearch.searchForIssuesUsingJql({ @@ -239,15 +264,11 @@ export class JiraV2Service { // Verified async getPaginatedIssueTypes(projectId: string, startAt?: number, maxResults?: number) { try { - return axios - .get( - `${this.hostname}/rest/api/2/issuetype/page?projectIds=${projectId}&startAt=${startAt}&maxResults=${maxResults}`, - { - headers: { - Authorization: `Bearer ${this.patToken}`, - }, - } - ) + return await this.jiraClient + .sendRequestFullResponse>({ + method: "GET", + url: `${this.hostname}/rest/api/2/issuetype/page?projectIds=${projectId}&startAt=${startAt}&maxResults=${maxResults}`, + }) .then((res) => res.data as Paginated); } catch (e) { console.error("error getProjectIssueTypes", e); @@ -264,15 +285,13 @@ export class JiraV2Service { async getCustomFieldsWithContext(projectId?: string) { try { - return await fetchPaginatedData((startAt) => - axios - .get(`${this.hostname}/rest/api/2/customFields?startAt=${startAt}&projectIds=${projectId ?? ""}`, { - headers: { - Authorization: `Bearer ${this.patToken}`, - }, - }) - .then((res) => res.data as Paginated) - ); + return await fetchPaginatedData(async (startAt) => { + const result = await this.jiraClient.sendRequestFullResponse>({ + method: "GET", + url: `${this.hostname}/rest/api/2/customFields?startAt=${startAt}&projectIds=${projectId ?? ""}`, + }); + return result.data; + }); } catch (e) { console.error("error getCustomFieldsWithContext", e); throw e; diff --git a/packages/etl/src/jira-server/types/index.ts b/packages/etl/src/jira-server/types/index.ts index 7c3c8752b0..56a9202e0b 100644 --- a/packages/etl/src/jira-server/types/index.ts +++ b/packages/etl/src/jira-server/types/index.ts @@ -15,8 +15,15 @@ import type { ExProject, ExState } from "@plane/sdk"; export type JiraProps = { hostname: string; patToken: string; + email: string; + authenticationType: EJiraAuthenticationType; }; +export enum EJiraAuthenticationType { + BASIC = "basic", + PERSONAL_ACCESS_TOKEN = "personalAccessToken", +} + export type ImportedJiraUser = { user_id: string; user_name: string; @@ -33,6 +40,7 @@ export type JiraApiUser = { key: string; name: string; emailAddress?: string; + accountType?: string; avatarUrls: { "16x16": string; "24x24": string; diff --git a/packages/etl/src/jira/etl/pull.ts b/packages/etl/src/jira/etl/pull.ts index 3e9fde6aa5..6235cfb1fb 100644 --- a/packages/etl/src/jira/etl/pull.ts +++ b/packages/etl/src/jira/etl/pull.ts @@ -64,6 +64,37 @@ export async function pullIssues(client: JiraService, projectKey: string, from?: return issues; } +/** + * Pull issues using nextPageToken pagination (Jira Cloud Enhanced Search API) + * Note: Enhanced Search API does not return total count, only nextPageToken for pagination + */ +export async function pullIssuesV2( + ctx: { + client: JiraService; + nextPageToken?: string; + maxResults?: number; + }, + projectKey: string, + // We are using this property for the pagination context + total = 0, + from?: Date +): Promise<{ + items: IJiraIssue[]; + hasMore: boolean; + total?: number; + nextPageToken?: string; +}> { + const { client, nextPageToken } = ctx; + const result = await client.getProjectIssues(projectKey, nextPageToken, from ? formatDateStringForHHMM(from) : ""); + + return { + items: result.issues || [], + hasMore: !!result.nextPageToken, + total: total, + nextPageToken: result.nextPageToken, + }; +} + export async function pullComments(issues: IJiraIssue[], client: JiraService): Promise { const comments: JiraComment[] = []; @@ -168,10 +199,11 @@ export const pullIssueFields = async ( const customFields: JiraIssueField[] = []; try { // initialize fields - const fields: FieldDetails[] = await client.getCustomFields(); + const allFields: FieldDetails[] = await client.getCustomFields(); + const filteredFields = allFields.filter((field) => field.custom); // get all field contexts - for (const field of fields) { + for (const field of filteredFields) { // skip if field has no id if (!field.id) continue; @@ -226,7 +258,11 @@ export const pullIssueFields = async ( (startAt) => client.getIssueFieldOptions(field.id as string, Number(issueTypeContext.contextId), startAt), (values: CustomFieldContextOption[]) => { values.map((value) => { - if (field.id) fieldOptions.push({ ...value, fieldId: field.id }); + if (field.id) + fieldOptions.push({ + ...value, + fieldId: field.id.includes("customfield_") ? field.id.split("_").pop()! : field.id, + }); }); }, "values" diff --git a/packages/sdk/src/services/asset.service.ts b/packages/sdk/src/services/asset.service.ts index a393fd6a88..7771a1aae2 100644 --- a/packages/sdk/src/services/asset.service.ts +++ b/packages/sdk/src/services/asset.service.ts @@ -67,10 +67,16 @@ export class AssetService extends APIService { external_source?: string; } ): Promise { + let fileType = file.type; + + // If the file type includes a semicolon, we need to remove the charset + if (fileType.includes(";")) { + fileType = fileType.split(";")[0]; + } // First get the presigned URL const uploadResponse = await this.createAsset(workspaceSlug, { name, - type: file.type, + type: fileType, size, project_id: options?.project_id, external_id: options?.external_id,