[SILO-724] feat: Jira Cloud Importer V2 (#4888)

* fix: ratelimit issues and fails with jira cloud

* fix: added rate limit handler for jira server

* fix: review comments

* fix: retry logic with axious

* fix: issue association with comments

* fix: label import issue with data_import_task

* fix: linter errors

* fix: added user email for jira service

* fix: updated tranformation logic

* fix: added steps for jira cloud migration steps

* fix: moved jira-server to shared steps

* fix: renamed types

* fix: config coming as false for skipUserImport

* fix: return format for pull issues for jira cloud

* fix: pull and transform functions for jira server

* fix: optimized orchestrator to use internal api client

* fix: comment ordering for jira cloud

* fix: bug with pulling issue types

* fix: comments from bugbot

* fix: attachment issue for jira and jira srv

* fix: reviews and formatting of code

* fix: build

* fix: worklog and authentication type

* fix: step failure protection for orchestrator

* fix: fetch sprints only for scrum boards
This commit is contained in:
Henit Chobisa
2025-12-03 16:27:26 +05:30
committed by GitHub
parent ada9f1acc8
commit 33249d5c99
42 changed files with 1203 additions and 405 deletions
@@ -2,9 +2,8 @@ import type { Request, Response } from "express";
// etl
import { Controller, Get, Post } from "@plane/decorators";
import { E_IMPORTER_KEYS } from "@plane/etl/core";
import type { JiraResource } from "@plane/etl/jira";
import type { JiraProject, JiraV2Service } from "@plane/etl/jira-server";
import { createJiraService } from "@plane/etl/jira-server";
import { createJiraService, EJiraAuthenticationType } from "@plane/etl/jira-server";
// db
// helpers
import { compareAndGetAdditionalUsers } from "@/helpers/additional-users";
@@ -24,14 +23,18 @@ class JiraDataCenterController {
async upsertCredentials(req: Request, res: Response) {
try {
const { workspaceId, userId, apiToken, personalAccessToken, userEmail, hostname } = req.body;
if (!workspaceId || !userId || !apiToken || !personalAccessToken) {
res.status(400).json({ message: "Workspace ID, User ID, API Token and Personal Access Token are required" });
if (!workspaceId || !userId || !apiToken || !personalAccessToken || !userEmail) {
return res
.status(400)
.json({ message: "Workspace ID, User ID, API Token, Personal Access Token and User Email are required" });
}
try {
const jiraService = createJiraService({
patToken: personalAccessToken,
hostname: hostname,
email: (userEmail as string).trim(),
authenticationType: EJiraAuthenticationType.PERSONAL_ACCESS_TOKEN,
});
await jiraService.getCurrentUser();
@@ -64,8 +67,7 @@ class JiraDataCenterController {
try {
const jiraClient = await createJiraServerClient(workspaceId, userId);
const serverInfo = await jiraClient.getServerInfo();
// @ts-expect-error
const resource: JiraResource = {
const resource = {
id: serverInfo.scmInfo ?? "",
url: serverInfo.baseUrl ?? "",
name: serverInfo.serverTitle ?? "",
@@ -185,6 +187,8 @@ export const createJiraServerClient = async (workspaceId: string, userId: string
return createJiraService({
hostname: jiraCredentials.source_hostname,
patToken: jiraCredentials.source_access_token,
email: jiraCredentials.source_auth_email,
authenticationType: EJiraAuthenticationType.PERSONAL_ACCESS_TOKEN,
});
};
@@ -6,7 +6,7 @@ import type {
} from "jira.js/out/version2/models";
import { E_IMPORTER_KEYS } from "@plane/etl/core";
import type { IPriorityConfig, IStateConfig, JiraComponent, JiraConfig, JiraSprint } from "@plane/etl/jira-server";
import { JiraV2Service } from "@plane/etl/jira-server";
import { EJiraAuthenticationType, JiraV2Service } from "@plane/etl/jira-server";
import type { ExIssueAttachment, ExState } from "@plane/sdk";
import type { TImportJob, TWorkspaceCredential } from "@plane/types";
@@ -92,8 +92,14 @@ export const createJiraClient = (
if (!credentials.source_access_token || !credentials.source_hostname || !credentials.source_auth_email) {
throw new Error(`Missing credentials in job config for job ${job.id}`);
}
const authenticationType =
job.source === E_IMPORTER_KEYS.JIRA ? EJiraAuthenticationType.BASIC : EJiraAuthenticationType.PERSONAL_ACCESS_TOKEN;
return new JiraV2Service({
email: credentials.source_auth_email,
patToken: credentials.source_access_token!,
hostname: credentials.source_hostname,
authenticationType,
});
};
@@ -1,5 +1,5 @@
import { v4 as uuid } from "uuid";
import type { TIssuePropertyValuesPayload } from "@plane/etl/core";
import type { E_IMPORTER_KEYS, TIssuePropertyValuesPayload } from "@plane/etl/core";
import type {
JiraCustomFieldKeys,
IJiraIssue,
@@ -48,7 +48,17 @@ export const getTransformedIssues = (
const resourceId = job.config.resource ? job.config.resource.id : uuid();
return entities.issues.map((issue: IJiraIssue): Partial<PlaneIssue> => {
const transformedIssue = transformIssue(resourceId, job.project_id, issue, resourceUrl, stateMap, priorityMap);
const transformedIssue = transformIssue(
{
resourceId,
projectId: job.project_id,
source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA,
},
issue,
resourceUrl,
stateMap,
priorityMap
);
if (job.config?.issueType && issue.fields.issuetype?.name) {
const issueTypeValue = job.config.issueType;
@@ -71,7 +81,16 @@ export const getTransformedComments = (
entities: JiraEntity
): Partial<ExIssueComment>[] => {
const resourceId = job.config.resource ? job.config.resource.id : uuid();
return entities.issue_comments.map((comment) => transformComment(resourceId, job.project_id, comment));
return entities.issue_comments.map((comment) =>
transformComment(
{
resourceId,
projectId: job.project_id,
source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA,
},
comment
)
);
};
export const getTransformedUsers = (_job: TImportJob<JiraConfig>, entities: JiraEntity): Partial<PlaneUser>[] =>
@@ -79,17 +98,44 @@ export const getTransformedUsers = (_job: TImportJob<JiraConfig>, entities: Jira
export const getTransformedSprints = (job: TImportJob<JiraConfig>, entities: JiraEntity): Partial<ExCycle>[] => {
const resourceId = job.config.resource ? job.config.resource.id : uuid();
return entities.sprints.map((sprint) => transformSprint(resourceId, job.project_id, sprint));
return entities.sprints.map((sprint) =>
transformSprint(
{
resourceId,
projectId: job.project_id,
source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA,
},
sprint
)
);
};
export const getTransformedComponents = (job: TImportJob<JiraConfig>, entities: JiraEntity): Partial<ExModule>[] => {
const resourceId = job.config.resource ? job.config.resource.id : uuid();
return entities.components.map((component) => transformComponent(resourceId, job.project_id, component));
return entities.components.map((component) =>
transformComponent(
{
resourceId,
projectId: job.project_id,
source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA,
},
component
)
);
};
export const getTransformedIssueTypes = (job: TImportJob<JiraConfig>, entities: JiraEntity): Partial<ExIssueType>[] => {
const resourceId = job.config.resource ? job.config.resource.id : uuid();
return entities.issueTypes.map((issueType) => transformIssueType(resourceId, job.project_id, issueType));
return entities.issueTypes.map((issueType) =>
transformIssueType(
{
resourceId,
projectId: job.project_id,
source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA,
},
issueType
)
);
};
export const getTransformedIssueFields = (
@@ -98,7 +144,16 @@ export const getTransformedIssueFields = (
): Partial<ExIssueProperty>[] => {
const resourceId = job.config.resource ? job.config.resource.id : uuid();
return entities.issueFields
.map((issueField) => transformIssueFields(resourceId, job.project_id, issueField))
.map((issueField) =>
transformIssueFields(
{
resourceId,
projectId: job.project_id,
source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA,
},
issueField
)
)
.filter((field) => field && field.property_type) as Partial<ExIssueProperty>[];
};
@@ -114,7 +169,16 @@ export const getTransformedIssueFieldOptions = (
OPTION_CUSTOM_FIELD_TYPES.includes(issueField.schema?.custom as JiraCustomFieldKeys)
)
.flatMap((issueField) =>
issueField?.options?.map((fieldOption) => transformIssueFieldOptions(resourceId, job.project_id, fieldOption))
issueField?.options?.map((fieldOption) =>
transformIssueFieldOptions(
{
resourceId,
projectId: job.project_id,
source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA,
},
fieldOption
)
)
)
.filter(Boolean) as Partial<ExIssuePropertyOption>[];
};
@@ -146,8 +210,7 @@ export const getTransformedIssuePropertyValues = (
entities.issues.forEach((issue: IJiraIssue) => {
if (issue.id && issue.fields) {
transformedIssuePropertyValues[`${projectId}_${resourceId}_${issue.id}`] = transformIssuePropertyValues(
resourceId,
projectId,
{ resourceId, projectId, source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA },
issue,
planeIssuePropertiesMap,
jiraCustomFieldMap
@@ -194,8 +257,7 @@ export const getTransformedIssuePropertyValuesV2 = (
// Transform custom field property values
const customPropertyValues = transformIssuePropertyValues(
resourceId,
projectId,
{ resourceId, projectId, source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA },
issue,
planeIssuePropertiesMap,
jiraCustomFieldMap
@@ -203,7 +265,11 @@ export const getTransformedIssuePropertyValuesV2 = (
// Transform default property values (fix versions, affected versions, reporter)
const defaultPropertyValues = issueTypeId
? transformDefaultPropertyValues(resourceId, projectId, issue, issueTypeId)
? transformDefaultPropertyValues(
{ resourceId, projectId, source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA },
issue,
issueTypeId
)
: {};
// Merge both custom and default property values
@@ -1,4 +1,4 @@
import { E_IMPORTER_KEYS } from "@plane/etl/core";
import type { E_IMPORTER_KEYS } from "@plane/etl/core";
import type { ExIssueProperty } from "@plane/sdk";
export enum E_DEFAULT_PROPERTY_TYPES {
@@ -23,12 +23,13 @@ export const getSupportedDefaultProperties = (
resourceId: string,
projectId: string,
issueTypeId: string,
issueTypeExternalId: string
issueTypeExternalId: string,
source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA
): Partial<ExIssueProperty>[] => [
{
type_id: issueTypeExternalId,
external_id: `${resourceId}-${projectId}-${issueTypeId}-${E_DEFAULT_PROPERTY_TYPES.FIX_VERSION}`,
external_source: E_IMPORTER_KEYS.JIRA_SERVER,
external_id: getDefaultPropertyExternalId(resourceId, projectId, issueTypeId, E_DEFAULT_PROPERTY_TYPES.FIX_VERSION),
external_source: source,
display_name: "Fix Version",
property_type: "TEXT",
settings: {
@@ -38,8 +39,13 @@ export const getSupportedDefaultProperties = (
},
{
type_id: issueTypeExternalId,
external_id: `${resourceId}-${projectId}-${issueTypeId}-${E_DEFAULT_PROPERTY_TYPES.AFFECTED_VERSION}`,
external_source: E_IMPORTER_KEYS.JIRA_SERVER,
external_id: getDefaultPropertyExternalId(
resourceId,
projectId,
issueTypeId,
E_DEFAULT_PROPERTY_TYPES.AFFECTED_VERSION
),
external_source: source,
display_name: "Affected Version",
property_type: "TEXT",
settings: {
@@ -49,8 +55,8 @@ export const getSupportedDefaultProperties = (
},
{
type_id: issueTypeExternalId,
external_id: `${resourceId}-${projectId}-${issueTypeId}-${E_DEFAULT_PROPERTY_TYPES.REPORTER}`,
external_source: E_IMPORTER_KEYS.JIRA_SERVER,
external_id: getDefaultPropertyExternalId(resourceId, projectId, issueTypeId, E_DEFAULT_PROPERTY_TYPES.REPORTER),
external_source: source,
display_name: "Reporter",
property_type: "RELATION",
relation_type: "USER",
@@ -0,0 +1,10 @@
import type { MQ, Store } from "@/worker/base";
import { JiraImportOrchestrator } from "./orchestrator";
import JIRA_CLOUD_STEPS from "./steps/cloud";
import JIRA_SERVER_STEPS from "./steps/server";
export const getJiraCloudImportOrchestrator = (mq: MQ, store: Store) =>
new JiraImportOrchestrator(mq, store, JIRA_CLOUD_STEPS);
export const getJiraServerImportOrchestrator = (mq: MQ, store: Store) =>
new JiraImportOrchestrator(mq, store, JIRA_SERVER_STEPS);
@@ -4,7 +4,6 @@ import { logger } from "@plane/logger";
import type { TImportJob } from "@plane/types";
import { createJiraClient } from "@/apps/jira-server-importer/helpers/migration-helpers";
import { flushJob } from "@/apps/jira-server-importer/v2/helpers/cache";
import JIRA_SERVER_STEPS from "@/apps/jira-server-importer/v2/migrator/steps";
import type {
IStep,
IStorageService,
@@ -15,7 +14,7 @@ import type {
} from "@/apps/jira-server-importer/v2/types";
import { getJobCredentials, getJobData } from "@/helpers/job";
import { getPlaneAPIClient } from "@/helpers/plane-api-client";
import { getAPIClient } from "@/services/client";
import { getAPIClient, getAPIClientInternal } from "@/services/client";
import type { TaskHandler, TaskHeaders } from "@/types";
import type { MQ, Store } from "@/worker/base";
import { redisStorageService } from "../services/storage.service";
@@ -37,18 +36,16 @@ type OrchestratorTaskData = {
stepContext?: TStepExecutionContext;
};
const client = getAPIClient();
const client = getAPIClientInternal();
export class JiraServerImportOrchestrator implements TaskHandler {
private steps: IStep[];
export class JiraImportOrchestrator implements TaskHandler {
private storage: IStorageService = redisStorageService;
constructor(
private readonly mq: MQ,
private readonly store: Store
) {
this.steps = JIRA_SERVER_STEPS;
}
private readonly store: Store,
private readonly steps: IStep[]
) { }
/**
* Main task handler - called by worker
@@ -143,8 +140,74 @@ export class JiraServerImportOrchestrator implements TaskHandler {
// Load execution context
const { state, step, jobContext } = await this.loadStepExecution(jobId, data);
// Execute step and handle result
await this.executeAndHandleResult(headers, state, step, jobContext);
try {
// Execute step and handle result
await this.executeAndHandleResult(headers, state, step, jobContext);
} catch (error) {
// Log error and move to next step instead of failing the job
await this.handleStepError(headers, state, step, error);
}
}
/**
* Handle step error: Log error and move to next step
*/
private async handleStepError(
headers: TaskHeaders,
state: TOrchestratorState,
step: IStep,
error: unknown
): Promise<void> {
const errorMessage = error instanceof Error ? error.message : String(error);
logger.error(`[ORCHESTRATOR] Step failed, continuing to next step`, {
jobId: state.jobId,
step: step.name,
stepIndex: state.currentStepIndex + 1,
totalSteps: state.totalSteps,
error: errorMessage,
});
// Track failed step
if (!state.failedSteps) {
state.failedSteps = [];
}
state.failedSteps.push({
name: step.name,
error: errorMessage,
failedAt: new Date().toISOString(),
});
// Move to next step
state.currentStepIndex++;
state.stepContext = undefined;
await this.saveState(state);
if (state.currentStepIndex >= this.steps.length) {
// All steps processed (some may have failed)
await this.mq.sendMessage(
{},
{
headers: {
...headers,
type: EOrchestratorTaskType.COMPLETE,
},
}
);
} else {
// Move to next step
state.currentStepName = this.steps[state.currentStepIndex].name;
await this.saveState(state);
await this.mq.sendMessage(
{ state },
{
headers: {
...headers,
type: EOrchestratorTaskType.EXECUTE_STEP,
},
}
);
}
}
/**
@@ -178,6 +241,18 @@ export class JiraServerImportOrchestrator implements TaskHandler {
return { state, step, jobContext };
}
/**
* Initialize job context
*/
private async initializeJobContext(jobId: string): Promise<TJobContext> {
const job = await getJobData(jobId);
const credentials = await getJobCredentials(job as TImportJob<JiraConfig>);
const planeClient = await getPlaneAPIClient(credentials, E_IMPORTER_KEYS.IMPORTER);
const sourceClient = createJiraClient(job as TImportJob<JiraConfig>, credentials);
return { job, credentials, planeClient, sourceClient };
}
/**
* Execute step and handle the result
*/
@@ -298,7 +373,7 @@ export class JiraServerImportOrchestrator implements TaskHandler {
// Delete orchestrator state
await this.store.del(stateKey);
logger.info(`[ORCHESTRATOR] Job completed successfully`, { jobId });
logger.info(`[ORCHESTRATOR] Job completed`, { jobId });
}
/**
@@ -362,18 +437,6 @@ export class JiraServerImportOrchestrator implements TaskHandler {
return dependencyData;
}
/**
* Initialize job context
*/
private async initializeJobContext(jobId: string): Promise<TJobContext> {
const job = await getJobData(jobId);
const credentials = await getJobCredentials(job as TImportJob<JiraConfig>);
const planeClient = await getPlaneAPIClient(credentials, E_IMPORTER_KEYS.IMPORTER);
const sourceClient = createJiraClient(job as TImportJob<JiraConfig>, credentials);
return { job, credentials, planeClient, sourceClient };
}
/**
* Check if job is cancelled
*/
@@ -0,0 +1,3 @@
export * from "./issue-properties.step";
export * from "./users.step";
export * from "./issue-types.step";
@@ -0,0 +1,29 @@
import { pullIssueFields } from "@plane/etl/jira";
import type { JiraIssueField } from "@plane/etl/jira-server";
import { createJiraClient } from "@/apps/jira-importer/helpers/migration-helpers";
import { getJobCredentials } from "@/helpers/job";
import type { TJobContext, TIssueTypesData } from "../../../../types";
import { JiraIssuePropertiesStep } from "../../shared/entities";
export class JiraCloudIssuePropertiesStep extends JiraIssuePropertiesStep {
protected async pull(
jobCtx: TJobContext,
projectId: string,
issueTypesData: TIssueTypesData
): Promise<JiraIssueField[]> {
const { job } = jobCtx;
const credentials = await getJobCredentials(job);
const client = createJiraClient(job, credentials);
const issueTypes = issueTypesData
.map((type) => {
const id = type.external_id.split("_").pop();
if (typeof id !== "string" || id.length === 0) {
return null;
}
return { id, name: type.name };
})
.filter((item): item is { id: string; name: string } => item !== null);
return pullIssueFields(client, issueTypes, projectId);
}
}
@@ -0,0 +1,29 @@
import { pullIssueTypes } from "@plane/etl/jira";
import { logger } from "@plane/logger";
import { createJiraClient } from "@/apps/jira-importer/helpers/migration-helpers";
import type { TJobContext } from "@/apps/jira-server-importer/v2/types";
import { getJobCredentials } from "@/helpers/job";
import { JiraIssueTypesStep } from "../../shared";
export class JiraCloudIssueTypesStep extends JiraIssueTypesStep {
protected async pull(jobContext: TJobContext, projectId: string, startAt: number) {
const { job } = jobContext;
const credentials = await getJobCredentials(job);
const client = createJiraClient(job, credentials);
const result = await pullIssueTypes(client, projectId);
logger.info(`[${jobContext.job.id}] [${this.name}] Pulled issue types from Jira Cloud`, {
jobId: jobContext.job.id,
count: result.length,
hasMore: false,
startAt,
});
return {
items: result,
hasMore: false,
startAt,
maxResults: result.length,
};
}
}
@@ -0,0 +1,51 @@
import { pullUsers } from "@plane/etl/jira";
import type { JiraV2Service, PaginatedResult, ImportedJiraUser } from "@plane/etl/jira-server";
import { logger } from "@plane/logger";
import type { TJobContext } from "../../../../types";
import { JiraUsersStep } from "../../shared/entities";
export class JiraCloudUserStep extends JiraUsersStep {
/*
* For jira cloud, we don't have access to user emails, hence
* we take a csv in order to import the users, and we use a different
* pull function, hence we need to overwrite this.
*/
protected async pull(
jobContext: TJobContext,
_client: JiraV2Service,
_startAt: number,
_jobId: string
): Promise<PaginatedResult<ImportedJiraUser>> {
const { job } = jobContext;
if (!job.config.users) {
logger.error(`[${job.id}] [${this.name}] No users found in config`, { jobId: job.id });
return {
items: [],
hasMore: false,
total: 0,
startAt: 0,
maxResults: 0,
};
}
const users = pullUsers(job.config.users);
const items = users.map((user) => ({
avatarUrl: "",
user_id: user.user_id,
email: user.email,
full_name: "",
user_name: user.user_name,
added_to_org: user.added_to_org,
org_role: user.org_role,
user_status: "Active",
}));
return {
items,
hasMore: false,
total: items.length,
startAt: 0,
maxResults: items.length,
};
}
}
@@ -0,0 +1,37 @@
import { E_IMPORTER_KEYS } from "@plane/etl/core";
import {
// Pre-run steps
PlaneProjectConfigurationStep,
// Entity steps
JiraBoardsStep,
JiraCyclesStep,
JiraDefaultPropertiesStep,
JiraIssuePropertyOptionsStep,
JiraModulesStep,
WaitForCeleryStep,
// Association steps
JiraRelationsStep,
} from "../shared";
import { JiraCloudUserStep, JiraCloudIssuePropertiesStep, JiraCloudIssueTypesStep } from "./entities";
import { JiraCloudIssuesStep } from "./issues";
const JIRA_CLOUD_STEPS = [
// Pre-run steps
new PlaneProjectConfigurationStep(),
// Entity steps
new JiraCloudUserStep(),
new JiraModulesStep(E_IMPORTER_KEYS.JIRA),
new JiraBoardsStep(),
new JiraCyclesStep(E_IMPORTER_KEYS.JIRA),
new JiraCloudIssueTypesStep(E_IMPORTER_KEYS.JIRA),
new JiraCloudIssuePropertiesStep(E_IMPORTER_KEYS.JIRA),
new JiraDefaultPropertiesStep(E_IMPORTER_KEYS.JIRA),
new JiraIssuePropertyOptionsStep(E_IMPORTER_KEYS.JIRA),
// Issue steps
new JiraCloudIssuesStep(E_IMPORTER_KEYS.JIRA),
new WaitForCeleryStep(),
// Association steps
new JiraRelationsStep(E_IMPORTER_KEYS.JIRA),
];
export default JIRA_CLOUD_STEPS;
@@ -0,0 +1 @@
export * from "./issues.step";
@@ -0,0 +1,155 @@
import { pullIssuesV2 } from "@plane/etl/jira";
import { logger } from "@plane/logger";
import type { TImportJob } from "@plane/types";
import { createJiraClient } from "@/apps/jira-importer/helpers/migration-helpers";
import type { TStepExecutionContext } from "@/apps/jira-server-importer/v2/types";
import { getJobCredentials } from "@/helpers/job";
import { withCache } from "../../../../helpers/cache";
import { buildExternalId } from "../../../../helpers/job";
import type { TJobContext } from "../../../../types";
import { JiraIssuesStep } from "../../shared/issues";
/**
* Jira Cloud Issues Step
*
* Extends JiraServerIssuesStep to use nextPageToken pagination
* instead of startAt pagination for Jira Cloud API
*/
export class JiraCloudIssuesStep extends JiraIssuesStep {
/**
* Override pagination context to use nextPageToken for Jira Cloud
*/
protected getPaginationContext(previousContext?: TStepExecutionContext) {
return {
startAt: 0, // Not used for Cloud pagination
totalProcessed: previousContext?.pageCtx.totalProcessed ?? 0,
nextPageToken: previousContext?.state?.nextPageToken as string | undefined,
};
}
/**
* Pull paginated issues from Jira Cloud using nextPageToken
*/
protected async pull(props: {
jobContext: TJobContext;
projectKey: string;
paginationCtx: { startAt: number; totalProcessed: number; nextPageToken?: string };
}) {
const { job } = props.jobContext;
const credentials = await getJobCredentials(job);
const client = createJiraClient(job, credentials);
// Get total number of issues first
const total = await withCache(
`jira-cloud-issues-total-${props.projectKey}`,
job,
async () => await client.getNumberOfIssues(props.projectKey)
);
const result = await pullIssuesV2(
{
client,
nextPageToken: props.paginationCtx.nextPageToken,
},
props.projectKey,
total
);
logger.info(`[${job.id}] [${this.name}] Pulled issues from Jira Cloud`, {
jobId: job.id,
count: result.items.length,
hasMore: result.hasMore,
nextPageToken: result.nextPageToken ? "present" : "none",
});
return {
items: result.items as any[],
hasMore: result.hasMore,
total: total ?? 0,
nextPageToken: result.nextPageToken,
};
}
/**
* Build next context with nextPageToken for Jira Cloud pagination
*/
protected buildNextContext(
issuesResult: { items: any[]; hasMore: boolean; total: number; nextPageToken?: string },
paginationCtx: { startAt: number; totalProcessed: number; nextPageToken?: string },
pulled: number,
pushed: number
) {
const newTotalProcessed = paginationCtx.totalProcessed + pulled;
if (issuesResult.hasMore && issuesResult.nextPageToken) {
return {
pageCtx: {
startAt: 0, // Not used for Cloud
hasMore: true,
totalProcessed: newTotalProcessed,
},
results: {
pulled,
pushed,
errors: [],
},
state: {
nextPageToken: issuesResult.nextPageToken,
},
};
}
return {
pageCtx: {
startAt: 0,
hasMore: false,
totalProcessed: newTotalProcessed,
},
results: {
pulled,
pushed,
errors: [],
},
};
}
/**
* Override shouldReturnEmpty to check first page correctly for Cloud
*/
protected shouldReturnEmpty(
issuesResult: { items: any[]; hasMore: boolean; total: number },
paginationCtx: { startAt: number; totalProcessed: number; nextPageToken?: string }
): boolean {
// First page in Cloud is when there's no nextPageToken in context
return issuesResult.items.length === 0 && !paginationCtx.nextPageToken;
}
protected extractSprints(job: TImportJob<any>, issue: any): string[] {
/* We go through the issue fields and find the custom field that matches
* the definition of a sprint, which should be an array of objects, and
* each object should contain boardId, name, startDate, endDate, and state
*/
const isSprintField = (value: any): boolean =>
Array.isArray(value) &&
value.length > 0 &&
value[0].boardId &&
value[0].name &&
value[0].startDate &&
value[0].endDate &&
value[0].state;
const config = job.config;
const resourceId = config.resource?.id || "";
const projectId = job.project_id || "";
for (const [fieldKey, fieldValue] of Object.entries(issue.fields)) {
if (!fieldKey.startsWith("customfield_")) continue;
if (isSprintField(fieldValue)) {
return (fieldValue as any[]).map((s: any) => buildExternalId(projectId, resourceId, s.id.toString()));
}
}
return [];
}
}
@@ -1,34 +0,0 @@
import {
JiraServerUsersStep,
JiraServerModulesStep,
JiraServerBoardsStep,
JiraServerCyclesStep,
JiraServerIssueTypesStep,
JiraServerIssuePropertiesStep,
JiraServerDefaultPropertiesStep,
JiraServerIssuePropertyOptionsStep,
} from "./entities";
import { JiraServerIssuesStep, WaitForCeleryStep } from "./issues";
import { PlaneProjectConfigurationStep } from "./pre-run";
import { JiraServerRelationsStep } from "./relations";
const JIRA_SERVER_STEPS = [
// Pre-run steps
new PlaneProjectConfigurationStep(),
// Entity steps
new JiraServerUsersStep(),
new JiraServerModulesStep(),
new JiraServerBoardsStep(),
new JiraServerCyclesStep(),
new JiraServerIssueTypesStep(),
new JiraServerIssuePropertiesStep(),
new JiraServerDefaultPropertiesStep(),
new JiraServerIssuePropertyOptionsStep(),
// Issue steps
new JiraServerIssuesStep(),
new WaitForCeleryStep(),
// Association steps
new JiraServerRelationsStep(),
];
export default JIRA_SERVER_STEPS;
@@ -0,0 +1,40 @@
import { E_IMPORTER_KEYS } from "@plane/etl/core";
import {
// Pre-run steps
PlaneProjectConfigurationStep,
// Entity steps
JiraBoardsStep,
JiraCyclesStep,
JiraDefaultPropertiesStep,
JiraIssuePropertiesStep,
JiraIssuePropertyOptionsStep,
JiraIssueTypesStep,
JiraModulesStep,
JiraUsersStep,
// Issue steps
JiraIssuesStep,
WaitForCeleryStep,
// Association steps
JiraRelationsStep,
} from "../shared";
const JIRA_SERVER_STEPS = [
// Pre-run steps
new PlaneProjectConfigurationStep(),
// Entity steps
new JiraUsersStep(),
new JiraModulesStep(E_IMPORTER_KEYS.JIRA_SERVER),
new JiraBoardsStep(),
new JiraCyclesStep(E_IMPORTER_KEYS.JIRA_SERVER),
new JiraIssueTypesStep(E_IMPORTER_KEYS.JIRA_SERVER),
new JiraIssuePropertiesStep(E_IMPORTER_KEYS.JIRA_SERVER),
new JiraDefaultPropertiesStep(E_IMPORTER_KEYS.JIRA_SERVER),
new JiraIssuePropertyOptionsStep(E_IMPORTER_KEYS.JIRA_SERVER),
// Issue steps
new JiraIssuesStep(E_IMPORTER_KEYS.JIRA_SERVER),
new WaitForCeleryStep(),
// Association steps
new JiraRelationsStep(E_IMPORTER_KEYS.JIRA_SERVER),
];
export default JIRA_SERVER_STEPS;
@@ -15,7 +15,7 @@ import type {
TStepExecutionContext,
TStepExecutionInput,
} from "@/apps/jira-server-importer/v2/types";
import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types";
import { EJiraStep } from "@/apps/jira-server-importer/v2/types";
/**
* Handles the extraction of boards from Jira Server.
@@ -28,8 +28,8 @@ import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types";
* for mapping issues to modules/views. If future requirements need board entities
* in Plane, add transform/push logic here.
*/
export class JiraServerBoardsStep implements IStep {
name = EJiraServerStep.BOARDS;
export class JiraBoardsStep implements IStep {
name = EJiraStep.BOARDS;
dependencies = [];
private readonly PAGE_SIZE = 100;
@@ -52,12 +52,13 @@ export class JiraServerBoardsStep implements IStep {
logger.info(`[${job.id}] [${this.name}] Pulling boards`, { startAt, totalProcessed });
const result = await this.pullBoards(sourceClient, projectId, startAt, job.id);
const scrumBoards = result.items.filter((board) => board.type === "scrum");
if (this.shouldReturnEmpty(result, startAt)) {
return createEmptyContext();
}
await this.storeBoards(storage, job.id, result.items);
await this.storeBoards(storage, job.id, scrumBoards);
const newTotalProcessed = totalProcessed + result.items.length;
@@ -1,5 +1,6 @@
import type { Sprint } from "jira.js/out/agile/models";
import { v4 as uuid } from "uuid";
import type { E_IMPORTER_KEYS } from "@plane/etl/core";
import type { JiraConfig, JiraSprint } from "@plane/etl/jira-server";
import { pullSprintsForBoardV2, transformSprint } from "@plane/etl/jira-server";
import { logger } from "@plane/logger";
@@ -14,7 +15,7 @@ import type {
TStepExecutionContext,
TStepExecutionInput,
} from "@/apps/jira-server-importer/v2/types";
import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types";
import { EJiraStep } from "@/apps/jira-server-importer/v2/types";
import { createAllCyclesV2 } from "@/etl/migrator/cycles.migrator";
/**
@@ -23,12 +24,14 @@ import { createAllCyclesV2 } from "@/etl/migrator/cycles.migrator";
*
* Dependencies: boards - board IDs to iterate through for sprint collection
*/
export class JiraServerCyclesStep implements IStep {
name = EJiraServerStep.CYCLES;
dependencies = [EJiraServerStep.BOARDS];
export class JiraCyclesStep implements IStep {
name = EJiraStep.CYCLES;
dependencies = [EJiraStep.BOARDS];
private readonly PAGE_SIZE = 100;
constructor(private readonly source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA) { }
/**
* Executes the sprint extraction process:
* 1. Extracts current board ID from dependency data
@@ -64,7 +67,7 @@ export class JiraServerCyclesStep implements IStep {
} catch (error) {
logger.error(`[${jobContext.job.id}] [${this.name}] Step failed`, {
jobId: jobContext.job.id,
error: error instanceof Error ? error.message : String(error),
error: error
});
throw error;
}
@@ -152,7 +155,16 @@ export class JiraServerCyclesStep implements IStep {
issues: [], // Issues are handled in separate step
}));
return formattedSprints.map((sprint) => transformSprint(resourceId, job.project_id, sprint));
return formattedSprints.map((sprint) =>
transformSprint(
{
resourceId,
projectId: job.project_id,
source: this.source,
},
sprint
)
);
}
/**
@@ -2,8 +2,8 @@ import { createSuccessContext } from "@/apps/jira-server-importer/v2/helpers/ctx
import { extractJobData } from "@/apps/jira-server-importer/v2/helpers/job";
import { getSupportedDefaultProperties } from "@/apps/jira-server-importer/v2/helpers/properties";
import type { TIssueTypesData, TStepExecutionContext, TStepExecutionInput } from "@/apps/jira-server-importer/v2/types";
import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types";
import { JiraServerIssuePropertiesStep } from "./issue-properties.step";
import { EJiraStep } from "@/apps/jira-server-importer/v2/types";
import { JiraIssuePropertiesStep } from "./issue-properties.step";
/*
* @overview
@@ -14,9 +14,9 @@ import { JiraServerIssuePropertiesStep } from "./issue-properties.step";
* to all the available issue types, which will allow us to map those properties
* to the corresponding plane properties.
*/
export class JiraServerDefaultPropertiesStep extends JiraServerIssuePropertiesStep {
name: EJiraServerStep = EJiraServerStep.DEFAULT_PROPERTIES;
dependencies: EJiraServerStep[] = [EJiraServerStep.ISSUE_TYPES];
export class JiraDefaultPropertiesStep extends JiraIssuePropertiesStep {
name: EJiraStep = EJiraStep.DEFAULT_PROPERTIES;
dependencies: EJiraStep[] = [EJiraStep.ISSUE_TYPES];
async execute(input: TStepExecutionInput): Promise<TStepExecutionContext> {
// Get the issue types from the dependency data and for each issue type, populate
@@ -37,13 +37,14 @@ export class JiraServerDefaultPropertiesStep extends JiraServerIssuePropertiesSt
resourceId,
projectId,
issueType.id,
issueType.external_id
issueType.external_id,
this.source
);
defaultPropertiesToCreate.push(...defaultProperties);
}
const pushed = await this.push(jobContext, defaultPropertiesToCreate, issueTypes);
await this.storePropertiesData(EJiraServerStep.ISSUE_PROPERTIES, job, pushed, storage);
await this.storePropertiesData(EJiraStep.ISSUE_PROPERTIES, job, pushed, storage);
return createSuccessContext({
pulled: defaultPropertiesToCreate.length,
@@ -1,4 +1,5 @@
import { v4 as uuid } from "uuid";
import type { E_IMPORTER_KEYS } from "@plane/etl/core";
import type { JiraConfig, JiraIssueField } from "@plane/etl/jira-server";
import { pullIssueFieldsV2, transformIssueFields } from "@plane/etl/jira-server";
import { logger } from "@plane/logger";
@@ -15,7 +16,7 @@ import type {
TStepExecutionContext,
TStepExecutionInput,
} from "@/apps/jira-server-importer/v2/types";
import { E_ADDITIONAL_STORAGE_KEYS, EJiraServerStep } from "@/apps/jira-server-importer/v2/types";
import { E_ADDITIONAL_STORAGE_KEYS, EJiraStep } from "@/apps/jira-server-importer/v2/types";
import { createOrUpdateIssueProperties } from "@/etl/migrator/issue-types/issue-property.migrator";
/**
@@ -24,9 +25,11 @@ import { createOrUpdateIssueProperties } from "@/etl/migrator/issue-types/issue-
*
* Note: No pagination needed - fields are a small dataset
*/
export class JiraServerIssuePropertiesStep implements IStep {
name = EJiraServerStep.ISSUE_PROPERTIES;
dependencies = [EJiraServerStep.ISSUE_TYPES];
export class JiraIssuePropertiesStep implements IStep {
name = EJiraStep.ISSUE_PROPERTIES;
dependencies = [EJiraStep.ISSUE_TYPES];
constructor(protected readonly source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA) {}
async execute(input: TStepExecutionInput): Promise<TStepExecutionContext> {
const { jobContext, storage, dependencyData } = input;
@@ -91,7 +94,7 @@ export class JiraServerIssuePropertiesStep implements IStep {
/**
* Pull custom fields from Jira Server
*/
private async pull(
protected async pull(
jobCtx: TJobContext,
projectId: string,
issueTypesData: TIssueTypesData
@@ -119,7 +122,7 @@ export class JiraServerIssuePropertiesStep implements IStep {
const resourceId = job.config.resource ? job.config.resource.id : uuid();
return jiraFields
.map((field) => transformIssueFields(resourceId, job.project_id, field))
.map((field) => transformIssueFields({ resourceId, projectId: job.project_id, source: this.source }, field))
.filter((field) => field && field.property_type) as Partial<ExIssueProperty>[];
}
@@ -261,7 +264,7 @@ export class JiraServerIssuePropertiesStep implements IStep {
*/
protected async storePropertiesData(
// We need to pass the name of the step, as default properties step uses this function to store data
name: EJiraServerStep,
name: EJiraStep,
job: TImportJob,
properties: ExIssueProperty[],
storage: IStorageService
@@ -1,4 +1,5 @@
import { v4 as uuid } from "uuid";
import type { E_IMPORTER_KEYS } from "@plane/etl/core";
import type { JiraConfig, JiraCustomFieldKeys, JiraIssueField } from "@plane/etl/jira-server";
import { OPTION_CUSTOM_FIELD_TYPES, transformIssueFieldOptions } from "@plane/etl/jira-server";
import { logger } from "@plane/logger";
@@ -14,7 +15,7 @@ import type {
TStepExecutionContext,
TStepExecutionInput,
} from "@/apps/jira-server-importer/v2/types";
import { E_ADDITIONAL_STORAGE_KEYS, EJiraServerStep } from "@/apps/jira-server-importer/v2/types";
import { E_ADDITIONAL_STORAGE_KEYS, EJiraStep } from "@/apps/jira-server-importer/v2/types";
import { createOrUpdateIssuePropertiesOptions } from "@/etl/migrator/issue-types/issue-property.migrator";
/**
@@ -23,9 +24,11 @@ import { createOrUpdateIssuePropertiesOptions } from "@/etl/migrator/issue-types
*
* Note: Only processes select/multi-select fields
*/
export class JiraServerIssuePropertyOptionsStep implements IStep {
name = EJiraServerStep.ISSUE_PROPERTY_OPTIONS;
dependencies = [EJiraServerStep.ISSUE_PROPERTIES];
export class JiraIssuePropertyOptionsStep implements IStep {
name = EJiraStep.ISSUE_PROPERTY_OPTIONS;
dependencies = [EJiraStep.ISSUE_PROPERTIES];
constructor(private readonly source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA) {}
async execute(input: TStepExecutionInput): Promise<TStepExecutionContext> {
const { jobContext, storage, dependencyData } = input;
@@ -85,7 +88,10 @@ export class JiraServerIssuePropertyOptionsStep implements IStep {
field.schema?.custom && OPTION_CUSTOM_FIELD_TYPES.includes(field.schema.custom as JiraCustomFieldKeys)
)
.flatMap(
(field) => field.options?.map((option) => transformIssueFieldOptions(resourceId, job.project_id, option)) || []
(field) =>
field.options?.map((option) =>
transformIssueFieldOptions({ resourceId, projectId: job.project_id, source: this.source }, option)
) || []
)
.filter(Boolean) as Partial<ExIssuePropertyOption>[];
@@ -1,6 +1,7 @@
import type { IssueTypeDetails } from "jira.js/out/version2/models";
import { v4 as uuid } from "uuid";
import { E_FEATURE_FLAGS } from "@plane/constants";
import type { E_IMPORTER_KEYS } from "@plane/etl/core";
import type { JiraConfig } from "@plane/etl/jira-server";
import { pullIssueTypesV2, transformIssueType } from "@plane/etl/jira-server";
import { logger } from "@plane/logger";
@@ -16,7 +17,7 @@ import type {
TStepExecutionContext,
TStepExecutionInput,
} from "@/apps/jira-server-importer/v2/types";
import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types";
import { EJiraStep } from "@/apps/jira-server-importer/v2/types";
import { createOrUpdateIssueTypes } from "@/etl/migrator/issue-types/issue-type.migrator";
import { getPlaneFeatureFlagService } from "@/helpers/plane-api-client";
import { protect } from "@/lib";
@@ -27,10 +28,12 @@ import { protect } from "@/lib";
*
* Handles Epic types and conflict resolution automatically
*/
export class JiraServerIssueTypesStep implements IStep {
name = EJiraServerStep.ISSUE_TYPES;
export class JiraIssueTypesStep implements IStep {
name = EJiraStep.ISSUE_TYPES;
dependencies = [];
constructor(private readonly source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA) {}
private readonly PAGE_SIZE = 50;
/**
@@ -135,7 +138,7 @@ export class JiraServerIssueTypesStep implements IStep {
/**
* Pull one page of issue types from Jira Server
*/
private async pull(jobContext: TJobContext, projectId: string, startAt: number) {
protected async pull(jobContext: TJobContext, projectId: string, startAt: number) {
const result = await pullIssueTypesV2(
{
client: jobContext.sourceClient,
@@ -161,7 +164,9 @@ export class JiraServerIssueTypesStep implements IStep {
private transform(job: TImportJob<JiraConfig>, jiraIssueTypes: IssueTypeDetails[]): Partial<ExIssueType>[] {
const resourceId = job.config.resource ? job.config.resource.id : uuid();
return jiraIssueTypes.map((issueType) => transformIssueType(resourceId, job.project_id, issueType));
return jiraIssueTypes.map((issueType) =>
transformIssueType({ resourceId, projectId: job.project_id, source: this.source }, issueType)
);
}
/**
@@ -1,6 +1,7 @@
/* ----------------- Modules Migrator Step ----------------- */
import type { ComponentWithIssueCount } from "jira.js/out/version2/models";
import { v4 as uuid } from "uuid";
import type { E_IMPORTER_KEYS } from "@plane/etl/core";
import type { JiraConfig, JiraV2Service } from "@plane/etl/jira-server";
import { pullComponentsV2, transformComponentV2 } from "@plane/etl/jira-server";
import { logger } from "@plane/logger";
@@ -14,19 +15,21 @@ import type {
TStepExecutionContext,
TStepExecutionInput,
} from "@/apps/jira-server-importer/v2/types";
import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types";
import { EJiraStep } from "@/apps/jira-server-importer/v2/types";
import { createAllModulesV2 } from "@/etl/migrator/modules.migrator";
/**
* Jira Server Modules Step (Components in Jira)
* Pulls components from Jira Server with pagination, transforms to modules, and pushes to Plane
*/
export class JiraServerModulesStep implements IStep {
name = EJiraServerStep.MODULES;
export class JiraModulesStep implements IStep {
name = EJiraStep.MODULES;
dependencies = [];
private readonly PAGE_SIZE = 100;
constructor(private readonly source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA) {}
async execute(input: TStepExecutionInput): Promise<TStepExecutionContext> {
const { jobContext, storage, previousContext } = input;
const { job, sourceClient } = jobContext;
@@ -102,7 +105,9 @@ export class JiraServerModulesStep implements IStep {
*/
private transform(job: TImportJob<JiraConfig>, jiraComponents: ComponentWithIssueCount[]) {
const resourceId = job.config.resource ? job.config.resource.id : uuid();
return jiraComponents.map((component) => transformComponentV2(resourceId, job.project_id, component));
return jiraComponents.map((component) =>
transformComponentV2({ resourceId, projectId: job.project_id, source: this.source }, component)
);
}
/**
@@ -12,7 +12,7 @@ import type {
TStepExecutionContext,
TStepExecutionInput,
} from "@/apps/jira-server-importer/v2/types";
import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types";
import { EJiraStep } from "@/apps/jira-server-importer/v2/types";
import { createUsers } from "@/etl/migrator/users.migrator";
import { protect } from "@/lib";
@@ -25,8 +25,8 @@ import { protect } from "@/lib";
*
* User avatars are also imported to Plane during this process.
*/
export class JiraServerUsersStep implements IStep {
name = EJiraServerStep.USERS;
export class JiraUsersStep implements IStep {
name = EJiraStep.USERS;
dependencies = [];
private readonly PAGE_SIZE = 100;
@@ -37,7 +37,7 @@ export class JiraServerUsersStep implements IStep {
private shouldExecute(input: TStepExecutionInput): boolean {
const { jobContext } = input;
const job = jobContext.job as TImportJob<JiraConfig>;
return !job.config?.skipUserImport;
return !job.config.skipUserImport;
}
/**
@@ -81,7 +81,7 @@ export class JiraServerUsersStep implements IStep {
});
// Pull users from Jira Server (paginated)
const pulledUsers = await this.pull(sourceClient, startAt, job.id);
const pulledUsers = await this.pull(jobContext, sourceClient, startAt, job.id);
if (pulledUsers.items.length === 0) {
logger.info(`[${job.id}] [${this.name}] No users found`, { jobId: job.id });
@@ -119,14 +119,14 @@ export class JiraServerUsersStep implements IStep {
* @param jobId - The import job ID for logging purposes
* @returns Paginated result containing user items and hasMore flag
*/
private async pull(client: JiraV2Service, startAt: number, jobId: string) {
protected async pull(_jobContext: TJobContext, client: JiraV2Service, startAt: number, jobId: string) {
const result = await pullUsersV2({
client,
startAt,
maxResults: this.PAGE_SIZE,
});
logger.info(`[${jobId}] [${this.name}] Pulled users from Jira Server`, {
logger.info(`[${jobId}] [${this.name}] Pulled users`, {
jobId,
count: result.items.length,
hasMore: result.hasMore,
@@ -256,18 +256,36 @@ export class JiraServerUsersStep implements IStep {
storage: IStorageService
): Promise<void> {
const allUsers = [...existingUsers, ...createdUsers];
/*
* In jira cloud, we don't have access to user emails, hence insead
* of poluting the space with multiple mappings and if else conditions
* we are using a merge of displayName and email, in case of transformation
* if the email is not present, we can fallback to displayName, which can be
* expected to be present in the mapping.
*/
const mappings = allUsers
.filter((user) => user.email && user.id)
.map((user) => ({
externalId: user.email!,
planeId: user.id!,
}));
.flatMap((user) => {
const displayName = user.display_name;
const email = user.email;
return [
...(email
? [
{
externalId: email,
planeId: user.id!,
},
]
: []),
{
externalId: displayName,
planeId: user.id!,
},
];
});
await storage.storeMapping(jobId, this.name, mappings);
logger.info(`[${jobId}] [${this.name}] Stored mappings`, {
jobId,
mappingCount: mappings.length,
});
}
}
@@ -0,0 +1,4 @@
export * from "./entities";
export * from "./issues";
export * from "./pre-run";
export * from "./relations";
@@ -1,7 +1,14 @@
import type { Worklog } from "jira.js/out/version2/models";
import { v4 as uuidv4 } from "uuid";
import type { TIssuePropertyValuesPayload } from "@plane/etl/core";
import type { IJiraIssue, JiraConfig, JiraIssueField } from "@plane/etl/jira-server";
import { pullIssuesV2, transformComment, transformIssueV2 } from "@plane/etl/jira-server";
import type { E_IMPORTER_KEYS, TIssuePropertyValuesPayload } from "@plane/etl/core";
import type { IJiraIssue, JiraConfig, JiraIssueField, JiraV2Service } from "@plane/etl/jira-server";
import {
pullAllCommentsForIssue,
pullAllWorklogsForIssue,
pullIssuesV2,
transformComment,
transformIssueV2,
} from "@plane/etl/jira-server";
import { logger } from "@plane/logger";
import type { ExIssue, ExIssueComment, ExIssueProperty, ExIssuePropertyOption, TWorklog } from "@plane/sdk";
import type { TImportJob } from "@plane/types";
@@ -20,7 +27,7 @@ import type {
TStepExecutionContext,
TStepExecutionInput,
} from "@/apps/jira-server-importer/v2/types";
import { E_ADDITIONAL_STORAGE_KEYS, EJiraServerStep } from "@/apps/jira-server-importer/v2/types";
import { E_ADDITIONAL_STORAGE_KEYS, EJiraStep } from "@/apps/jira-server-importer/v2/types";
import { generateIssuePayloadV2 } from "@/etl/migrator/issues.migrator";
import { getAPIClient } from "@/services/client";
import type { BulkIssuePayload } from "@/types";
@@ -41,90 +48,71 @@ import { celeryProducer } from "@/worker";
* NOT dependencies (accessed via storage.lookupMapping):
* - users, labels, issue_types
*/
export class JiraServerIssuesStep implements IStep {
name = EJiraServerStep.ISSUES;
export class JiraIssuesStep implements IStep {
name = EJiraStep.ISSUES;
dependencies = [
EJiraServerStep.ISSUE_TYPES, // Provides issue types
EJiraServerStep.ISSUE_PROPERTIES, // Provides properties + rawFields
EJiraServerStep.ISSUE_PROPERTY_OPTIONS, // Provides options
EJiraStep.ISSUE_TYPES, // Provides issue types
EJiraStep.ISSUE_PROPERTIES, // Provides properties + rawFields
EJiraStep.ISSUE_PROPERTY_OPTIONS, // Provides options
];
constructor(private readonly source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA) { }
private readonly PAGE_SIZE = 50;
/**
* Initialize report batch count on first page
* Automatically detects if it's the first page and calculates total batches
*/
private async initializeReportBatchCount(input: TStepExecutionInput, totalIssues: number): Promise<void> {
const { jobContext, previousContext } = input;
const startAt = previousContext?.pageCtx.startAt || 0;
// Only initialize on first page
if (startAt !== 0) return;
const totalBatches = Math.ceil(totalIssues / this.PAGE_SIZE);
const client = getAPIClient();
await client.importReport.updateImportReport(jobContext.job.report_id, {
total_batch_count: totalBatches,
});
logger.info(`[${jobContext.job.id}] [${this.name}] Initialized report batch count`, {
jobId: jobContext.job.id,
totalIssues,
pageSize: this.PAGE_SIZE,
totalBatches,
});
}
async execute(input: TStepExecutionInput): Promise<TStepExecutionContext> {
const { jobContext, storage, previousContext, dependencyData } = input;
const { job } = jobContext;
try {
const projectKey = job.config?.project?.key;
if (!projectKey) {
throw new Error("Project key not found in job config");
}
const projectKey = this.getProjectKey(job);
const paginationCtx = this.getPaginationContext(previousContext);
const startAt = previousContext?.pageCtx.startAt || 0;
const totalProcessed = previousContext?.pageCtx.totalProcessed || 0;
logger.info(`[${jobContext.job.id}] [${this.name}] Starting execution`, {
logger.info(`[${job.id}] [${this.name}] Starting execution`, {
jobId: job.id,
startAt,
totalProcessed,
...paginationCtx,
});
// Get property data from dependencies (needed for property value transformation)
const propertyData = this.getPropertyData(dependencyData);
const additionalData = await this.getAdditionalData(storage, job);
// Pull paginated issues, comments, and transform property values
const pulled = await this.pull(jobContext, projectKey, startAt, propertyData, additionalData);
// Pull paginated issues from Jira (can be overridden in subclasses)
const issuesResult = await this.pull({
jobContext,
projectKey,
paginationCtx,
});
if (pulled.total !== undefined) {
await this.initializeReportBatchCount(input, pulled.total);
}
await this.initializeReportBatchCount(paginationCtx, issuesResult.total, job);
const associations = this.extractAssociations(job, pulled.issues);
if (pulled.issues.length === 0) {
logger.info(`[${jobContext.job.id}] [${this.name}] No issues found`, { jobId: job.id });
if (this.shouldReturnEmpty(issuesResult, paginationCtx)) {
logger.info(`[${job.id}] [${this.name}] No issues found`, { jobId: job.id });
return createEmptyContext();
}
// Process issues: extract comments and transform property values
const processed = await this.processIssuesData({
jobContext,
issuesResult,
propertyData,
additionalData,
});
const associations = await this.extractAssociations(job, jobContext.sourceClient, processed.issues);
// Load entity mappings from storage
const mappings = await this.loadMappings(job, pulled.issues, associations, storage);
const mappings = await this.loadMappings(job, processed.issues, associations, storage);
// Transform issues (uses existing transformIssue function)
const transformed = await this.transform(job, pulled.issues);
const transformed = await this.transform(job, processed.issues);
// Generate payload and send to Celery
const pushed = await this.push(
transformed,
pulled.comments,
pulled.propertyValues,
processed.comments,
processed.propertyValues,
mappings,
associations,
propertyData,
@@ -132,26 +120,19 @@ export class JiraServerIssuesStep implements IStep {
);
// Store relations for Relations Step
const relations = this.extractRelations(job, pulled.issues);
const relations = this.extractRelations(job, processed.issues);
await this.storeRelations(relations, storage, job.id);
logger.info(`[${jobContext.job.id}] [${this.name}] Completed page`, {
logger.info(`[${job.id}] [${this.name}] Completed page`, {
jobId: job.id,
issues: pulled.issues.length,
comments: pulled.comments.length,
issues: processed.issues.length,
comments: processed.comments.length,
pushed: pushed.length,
});
return createPaginationContext({
hasMore: pulled.hasMore,
startAt: startAt,
pageSize: this.PAGE_SIZE,
pulled: pulled.issues.length,
pushed: pushed.length,
totalProcessed: totalProcessed + pulled.issues.length,
});
return this.buildNextContext(issuesResult, paginationCtx, processed.issues.length, pushed.length);
} catch (error) {
logger.error(`[${jobContext.job.id}] [${this.name}] Step failed`, {
logger.error(`[${job.id}] [${this.name}] Step failed`, {
jobId: job.id,
error: error instanceof Error ? error.message : String(error),
});
@@ -160,50 +141,152 @@ export class JiraServerIssuesStep implements IStep {
}
/**
* Pull issues and comments, then transform property values
* Uses existing transformIssuePropertyValues for proper conversion
* Extract pagination state from previous context
* Override this in subclasses to support different pagination strategies
* (e.g., nextPageToken for Jira Cloud)
*/
private async pull(
jobContext: TJobContext,
projectKey: string,
startAt: number,
propertyData: {
issueTypes: TIssueTypesData;
planeIssueProperties: ExIssueProperty[];
planeIssuePropertiesOptions: ExIssuePropertyOption[];
},
additionalData: {
rawFields: JiraIssueField[];
protected getPaginationContext(previousContext?: TStepExecutionContext) {
return {
startAt: previousContext?.pageCtx.startAt ?? 0,
totalProcessed: previousContext?.pageCtx.totalProcessed ?? 0,
};
}
/**
* Extract and validate project key from job configuration
*/
protected getProjectKey(job: TImportJob<JiraConfig>): string {
const projectKey = job.config?.project?.key;
if (!projectKey) {
throw new Error("Project key not found in job config");
}
): Promise<{
issues: IJiraIssue[];
comments: ExIssueComment[];
propertyValues: TIssuePropertyValuesPayload;
return projectKey;
}
/**
* Initialize report batch count on first page
*/
protected async initializeReportBatchCount(
paginationCtx: { startAt: number; totalProcessed: number },
totalIssues: number | undefined,
job: TImportJob<JiraConfig>
): Promise<void> {
// Only initialize on first page
if (paginationCtx.startAt !== 0 || !totalIssues) return;
const totalBatches = Math.ceil(totalIssues / this.PAGE_SIZE);
const client = getAPIClient();
await client.importReport.updateImportReport(job.report_id, {
total_batch_count: totalBatches,
});
logger.info(`[${job.id}] [${this.name}] Initialized report batch count`, {
jobId: job.id,
totalIssues,
pageSize: this.PAGE_SIZE,
totalBatches,
});
}
/**
* Check if step should return empty context
*/
protected shouldReturnEmpty(
issuesResult: { items: IJiraIssue[]; hasMore: boolean; total: number },
paginationCtx: { startAt: number; totalProcessed: number }
): boolean {
return issuesResult.items.length === 0 && paginationCtx.startAt === 0;
}
/**
* Build the next context based on pagination state
* Override this in subclasses to support different pagination strategies
*/
protected buildNextContext(
issuesResult: { items: IJiraIssue[]; hasMore: boolean; total: number },
paginationCtx: { startAt: number; totalProcessed: number },
pulled: number,
pushed: number
): TStepExecutionContext {
return createPaginationContext({
hasMore: issuesResult.hasMore,
startAt: paginationCtx.startAt,
pageSize: this.PAGE_SIZE,
pulled,
pushed,
totalProcessed: paginationCtx.totalProcessed + pulled,
});
}
/**
* Pull paginated issues from Jira
* Override this in subclasses to customize issue fetching behavior
* (e.g., use nextPageToken for Jira Cloud)
*/
protected async pull(props: {
jobContext: TJobContext;
projectKey: string;
paginationCtx: { startAt: number; totalProcessed: number };
}): Promise<{
items: IJiraIssue[];
hasMore: boolean;
total: number;
}> {
const { projectId, resourceId } = extractJobData(jobContext.job);
// Pull paginated issues
const issuesResult = await pullIssuesV2(
{
client: jobContext.sourceClient,
startAt,
client: props.jobContext.sourceClient,
startAt: props.paginationCtx.startAt,
maxResults: this.PAGE_SIZE,
},
projectKey
props.projectKey
);
// Pull comments for these issues
// Merge: timestamps from fields.comment, HTML body from renderedFields.comment
const comments: Partial<ExIssueComment>[] = issuesResult.items
.map((issue) => {
const fieldsComments = issue.fields?.comment?.comments || [];
const renderedComments = issue.renderedFields?.comment?.comments || [];
logger.info(`[${props.jobContext.job.id}] [${this.name}] Pulled issues`, {
jobId: props.jobContext.job.id,
count: issuesResult.items.length,
hasMore: issuesResult.hasMore,
startAt: props.paginationCtx.startAt,
});
return {
items: issuesResult.items,
hasMore: issuesResult.hasMore,
total: issuesResult.total ?? 0,
};
}
/**
* Extract and transform comments from issues
* Merges timestamps from fields.comment with HTML body from renderedFields.comment
*/
protected async extractCommentsFromIssues(props: {
_jobContext: TJobContext;
issues: IJiraIssue[];
resourceId: string;
projectId: string;
}): Promise<ExIssueComment[]> {
const comments = [];
for (const issue of props.issues) {
const fieldsComments = issue.fields?.comment?.comments || [];
const renderedComments = issue.renderedFields?.comment?.comments || [];
/*
* Jira provides us with the comments, but those comments are paginated, in
* case the number of comments are more than what we have in the comments property
* we will need to fetch the remaining comments.
*/
const shouldFetchMoreComments = issue.fields.comment?.total > issue.fields.comment?.comments?.length;
if (shouldFetchMoreComments) {
const allComments = await pullAllCommentsForIssue(issue, props._jobContext.sourceClient);
const transformedComments = allComments.map((comment) =>
transformComment({ resourceId: props.resourceId, projectId: props.projectId, source: this.source }, comment)
);
comments.push(transformedComments);
} else {
// Create a map of rendered comments by ID for quick lookup
const renderedCommentsMap = new Map(renderedComments.map((rendered) => [rendered.id, rendered]));
return fieldsComments.map((comment) => {
const transformedComments = fieldsComments.map((comment) => {
// Get the rendered version for HTML body
const renderedComment = renderedCommentsMap.get(comment.id);
@@ -217,35 +300,72 @@ export class JiraServerIssuesStep implements IStep {
issue_id: issue.id,
};
return transformComment(resourceId, projectId, mergedComment);
return transformComment(
{ resourceId: props.resourceId, projectId: props.projectId, source: this.source },
mergedComment
);
});
})
.flat();
comments.push(transformedComments);
}
}
return comments.flat() as ExIssueComment[];
}
/**
* Process pulled issues: extract comments and transform property values
* Uses existing transformIssuePropertyValues for proper conversion
*/
protected async processIssuesData(props: {
jobContext: TJobContext;
issuesResult: {
items: IJiraIssue[];
hasMore: boolean;
total: number;
};
propertyData: {
issueTypes: TIssueTypesData;
planeIssueProperties: ExIssueProperty[];
planeIssuePropertiesOptions: ExIssuePropertyOption[];
};
additionalData: {
rawFields: JiraIssueField[];
};
}): Promise<{
issues: IJiraIssue[];
comments: ExIssueComment[];
propertyValues: TIssuePropertyValuesPayload;
}> {
const { projectId, resourceId } = extractJobData(props.jobContext.job);
// Extract comments from issues
const comments = await this.extractCommentsFromIssues({
_jobContext: props.jobContext,
issues: props.issuesResult.items,
resourceId,
projectId,
});
// Transform property values for each issue using existing function
const propertyValues = getTransformedIssuePropertyValuesV2(
jobContext.job,
issuesResult.items,
additionalData.rawFields,
propertyData.planeIssueProperties,
propertyData.issueTypes
props.jobContext.job,
props.issuesResult.items,
props.additionalData.rawFields,
props.propertyData.planeIssueProperties,
props.propertyData.issueTypes
);
logger.info(`[${jobContext.job.id}] [${this.name}] Pulled from Jira`, {
jobId: jobContext.job.id,
issues: issuesResult.items.length,
logger.info(`[${props.jobContext.job.id}] [${this.name}] Processed issues data`, {
jobId: props.jobContext.job.id,
issues: props.issuesResult.items.length,
comments: comments.length,
issuesWithPropertyValues: Object.keys(propertyValues).length,
hasMore: issuesResult.hasMore,
startAt,
});
return {
issues: issuesResult.items,
comments: comments as ExIssueComment[],
issues: props.issuesResult.items,
comments,
propertyValues,
hasMore: issuesResult.hasMore,
total: issuesResult.total ?? 0,
};
}
@@ -265,9 +385,9 @@ export class JiraServerIssuesStep implements IStep {
};
}
const issueTypes = dependencyData[EJiraServerStep.ISSUE_TYPES] as TIssueTypesData;
const propertiesData = dependencyData[EJiraServerStep.ISSUE_PROPERTIES] as TIssuePropertiesData;
const optionsData = dependencyData[EJiraServerStep.ISSUE_PROPERTY_OPTIONS];
const issueTypes = dependencyData[EJiraStep.ISSUE_TYPES] as TIssueTypesData;
const propertiesData = dependencyData[EJiraStep.ISSUE_PROPERTIES] as TIssuePropertiesData;
const optionsData = dependencyData[EJiraStep.ISSUE_PROPERTY_OPTIONS];
return {
issueTypes,
@@ -336,10 +456,10 @@ export class JiraServerIssuesStep implements IStep {
// Load mappings in parallel from storage
const [userMap, issueTypeMap, cycleMap, moduleMap] = await Promise.all([
// We are retrieving all users instead of associated users, as there can be custom fields that associates with users, and we can't select those
storage.retrieveMapping(jobId, EJiraServerStep.USERS),
storage.lookupMapping(jobId, EJiraServerStep.ISSUE_TYPES, Array.from(issueTypeIds)),
storage.lookupMapping(jobId, EJiraServerStep.CYCLES, Array.from(sprintExternalIds)),
storage.lookupMapping(jobId, EJiraServerStep.MODULES, Array.from(componentExternalIds)),
storage.retrieveMapping(jobId, EJiraStep.USERS),
storage.lookupMapping(jobId, EJiraStep.ISSUE_TYPES, Array.from(issueTypeIds)),
storage.lookupMapping(jobId, EJiraStep.CYCLES, Array.from(sprintExternalIds)),
storage.lookupMapping(jobId, EJiraStep.MODULES, Array.from(componentExternalIds)),
]);
logger.info(`[${jobId}] [${this.name}] Loaded mappings`, {
@@ -362,7 +482,13 @@ export class JiraServerIssuesStep implements IStep {
const priorityMapping = job.config?.priority || {};
const transformed = issues.map((issue) =>
transformIssueV2(resourceId, job.project_id, issue, resourceUrl, stateMapping, priorityMapping)
transformIssueV2(
{ resourceId, projectId: job.project_id, source: this.source },
issue,
resourceUrl,
stateMapping,
priorityMapping
)
);
return transformed;
@@ -395,7 +521,11 @@ export class JiraServerIssuesStep implements IStep {
);
}
private extractAssociations(job: TImportJob<JiraConfig>, issues: IJiraIssue[]): TIssuesAssociationsData {
private async extractAssociations(
job: TImportJob<JiraConfig>,
client: JiraV2Service,
issues: IJiraIssue[]
): Promise<TIssuesAssociationsData> {
const { projectId, resourceId } = extractJobData(job);
const cycles = new Map<string, string[]>();
const modules = new Map<string, string[]>();
@@ -404,7 +534,7 @@ export class JiraServerIssuesStep implements IStep {
const issueExternalId = buildExternalId(projectId, resourceId, issue.id);
const sprintExternalIds = this.extractSprints(job, issue);
const componentExternalIds = this.extractComponents(job, issue);
const issueWorklogs = this.extractWorklogs(job, issue);
const issueWorklogs = await this.extractWorklogs(job, client, issue);
cycles.set(issueExternalId, sprintExternalIds);
modules.set(issueExternalId, componentExternalIds);
worklogs.set(issueExternalId, issueWorklogs);
@@ -412,7 +542,7 @@ export class JiraServerIssuesStep implements IStep {
return { cycles, modules, worklogs };
}
private extractSprints(job: TImportJob<JiraConfig>, issue: IJiraIssue): string[] {
protected extractSprints(job: TImportJob<JiraConfig>, issue: IJiraIssue): string[] {
const { projectId, resourceId } = extractJobData(job);
const sprintFieldKey = detectSprintFieldId(issue);
@@ -424,8 +554,8 @@ export class JiraServerIssuesStep implements IStep {
: null;
return sprintObjects
? sprintObjects
.map((s) => (s ? buildExternalId(projectId, resourceId, s.id.toString()) : null))
.filter((s) => s !== null)
.map((s) => (s ? buildExternalId(projectId, resourceId, s.id.toString()) : null))
.filter((s) => s !== null)
: [];
}
@@ -467,15 +597,34 @@ export class JiraServerIssuesStep implements IStep {
/**
* Extract worklogs from issue
*/
private extractWorklogs(_job: TImportJob<JiraConfig>, issue: IJiraIssue): Partial<TWorklog>[] {
protected async extractWorklogs(
_job: TImportJob<JiraConfig>,
client: JiraV2Service,
issue: IJiraIssue
): Promise<Partial<TWorklog>[]> {
const transformWorklog = (worklog: Worklog) => ({
description: worklog.comment ?? "",
duration: worklog.timeSpentSeconds ? worklog.timeSpentSeconds / 60 : 0,
logged_by: worklog.author?.emailAddress,
created_at: worklog.created,
updated_at: worklog.updated,
});
const shouldPullMoreWorklogs = issue.fields?.worklog?.total > issue.fields?.worklog?.worklogs?.length;
if (shouldPullMoreWorklogs) {
const worklogs = await pullAllWorklogsForIssue(issue, client);
return worklogs.map(transformWorklog);
}
return (
issue.fields.worklog?.worklogs?.map((worklog) => ({
description: worklog.comment ?? "",
duration: worklog.timeSpentSeconds ? worklog.timeSpentSeconds / 60 : 0,
logged_by: worklog.author?.emailAddress,
created_at: worklog.created,
updated_at: worklog.updated,
})) || []
issue.fields?.worklog?.worklogs?.map((worklog, index) => {
const comment =
typeof worklog.comment === "string"
? worklog.comment
: issue.renderedFields?.worklog?.worklogs?.[index]?.comment;
return transformWorklog({ ...worklog, comment });
}) || []
);
}
@@ -1,12 +1,12 @@
import { logger } from "@plane/logger";
import { createEmptyContext } from "@/apps/jira-server-importer/v2/helpers/ctx";
import type { IStep, TStepExecutionContext, TStepExecutionInput } from "@/apps/jira-server-importer/v2/types";
import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types";
import { EJiraStep } from "@/apps/jira-server-importer/v2/types";
import { wait } from "@/helpers/delay";
import { getAPIClient } from "@/services/client";
export class WaitForCeleryStep implements IStep {
name = EJiraServerStep.WAIT_FOR_CELERY;
name = EJiraStep.WAIT_FOR_CELERY;
dependencies = [];
async execute(input: TStepExecutionInput): Promise<TStepExecutionContext> {
@@ -8,7 +8,7 @@ import { E_FEATURE_FLAGS } from "@plane/constants";
import { logger } from "@plane/logger";
import { createEmptyContext } from "@/apps/jira-server-importer/v2/helpers/ctx";
import type { IStep, TStepExecutionContext, TStepExecutionInput } from "@/apps/jira-server-importer/v2/types";
import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types";
import { EJiraStep } from "@/apps/jira-server-importer/v2/types";
import { getPlaneFeatureFlagService } from "@/helpers/plane-api-client";
export type TRequiredFlags = {
@@ -18,8 +18,8 @@ export type TRequiredFlags = {
};
export class PlaneProjectConfigurationStep implements IStep {
name = EJiraServerStep.PLANE_PROJECT_CONFIGURATION;
dependencies: EJiraServerStep[] = [];
name = EJiraStep.PLANE_PROJECT_CONFIGURATION;
dependencies: EJiraStep[] = [];
async execute(input: TStepExecutionInput): Promise<TStepExecutionContext> {
const { jobContext } = input;
@@ -1,4 +1,5 @@
import { v4 as uuidv4 } from "uuid";
import type { E_IMPORTER_KEYS } from "@plane/etl/core";
import { logger } from "@plane/logger";
import { createEmptyContext } from "@/apps/jira-server-importer/v2/helpers/ctx";
import type {
@@ -7,7 +8,7 @@ import type {
TStepExecutionContext,
TStepExecutionInput,
} from "@/apps/jira-server-importer/v2/types";
import { E_ADDITIONAL_STORAGE_KEYS, EJiraServerStep } from "@/apps/jira-server-importer/v2/types";
import { E_ADDITIONAL_STORAGE_KEYS, EJiraStep } from "@/apps/jira-server-importer/v2/types";
import { celeryProducer } from "@/worker";
/**
@@ -16,9 +17,10 @@ import { celeryProducer } from "@/worker";
* Retrieves all accumulated relations from storage and sends to Celery in one batch
* Only handles relationships (parent, blocking, etc) - cycles/modules already handled
*/
export class JiraServerRelationsStep implements IStep {
name = EJiraServerStep.RELATIONS;
export class JiraRelationsStep implements IStep {
name = EJiraStep.RELATIONS;
dependencies = [];
constructor(private readonly source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA) {}
async execute(input: TStepExecutionInput): Promise<TStepExecutionContext> {
const { jobContext, storage } = input;
@@ -68,7 +70,7 @@ export class JiraServerRelationsStep implements IStep {
job_id: job.id,
workspace_id: job.workspace_id,
project_id: job.project_id,
source: job.source,
source: this.source,
user_id: credentials.user_id,
},
job.workspace_slug,
@@ -2,7 +2,7 @@ import type { JiraV2Service } from "@plane/etl/jira-server";
import type { Client as PlaneClient, TWorklog } from "@plane/sdk";
import type { TImportJob, TWorkspaceCredential } from "@plane/types";
export enum EJiraServerStep {
export enum EJiraStep {
// Pre-run
PLANE_PROJECT_CONFIGURATION = "plane_project_configuration",
@@ -45,6 +45,15 @@ export type TIssuesAssociationsData = {
worklogs: Map<string, Partial<TWorklog>[]>;
};
/**
* Failed step tracking structure
*/
export type TFailedStep = {
name: string;
error: string;
failedAt: string;
};
export type TOrchestratorState = {
jobId: string;
@@ -54,6 +63,9 @@ export type TOrchestratorState = {
totalSteps: number;
completedSteps: string[];
// Failed steps (logged and skipped)
failedSteps?: TFailedStep[];
// Timestamps
startedAt: string;
lastUpdatedAt: string;
@@ -103,10 +115,10 @@ export type TIssuePropertiesData = Array<{
*/
export interface IStep {
/** Step name (e.g., 'users', 'issues') */
name: EJiraServerStep;
name: EJiraStep;
/** data that should be loaded before handling this step */
dependencies: EJiraServerStep[];
dependencies: EJiraStep[];
shouldFail?: boolean;
/** Execute pull, transform, and push for this step */
+6 -5
View File
@@ -13,8 +13,10 @@ import { FlatfileMigrator } from "@/apps/flatfile/migrator/flatfile.migrator";
import { GithubWebhookWorker } from "@/apps/github/workers";
import { PlaneGithubWebhookWorker } from "@/apps/github/workers/plane";
import { GitlabWebhookWorker } from "@/apps/gitlab";
import { JiraDataMigrator } from "@/apps/jira-importer/migrator/jira.migrator";
import { JiraServerImportOrchestrator } from "@/apps/jira-server-importer/v2/migrator/orchestrator";
import {
getJiraCloudImportOrchestrator,
getJiraServerImportOrchestrator,
} from "@/apps/jira-server-importer/v2/migrator";
import { LinearDocsMigrator } from "@/apps/linear-importer/migrator/linear-docs.migrator";
import { LinearDataMigrator } from "@/apps/linear-importer/migrator/linear.migrator";
import { NotionDataMigrator } from "@/apps/notion-importer/worker";
@@ -46,10 +48,9 @@ class WorkerFactory {
static createWorker(type: string, mq: MQ, store: Store): TaskHandler {
switch (type) {
case "jira":
return new JiraDataMigrator(mq, store);
return getJiraCloudImportOrchestrator(mq, store);
case "jira_server":
// return new JiraDataCenterMigrator(mq, store);
return new JiraServerImportOrchestrator(mq, store);
return getJiraServerImportOrchestrator(mq, store);
case "linear":
return new LinearDataMigrator(mq, store);
case "linear_docs":
@@ -64,7 +64,7 @@ export const ImportUsersFromJira: FC = observer(() => {
}
if (key === "userData" && !formData.userSkipToggle && typeof value === "string") {
handleSyncJobConfig("users", value);
handleSyncJobConfig("skipUserImport", true);
handleSyncJobConfig("skipUserImport", formData.userSkipToggle);
}
};
+30 -1
View File
@@ -5,6 +5,7 @@ import type {
Comment as JComment,
IssueTypeDetails as JiraIssueTypeDetails,
FieldDetails,
Worklog
} from "jira.js/out/version2/models";
import type {
ImportedJiraUser,
@@ -15,7 +16,7 @@ import type {
JiraCustomFieldKeys,
JiraV2Service,
} from "..";
import { formatDateStringForHHMM, OPTION_CUSTOM_FIELD_TYPES } from "../helpers";
import { fetchPaginatedDataByKey, formatDateStringForHHMM, OPTION_CUSTOM_FIELD_TYPES } from "../helpers";
type BasePaginationContext = {
client: JiraV2Service;
@@ -191,6 +192,34 @@ export async function pullComponentIssuesV2(
};
}
export const pullAllCommentsForIssue = async (issue: IJiraIssue, client: JiraV2Service): Promise<JiraComment[]> => {
const values = await fetchPaginatedDataByKey<JComment>(
(startAt) => client.getIssueComments(issue.id, startAt, 500),
"comments"
);
return values.map(
(comment): JiraComment => ({
...comment,
issue_id: issue.id,
})
);
};
export const pullAllWorklogsForIssue = async (issue: IJiraIssue, client: JiraV2Service): Promise<Worklog[]> => {
const values = await fetchPaginatedDataByKey<Worklog>(
(startAt) => client.getIssueWorklogs(issue.id, startAt, 500),
"worklogs"
);
return values.map(
(worklog) => ({
...worklog,
issue_id: issue.id,
})
);
};
export async function pullCommentsForIssueV2(
ctx: BasePaginationContext,
issue: IJiraIssue
+86 -71
View File
@@ -35,14 +35,20 @@ import type {
JiraIssueFieldOptions,
} from "../types";
export type TTransformationContext = {
resourceId: string;
projectId: string;
source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA;
};
export const transformIssue = (
resourceId: string,
projectId: string,
ctx: TTransformationContext,
issue: IJiraIssue,
resourceUrl: string,
stateMap: IStateConfig[],
priorityMap: IPriorityConfig[]
): Partial<PlaneIssue> => {
const { resourceId, projectId, source } = ctx;
const targetState = getTargetState(stateMap, issue.fields.status);
const targetPriority = getTargetPriority(priorityMap, issue.fields.priority);
const attachments = getTargetAttachments(resourceId, projectId, issue.fields.attachment);
@@ -66,7 +72,7 @@ export const transformIssue = (
assignees: issue.fields.assignee?.name ? [issue.fields.assignee.name] : [],
links,
external_id: `${projectId}_${resourceId}_${issue.id}`,
external_source: E_IMPORTER_KEYS.JIRA_SERVER,
external_source: source,
created_by: issue.fields.creator?.name,
name: issue.fields.summary ?? "Untitled",
description_html: description,
@@ -84,13 +90,13 @@ export const transformIssue = (
};
export const transformIssueV2 = (
resourceId: string,
projectId: string,
ctx: TTransformationContext,
issue: IJiraIssue,
resourceUrl: string,
stateMap: IStateConfig[],
priorityMap: IPriorityConfig[]
): Partial<PlaneIssue> => {
const { resourceId, projectId, source } = ctx;
const targetState = getTargetState(stateMap, issue.fields.status);
const targetPriority = getTargetPriority(priorityMap, issue.fields.priority);
const attachments = getTargetAttachments(resourceId, projectId, issue.fields.attachment);
@@ -111,11 +117,15 @@ export const transformIssueV2 = (
issue.fields.labels.push("JIRA IMPORTED");
return {
assignees: issue.fields.assignee?.emailAddress ? [issue.fields.assignee.emailAddress] : [],
assignees: issue.fields.assignee?.emailAddress
? [issue.fields.assignee.emailAddress]
: issue.fields.assignee?.displayName
? [issue.fields.assignee.displayName]
: [],
links,
external_id: `${projectId}_${resourceId}_${issue.id}`,
external_source: E_IMPORTER_KEYS.JIRA_SERVER,
created_by: issue.fields.creator?.emailAddress,
external_source: source,
created_by: issue.fields.creator?.emailAddress || issue.fields.creator?.displayName,
name: issue.fields.summary ?? "Untitled",
description_html: description,
target_date: issue.fields.duedate,
@@ -136,20 +146,19 @@ export const transformLabel = (label: string): Partial<ExIssueLabel> => ({
color: getRandomColor(),
});
export const transformComment = (
resourceId: string,
projectId: string,
comment: JiraComment
): Partial<ExIssueComment> => ({
external_id: `${projectId}_${resourceId}_${comment.id}`,
external_source: E_IMPORTER_KEYS.JIRA_SERVER,
created_at: getFormattedDate(comment.created),
created_by: comment.author?.emailAddress,
// @ts-expect-error
comment_html: comment.body ?? "<p></p>",
actor: comment.author?.emailAddress,
issue: `${projectId}_${resourceId}_${comment.issue_id}`,
});
export const transformComment = (ctx: TTransformationContext, comment: JiraComment): Partial<ExIssueComment> => {
const { resourceId, projectId, source } = ctx;
return {
external_id: `${projectId}_${resourceId}_${comment.id}`,
external_source: source,
created_at: comment.created,
created_by: comment.author?.emailAddress || comment.author?.displayName,
// @ts-expect-error
comment_html: comment.renderedBody ? comment.renderedBody : (comment.body ?? "<p></p>"),
actor: comment.author?.emailAddress || comment.author?.displayName,
issue: `${projectId}_${resourceId}_${comment.issue_id}`,
};
};
export const transformUser = (user: ImportedJiraUser): Partial<PlaneUser> => {
const [first_name, last_name] = user.full_name.split(" ");
@@ -165,42 +174,46 @@ export const transformUser = (user: ImportedJiraUser): Partial<PlaneUser> => {
};
};
export const transformSprint = (resourceId: string, projectId: string, sprint: JiraSprint): Partial<ExCycle> => ({
external_id: `${projectId}_${resourceId}_${sprint.sprint.id.toString()}`,
external_source: E_IMPORTER_KEYS.JIRA_SERVER,
name: sprint.sprint.name,
start_date: getFormattedDate(sprint.sprint.startDate),
end_date: getFormattedDate(sprint.sprint.endDate),
created_at: getFormattedDate(sprint.sprint.createdDate),
issues: sprint.issues.map((issue) => `${projectId}_${resourceId}_${issue.id}`),
});
export const transformSprint = (ctx: TTransformationContext, sprint: JiraSprint): Partial<ExCycle> => {
const { resourceId, projectId, source } = ctx;
return {
external_id: `${projectId}_${resourceId}_${sprint.sprint.id.toString()}`,
external_source: source,
name: sprint.sprint.name,
start_date: getFormattedDate(sprint.sprint.startDate),
end_date: getFormattedDate(sprint.sprint.endDate),
created_at: getFormattedDate(sprint.sprint.createdDate),
issues: sprint.issues.map((issue) => `${projectId}_${resourceId}_${issue.id}`),
};
};
export const transformComponent = (
resourceId: string,
projectId: string,
component: JiraComponent
): Partial<ExModule> => ({
external_id: `${projectId}_${resourceId}_${component.component.id}`,
external_source: E_IMPORTER_KEYS.JIRA_SERVER,
name: component.component.name,
issues: component.issues.map((issue) => `${projectId}_${resourceId}_${issue.id}`),
});
export const transformComponent = (ctx: TTransformationContext, component: JiraComponent): Partial<ExModule> => {
const { resourceId, projectId, source } = ctx;
return {
external_id: `${projectId}_${resourceId}_${component.component.id}`,
external_source: source,
name: component.component.name,
issues: component.issues.map((issue) => `${projectId}_${resourceId}_${issue.id}`),
};
};
export const transformComponentV2 = (
resourceId: string,
projectId: string,
ctx: TTransformationContext,
component: ComponentWithIssueCount
): Partial<ExModule> => ({
external_id: `${projectId}_${resourceId}_${component.id}`,
external_source: E_IMPORTER_KEYS.JIRA_SERVER,
name: component.name,
});
): Partial<ExModule> => {
const { resourceId, projectId, source } = ctx;
return {
external_id: `${projectId}_${resourceId}_${component.id}`,
external_source: source,
name: component.name,
};
};
export const transformIssueType = (
resourceId: string,
projectId: string,
ctx: TTransformationContext,
issueType: JiraIssueTypeDetails
): Partial<ExIssueType> => {
const { resourceId, projectId, source } = ctx;
const isEpic = issueType.name?.toLowerCase().includes("epic");
return {
@@ -209,15 +222,15 @@ export const transformIssueType = (
is_active: true,
is_epic: isEpic,
external_id: `${projectId}_${resourceId}_${issueType.id}`,
external_source: E_IMPORTER_KEYS.JIRA_SERVER,
external_source: source,
};
};
export const transformIssueFields = (
resourceId: string,
projectId: string,
ctx: TTransformationContext,
issueField: JiraIssueField
): Partial<ExIssueProperty> | undefined => {
const { resourceId, projectId, source } = ctx;
if (
!issueField.schema ||
!issueField.scope?.type ||
@@ -231,7 +244,7 @@ export const transformIssueFields = (
return {
external_id: `${projectId}_${resourceId}_${fieldId}`,
external_source: E_IMPORTER_KEYS.JIRA_SERVER,
external_source: source,
display_name: issueField.name,
type_id: issueField.scope?.type ? `${projectId}_${resourceId}_${issueField.scope?.type}` : undefined,
is_required: false,
@@ -241,26 +254,28 @@ export const transformIssueFields = (
};
export const transformIssueFieldOptions = (
resourceId: string,
projectId: string,
ctx: TTransformationContext,
issueFieldOption: JiraIssueFieldOptions
): Partial<ExIssuePropertyOption> => ({
external_id: `${projectId}_${resourceId}_${issueFieldOption.id}`,
external_source: E_IMPORTER_KEYS.JIRA_SERVER,
name: issueFieldOption.value,
is_active: issueFieldOption.disabled ? false : true,
property_id: `${projectId}_${resourceId}_${issueFieldOption.fieldId}`,
});
): Partial<ExIssuePropertyOption> => {
const { resourceId, projectId, source } = ctx;
return {
external_id: `${projectId}_${resourceId}_${issueFieldOption.id}`,
external_source: source,
name: issueFieldOption.value,
is_active: issueFieldOption.disabled ? false : true,
property_id: `${projectId}_${resourceId}_${issueFieldOption.fieldId}`,
};
};
export const transformIssuePropertyValues = (
resourceId: string,
projectId: string,
ctx: TTransformationContext,
issue: IJiraIssue,
// eslint-disable-next-line no-undef
planeIssueProperties: Map<string, Partial<ExIssueProperty>>, // TODO: replace Map with Record<string, Partial<ExIssueProperty>> in the future
// eslint-disable-next-line no-undef
jiraCustomFieldMap: Map<string, string> // TODO: replace Map with Record<string, string> in the future
): TPropertyValuesPayload => {
const { resourceId, projectId } = ctx;
// Get all custom fields that are present in the issue and are also present in the plane issue properties
const customFieldKeysToTransform = Object.keys(issue.fields).filter(
(key) => key.startsWith("customfield_") && planeIssueProperties.has(key)
@@ -283,18 +298,18 @@ export const transformIssuePropertyValues = (
};
export const transformDefaultPropertyValues = (
resourceId: string,
projectId: string,
ctx: TTransformationContext,
issue: IJiraIssue,
issueTypeId: string
): TPropertyValuesPayload => {
const { resourceId, projectId, source } = ctx;
const propertyValuesPayload: TPropertyValuesPayload = {};
// Fix Versions
if (issue.fields.fixVersions && Array.isArray(issue.fields.fixVersions)) {
const fixVersionExternalId = `${resourceId}-${projectId}-${issueTypeId}-fix-version`;
propertyValuesPayload[fixVersionExternalId] = issue.fields.fixVersions.map((version: any) => ({
external_source: "JIRA_SERVER",
external_source: source,
external_id: version.id ? String(version.id) : undefined,
value: version.name || "",
}));
@@ -304,7 +319,7 @@ export const transformDefaultPropertyValues = (
if (issue.fields.versions && Array.isArray(issue.fields.versions)) {
const affectedVersionExternalId = `${resourceId}-${projectId}-${issueTypeId}-affected-version`;
propertyValuesPayload[affectedVersionExternalId] = issue.fields.versions.map((version: any) => ({
external_source: "JIRA_SERVER",
external_source: source,
external_id: version.id ? String(version.id) : undefined,
value: version.name || "",
}));
@@ -315,8 +330,8 @@ export const transformDefaultPropertyValues = (
const reporterExternalId = `${resourceId}-${projectId}-${issueTypeId}-reporter`;
propertyValuesPayload[reporterExternalId] = [
{
external_source: "JIRA_SERVER",
external_id: issue.fields.reporter.emailAddress || issue.fields.reporter.name,
external_source: source,
external_id: issue.fields.reporter.emailAddress || issue.fields.reporter.displayName || "",
value: issue.fields.reporter.emailAddress || issue.fields.reporter.displayName || "",
},
];
+6 -6
View File
@@ -191,11 +191,11 @@ export const getPropertyValues = (
break;
case "com.atlassian.jira.plugin.system.customfieldtypes:userpicker":
// Handle userpicker
if (value.emailAddress) {
if (value.emailAddress || value.displayName) {
propertyValues.push({
...commonPropertyProp,
external_id: value.emailAddress,
value: value.emailAddress,
external_id: value.emailAddress ?? value.displayName,
value: value.emailAddress ?? value.displayName,
});
}
break;
@@ -277,11 +277,11 @@ export const getPropertyValues = (
// Handle multiuserpicker
if (Array.isArray(value)) {
value.forEach((val) => {
if (val.emailAddress) {
if (val.emailAddress || val.displayName) {
propertyValues.push({
...commonPropertyProp,
external_id: val.emailAddress,
value: val.emailAddress,
external_id: val.emailAddress ?? val.displayName,
value: val.emailAddress ?? val.displayName,
});
}
});
@@ -1,5 +1,6 @@
// services
import axios, { AxiosError } from "axios";
import type { AxiosError } from "axios";
import axios from "axios";
import type { Paginated } from "jira.js";
import { Board as BoardClient } from "jira.js/out/agile";
import { Version2Client } from "jira.js/out/version2";
@@ -7,29 +8,45 @@ import type {
CustomFieldContextOption,
FieldDetails,
Issue,
IssueTypeDetails,
JiraStatus,
Project,
IssueTypeDetails,
} from "jira.js/out/version2/models";
import type { JiraCustomFieldWithCtx } from "@/jira-server/types/custom-fields";
import type { JiraApiUser, JiraProps } from "..";
import { fetchPaginatedData } from "..";
import { fetchPaginatedData, EJiraAuthenticationType } from "..";
export class JiraV2Service {
private jiraClient: Version2Client;
private hostname: string;
private patToken: string;
private email: string;
private authenticationType: EJiraAuthenticationType = EJiraAuthenticationType.PERSONAL_ACCESS_TOKEN;
constructor(props: JiraProps) {
this.hostname = props.hostname;
this.patToken = props.patToken;
this.email = props.email;
this.authenticationType = props.authenticationType;
this.jiraClient = new Version2Client({
host: props.hostname,
authentication: {
personalAccessToken: props.patToken,
},
});
if (this.authenticationType === EJiraAuthenticationType.BASIC) {
this.jiraClient = new Version2Client({
host: props.hostname,
authentication: {
basic: {
email: this.email,
apiToken: props.patToken,
},
},
});
} else {
this.jiraClient = new Version2Client({
host: props.hostname,
authentication: {
personalAccessToken: props.patToken,
},
});
}
this.jiraClient.handleFailedResponse = async (request) => {
const error = request as AxiosError;
@@ -70,6 +87,14 @@ export class JiraV2Service {
})) as JiraApiUser[];
}
async getIssueWorklogs(issueId: string, startAt: number, maxResults: number) {
return this.jiraClient.issueWorklogs.getIssueWorklog({
issueIdOrKey: issueId,
startAt: startAt,
maxResults: maxResults,
});
}
// Verified
async getNumberOfIssues(projectKey: string) {
const issues = await this.jiraClient.issueSearch.searchForIssuesUsingJql({
@@ -239,15 +264,11 @@ export class JiraV2Service {
// Verified
async getPaginatedIssueTypes(projectId: string, startAt?: number, maxResults?: number) {
try {
return axios
.get(
`${this.hostname}/rest/api/2/issuetype/page?projectIds=${projectId}&startAt=${startAt}&maxResults=${maxResults}`,
{
headers: {
Authorization: `Bearer ${this.patToken}`,
},
}
)
return await this.jiraClient
.sendRequestFullResponse<Paginated<IssueTypeDetails>>({
method: "GET",
url: `${this.hostname}/rest/api/2/issuetype/page?projectIds=${projectId}&startAt=${startAt}&maxResults=${maxResults}`,
})
.then((res) => res.data as Paginated<IssueTypeDetails>);
} catch (e) {
console.error("error getProjectIssueTypes", e);
@@ -264,15 +285,13 @@ export class JiraV2Service {
async getCustomFieldsWithContext(projectId?: string) {
try {
return await fetchPaginatedData<JiraCustomFieldWithCtx>((startAt) =>
axios
.get(`${this.hostname}/rest/api/2/customFields?startAt=${startAt}&projectIds=${projectId ?? ""}`, {
headers: {
Authorization: `Bearer ${this.patToken}`,
},
})
.then((res) => res.data as Paginated<JiraCustomFieldWithCtx>)
);
return await fetchPaginatedData<JiraCustomFieldWithCtx>(async (startAt) => {
const result = await this.jiraClient.sendRequestFullResponse<Paginated<JiraCustomFieldWithCtx>>({
method: "GET",
url: `${this.hostname}/rest/api/2/customFields?startAt=${startAt}&projectIds=${projectId ?? ""}`,
});
return result.data;
});
} catch (e) {
console.error("error getCustomFieldsWithContext", e);
throw e;
@@ -15,8 +15,15 @@ import type { ExProject, ExState } from "@plane/sdk";
export type JiraProps = {
hostname: string;
patToken: string;
email: string;
authenticationType: EJiraAuthenticationType;
};
export enum EJiraAuthenticationType {
BASIC = "basic",
PERSONAL_ACCESS_TOKEN = "personalAccessToken",
}
export type ImportedJiraUser = {
user_id: string;
user_name: string;
@@ -33,6 +40,7 @@ export type JiraApiUser = {
key: string;
name: string;
emailAddress?: string;
accountType?: string;
avatarUrls: {
"16x16": string;
"24x24": string;
+39 -3
View File
@@ -64,6 +64,37 @@ export async function pullIssues(client: JiraService, projectKey: string, from?:
return issues;
}
/**
* Pull issues using nextPageToken pagination (Jira Cloud Enhanced Search API)
* Note: Enhanced Search API does not return total count, only nextPageToken for pagination
*/
export async function pullIssuesV2(
ctx: {
client: JiraService;
nextPageToken?: string;
maxResults?: number;
},
projectKey: string,
// We are using this property for the pagination context
total = 0,
from?: Date
): Promise<{
items: IJiraIssue[];
hasMore: boolean;
total?: number;
nextPageToken?: string;
}> {
const { client, nextPageToken } = ctx;
const result = await client.getProjectIssues(projectKey, nextPageToken, from ? formatDateStringForHHMM(from) : "");
return {
items: result.issues || [],
hasMore: !!result.nextPageToken,
total: total,
nextPageToken: result.nextPageToken,
};
}
export async function pullComments(issues: IJiraIssue[], client: JiraService): Promise<any[]> {
const comments: JiraComment[] = [];
@@ -168,10 +199,11 @@ export const pullIssueFields = async (
const customFields: JiraIssueField[] = [];
try {
// initialize fields
const fields: FieldDetails[] = await client.getCustomFields();
const allFields: FieldDetails[] = await client.getCustomFields();
const filteredFields = allFields.filter((field) => field.custom);
// get all field contexts
for (const field of fields) {
for (const field of filteredFields) {
// skip if field has no id
if (!field.id) continue;
@@ -226,7 +258,11 @@ export const pullIssueFields = async (
(startAt) => client.getIssueFieldOptions(field.id as string, Number(issueTypeContext.contextId), startAt),
(values: CustomFieldContextOption[]) => {
values.map((value) => {
if (field.id) fieldOptions.push({ ...value, fieldId: field.id });
if (field.id)
fieldOptions.push({
...value,
fieldId: field.id.includes("customfield_") ? field.id.split("_").pop()! : field.id,
});
});
},
"values"
+7 -1
View File
@@ -67,10 +67,16 @@ export class AssetService extends APIService {
external_source?: string;
}
): Promise<string> {
let fileType = file.type;
// If the file type includes a semicolon, we need to remove the charset
if (fileType.includes(";")) {
fileType = fileType.split(";")[0];
}
// First get the presigned URL
const uploadResponse = await this.createAsset(workspaceSlug, {
name,
type: file.type,
type: fileType,
size,
project_id: options?.project_id,
external_id: options?.external_id,