[SILO-724] feat: Jira Cloud Importer V2 (#4888)
* fix: ratelimit issues and fails with jira cloud * fix: added rate limit handler for jira server * fix: review comments * fix: retry logic with axious * fix: issue association with comments * fix: label import issue with data_import_task * fix: linter errors * fix: added user email for jira service * fix: updated tranformation logic * fix: added steps for jira cloud migration steps * fix: moved jira-server to shared steps * fix: renamed types * fix: config coming as false for skipUserImport * fix: return format for pull issues for jira cloud * fix: pull and transform functions for jira server * fix: optimized orchestrator to use internal api client * fix: comment ordering for jira cloud * fix: bug with pulling issue types * fix: comments from bugbot * fix: attachment issue for jira and jira srv * fix: reviews and formatting of code * fix: build * fix: worklog and authentication type * fix: step failure protection for orchestrator * fix: fetch sprints only for scrum boards
This commit is contained in:
@@ -2,9 +2,8 @@ import type { Request, Response } from "express";
|
||||
// etl
|
||||
import { Controller, Get, Post } from "@plane/decorators";
|
||||
import { E_IMPORTER_KEYS } from "@plane/etl/core";
|
||||
import type { JiraResource } from "@plane/etl/jira";
|
||||
import type { JiraProject, JiraV2Service } from "@plane/etl/jira-server";
|
||||
import { createJiraService } from "@plane/etl/jira-server";
|
||||
import { createJiraService, EJiraAuthenticationType } from "@plane/etl/jira-server";
|
||||
// db
|
||||
// helpers
|
||||
import { compareAndGetAdditionalUsers } from "@/helpers/additional-users";
|
||||
@@ -24,14 +23,18 @@ class JiraDataCenterController {
|
||||
async upsertCredentials(req: Request, res: Response) {
|
||||
try {
|
||||
const { workspaceId, userId, apiToken, personalAccessToken, userEmail, hostname } = req.body;
|
||||
if (!workspaceId || !userId || !apiToken || !personalAccessToken) {
|
||||
res.status(400).json({ message: "Workspace ID, User ID, API Token and Personal Access Token are required" });
|
||||
if (!workspaceId || !userId || !apiToken || !personalAccessToken || !userEmail) {
|
||||
return res
|
||||
.status(400)
|
||||
.json({ message: "Workspace ID, User ID, API Token, Personal Access Token and User Email are required" });
|
||||
}
|
||||
|
||||
try {
|
||||
const jiraService = createJiraService({
|
||||
patToken: personalAccessToken,
|
||||
hostname: hostname,
|
||||
email: (userEmail as string).trim(),
|
||||
authenticationType: EJiraAuthenticationType.PERSONAL_ACCESS_TOKEN,
|
||||
});
|
||||
|
||||
await jiraService.getCurrentUser();
|
||||
@@ -64,8 +67,7 @@ class JiraDataCenterController {
|
||||
try {
|
||||
const jiraClient = await createJiraServerClient(workspaceId, userId);
|
||||
const serverInfo = await jiraClient.getServerInfo();
|
||||
// @ts-expect-error
|
||||
const resource: JiraResource = {
|
||||
const resource = {
|
||||
id: serverInfo.scmInfo ?? "",
|
||||
url: serverInfo.baseUrl ?? "",
|
||||
name: serverInfo.serverTitle ?? "",
|
||||
@@ -185,6 +187,8 @@ export const createJiraServerClient = async (workspaceId: string, userId: string
|
||||
return createJiraService({
|
||||
hostname: jiraCredentials.source_hostname,
|
||||
patToken: jiraCredentials.source_access_token,
|
||||
email: jiraCredentials.source_auth_email,
|
||||
authenticationType: EJiraAuthenticationType.PERSONAL_ACCESS_TOKEN,
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ import type {
|
||||
} from "jira.js/out/version2/models";
|
||||
import { E_IMPORTER_KEYS } from "@plane/etl/core";
|
||||
import type { IPriorityConfig, IStateConfig, JiraComponent, JiraConfig, JiraSprint } from "@plane/etl/jira-server";
|
||||
import { JiraV2Service } from "@plane/etl/jira-server";
|
||||
import { EJiraAuthenticationType, JiraV2Service } from "@plane/etl/jira-server";
|
||||
|
||||
import type { ExIssueAttachment, ExState } from "@plane/sdk";
|
||||
import type { TImportJob, TWorkspaceCredential } from "@plane/types";
|
||||
@@ -92,8 +92,14 @@ export const createJiraClient = (
|
||||
if (!credentials.source_access_token || !credentials.source_hostname || !credentials.source_auth_email) {
|
||||
throw new Error(`Missing credentials in job config for job ${job.id}`);
|
||||
}
|
||||
|
||||
const authenticationType =
|
||||
job.source === E_IMPORTER_KEYS.JIRA ? EJiraAuthenticationType.BASIC : EJiraAuthenticationType.PERSONAL_ACCESS_TOKEN;
|
||||
|
||||
return new JiraV2Service({
|
||||
email: credentials.source_auth_email,
|
||||
patToken: credentials.source_access_token!,
|
||||
hostname: credentials.source_hostname,
|
||||
authenticationType,
|
||||
});
|
||||
};
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { v4 as uuid } from "uuid";
|
||||
import type { TIssuePropertyValuesPayload } from "@plane/etl/core";
|
||||
import type { E_IMPORTER_KEYS, TIssuePropertyValuesPayload } from "@plane/etl/core";
|
||||
import type {
|
||||
JiraCustomFieldKeys,
|
||||
IJiraIssue,
|
||||
@@ -48,7 +48,17 @@ export const getTransformedIssues = (
|
||||
const resourceId = job.config.resource ? job.config.resource.id : uuid();
|
||||
|
||||
return entities.issues.map((issue: IJiraIssue): Partial<PlaneIssue> => {
|
||||
const transformedIssue = transformIssue(resourceId, job.project_id, issue, resourceUrl, stateMap, priorityMap);
|
||||
const transformedIssue = transformIssue(
|
||||
{
|
||||
resourceId,
|
||||
projectId: job.project_id,
|
||||
source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA,
|
||||
},
|
||||
issue,
|
||||
resourceUrl,
|
||||
stateMap,
|
||||
priorityMap
|
||||
);
|
||||
|
||||
if (job.config?.issueType && issue.fields.issuetype?.name) {
|
||||
const issueTypeValue = job.config.issueType;
|
||||
@@ -71,7 +81,16 @@ export const getTransformedComments = (
|
||||
entities: JiraEntity
|
||||
): Partial<ExIssueComment>[] => {
|
||||
const resourceId = job.config.resource ? job.config.resource.id : uuid();
|
||||
return entities.issue_comments.map((comment) => transformComment(resourceId, job.project_id, comment));
|
||||
return entities.issue_comments.map((comment) =>
|
||||
transformComment(
|
||||
{
|
||||
resourceId,
|
||||
projectId: job.project_id,
|
||||
source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA,
|
||||
},
|
||||
comment
|
||||
)
|
||||
);
|
||||
};
|
||||
|
||||
export const getTransformedUsers = (_job: TImportJob<JiraConfig>, entities: JiraEntity): Partial<PlaneUser>[] =>
|
||||
@@ -79,17 +98,44 @@ export const getTransformedUsers = (_job: TImportJob<JiraConfig>, entities: Jira
|
||||
|
||||
export const getTransformedSprints = (job: TImportJob<JiraConfig>, entities: JiraEntity): Partial<ExCycle>[] => {
|
||||
const resourceId = job.config.resource ? job.config.resource.id : uuid();
|
||||
return entities.sprints.map((sprint) => transformSprint(resourceId, job.project_id, sprint));
|
||||
return entities.sprints.map((sprint) =>
|
||||
transformSprint(
|
||||
{
|
||||
resourceId,
|
||||
projectId: job.project_id,
|
||||
source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA,
|
||||
},
|
||||
sprint
|
||||
)
|
||||
);
|
||||
};
|
||||
|
||||
export const getTransformedComponents = (job: TImportJob<JiraConfig>, entities: JiraEntity): Partial<ExModule>[] => {
|
||||
const resourceId = job.config.resource ? job.config.resource.id : uuid();
|
||||
return entities.components.map((component) => transformComponent(resourceId, job.project_id, component));
|
||||
return entities.components.map((component) =>
|
||||
transformComponent(
|
||||
{
|
||||
resourceId,
|
||||
projectId: job.project_id,
|
||||
source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA,
|
||||
},
|
||||
component
|
||||
)
|
||||
);
|
||||
};
|
||||
|
||||
export const getTransformedIssueTypes = (job: TImportJob<JiraConfig>, entities: JiraEntity): Partial<ExIssueType>[] => {
|
||||
const resourceId = job.config.resource ? job.config.resource.id : uuid();
|
||||
return entities.issueTypes.map((issueType) => transformIssueType(resourceId, job.project_id, issueType));
|
||||
return entities.issueTypes.map((issueType) =>
|
||||
transformIssueType(
|
||||
{
|
||||
resourceId,
|
||||
projectId: job.project_id,
|
||||
source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA,
|
||||
},
|
||||
issueType
|
||||
)
|
||||
);
|
||||
};
|
||||
|
||||
export const getTransformedIssueFields = (
|
||||
@@ -98,7 +144,16 @@ export const getTransformedIssueFields = (
|
||||
): Partial<ExIssueProperty>[] => {
|
||||
const resourceId = job.config.resource ? job.config.resource.id : uuid();
|
||||
return entities.issueFields
|
||||
.map((issueField) => transformIssueFields(resourceId, job.project_id, issueField))
|
||||
.map((issueField) =>
|
||||
transformIssueFields(
|
||||
{
|
||||
resourceId,
|
||||
projectId: job.project_id,
|
||||
source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA,
|
||||
},
|
||||
issueField
|
||||
)
|
||||
)
|
||||
.filter((field) => field && field.property_type) as Partial<ExIssueProperty>[];
|
||||
};
|
||||
|
||||
@@ -114,7 +169,16 @@ export const getTransformedIssueFieldOptions = (
|
||||
OPTION_CUSTOM_FIELD_TYPES.includes(issueField.schema?.custom as JiraCustomFieldKeys)
|
||||
)
|
||||
.flatMap((issueField) =>
|
||||
issueField?.options?.map((fieldOption) => transformIssueFieldOptions(resourceId, job.project_id, fieldOption))
|
||||
issueField?.options?.map((fieldOption) =>
|
||||
transformIssueFieldOptions(
|
||||
{
|
||||
resourceId,
|
||||
projectId: job.project_id,
|
||||
source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA,
|
||||
},
|
||||
fieldOption
|
||||
)
|
||||
)
|
||||
)
|
||||
.filter(Boolean) as Partial<ExIssuePropertyOption>[];
|
||||
};
|
||||
@@ -146,8 +210,7 @@ export const getTransformedIssuePropertyValues = (
|
||||
entities.issues.forEach((issue: IJiraIssue) => {
|
||||
if (issue.id && issue.fields) {
|
||||
transformedIssuePropertyValues[`${projectId}_${resourceId}_${issue.id}`] = transformIssuePropertyValues(
|
||||
resourceId,
|
||||
projectId,
|
||||
{ resourceId, projectId, source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA },
|
||||
issue,
|
||||
planeIssuePropertiesMap,
|
||||
jiraCustomFieldMap
|
||||
@@ -194,8 +257,7 @@ export const getTransformedIssuePropertyValuesV2 = (
|
||||
|
||||
// Transform custom field property values
|
||||
const customPropertyValues = transformIssuePropertyValues(
|
||||
resourceId,
|
||||
projectId,
|
||||
{ resourceId, projectId, source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA },
|
||||
issue,
|
||||
planeIssuePropertiesMap,
|
||||
jiraCustomFieldMap
|
||||
@@ -203,7 +265,11 @@ export const getTransformedIssuePropertyValuesV2 = (
|
||||
|
||||
// Transform default property values (fix versions, affected versions, reporter)
|
||||
const defaultPropertyValues = issueTypeId
|
||||
? transformDefaultPropertyValues(resourceId, projectId, issue, issueTypeId)
|
||||
? transformDefaultPropertyValues(
|
||||
{ resourceId, projectId, source: job.source as E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA },
|
||||
issue,
|
||||
issueTypeId
|
||||
)
|
||||
: {};
|
||||
|
||||
// Merge both custom and default property values
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { E_IMPORTER_KEYS } from "@plane/etl/core";
|
||||
import type { E_IMPORTER_KEYS } from "@plane/etl/core";
|
||||
import type { ExIssueProperty } from "@plane/sdk";
|
||||
|
||||
export enum E_DEFAULT_PROPERTY_TYPES {
|
||||
@@ -23,12 +23,13 @@ export const getSupportedDefaultProperties = (
|
||||
resourceId: string,
|
||||
projectId: string,
|
||||
issueTypeId: string,
|
||||
issueTypeExternalId: string
|
||||
issueTypeExternalId: string,
|
||||
source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA
|
||||
): Partial<ExIssueProperty>[] => [
|
||||
{
|
||||
type_id: issueTypeExternalId,
|
||||
external_id: `${resourceId}-${projectId}-${issueTypeId}-${E_DEFAULT_PROPERTY_TYPES.FIX_VERSION}`,
|
||||
external_source: E_IMPORTER_KEYS.JIRA_SERVER,
|
||||
external_id: getDefaultPropertyExternalId(resourceId, projectId, issueTypeId, E_DEFAULT_PROPERTY_TYPES.FIX_VERSION),
|
||||
external_source: source,
|
||||
display_name: "Fix Version",
|
||||
property_type: "TEXT",
|
||||
settings: {
|
||||
@@ -38,8 +39,13 @@ export const getSupportedDefaultProperties = (
|
||||
},
|
||||
{
|
||||
type_id: issueTypeExternalId,
|
||||
external_id: `${resourceId}-${projectId}-${issueTypeId}-${E_DEFAULT_PROPERTY_TYPES.AFFECTED_VERSION}`,
|
||||
external_source: E_IMPORTER_KEYS.JIRA_SERVER,
|
||||
external_id: getDefaultPropertyExternalId(
|
||||
resourceId,
|
||||
projectId,
|
||||
issueTypeId,
|
||||
E_DEFAULT_PROPERTY_TYPES.AFFECTED_VERSION
|
||||
),
|
||||
external_source: source,
|
||||
display_name: "Affected Version",
|
||||
property_type: "TEXT",
|
||||
settings: {
|
||||
@@ -49,8 +55,8 @@ export const getSupportedDefaultProperties = (
|
||||
},
|
||||
{
|
||||
type_id: issueTypeExternalId,
|
||||
external_id: `${resourceId}-${projectId}-${issueTypeId}-${E_DEFAULT_PROPERTY_TYPES.REPORTER}`,
|
||||
external_source: E_IMPORTER_KEYS.JIRA_SERVER,
|
||||
external_id: getDefaultPropertyExternalId(resourceId, projectId, issueTypeId, E_DEFAULT_PROPERTY_TYPES.REPORTER),
|
||||
external_source: source,
|
||||
display_name: "Reporter",
|
||||
property_type: "RELATION",
|
||||
relation_type: "USER",
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
import type { MQ, Store } from "@/worker/base";
|
||||
import { JiraImportOrchestrator } from "./orchestrator";
|
||||
import JIRA_CLOUD_STEPS from "./steps/cloud";
|
||||
import JIRA_SERVER_STEPS from "./steps/server";
|
||||
|
||||
export const getJiraCloudImportOrchestrator = (mq: MQ, store: Store) =>
|
||||
new JiraImportOrchestrator(mq, store, JIRA_CLOUD_STEPS);
|
||||
|
||||
export const getJiraServerImportOrchestrator = (mq: MQ, store: Store) =>
|
||||
new JiraImportOrchestrator(mq, store, JIRA_SERVER_STEPS);
|
||||
|
||||
@@ -4,7 +4,6 @@ import { logger } from "@plane/logger";
|
||||
import type { TImportJob } from "@plane/types";
|
||||
import { createJiraClient } from "@/apps/jira-server-importer/helpers/migration-helpers";
|
||||
import { flushJob } from "@/apps/jira-server-importer/v2/helpers/cache";
|
||||
import JIRA_SERVER_STEPS from "@/apps/jira-server-importer/v2/migrator/steps";
|
||||
import type {
|
||||
IStep,
|
||||
IStorageService,
|
||||
@@ -15,7 +14,7 @@ import type {
|
||||
} from "@/apps/jira-server-importer/v2/types";
|
||||
import { getJobCredentials, getJobData } from "@/helpers/job";
|
||||
import { getPlaneAPIClient } from "@/helpers/plane-api-client";
|
||||
import { getAPIClient } from "@/services/client";
|
||||
import { getAPIClient, getAPIClientInternal } from "@/services/client";
|
||||
import type { TaskHandler, TaskHeaders } from "@/types";
|
||||
import type { MQ, Store } from "@/worker/base";
|
||||
import { redisStorageService } from "../services/storage.service";
|
||||
@@ -37,18 +36,16 @@ type OrchestratorTaskData = {
|
||||
stepContext?: TStepExecutionContext;
|
||||
};
|
||||
|
||||
const client = getAPIClient();
|
||||
const client = getAPIClientInternal();
|
||||
|
||||
export class JiraServerImportOrchestrator implements TaskHandler {
|
||||
private steps: IStep[];
|
||||
export class JiraImportOrchestrator implements TaskHandler {
|
||||
private storage: IStorageService = redisStorageService;
|
||||
|
||||
constructor(
|
||||
private readonly mq: MQ,
|
||||
private readonly store: Store
|
||||
) {
|
||||
this.steps = JIRA_SERVER_STEPS;
|
||||
}
|
||||
private readonly store: Store,
|
||||
private readonly steps: IStep[]
|
||||
) { }
|
||||
|
||||
/**
|
||||
* Main task handler - called by worker
|
||||
@@ -143,8 +140,74 @@ export class JiraServerImportOrchestrator implements TaskHandler {
|
||||
// Load execution context
|
||||
const { state, step, jobContext } = await this.loadStepExecution(jobId, data);
|
||||
|
||||
// Execute step and handle result
|
||||
await this.executeAndHandleResult(headers, state, step, jobContext);
|
||||
try {
|
||||
// Execute step and handle result
|
||||
await this.executeAndHandleResult(headers, state, step, jobContext);
|
||||
} catch (error) {
|
||||
// Log error and move to next step instead of failing the job
|
||||
await this.handleStepError(headers, state, step, error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle step error: Log error and move to next step
|
||||
*/
|
||||
private async handleStepError(
|
||||
headers: TaskHeaders,
|
||||
state: TOrchestratorState,
|
||||
step: IStep,
|
||||
error: unknown
|
||||
): Promise<void> {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
|
||||
logger.error(`[ORCHESTRATOR] Step failed, continuing to next step`, {
|
||||
jobId: state.jobId,
|
||||
step: step.name,
|
||||
stepIndex: state.currentStepIndex + 1,
|
||||
totalSteps: state.totalSteps,
|
||||
error: errorMessage,
|
||||
});
|
||||
|
||||
// Track failed step
|
||||
if (!state.failedSteps) {
|
||||
state.failedSteps = [];
|
||||
}
|
||||
state.failedSteps.push({
|
||||
name: step.name,
|
||||
error: errorMessage,
|
||||
failedAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
// Move to next step
|
||||
state.currentStepIndex++;
|
||||
state.stepContext = undefined;
|
||||
await this.saveState(state);
|
||||
|
||||
if (state.currentStepIndex >= this.steps.length) {
|
||||
// All steps processed (some may have failed)
|
||||
await this.mq.sendMessage(
|
||||
{},
|
||||
{
|
||||
headers: {
|
||||
...headers,
|
||||
type: EOrchestratorTaskType.COMPLETE,
|
||||
},
|
||||
}
|
||||
);
|
||||
} else {
|
||||
// Move to next step
|
||||
state.currentStepName = this.steps[state.currentStepIndex].name;
|
||||
await this.saveState(state);
|
||||
await this.mq.sendMessage(
|
||||
{ state },
|
||||
{
|
||||
headers: {
|
||||
...headers,
|
||||
type: EOrchestratorTaskType.EXECUTE_STEP,
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -178,6 +241,18 @@ export class JiraServerImportOrchestrator implements TaskHandler {
|
||||
return { state, step, jobContext };
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize job context
|
||||
*/
|
||||
private async initializeJobContext(jobId: string): Promise<TJobContext> {
|
||||
const job = await getJobData(jobId);
|
||||
const credentials = await getJobCredentials(job as TImportJob<JiraConfig>);
|
||||
const planeClient = await getPlaneAPIClient(credentials, E_IMPORTER_KEYS.IMPORTER);
|
||||
const sourceClient = createJiraClient(job as TImportJob<JiraConfig>, credentials);
|
||||
|
||||
return { job, credentials, planeClient, sourceClient };
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute step and handle the result
|
||||
*/
|
||||
@@ -298,7 +373,7 @@ export class JiraServerImportOrchestrator implements TaskHandler {
|
||||
// Delete orchestrator state
|
||||
await this.store.del(stateKey);
|
||||
|
||||
logger.info(`[ORCHESTRATOR] Job completed successfully`, { jobId });
|
||||
logger.info(`[ORCHESTRATOR] Job completed`, { jobId });
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -362,18 +437,6 @@ export class JiraServerImportOrchestrator implements TaskHandler {
|
||||
return dependencyData;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize job context
|
||||
*/
|
||||
private async initializeJobContext(jobId: string): Promise<TJobContext> {
|
||||
const job = await getJobData(jobId);
|
||||
const credentials = await getJobCredentials(job as TImportJob<JiraConfig>);
|
||||
const planeClient = await getPlaneAPIClient(credentials, E_IMPORTER_KEYS.IMPORTER);
|
||||
const sourceClient = createJiraClient(job as TImportJob<JiraConfig>, credentials);
|
||||
|
||||
return { job, credentials, planeClient, sourceClient };
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if job is cancelled
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
export * from "./issue-properties.step";
|
||||
export * from "./users.step";
|
||||
export * from "./issue-types.step";
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
import { pullIssueFields } from "@plane/etl/jira";
|
||||
import type { JiraIssueField } from "@plane/etl/jira-server";
|
||||
import { createJiraClient } from "@/apps/jira-importer/helpers/migration-helpers";
|
||||
import { getJobCredentials } from "@/helpers/job";
|
||||
import type { TJobContext, TIssueTypesData } from "../../../../types";
|
||||
import { JiraIssuePropertiesStep } from "../../shared/entities";
|
||||
|
||||
export class JiraCloudIssuePropertiesStep extends JiraIssuePropertiesStep {
|
||||
protected async pull(
|
||||
jobCtx: TJobContext,
|
||||
projectId: string,
|
||||
issueTypesData: TIssueTypesData
|
||||
): Promise<JiraIssueField[]> {
|
||||
const { job } = jobCtx;
|
||||
const credentials = await getJobCredentials(job);
|
||||
const client = createJiraClient(job, credentials);
|
||||
const issueTypes = issueTypesData
|
||||
.map((type) => {
|
||||
const id = type.external_id.split("_").pop();
|
||||
if (typeof id !== "string" || id.length === 0) {
|
||||
return null;
|
||||
}
|
||||
return { id, name: type.name };
|
||||
})
|
||||
.filter((item): item is { id: string; name: string } => item !== null);
|
||||
|
||||
return pullIssueFields(client, issueTypes, projectId);
|
||||
}
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
import { pullIssueTypes } from "@plane/etl/jira";
|
||||
import { logger } from "@plane/logger";
|
||||
import { createJiraClient } from "@/apps/jira-importer/helpers/migration-helpers";
|
||||
import type { TJobContext } from "@/apps/jira-server-importer/v2/types";
|
||||
import { getJobCredentials } from "@/helpers/job";
|
||||
import { JiraIssueTypesStep } from "../../shared";
|
||||
|
||||
export class JiraCloudIssueTypesStep extends JiraIssueTypesStep {
|
||||
protected async pull(jobContext: TJobContext, projectId: string, startAt: number) {
|
||||
const { job } = jobContext;
|
||||
const credentials = await getJobCredentials(job);
|
||||
const client = createJiraClient(job, credentials);
|
||||
const result = await pullIssueTypes(client, projectId);
|
||||
|
||||
logger.info(`[${jobContext.job.id}] [${this.name}] Pulled issue types from Jira Cloud`, {
|
||||
jobId: jobContext.job.id,
|
||||
count: result.length,
|
||||
hasMore: false,
|
||||
startAt,
|
||||
});
|
||||
|
||||
return {
|
||||
items: result,
|
||||
hasMore: false,
|
||||
startAt,
|
||||
maxResults: result.length,
|
||||
};
|
||||
}
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
import { pullUsers } from "@plane/etl/jira";
|
||||
import type { JiraV2Service, PaginatedResult, ImportedJiraUser } from "@plane/etl/jira-server";
|
||||
import { logger } from "@plane/logger";
|
||||
import type { TJobContext } from "../../../../types";
|
||||
import { JiraUsersStep } from "../../shared/entities";
|
||||
|
||||
export class JiraCloudUserStep extends JiraUsersStep {
|
||||
/*
|
||||
* For jira cloud, we don't have access to user emails, hence
|
||||
* we take a csv in order to import the users, and we use a different
|
||||
* pull function, hence we need to overwrite this.
|
||||
*/
|
||||
protected async pull(
|
||||
jobContext: TJobContext,
|
||||
_client: JiraV2Service,
|
||||
_startAt: number,
|
||||
_jobId: string
|
||||
): Promise<PaginatedResult<ImportedJiraUser>> {
|
||||
const { job } = jobContext;
|
||||
if (!job.config.users) {
|
||||
logger.error(`[${job.id}] [${this.name}] No users found in config`, { jobId: job.id });
|
||||
return {
|
||||
items: [],
|
||||
hasMore: false,
|
||||
total: 0,
|
||||
startAt: 0,
|
||||
maxResults: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const users = pullUsers(job.config.users);
|
||||
const items = users.map((user) => ({
|
||||
avatarUrl: "",
|
||||
user_id: user.user_id,
|
||||
email: user.email,
|
||||
full_name: "",
|
||||
user_name: user.user_name,
|
||||
added_to_org: user.added_to_org,
|
||||
org_role: user.org_role,
|
||||
user_status: "Active",
|
||||
}));
|
||||
|
||||
return {
|
||||
items,
|
||||
hasMore: false,
|
||||
total: items.length,
|
||||
startAt: 0,
|
||||
maxResults: items.length,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import { E_IMPORTER_KEYS } from "@plane/etl/core";
|
||||
import {
|
||||
// Pre-run steps
|
||||
PlaneProjectConfigurationStep,
|
||||
// Entity steps
|
||||
JiraBoardsStep,
|
||||
JiraCyclesStep,
|
||||
JiraDefaultPropertiesStep,
|
||||
JiraIssuePropertyOptionsStep,
|
||||
JiraModulesStep,
|
||||
WaitForCeleryStep,
|
||||
// Association steps
|
||||
JiraRelationsStep,
|
||||
} from "../shared";
|
||||
import { JiraCloudUserStep, JiraCloudIssuePropertiesStep, JiraCloudIssueTypesStep } from "./entities";
|
||||
import { JiraCloudIssuesStep } from "./issues";
|
||||
|
||||
const JIRA_CLOUD_STEPS = [
|
||||
// Pre-run steps
|
||||
new PlaneProjectConfigurationStep(),
|
||||
// Entity steps
|
||||
new JiraCloudUserStep(),
|
||||
new JiraModulesStep(E_IMPORTER_KEYS.JIRA),
|
||||
new JiraBoardsStep(),
|
||||
new JiraCyclesStep(E_IMPORTER_KEYS.JIRA),
|
||||
new JiraCloudIssueTypesStep(E_IMPORTER_KEYS.JIRA),
|
||||
new JiraCloudIssuePropertiesStep(E_IMPORTER_KEYS.JIRA),
|
||||
new JiraDefaultPropertiesStep(E_IMPORTER_KEYS.JIRA),
|
||||
new JiraIssuePropertyOptionsStep(E_IMPORTER_KEYS.JIRA),
|
||||
// Issue steps
|
||||
new JiraCloudIssuesStep(E_IMPORTER_KEYS.JIRA),
|
||||
new WaitForCeleryStep(),
|
||||
// Association steps
|
||||
new JiraRelationsStep(E_IMPORTER_KEYS.JIRA),
|
||||
];
|
||||
|
||||
export default JIRA_CLOUD_STEPS;
|
||||
@@ -0,0 +1 @@
|
||||
export * from "./issues.step";
|
||||
+155
@@ -0,0 +1,155 @@
|
||||
import { pullIssuesV2 } from "@plane/etl/jira";
|
||||
import { logger } from "@plane/logger";
|
||||
import type { TImportJob } from "@plane/types";
|
||||
import { createJiraClient } from "@/apps/jira-importer/helpers/migration-helpers";
|
||||
import type { TStepExecutionContext } from "@/apps/jira-server-importer/v2/types";
|
||||
import { getJobCredentials } from "@/helpers/job";
|
||||
import { withCache } from "../../../../helpers/cache";
|
||||
import { buildExternalId } from "../../../../helpers/job";
|
||||
import type { TJobContext } from "../../../../types";
|
||||
import { JiraIssuesStep } from "../../shared/issues";
|
||||
|
||||
/**
|
||||
* Jira Cloud Issues Step
|
||||
*
|
||||
* Extends JiraServerIssuesStep to use nextPageToken pagination
|
||||
* instead of startAt pagination for Jira Cloud API
|
||||
*/
|
||||
export class JiraCloudIssuesStep extends JiraIssuesStep {
|
||||
/**
|
||||
* Override pagination context to use nextPageToken for Jira Cloud
|
||||
*/
|
||||
protected getPaginationContext(previousContext?: TStepExecutionContext) {
|
||||
return {
|
||||
startAt: 0, // Not used for Cloud pagination
|
||||
totalProcessed: previousContext?.pageCtx.totalProcessed ?? 0,
|
||||
nextPageToken: previousContext?.state?.nextPageToken as string | undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull paginated issues from Jira Cloud using nextPageToken
|
||||
*/
|
||||
protected async pull(props: {
|
||||
jobContext: TJobContext;
|
||||
projectKey: string;
|
||||
paginationCtx: { startAt: number; totalProcessed: number; nextPageToken?: string };
|
||||
}) {
|
||||
const { job } = props.jobContext;
|
||||
const credentials = await getJobCredentials(job);
|
||||
const client = createJiraClient(job, credentials);
|
||||
|
||||
// Get total number of issues first
|
||||
const total = await withCache(
|
||||
`jira-cloud-issues-total-${props.projectKey}`,
|
||||
job,
|
||||
async () => await client.getNumberOfIssues(props.projectKey)
|
||||
);
|
||||
|
||||
const result = await pullIssuesV2(
|
||||
{
|
||||
client,
|
||||
nextPageToken: props.paginationCtx.nextPageToken,
|
||||
},
|
||||
props.projectKey,
|
||||
total
|
||||
);
|
||||
|
||||
logger.info(`[${job.id}] [${this.name}] Pulled issues from Jira Cloud`, {
|
||||
jobId: job.id,
|
||||
count: result.items.length,
|
||||
hasMore: result.hasMore,
|
||||
nextPageToken: result.nextPageToken ? "present" : "none",
|
||||
});
|
||||
|
||||
return {
|
||||
items: result.items as any[],
|
||||
hasMore: result.hasMore,
|
||||
total: total ?? 0,
|
||||
nextPageToken: result.nextPageToken,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build next context with nextPageToken for Jira Cloud pagination
|
||||
*/
|
||||
protected buildNextContext(
|
||||
issuesResult: { items: any[]; hasMore: boolean; total: number; nextPageToken?: string },
|
||||
paginationCtx: { startAt: number; totalProcessed: number; nextPageToken?: string },
|
||||
pulled: number,
|
||||
pushed: number
|
||||
) {
|
||||
const newTotalProcessed = paginationCtx.totalProcessed + pulled;
|
||||
|
||||
if (issuesResult.hasMore && issuesResult.nextPageToken) {
|
||||
return {
|
||||
pageCtx: {
|
||||
startAt: 0, // Not used for Cloud
|
||||
hasMore: true,
|
||||
totalProcessed: newTotalProcessed,
|
||||
},
|
||||
results: {
|
||||
pulled,
|
||||
pushed,
|
||||
errors: [],
|
||||
},
|
||||
state: {
|
||||
nextPageToken: issuesResult.nextPageToken,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
pageCtx: {
|
||||
startAt: 0,
|
||||
hasMore: false,
|
||||
totalProcessed: newTotalProcessed,
|
||||
},
|
||||
results: {
|
||||
pulled,
|
||||
pushed,
|
||||
errors: [],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Override shouldReturnEmpty to check first page correctly for Cloud
|
||||
*/
|
||||
protected shouldReturnEmpty(
|
||||
issuesResult: { items: any[]; hasMore: boolean; total: number },
|
||||
paginationCtx: { startAt: number; totalProcessed: number; nextPageToken?: string }
|
||||
): boolean {
|
||||
// First page in Cloud is when there's no nextPageToken in context
|
||||
return issuesResult.items.length === 0 && !paginationCtx.nextPageToken;
|
||||
}
|
||||
|
||||
protected extractSprints(job: TImportJob<any>, issue: any): string[] {
|
||||
/* We go through the issue fields and find the custom field that matches
|
||||
* the definition of a sprint, which should be an array of objects, and
|
||||
* each object should contain boardId, name, startDate, endDate, and state
|
||||
*/
|
||||
|
||||
const isSprintField = (value: any): boolean =>
|
||||
Array.isArray(value) &&
|
||||
value.length > 0 &&
|
||||
value[0].boardId &&
|
||||
value[0].name &&
|
||||
value[0].startDate &&
|
||||
value[0].endDate &&
|
||||
value[0].state;
|
||||
|
||||
const config = job.config;
|
||||
const resourceId = config.resource?.id || "";
|
||||
const projectId = job.project_id || "";
|
||||
|
||||
for (const [fieldKey, fieldValue] of Object.entries(issue.fields)) {
|
||||
if (!fieldKey.startsWith("customfield_")) continue;
|
||||
if (isSprintField(fieldValue)) {
|
||||
return (fieldValue as any[]).map((s: any) => buildExternalId(projectId, resourceId, s.id.toString()));
|
||||
}
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
import {
|
||||
JiraServerUsersStep,
|
||||
JiraServerModulesStep,
|
||||
JiraServerBoardsStep,
|
||||
JiraServerCyclesStep,
|
||||
JiraServerIssueTypesStep,
|
||||
JiraServerIssuePropertiesStep,
|
||||
JiraServerDefaultPropertiesStep,
|
||||
JiraServerIssuePropertyOptionsStep,
|
||||
} from "./entities";
|
||||
import { JiraServerIssuesStep, WaitForCeleryStep } from "./issues";
|
||||
import { PlaneProjectConfigurationStep } from "./pre-run";
|
||||
import { JiraServerRelationsStep } from "./relations";
|
||||
|
||||
const JIRA_SERVER_STEPS = [
|
||||
// Pre-run steps
|
||||
new PlaneProjectConfigurationStep(),
|
||||
// Entity steps
|
||||
new JiraServerUsersStep(),
|
||||
new JiraServerModulesStep(),
|
||||
new JiraServerBoardsStep(),
|
||||
new JiraServerCyclesStep(),
|
||||
new JiraServerIssueTypesStep(),
|
||||
new JiraServerIssuePropertiesStep(),
|
||||
new JiraServerDefaultPropertiesStep(),
|
||||
new JiraServerIssuePropertyOptionsStep(),
|
||||
// Issue steps
|
||||
new JiraServerIssuesStep(),
|
||||
new WaitForCeleryStep(),
|
||||
// Association steps
|
||||
new JiraServerRelationsStep(),
|
||||
];
|
||||
|
||||
export default JIRA_SERVER_STEPS;
|
||||
@@ -0,0 +1,40 @@
|
||||
import { E_IMPORTER_KEYS } from "@plane/etl/core";
|
||||
import {
|
||||
// Pre-run steps
|
||||
PlaneProjectConfigurationStep,
|
||||
// Entity steps
|
||||
JiraBoardsStep,
|
||||
JiraCyclesStep,
|
||||
JiraDefaultPropertiesStep,
|
||||
JiraIssuePropertiesStep,
|
||||
JiraIssuePropertyOptionsStep,
|
||||
JiraIssueTypesStep,
|
||||
JiraModulesStep,
|
||||
JiraUsersStep,
|
||||
// Issue steps
|
||||
JiraIssuesStep,
|
||||
WaitForCeleryStep,
|
||||
// Association steps
|
||||
JiraRelationsStep,
|
||||
} from "../shared";
|
||||
|
||||
const JIRA_SERVER_STEPS = [
|
||||
// Pre-run steps
|
||||
new PlaneProjectConfigurationStep(),
|
||||
// Entity steps
|
||||
new JiraUsersStep(),
|
||||
new JiraModulesStep(E_IMPORTER_KEYS.JIRA_SERVER),
|
||||
new JiraBoardsStep(),
|
||||
new JiraCyclesStep(E_IMPORTER_KEYS.JIRA_SERVER),
|
||||
new JiraIssueTypesStep(E_IMPORTER_KEYS.JIRA_SERVER),
|
||||
new JiraIssuePropertiesStep(E_IMPORTER_KEYS.JIRA_SERVER),
|
||||
new JiraDefaultPropertiesStep(E_IMPORTER_KEYS.JIRA_SERVER),
|
||||
new JiraIssuePropertyOptionsStep(E_IMPORTER_KEYS.JIRA_SERVER),
|
||||
// Issue steps
|
||||
new JiraIssuesStep(E_IMPORTER_KEYS.JIRA_SERVER),
|
||||
new WaitForCeleryStep(),
|
||||
// Association steps
|
||||
new JiraRelationsStep(E_IMPORTER_KEYS.JIRA_SERVER),
|
||||
];
|
||||
|
||||
export default JIRA_SERVER_STEPS;
|
||||
+5
-4
@@ -15,7 +15,7 @@ import type {
|
||||
TStepExecutionContext,
|
||||
TStepExecutionInput,
|
||||
} from "@/apps/jira-server-importer/v2/types";
|
||||
import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types";
|
||||
import { EJiraStep } from "@/apps/jira-server-importer/v2/types";
|
||||
|
||||
/**
|
||||
* Handles the extraction of boards from Jira Server.
|
||||
@@ -28,8 +28,8 @@ import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types";
|
||||
* for mapping issues to modules/views. If future requirements need board entities
|
||||
* in Plane, add transform/push logic here.
|
||||
*/
|
||||
export class JiraServerBoardsStep implements IStep {
|
||||
name = EJiraServerStep.BOARDS;
|
||||
export class JiraBoardsStep implements IStep {
|
||||
name = EJiraStep.BOARDS;
|
||||
dependencies = [];
|
||||
|
||||
private readonly PAGE_SIZE = 100;
|
||||
@@ -52,12 +52,13 @@ export class JiraServerBoardsStep implements IStep {
|
||||
logger.info(`[${job.id}] [${this.name}] Pulling boards`, { startAt, totalProcessed });
|
||||
|
||||
const result = await this.pullBoards(sourceClient, projectId, startAt, job.id);
|
||||
const scrumBoards = result.items.filter((board) => board.type === "scrum");
|
||||
|
||||
if (this.shouldReturnEmpty(result, startAt)) {
|
||||
return createEmptyContext();
|
||||
}
|
||||
|
||||
await this.storeBoards(storage, job.id, result.items);
|
||||
await this.storeBoards(storage, job.id, scrumBoards);
|
||||
|
||||
const newTotalProcessed = totalProcessed + result.items.length;
|
||||
|
||||
+18
-6
@@ -1,5 +1,6 @@
|
||||
import type { Sprint } from "jira.js/out/agile/models";
|
||||
import { v4 as uuid } from "uuid";
|
||||
import type { E_IMPORTER_KEYS } from "@plane/etl/core";
|
||||
import type { JiraConfig, JiraSprint } from "@plane/etl/jira-server";
|
||||
import { pullSprintsForBoardV2, transformSprint } from "@plane/etl/jira-server";
|
||||
import { logger } from "@plane/logger";
|
||||
@@ -14,7 +15,7 @@ import type {
|
||||
TStepExecutionContext,
|
||||
TStepExecutionInput,
|
||||
} from "@/apps/jira-server-importer/v2/types";
|
||||
import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types";
|
||||
import { EJiraStep } from "@/apps/jira-server-importer/v2/types";
|
||||
import { createAllCyclesV2 } from "@/etl/migrator/cycles.migrator";
|
||||
|
||||
/**
|
||||
@@ -23,12 +24,14 @@ import { createAllCyclesV2 } from "@/etl/migrator/cycles.migrator";
|
||||
*
|
||||
* Dependencies: boards - board IDs to iterate through for sprint collection
|
||||
*/
|
||||
export class JiraServerCyclesStep implements IStep {
|
||||
name = EJiraServerStep.CYCLES;
|
||||
dependencies = [EJiraServerStep.BOARDS];
|
||||
export class JiraCyclesStep implements IStep {
|
||||
name = EJiraStep.CYCLES;
|
||||
dependencies = [EJiraStep.BOARDS];
|
||||
|
||||
private readonly PAGE_SIZE = 100;
|
||||
|
||||
constructor(private readonly source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA) { }
|
||||
|
||||
/**
|
||||
* Executes the sprint extraction process:
|
||||
* 1. Extracts current board ID from dependency data
|
||||
@@ -64,7 +67,7 @@ export class JiraServerCyclesStep implements IStep {
|
||||
} catch (error) {
|
||||
logger.error(`[${jobContext.job.id}] [${this.name}] Step failed`, {
|
||||
jobId: jobContext.job.id,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
error: error
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
@@ -152,7 +155,16 @@ export class JiraServerCyclesStep implements IStep {
|
||||
issues: [], // Issues are handled in separate step
|
||||
}));
|
||||
|
||||
return formattedSprints.map((sprint) => transformSprint(resourceId, job.project_id, sprint));
|
||||
return formattedSprints.map((sprint) =>
|
||||
transformSprint(
|
||||
{
|
||||
resourceId,
|
||||
projectId: job.project_id,
|
||||
source: this.source,
|
||||
},
|
||||
sprint
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
+8
-7
@@ -2,8 +2,8 @@ import { createSuccessContext } from "@/apps/jira-server-importer/v2/helpers/ctx
|
||||
import { extractJobData } from "@/apps/jira-server-importer/v2/helpers/job";
|
||||
import { getSupportedDefaultProperties } from "@/apps/jira-server-importer/v2/helpers/properties";
|
||||
import type { TIssueTypesData, TStepExecutionContext, TStepExecutionInput } from "@/apps/jira-server-importer/v2/types";
|
||||
import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types";
|
||||
import { JiraServerIssuePropertiesStep } from "./issue-properties.step";
|
||||
import { EJiraStep } from "@/apps/jira-server-importer/v2/types";
|
||||
import { JiraIssuePropertiesStep } from "./issue-properties.step";
|
||||
|
||||
/*
|
||||
* @overview
|
||||
@@ -14,9 +14,9 @@ import { JiraServerIssuePropertiesStep } from "./issue-properties.step";
|
||||
* to all the available issue types, which will allow us to map those properties
|
||||
* to the corresponding plane properties.
|
||||
*/
|
||||
export class JiraServerDefaultPropertiesStep extends JiraServerIssuePropertiesStep {
|
||||
name: EJiraServerStep = EJiraServerStep.DEFAULT_PROPERTIES;
|
||||
dependencies: EJiraServerStep[] = [EJiraServerStep.ISSUE_TYPES];
|
||||
export class JiraDefaultPropertiesStep extends JiraIssuePropertiesStep {
|
||||
name: EJiraStep = EJiraStep.DEFAULT_PROPERTIES;
|
||||
dependencies: EJiraStep[] = [EJiraStep.ISSUE_TYPES];
|
||||
|
||||
async execute(input: TStepExecutionInput): Promise<TStepExecutionContext> {
|
||||
// Get the issue types from the dependency data and for each issue type, populate
|
||||
@@ -37,13 +37,14 @@ export class JiraServerDefaultPropertiesStep extends JiraServerIssuePropertiesSt
|
||||
resourceId,
|
||||
projectId,
|
||||
issueType.id,
|
||||
issueType.external_id
|
||||
issueType.external_id,
|
||||
this.source
|
||||
);
|
||||
defaultPropertiesToCreate.push(...defaultProperties);
|
||||
}
|
||||
|
||||
const pushed = await this.push(jobContext, defaultPropertiesToCreate, issueTypes);
|
||||
await this.storePropertiesData(EJiraServerStep.ISSUE_PROPERTIES, job, pushed, storage);
|
||||
await this.storePropertiesData(EJiraStep.ISSUE_PROPERTIES, job, pushed, storage);
|
||||
|
||||
return createSuccessContext({
|
||||
pulled: defaultPropertiesToCreate.length,
|
||||
+10
-7
@@ -1,4 +1,5 @@
|
||||
import { v4 as uuid } from "uuid";
|
||||
import type { E_IMPORTER_KEYS } from "@plane/etl/core";
|
||||
import type { JiraConfig, JiraIssueField } from "@plane/etl/jira-server";
|
||||
import { pullIssueFieldsV2, transformIssueFields } from "@plane/etl/jira-server";
|
||||
import { logger } from "@plane/logger";
|
||||
@@ -15,7 +16,7 @@ import type {
|
||||
TStepExecutionContext,
|
||||
TStepExecutionInput,
|
||||
} from "@/apps/jira-server-importer/v2/types";
|
||||
import { E_ADDITIONAL_STORAGE_KEYS, EJiraServerStep } from "@/apps/jira-server-importer/v2/types";
|
||||
import { E_ADDITIONAL_STORAGE_KEYS, EJiraStep } from "@/apps/jira-server-importer/v2/types";
|
||||
import { createOrUpdateIssueProperties } from "@/etl/migrator/issue-types/issue-property.migrator";
|
||||
|
||||
/**
|
||||
@@ -24,9 +25,11 @@ import { createOrUpdateIssueProperties } from "@/etl/migrator/issue-types/issue-
|
||||
*
|
||||
* Note: No pagination needed - fields are a small dataset
|
||||
*/
|
||||
export class JiraServerIssuePropertiesStep implements IStep {
|
||||
name = EJiraServerStep.ISSUE_PROPERTIES;
|
||||
dependencies = [EJiraServerStep.ISSUE_TYPES];
|
||||
export class JiraIssuePropertiesStep implements IStep {
|
||||
name = EJiraStep.ISSUE_PROPERTIES;
|
||||
dependencies = [EJiraStep.ISSUE_TYPES];
|
||||
|
||||
constructor(protected readonly source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA) {}
|
||||
|
||||
async execute(input: TStepExecutionInput): Promise<TStepExecutionContext> {
|
||||
const { jobContext, storage, dependencyData } = input;
|
||||
@@ -91,7 +94,7 @@ export class JiraServerIssuePropertiesStep implements IStep {
|
||||
/**
|
||||
* Pull custom fields from Jira Server
|
||||
*/
|
||||
private async pull(
|
||||
protected async pull(
|
||||
jobCtx: TJobContext,
|
||||
projectId: string,
|
||||
issueTypesData: TIssueTypesData
|
||||
@@ -119,7 +122,7 @@ export class JiraServerIssuePropertiesStep implements IStep {
|
||||
const resourceId = job.config.resource ? job.config.resource.id : uuid();
|
||||
|
||||
return jiraFields
|
||||
.map((field) => transformIssueFields(resourceId, job.project_id, field))
|
||||
.map((field) => transformIssueFields({ resourceId, projectId: job.project_id, source: this.source }, field))
|
||||
.filter((field) => field && field.property_type) as Partial<ExIssueProperty>[];
|
||||
}
|
||||
|
||||
@@ -261,7 +264,7 @@ export class JiraServerIssuePropertiesStep implements IStep {
|
||||
*/
|
||||
protected async storePropertiesData(
|
||||
// We need to pass the name of the step, as default properties step uses this function to store data
|
||||
name: EJiraServerStep,
|
||||
name: EJiraStep,
|
||||
job: TImportJob,
|
||||
properties: ExIssueProperty[],
|
||||
storage: IStorageService
|
||||
+11
-5
@@ -1,4 +1,5 @@
|
||||
import { v4 as uuid } from "uuid";
|
||||
import type { E_IMPORTER_KEYS } from "@plane/etl/core";
|
||||
import type { JiraConfig, JiraCustomFieldKeys, JiraIssueField } from "@plane/etl/jira-server";
|
||||
import { OPTION_CUSTOM_FIELD_TYPES, transformIssueFieldOptions } from "@plane/etl/jira-server";
|
||||
import { logger } from "@plane/logger";
|
||||
@@ -14,7 +15,7 @@ import type {
|
||||
TStepExecutionContext,
|
||||
TStepExecutionInput,
|
||||
} from "@/apps/jira-server-importer/v2/types";
|
||||
import { E_ADDITIONAL_STORAGE_KEYS, EJiraServerStep } from "@/apps/jira-server-importer/v2/types";
|
||||
import { E_ADDITIONAL_STORAGE_KEYS, EJiraStep } from "@/apps/jira-server-importer/v2/types";
|
||||
import { createOrUpdateIssuePropertiesOptions } from "@/etl/migrator/issue-types/issue-property.migrator";
|
||||
|
||||
/**
|
||||
@@ -23,9 +24,11 @@ import { createOrUpdateIssuePropertiesOptions } from "@/etl/migrator/issue-types
|
||||
*
|
||||
* Note: Only processes select/multi-select fields
|
||||
*/
|
||||
export class JiraServerIssuePropertyOptionsStep implements IStep {
|
||||
name = EJiraServerStep.ISSUE_PROPERTY_OPTIONS;
|
||||
dependencies = [EJiraServerStep.ISSUE_PROPERTIES];
|
||||
export class JiraIssuePropertyOptionsStep implements IStep {
|
||||
name = EJiraStep.ISSUE_PROPERTY_OPTIONS;
|
||||
dependencies = [EJiraStep.ISSUE_PROPERTIES];
|
||||
|
||||
constructor(private readonly source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA) {}
|
||||
|
||||
async execute(input: TStepExecutionInput): Promise<TStepExecutionContext> {
|
||||
const { jobContext, storage, dependencyData } = input;
|
||||
@@ -85,7 +88,10 @@ export class JiraServerIssuePropertyOptionsStep implements IStep {
|
||||
field.schema?.custom && OPTION_CUSTOM_FIELD_TYPES.includes(field.schema.custom as JiraCustomFieldKeys)
|
||||
)
|
||||
.flatMap(
|
||||
(field) => field.options?.map((option) => transformIssueFieldOptions(resourceId, job.project_id, option)) || []
|
||||
(field) =>
|
||||
field.options?.map((option) =>
|
||||
transformIssueFieldOptions({ resourceId, projectId: job.project_id, source: this.source }, option)
|
||||
) || []
|
||||
)
|
||||
.filter(Boolean) as Partial<ExIssuePropertyOption>[];
|
||||
|
||||
+10
-5
@@ -1,6 +1,7 @@
|
||||
import type { IssueTypeDetails } from "jira.js/out/version2/models";
|
||||
import { v4 as uuid } from "uuid";
|
||||
import { E_FEATURE_FLAGS } from "@plane/constants";
|
||||
import type { E_IMPORTER_KEYS } from "@plane/etl/core";
|
||||
import type { JiraConfig } from "@plane/etl/jira-server";
|
||||
import { pullIssueTypesV2, transformIssueType } from "@plane/etl/jira-server";
|
||||
import { logger } from "@plane/logger";
|
||||
@@ -16,7 +17,7 @@ import type {
|
||||
TStepExecutionContext,
|
||||
TStepExecutionInput,
|
||||
} from "@/apps/jira-server-importer/v2/types";
|
||||
import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types";
|
||||
import { EJiraStep } from "@/apps/jira-server-importer/v2/types";
|
||||
import { createOrUpdateIssueTypes } from "@/etl/migrator/issue-types/issue-type.migrator";
|
||||
import { getPlaneFeatureFlagService } from "@/helpers/plane-api-client";
|
||||
import { protect } from "@/lib";
|
||||
@@ -27,10 +28,12 @@ import { protect } from "@/lib";
|
||||
*
|
||||
* Handles Epic types and conflict resolution automatically
|
||||
*/
|
||||
export class JiraServerIssueTypesStep implements IStep {
|
||||
name = EJiraServerStep.ISSUE_TYPES;
|
||||
export class JiraIssueTypesStep implements IStep {
|
||||
name = EJiraStep.ISSUE_TYPES;
|
||||
dependencies = [];
|
||||
|
||||
constructor(private readonly source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA) {}
|
||||
|
||||
private readonly PAGE_SIZE = 50;
|
||||
|
||||
/**
|
||||
@@ -135,7 +138,7 @@ export class JiraServerIssueTypesStep implements IStep {
|
||||
/**
|
||||
* Pull one page of issue types from Jira Server
|
||||
*/
|
||||
private async pull(jobContext: TJobContext, projectId: string, startAt: number) {
|
||||
protected async pull(jobContext: TJobContext, projectId: string, startAt: number) {
|
||||
const result = await pullIssueTypesV2(
|
||||
{
|
||||
client: jobContext.sourceClient,
|
||||
@@ -161,7 +164,9 @@ export class JiraServerIssueTypesStep implements IStep {
|
||||
private transform(job: TImportJob<JiraConfig>, jiraIssueTypes: IssueTypeDetails[]): Partial<ExIssueType>[] {
|
||||
const resourceId = job.config.resource ? job.config.resource.id : uuid();
|
||||
|
||||
return jiraIssueTypes.map((issueType) => transformIssueType(resourceId, job.project_id, issueType));
|
||||
return jiraIssueTypes.map((issueType) =>
|
||||
transformIssueType({ resourceId, projectId: job.project_id, source: this.source }, issueType)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
+9
-4
@@ -1,6 +1,7 @@
|
||||
/* ----------------- Modules Migrator Step ----------------- */
|
||||
import type { ComponentWithIssueCount } from "jira.js/out/version2/models";
|
||||
import { v4 as uuid } from "uuid";
|
||||
import type { E_IMPORTER_KEYS } from "@plane/etl/core";
|
||||
import type { JiraConfig, JiraV2Service } from "@plane/etl/jira-server";
|
||||
import { pullComponentsV2, transformComponentV2 } from "@plane/etl/jira-server";
|
||||
import { logger } from "@plane/logger";
|
||||
@@ -14,19 +15,21 @@ import type {
|
||||
TStepExecutionContext,
|
||||
TStepExecutionInput,
|
||||
} from "@/apps/jira-server-importer/v2/types";
|
||||
import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types";
|
||||
import { EJiraStep } from "@/apps/jira-server-importer/v2/types";
|
||||
import { createAllModulesV2 } from "@/etl/migrator/modules.migrator";
|
||||
|
||||
/**
|
||||
* Jira Server Modules Step (Components in Jira)
|
||||
* Pulls components from Jira Server with pagination, transforms to modules, and pushes to Plane
|
||||
*/
|
||||
export class JiraServerModulesStep implements IStep {
|
||||
name = EJiraServerStep.MODULES;
|
||||
export class JiraModulesStep implements IStep {
|
||||
name = EJiraStep.MODULES;
|
||||
dependencies = [];
|
||||
|
||||
private readonly PAGE_SIZE = 100;
|
||||
|
||||
constructor(private readonly source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA) {}
|
||||
|
||||
async execute(input: TStepExecutionInput): Promise<TStepExecutionContext> {
|
||||
const { jobContext, storage, previousContext } = input;
|
||||
const { job, sourceClient } = jobContext;
|
||||
@@ -102,7 +105,9 @@ export class JiraServerModulesStep implements IStep {
|
||||
*/
|
||||
private transform(job: TImportJob<JiraConfig>, jiraComponents: ComponentWithIssueCount[]) {
|
||||
const resourceId = job.config.resource ? job.config.resource.id : uuid();
|
||||
return jiraComponents.map((component) => transformComponentV2(resourceId, job.project_id, component));
|
||||
return jiraComponents.map((component) =>
|
||||
transformComponentV2({ resourceId, projectId: job.project_id, source: this.source }, component)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
+34
-16
@@ -12,7 +12,7 @@ import type {
|
||||
TStepExecutionContext,
|
||||
TStepExecutionInput,
|
||||
} from "@/apps/jira-server-importer/v2/types";
|
||||
import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types";
|
||||
import { EJiraStep } from "@/apps/jira-server-importer/v2/types";
|
||||
import { createUsers } from "@/etl/migrator/users.migrator";
|
||||
import { protect } from "@/lib";
|
||||
|
||||
@@ -25,8 +25,8 @@ import { protect } from "@/lib";
|
||||
*
|
||||
* User avatars are also imported to Plane during this process.
|
||||
*/
|
||||
export class JiraServerUsersStep implements IStep {
|
||||
name = EJiraServerStep.USERS;
|
||||
export class JiraUsersStep implements IStep {
|
||||
name = EJiraStep.USERS;
|
||||
dependencies = [];
|
||||
|
||||
private readonly PAGE_SIZE = 100;
|
||||
@@ -37,7 +37,7 @@ export class JiraServerUsersStep implements IStep {
|
||||
private shouldExecute(input: TStepExecutionInput): boolean {
|
||||
const { jobContext } = input;
|
||||
const job = jobContext.job as TImportJob<JiraConfig>;
|
||||
return !job.config?.skipUserImport;
|
||||
return !job.config.skipUserImport;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -81,7 +81,7 @@ export class JiraServerUsersStep implements IStep {
|
||||
});
|
||||
|
||||
// Pull users from Jira Server (paginated)
|
||||
const pulledUsers = await this.pull(sourceClient, startAt, job.id);
|
||||
const pulledUsers = await this.pull(jobContext, sourceClient, startAt, job.id);
|
||||
|
||||
if (pulledUsers.items.length === 0) {
|
||||
logger.info(`[${job.id}] [${this.name}] No users found`, { jobId: job.id });
|
||||
@@ -119,14 +119,14 @@ export class JiraServerUsersStep implements IStep {
|
||||
* @param jobId - The import job ID for logging purposes
|
||||
* @returns Paginated result containing user items and hasMore flag
|
||||
*/
|
||||
private async pull(client: JiraV2Service, startAt: number, jobId: string) {
|
||||
protected async pull(_jobContext: TJobContext, client: JiraV2Service, startAt: number, jobId: string) {
|
||||
const result = await pullUsersV2({
|
||||
client,
|
||||
startAt,
|
||||
maxResults: this.PAGE_SIZE,
|
||||
});
|
||||
|
||||
logger.info(`[${jobId}] [${this.name}] Pulled users from Jira Server`, {
|
||||
logger.info(`[${jobId}] [${this.name}] Pulled users`, {
|
||||
jobId,
|
||||
count: result.items.length,
|
||||
hasMore: result.hasMore,
|
||||
@@ -256,18 +256,36 @@ export class JiraServerUsersStep implements IStep {
|
||||
storage: IStorageService
|
||||
): Promise<void> {
|
||||
const allUsers = [...existingUsers, ...createdUsers];
|
||||
|
||||
/*
|
||||
* In jira cloud, we don't have access to user emails, hence insead
|
||||
* of poluting the space with multiple mappings and if else conditions
|
||||
* we are using a merge of displayName and email, in case of transformation
|
||||
* if the email is not present, we can fallback to displayName, which can be
|
||||
* expected to be present in the mapping.
|
||||
*/
|
||||
const mappings = allUsers
|
||||
.filter((user) => user.email && user.id)
|
||||
.map((user) => ({
|
||||
externalId: user.email!,
|
||||
planeId: user.id!,
|
||||
}));
|
||||
.flatMap((user) => {
|
||||
const displayName = user.display_name;
|
||||
const email = user.email;
|
||||
|
||||
return [
|
||||
...(email
|
||||
? [
|
||||
{
|
||||
externalId: email,
|
||||
planeId: user.id!,
|
||||
},
|
||||
]
|
||||
: []),
|
||||
{
|
||||
externalId: displayName,
|
||||
planeId: user.id!,
|
||||
},
|
||||
];
|
||||
});
|
||||
|
||||
await storage.storeMapping(jobId, this.name, mappings);
|
||||
|
||||
logger.info(`[${jobId}] [${this.name}] Stored mappings`, {
|
||||
jobId,
|
||||
mappingCount: mappings.length,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
export * from "./entities";
|
||||
export * from "./issues";
|
||||
export * from "./pre-run";
|
||||
export * from "./relations";
|
||||
+287
-138
@@ -1,7 +1,14 @@
|
||||
import type { Worklog } from "jira.js/out/version2/models";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import type { TIssuePropertyValuesPayload } from "@plane/etl/core";
|
||||
import type { IJiraIssue, JiraConfig, JiraIssueField } from "@plane/etl/jira-server";
|
||||
import { pullIssuesV2, transformComment, transformIssueV2 } from "@plane/etl/jira-server";
|
||||
import type { E_IMPORTER_KEYS, TIssuePropertyValuesPayload } from "@plane/etl/core";
|
||||
import type { IJiraIssue, JiraConfig, JiraIssueField, JiraV2Service } from "@plane/etl/jira-server";
|
||||
import {
|
||||
pullAllCommentsForIssue,
|
||||
pullAllWorklogsForIssue,
|
||||
pullIssuesV2,
|
||||
transformComment,
|
||||
transformIssueV2,
|
||||
} from "@plane/etl/jira-server";
|
||||
import { logger } from "@plane/logger";
|
||||
import type { ExIssue, ExIssueComment, ExIssueProperty, ExIssuePropertyOption, TWorklog } from "@plane/sdk";
|
||||
import type { TImportJob } from "@plane/types";
|
||||
@@ -20,7 +27,7 @@ import type {
|
||||
TStepExecutionContext,
|
||||
TStepExecutionInput,
|
||||
} from "@/apps/jira-server-importer/v2/types";
|
||||
import { E_ADDITIONAL_STORAGE_KEYS, EJiraServerStep } from "@/apps/jira-server-importer/v2/types";
|
||||
import { E_ADDITIONAL_STORAGE_KEYS, EJiraStep } from "@/apps/jira-server-importer/v2/types";
|
||||
import { generateIssuePayloadV2 } from "@/etl/migrator/issues.migrator";
|
||||
import { getAPIClient } from "@/services/client";
|
||||
import type { BulkIssuePayload } from "@/types";
|
||||
@@ -41,90 +48,71 @@ import { celeryProducer } from "@/worker";
|
||||
* NOT dependencies (accessed via storage.lookupMapping):
|
||||
* - users, labels, issue_types
|
||||
*/
|
||||
export class JiraServerIssuesStep implements IStep {
|
||||
name = EJiraServerStep.ISSUES;
|
||||
export class JiraIssuesStep implements IStep {
|
||||
name = EJiraStep.ISSUES;
|
||||
|
||||
dependencies = [
|
||||
EJiraServerStep.ISSUE_TYPES, // Provides issue types
|
||||
EJiraServerStep.ISSUE_PROPERTIES, // Provides properties + rawFields
|
||||
EJiraServerStep.ISSUE_PROPERTY_OPTIONS, // Provides options
|
||||
EJiraStep.ISSUE_TYPES, // Provides issue types
|
||||
EJiraStep.ISSUE_PROPERTIES, // Provides properties + rawFields
|
||||
EJiraStep.ISSUE_PROPERTY_OPTIONS, // Provides options
|
||||
];
|
||||
|
||||
constructor(private readonly source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA) { }
|
||||
|
||||
private readonly PAGE_SIZE = 50;
|
||||
|
||||
/**
|
||||
* Initialize report batch count on first page
|
||||
* Automatically detects if it's the first page and calculates total batches
|
||||
*/
|
||||
private async initializeReportBatchCount(input: TStepExecutionInput, totalIssues: number): Promise<void> {
|
||||
const { jobContext, previousContext } = input;
|
||||
const startAt = previousContext?.pageCtx.startAt || 0;
|
||||
|
||||
// Only initialize on first page
|
||||
if (startAt !== 0) return;
|
||||
|
||||
const totalBatches = Math.ceil(totalIssues / this.PAGE_SIZE);
|
||||
const client = getAPIClient();
|
||||
|
||||
await client.importReport.updateImportReport(jobContext.job.report_id, {
|
||||
total_batch_count: totalBatches,
|
||||
});
|
||||
|
||||
logger.info(`[${jobContext.job.id}] [${this.name}] Initialized report batch count`, {
|
||||
jobId: jobContext.job.id,
|
||||
totalIssues,
|
||||
pageSize: this.PAGE_SIZE,
|
||||
totalBatches,
|
||||
});
|
||||
}
|
||||
|
||||
async execute(input: TStepExecutionInput): Promise<TStepExecutionContext> {
|
||||
const { jobContext, storage, previousContext, dependencyData } = input;
|
||||
const { job } = jobContext;
|
||||
|
||||
try {
|
||||
const projectKey = job.config?.project?.key;
|
||||
if (!projectKey) {
|
||||
throw new Error("Project key not found in job config");
|
||||
}
|
||||
const projectKey = this.getProjectKey(job);
|
||||
const paginationCtx = this.getPaginationContext(previousContext);
|
||||
|
||||
const startAt = previousContext?.pageCtx.startAt || 0;
|
||||
const totalProcessed = previousContext?.pageCtx.totalProcessed || 0;
|
||||
|
||||
logger.info(`[${jobContext.job.id}] [${this.name}] Starting execution`, {
|
||||
logger.info(`[${job.id}] [${this.name}] Starting execution`, {
|
||||
jobId: job.id,
|
||||
startAt,
|
||||
totalProcessed,
|
||||
...paginationCtx,
|
||||
});
|
||||
|
||||
// Get property data from dependencies (needed for property value transformation)
|
||||
const propertyData = this.getPropertyData(dependencyData);
|
||||
const additionalData = await this.getAdditionalData(storage, job);
|
||||
|
||||
// Pull paginated issues, comments, and transform property values
|
||||
const pulled = await this.pull(jobContext, projectKey, startAt, propertyData, additionalData);
|
||||
// Pull paginated issues from Jira (can be overridden in subclasses)
|
||||
const issuesResult = await this.pull({
|
||||
jobContext,
|
||||
projectKey,
|
||||
paginationCtx,
|
||||
});
|
||||
|
||||
if (pulled.total !== undefined) {
|
||||
await this.initializeReportBatchCount(input, pulled.total);
|
||||
}
|
||||
await this.initializeReportBatchCount(paginationCtx, issuesResult.total, job);
|
||||
|
||||
const associations = this.extractAssociations(job, pulled.issues);
|
||||
|
||||
if (pulled.issues.length === 0) {
|
||||
logger.info(`[${jobContext.job.id}] [${this.name}] No issues found`, { jobId: job.id });
|
||||
if (this.shouldReturnEmpty(issuesResult, paginationCtx)) {
|
||||
logger.info(`[${job.id}] [${this.name}] No issues found`, { jobId: job.id });
|
||||
return createEmptyContext();
|
||||
}
|
||||
|
||||
// Process issues: extract comments and transform property values
|
||||
const processed = await this.processIssuesData({
|
||||
jobContext,
|
||||
issuesResult,
|
||||
propertyData,
|
||||
additionalData,
|
||||
});
|
||||
|
||||
const associations = await this.extractAssociations(job, jobContext.sourceClient, processed.issues);
|
||||
|
||||
// Load entity mappings from storage
|
||||
const mappings = await this.loadMappings(job, pulled.issues, associations, storage);
|
||||
const mappings = await this.loadMappings(job, processed.issues, associations, storage);
|
||||
|
||||
// Transform issues (uses existing transformIssue function)
|
||||
const transformed = await this.transform(job, pulled.issues);
|
||||
const transformed = await this.transform(job, processed.issues);
|
||||
|
||||
// Generate payload and send to Celery
|
||||
const pushed = await this.push(
|
||||
transformed,
|
||||
pulled.comments,
|
||||
pulled.propertyValues,
|
||||
processed.comments,
|
||||
processed.propertyValues,
|
||||
mappings,
|
||||
associations,
|
||||
propertyData,
|
||||
@@ -132,26 +120,19 @@ export class JiraServerIssuesStep implements IStep {
|
||||
);
|
||||
|
||||
// Store relations for Relations Step
|
||||
const relations = this.extractRelations(job, pulled.issues);
|
||||
const relations = this.extractRelations(job, processed.issues);
|
||||
await this.storeRelations(relations, storage, job.id);
|
||||
|
||||
logger.info(`[${jobContext.job.id}] [${this.name}] Completed page`, {
|
||||
logger.info(`[${job.id}] [${this.name}] Completed page`, {
|
||||
jobId: job.id,
|
||||
issues: pulled.issues.length,
|
||||
comments: pulled.comments.length,
|
||||
issues: processed.issues.length,
|
||||
comments: processed.comments.length,
|
||||
pushed: pushed.length,
|
||||
});
|
||||
|
||||
return createPaginationContext({
|
||||
hasMore: pulled.hasMore,
|
||||
startAt: startAt,
|
||||
pageSize: this.PAGE_SIZE,
|
||||
pulled: pulled.issues.length,
|
||||
pushed: pushed.length,
|
||||
totalProcessed: totalProcessed + pulled.issues.length,
|
||||
});
|
||||
return this.buildNextContext(issuesResult, paginationCtx, processed.issues.length, pushed.length);
|
||||
} catch (error) {
|
||||
logger.error(`[${jobContext.job.id}] [${this.name}] Step failed`, {
|
||||
logger.error(`[${job.id}] [${this.name}] Step failed`, {
|
||||
jobId: job.id,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
@@ -160,50 +141,152 @@ export class JiraServerIssuesStep implements IStep {
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull issues and comments, then transform property values
|
||||
* Uses existing transformIssuePropertyValues for proper conversion
|
||||
* Extract pagination state from previous context
|
||||
* Override this in subclasses to support different pagination strategies
|
||||
* (e.g., nextPageToken for Jira Cloud)
|
||||
*/
|
||||
private async pull(
|
||||
jobContext: TJobContext,
|
||||
projectKey: string,
|
||||
startAt: number,
|
||||
propertyData: {
|
||||
issueTypes: TIssueTypesData;
|
||||
planeIssueProperties: ExIssueProperty[];
|
||||
planeIssuePropertiesOptions: ExIssuePropertyOption[];
|
||||
},
|
||||
additionalData: {
|
||||
rawFields: JiraIssueField[];
|
||||
protected getPaginationContext(previousContext?: TStepExecutionContext) {
|
||||
return {
|
||||
startAt: previousContext?.pageCtx.startAt ?? 0,
|
||||
totalProcessed: previousContext?.pageCtx.totalProcessed ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract and validate project key from job configuration
|
||||
*/
|
||||
protected getProjectKey(job: TImportJob<JiraConfig>): string {
|
||||
const projectKey = job.config?.project?.key;
|
||||
if (!projectKey) {
|
||||
throw new Error("Project key not found in job config");
|
||||
}
|
||||
): Promise<{
|
||||
issues: IJiraIssue[];
|
||||
comments: ExIssueComment[];
|
||||
propertyValues: TIssuePropertyValuesPayload;
|
||||
return projectKey;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize report batch count on first page
|
||||
*/
|
||||
protected async initializeReportBatchCount(
|
||||
paginationCtx: { startAt: number; totalProcessed: number },
|
||||
totalIssues: number | undefined,
|
||||
job: TImportJob<JiraConfig>
|
||||
): Promise<void> {
|
||||
// Only initialize on first page
|
||||
if (paginationCtx.startAt !== 0 || !totalIssues) return;
|
||||
|
||||
const totalBatches = Math.ceil(totalIssues / this.PAGE_SIZE);
|
||||
const client = getAPIClient();
|
||||
|
||||
await client.importReport.updateImportReport(job.report_id, {
|
||||
total_batch_count: totalBatches,
|
||||
});
|
||||
|
||||
logger.info(`[${job.id}] [${this.name}] Initialized report batch count`, {
|
||||
jobId: job.id,
|
||||
totalIssues,
|
||||
pageSize: this.PAGE_SIZE,
|
||||
totalBatches,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if step should return empty context
|
||||
*/
|
||||
protected shouldReturnEmpty(
|
||||
issuesResult: { items: IJiraIssue[]; hasMore: boolean; total: number },
|
||||
paginationCtx: { startAt: number; totalProcessed: number }
|
||||
): boolean {
|
||||
return issuesResult.items.length === 0 && paginationCtx.startAt === 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the next context based on pagination state
|
||||
* Override this in subclasses to support different pagination strategies
|
||||
*/
|
||||
protected buildNextContext(
|
||||
issuesResult: { items: IJiraIssue[]; hasMore: boolean; total: number },
|
||||
paginationCtx: { startAt: number; totalProcessed: number },
|
||||
pulled: number,
|
||||
pushed: number
|
||||
): TStepExecutionContext {
|
||||
return createPaginationContext({
|
||||
hasMore: issuesResult.hasMore,
|
||||
startAt: paginationCtx.startAt,
|
||||
pageSize: this.PAGE_SIZE,
|
||||
pulled,
|
||||
pushed,
|
||||
totalProcessed: paginationCtx.totalProcessed + pulled,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull paginated issues from Jira
|
||||
* Override this in subclasses to customize issue fetching behavior
|
||||
* (e.g., use nextPageToken for Jira Cloud)
|
||||
*/
|
||||
protected async pull(props: {
|
||||
jobContext: TJobContext;
|
||||
projectKey: string;
|
||||
paginationCtx: { startAt: number; totalProcessed: number };
|
||||
}): Promise<{
|
||||
items: IJiraIssue[];
|
||||
hasMore: boolean;
|
||||
total: number;
|
||||
}> {
|
||||
const { projectId, resourceId } = extractJobData(jobContext.job);
|
||||
// Pull paginated issues
|
||||
const issuesResult = await pullIssuesV2(
|
||||
{
|
||||
client: jobContext.sourceClient,
|
||||
startAt,
|
||||
client: props.jobContext.sourceClient,
|
||||
startAt: props.paginationCtx.startAt,
|
||||
maxResults: this.PAGE_SIZE,
|
||||
},
|
||||
projectKey
|
||||
props.projectKey
|
||||
);
|
||||
|
||||
// Pull comments for these issues
|
||||
// Merge: timestamps from fields.comment, HTML body from renderedFields.comment
|
||||
const comments: Partial<ExIssueComment>[] = issuesResult.items
|
||||
.map((issue) => {
|
||||
const fieldsComments = issue.fields?.comment?.comments || [];
|
||||
const renderedComments = issue.renderedFields?.comment?.comments || [];
|
||||
logger.info(`[${props.jobContext.job.id}] [${this.name}] Pulled issues`, {
|
||||
jobId: props.jobContext.job.id,
|
||||
count: issuesResult.items.length,
|
||||
hasMore: issuesResult.hasMore,
|
||||
startAt: props.paginationCtx.startAt,
|
||||
});
|
||||
|
||||
return {
|
||||
items: issuesResult.items,
|
||||
hasMore: issuesResult.hasMore,
|
||||
total: issuesResult.total ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract and transform comments from issues
|
||||
* Merges timestamps from fields.comment with HTML body from renderedFields.comment
|
||||
*/
|
||||
protected async extractCommentsFromIssues(props: {
|
||||
_jobContext: TJobContext;
|
||||
issues: IJiraIssue[];
|
||||
resourceId: string;
|
||||
projectId: string;
|
||||
}): Promise<ExIssueComment[]> {
|
||||
const comments = [];
|
||||
for (const issue of props.issues) {
|
||||
const fieldsComments = issue.fields?.comment?.comments || [];
|
||||
const renderedComments = issue.renderedFields?.comment?.comments || [];
|
||||
/*
|
||||
* Jira provides us with the comments, but those comments are paginated, in
|
||||
* case the number of comments are more than what we have in the comments property
|
||||
* we will need to fetch the remaining comments.
|
||||
*/
|
||||
const shouldFetchMoreComments = issue.fields.comment?.total > issue.fields.comment?.comments?.length;
|
||||
if (shouldFetchMoreComments) {
|
||||
const allComments = await pullAllCommentsForIssue(issue, props._jobContext.sourceClient);
|
||||
const transformedComments = allComments.map((comment) =>
|
||||
transformComment({ resourceId: props.resourceId, projectId: props.projectId, source: this.source }, comment)
|
||||
);
|
||||
comments.push(transformedComments);
|
||||
} else {
|
||||
// Create a map of rendered comments by ID for quick lookup
|
||||
const renderedCommentsMap = new Map(renderedComments.map((rendered) => [rendered.id, rendered]));
|
||||
|
||||
return fieldsComments.map((comment) => {
|
||||
const transformedComments = fieldsComments.map((comment) => {
|
||||
// Get the rendered version for HTML body
|
||||
const renderedComment = renderedCommentsMap.get(comment.id);
|
||||
|
||||
@@ -217,35 +300,72 @@ export class JiraServerIssuesStep implements IStep {
|
||||
issue_id: issue.id,
|
||||
};
|
||||
|
||||
return transformComment(resourceId, projectId, mergedComment);
|
||||
return transformComment(
|
||||
{ resourceId: props.resourceId, projectId: props.projectId, source: this.source },
|
||||
mergedComment
|
||||
);
|
||||
});
|
||||
})
|
||||
.flat();
|
||||
comments.push(transformedComments);
|
||||
}
|
||||
}
|
||||
|
||||
return comments.flat() as ExIssueComment[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Process pulled issues: extract comments and transform property values
|
||||
* Uses existing transformIssuePropertyValues for proper conversion
|
||||
*/
|
||||
protected async processIssuesData(props: {
|
||||
jobContext: TJobContext;
|
||||
issuesResult: {
|
||||
items: IJiraIssue[];
|
||||
hasMore: boolean;
|
||||
total: number;
|
||||
};
|
||||
propertyData: {
|
||||
issueTypes: TIssueTypesData;
|
||||
planeIssueProperties: ExIssueProperty[];
|
||||
planeIssuePropertiesOptions: ExIssuePropertyOption[];
|
||||
};
|
||||
additionalData: {
|
||||
rawFields: JiraIssueField[];
|
||||
};
|
||||
}): Promise<{
|
||||
issues: IJiraIssue[];
|
||||
comments: ExIssueComment[];
|
||||
propertyValues: TIssuePropertyValuesPayload;
|
||||
}> {
|
||||
const { projectId, resourceId } = extractJobData(props.jobContext.job);
|
||||
|
||||
// Extract comments from issues
|
||||
const comments = await this.extractCommentsFromIssues({
|
||||
_jobContext: props.jobContext,
|
||||
issues: props.issuesResult.items,
|
||||
resourceId,
|
||||
projectId,
|
||||
});
|
||||
|
||||
// Transform property values for each issue using existing function
|
||||
const propertyValues = getTransformedIssuePropertyValuesV2(
|
||||
jobContext.job,
|
||||
issuesResult.items,
|
||||
additionalData.rawFields,
|
||||
propertyData.planeIssueProperties,
|
||||
propertyData.issueTypes
|
||||
props.jobContext.job,
|
||||
props.issuesResult.items,
|
||||
props.additionalData.rawFields,
|
||||
props.propertyData.planeIssueProperties,
|
||||
props.propertyData.issueTypes
|
||||
);
|
||||
|
||||
logger.info(`[${jobContext.job.id}] [${this.name}] Pulled from Jira`, {
|
||||
jobId: jobContext.job.id,
|
||||
issues: issuesResult.items.length,
|
||||
logger.info(`[${props.jobContext.job.id}] [${this.name}] Processed issues data`, {
|
||||
jobId: props.jobContext.job.id,
|
||||
issues: props.issuesResult.items.length,
|
||||
comments: comments.length,
|
||||
issuesWithPropertyValues: Object.keys(propertyValues).length,
|
||||
hasMore: issuesResult.hasMore,
|
||||
startAt,
|
||||
});
|
||||
|
||||
return {
|
||||
issues: issuesResult.items,
|
||||
comments: comments as ExIssueComment[],
|
||||
issues: props.issuesResult.items,
|
||||
comments,
|
||||
propertyValues,
|
||||
hasMore: issuesResult.hasMore,
|
||||
total: issuesResult.total ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -265,9 +385,9 @@ export class JiraServerIssuesStep implements IStep {
|
||||
};
|
||||
}
|
||||
|
||||
const issueTypes = dependencyData[EJiraServerStep.ISSUE_TYPES] as TIssueTypesData;
|
||||
const propertiesData = dependencyData[EJiraServerStep.ISSUE_PROPERTIES] as TIssuePropertiesData;
|
||||
const optionsData = dependencyData[EJiraServerStep.ISSUE_PROPERTY_OPTIONS];
|
||||
const issueTypes = dependencyData[EJiraStep.ISSUE_TYPES] as TIssueTypesData;
|
||||
const propertiesData = dependencyData[EJiraStep.ISSUE_PROPERTIES] as TIssuePropertiesData;
|
||||
const optionsData = dependencyData[EJiraStep.ISSUE_PROPERTY_OPTIONS];
|
||||
|
||||
return {
|
||||
issueTypes,
|
||||
@@ -336,10 +456,10 @@ export class JiraServerIssuesStep implements IStep {
|
||||
// Load mappings in parallel from storage
|
||||
const [userMap, issueTypeMap, cycleMap, moduleMap] = await Promise.all([
|
||||
// We are retrieving all users instead of associated users, as there can be custom fields that associates with users, and we can't select those
|
||||
storage.retrieveMapping(jobId, EJiraServerStep.USERS),
|
||||
storage.lookupMapping(jobId, EJiraServerStep.ISSUE_TYPES, Array.from(issueTypeIds)),
|
||||
storage.lookupMapping(jobId, EJiraServerStep.CYCLES, Array.from(sprintExternalIds)),
|
||||
storage.lookupMapping(jobId, EJiraServerStep.MODULES, Array.from(componentExternalIds)),
|
||||
storage.retrieveMapping(jobId, EJiraStep.USERS),
|
||||
storage.lookupMapping(jobId, EJiraStep.ISSUE_TYPES, Array.from(issueTypeIds)),
|
||||
storage.lookupMapping(jobId, EJiraStep.CYCLES, Array.from(sprintExternalIds)),
|
||||
storage.lookupMapping(jobId, EJiraStep.MODULES, Array.from(componentExternalIds)),
|
||||
]);
|
||||
|
||||
logger.info(`[${jobId}] [${this.name}] Loaded mappings`, {
|
||||
@@ -362,7 +482,13 @@ export class JiraServerIssuesStep implements IStep {
|
||||
const priorityMapping = job.config?.priority || {};
|
||||
|
||||
const transformed = issues.map((issue) =>
|
||||
transformIssueV2(resourceId, job.project_id, issue, resourceUrl, stateMapping, priorityMapping)
|
||||
transformIssueV2(
|
||||
{ resourceId, projectId: job.project_id, source: this.source },
|
||||
issue,
|
||||
resourceUrl,
|
||||
stateMapping,
|
||||
priorityMapping
|
||||
)
|
||||
);
|
||||
|
||||
return transformed;
|
||||
@@ -395,7 +521,11 @@ export class JiraServerIssuesStep implements IStep {
|
||||
);
|
||||
}
|
||||
|
||||
private extractAssociations(job: TImportJob<JiraConfig>, issues: IJiraIssue[]): TIssuesAssociationsData {
|
||||
private async extractAssociations(
|
||||
job: TImportJob<JiraConfig>,
|
||||
client: JiraV2Service,
|
||||
issues: IJiraIssue[]
|
||||
): Promise<TIssuesAssociationsData> {
|
||||
const { projectId, resourceId } = extractJobData(job);
|
||||
const cycles = new Map<string, string[]>();
|
||||
const modules = new Map<string, string[]>();
|
||||
@@ -404,7 +534,7 @@ export class JiraServerIssuesStep implements IStep {
|
||||
const issueExternalId = buildExternalId(projectId, resourceId, issue.id);
|
||||
const sprintExternalIds = this.extractSprints(job, issue);
|
||||
const componentExternalIds = this.extractComponents(job, issue);
|
||||
const issueWorklogs = this.extractWorklogs(job, issue);
|
||||
const issueWorklogs = await this.extractWorklogs(job, client, issue);
|
||||
cycles.set(issueExternalId, sprintExternalIds);
|
||||
modules.set(issueExternalId, componentExternalIds);
|
||||
worklogs.set(issueExternalId, issueWorklogs);
|
||||
@@ -412,7 +542,7 @@ export class JiraServerIssuesStep implements IStep {
|
||||
return { cycles, modules, worklogs };
|
||||
}
|
||||
|
||||
private extractSprints(job: TImportJob<JiraConfig>, issue: IJiraIssue): string[] {
|
||||
protected extractSprints(job: TImportJob<JiraConfig>, issue: IJiraIssue): string[] {
|
||||
const { projectId, resourceId } = extractJobData(job);
|
||||
|
||||
const sprintFieldKey = detectSprintFieldId(issue);
|
||||
@@ -424,8 +554,8 @@ export class JiraServerIssuesStep implements IStep {
|
||||
: null;
|
||||
return sprintObjects
|
||||
? sprintObjects
|
||||
.map((s) => (s ? buildExternalId(projectId, resourceId, s.id.toString()) : null))
|
||||
.filter((s) => s !== null)
|
||||
.map((s) => (s ? buildExternalId(projectId, resourceId, s.id.toString()) : null))
|
||||
.filter((s) => s !== null)
|
||||
: [];
|
||||
}
|
||||
|
||||
@@ -467,15 +597,34 @@ export class JiraServerIssuesStep implements IStep {
|
||||
/**
|
||||
* Extract worklogs from issue
|
||||
*/
|
||||
private extractWorklogs(_job: TImportJob<JiraConfig>, issue: IJiraIssue): Partial<TWorklog>[] {
|
||||
protected async extractWorklogs(
|
||||
_job: TImportJob<JiraConfig>,
|
||||
client: JiraV2Service,
|
||||
issue: IJiraIssue
|
||||
): Promise<Partial<TWorklog>[]> {
|
||||
const transformWorklog = (worklog: Worklog) => ({
|
||||
description: worklog.comment ?? "",
|
||||
duration: worklog.timeSpentSeconds ? worklog.timeSpentSeconds / 60 : 0,
|
||||
logged_by: worklog.author?.emailAddress,
|
||||
created_at: worklog.created,
|
||||
updated_at: worklog.updated,
|
||||
});
|
||||
|
||||
const shouldPullMoreWorklogs = issue.fields?.worklog?.total > issue.fields?.worklog?.worklogs?.length;
|
||||
if (shouldPullMoreWorklogs) {
|
||||
const worklogs = await pullAllWorklogsForIssue(issue, client);
|
||||
return worklogs.map(transformWorklog);
|
||||
}
|
||||
|
||||
return (
|
||||
issue.fields.worklog?.worklogs?.map((worklog) => ({
|
||||
description: worklog.comment ?? "",
|
||||
duration: worklog.timeSpentSeconds ? worklog.timeSpentSeconds / 60 : 0,
|
||||
logged_by: worklog.author?.emailAddress,
|
||||
created_at: worklog.created,
|
||||
updated_at: worklog.updated,
|
||||
})) || []
|
||||
issue.fields?.worklog?.worklogs?.map((worklog, index) => {
|
||||
const comment =
|
||||
typeof worklog.comment === "string"
|
||||
? worklog.comment
|
||||
: issue.renderedFields?.worklog?.worklogs?.[index]?.comment;
|
||||
|
||||
return transformWorklog({ ...worklog, comment });
|
||||
}) || []
|
||||
);
|
||||
}
|
||||
|
||||
+2
-2
@@ -1,12 +1,12 @@
|
||||
import { logger } from "@plane/logger";
|
||||
import { createEmptyContext } from "@/apps/jira-server-importer/v2/helpers/ctx";
|
||||
import type { IStep, TStepExecutionContext, TStepExecutionInput } from "@/apps/jira-server-importer/v2/types";
|
||||
import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types";
|
||||
import { EJiraStep } from "@/apps/jira-server-importer/v2/types";
|
||||
import { wait } from "@/helpers/delay";
|
||||
import { getAPIClient } from "@/services/client";
|
||||
|
||||
export class WaitForCeleryStep implements IStep {
|
||||
name = EJiraServerStep.WAIT_FOR_CELERY;
|
||||
name = EJiraStep.WAIT_FOR_CELERY;
|
||||
dependencies = [];
|
||||
|
||||
async execute(input: TStepExecutionInput): Promise<TStepExecutionContext> {
|
||||
+3
-3
@@ -8,7 +8,7 @@ import { E_FEATURE_FLAGS } from "@plane/constants";
|
||||
import { logger } from "@plane/logger";
|
||||
import { createEmptyContext } from "@/apps/jira-server-importer/v2/helpers/ctx";
|
||||
import type { IStep, TStepExecutionContext, TStepExecutionInput } from "@/apps/jira-server-importer/v2/types";
|
||||
import { EJiraServerStep } from "@/apps/jira-server-importer/v2/types";
|
||||
import { EJiraStep } from "@/apps/jira-server-importer/v2/types";
|
||||
import { getPlaneFeatureFlagService } from "@/helpers/plane-api-client";
|
||||
|
||||
export type TRequiredFlags = {
|
||||
@@ -18,8 +18,8 @@ export type TRequiredFlags = {
|
||||
};
|
||||
|
||||
export class PlaneProjectConfigurationStep implements IStep {
|
||||
name = EJiraServerStep.PLANE_PROJECT_CONFIGURATION;
|
||||
dependencies: EJiraServerStep[] = [];
|
||||
name = EJiraStep.PLANE_PROJECT_CONFIGURATION;
|
||||
dependencies: EJiraStep[] = [];
|
||||
|
||||
async execute(input: TStepExecutionInput): Promise<TStepExecutionContext> {
|
||||
const { jobContext } = input;
|
||||
+6
-4
@@ -1,4 +1,5 @@
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import type { E_IMPORTER_KEYS } from "@plane/etl/core";
|
||||
import { logger } from "@plane/logger";
|
||||
import { createEmptyContext } from "@/apps/jira-server-importer/v2/helpers/ctx";
|
||||
import type {
|
||||
@@ -7,7 +8,7 @@ import type {
|
||||
TStepExecutionContext,
|
||||
TStepExecutionInput,
|
||||
} from "@/apps/jira-server-importer/v2/types";
|
||||
import { E_ADDITIONAL_STORAGE_KEYS, EJiraServerStep } from "@/apps/jira-server-importer/v2/types";
|
||||
import { E_ADDITIONAL_STORAGE_KEYS, EJiraStep } from "@/apps/jira-server-importer/v2/types";
|
||||
import { celeryProducer } from "@/worker";
|
||||
|
||||
/**
|
||||
@@ -16,9 +17,10 @@ import { celeryProducer } from "@/worker";
|
||||
* Retrieves all accumulated relations from storage and sends to Celery in one batch
|
||||
* Only handles relationships (parent, blocking, etc) - cycles/modules already handled
|
||||
*/
|
||||
export class JiraServerRelationsStep implements IStep {
|
||||
name = EJiraServerStep.RELATIONS;
|
||||
export class JiraRelationsStep implements IStep {
|
||||
name = EJiraStep.RELATIONS;
|
||||
dependencies = [];
|
||||
constructor(private readonly source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA) {}
|
||||
|
||||
async execute(input: TStepExecutionInput): Promise<TStepExecutionContext> {
|
||||
const { jobContext, storage } = input;
|
||||
@@ -68,7 +70,7 @@ export class JiraServerRelationsStep implements IStep {
|
||||
job_id: job.id,
|
||||
workspace_id: job.workspace_id,
|
||||
project_id: job.project_id,
|
||||
source: job.source,
|
||||
source: this.source,
|
||||
user_id: credentials.user_id,
|
||||
},
|
||||
job.workspace_slug,
|
||||
@@ -2,7 +2,7 @@ import type { JiraV2Service } from "@plane/etl/jira-server";
|
||||
import type { Client as PlaneClient, TWorklog } from "@plane/sdk";
|
||||
import type { TImportJob, TWorkspaceCredential } from "@plane/types";
|
||||
|
||||
export enum EJiraServerStep {
|
||||
export enum EJiraStep {
|
||||
// Pre-run
|
||||
PLANE_PROJECT_CONFIGURATION = "plane_project_configuration",
|
||||
|
||||
@@ -45,6 +45,15 @@ export type TIssuesAssociationsData = {
|
||||
worklogs: Map<string, Partial<TWorklog>[]>;
|
||||
};
|
||||
|
||||
/**
|
||||
* Failed step tracking structure
|
||||
*/
|
||||
export type TFailedStep = {
|
||||
name: string;
|
||||
error: string;
|
||||
failedAt: string;
|
||||
};
|
||||
|
||||
export type TOrchestratorState = {
|
||||
jobId: string;
|
||||
|
||||
@@ -54,6 +63,9 @@ export type TOrchestratorState = {
|
||||
totalSteps: number;
|
||||
completedSteps: string[];
|
||||
|
||||
// Failed steps (logged and skipped)
|
||||
failedSteps?: TFailedStep[];
|
||||
|
||||
// Timestamps
|
||||
startedAt: string;
|
||||
lastUpdatedAt: string;
|
||||
@@ -103,10 +115,10 @@ export type TIssuePropertiesData = Array<{
|
||||
*/
|
||||
export interface IStep {
|
||||
/** Step name (e.g., 'users', 'issues') */
|
||||
name: EJiraServerStep;
|
||||
name: EJiraStep;
|
||||
|
||||
/** data that should be loaded before handling this step */
|
||||
dependencies: EJiraServerStep[];
|
||||
dependencies: EJiraStep[];
|
||||
shouldFail?: boolean;
|
||||
|
||||
/** Execute pull, transform, and push for this step */
|
||||
|
||||
@@ -13,8 +13,10 @@ import { FlatfileMigrator } from "@/apps/flatfile/migrator/flatfile.migrator";
|
||||
import { GithubWebhookWorker } from "@/apps/github/workers";
|
||||
import { PlaneGithubWebhookWorker } from "@/apps/github/workers/plane";
|
||||
import { GitlabWebhookWorker } from "@/apps/gitlab";
|
||||
import { JiraDataMigrator } from "@/apps/jira-importer/migrator/jira.migrator";
|
||||
import { JiraServerImportOrchestrator } from "@/apps/jira-server-importer/v2/migrator/orchestrator";
|
||||
import {
|
||||
getJiraCloudImportOrchestrator,
|
||||
getJiraServerImportOrchestrator,
|
||||
} from "@/apps/jira-server-importer/v2/migrator";
|
||||
import { LinearDocsMigrator } from "@/apps/linear-importer/migrator/linear-docs.migrator";
|
||||
import { LinearDataMigrator } from "@/apps/linear-importer/migrator/linear.migrator";
|
||||
import { NotionDataMigrator } from "@/apps/notion-importer/worker";
|
||||
@@ -46,10 +48,9 @@ class WorkerFactory {
|
||||
static createWorker(type: string, mq: MQ, store: Store): TaskHandler {
|
||||
switch (type) {
|
||||
case "jira":
|
||||
return new JiraDataMigrator(mq, store);
|
||||
return getJiraCloudImportOrchestrator(mq, store);
|
||||
case "jira_server":
|
||||
// return new JiraDataCenterMigrator(mq, store);
|
||||
return new JiraServerImportOrchestrator(mq, store);
|
||||
return getJiraServerImportOrchestrator(mq, store);
|
||||
case "linear":
|
||||
return new LinearDataMigrator(mq, store);
|
||||
case "linear_docs":
|
||||
|
||||
@@ -64,7 +64,7 @@ export const ImportUsersFromJira: FC = observer(() => {
|
||||
}
|
||||
if (key === "userData" && !formData.userSkipToggle && typeof value === "string") {
|
||||
handleSyncJobConfig("users", value);
|
||||
handleSyncJobConfig("skipUserImport", true);
|
||||
handleSyncJobConfig("skipUserImport", formData.userSkipToggle);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import type {
|
||||
Comment as JComment,
|
||||
IssueTypeDetails as JiraIssueTypeDetails,
|
||||
FieldDetails,
|
||||
Worklog
|
||||
} from "jira.js/out/version2/models";
|
||||
import type {
|
||||
ImportedJiraUser,
|
||||
@@ -15,7 +16,7 @@ import type {
|
||||
JiraCustomFieldKeys,
|
||||
JiraV2Service,
|
||||
} from "..";
|
||||
import { formatDateStringForHHMM, OPTION_CUSTOM_FIELD_TYPES } from "../helpers";
|
||||
import { fetchPaginatedDataByKey, formatDateStringForHHMM, OPTION_CUSTOM_FIELD_TYPES } from "../helpers";
|
||||
|
||||
type BasePaginationContext = {
|
||||
client: JiraV2Service;
|
||||
@@ -191,6 +192,34 @@ export async function pullComponentIssuesV2(
|
||||
};
|
||||
}
|
||||
|
||||
export const pullAllCommentsForIssue = async (issue: IJiraIssue, client: JiraV2Service): Promise<JiraComment[]> => {
|
||||
const values = await fetchPaginatedDataByKey<JComment>(
|
||||
(startAt) => client.getIssueComments(issue.id, startAt, 500),
|
||||
"comments"
|
||||
);
|
||||
|
||||
return values.map(
|
||||
(comment): JiraComment => ({
|
||||
...comment,
|
||||
issue_id: issue.id,
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
export const pullAllWorklogsForIssue = async (issue: IJiraIssue, client: JiraV2Service): Promise<Worklog[]> => {
|
||||
const values = await fetchPaginatedDataByKey<Worklog>(
|
||||
(startAt) => client.getIssueWorklogs(issue.id, startAt, 500),
|
||||
"worklogs"
|
||||
);
|
||||
|
||||
return values.map(
|
||||
(worklog) => ({
|
||||
...worklog,
|
||||
issue_id: issue.id,
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
export async function pullCommentsForIssueV2(
|
||||
ctx: BasePaginationContext,
|
||||
issue: IJiraIssue
|
||||
|
||||
@@ -35,14 +35,20 @@ import type {
|
||||
JiraIssueFieldOptions,
|
||||
} from "../types";
|
||||
|
||||
export type TTransformationContext = {
|
||||
resourceId: string;
|
||||
projectId: string;
|
||||
source: E_IMPORTER_KEYS.JIRA_SERVER | E_IMPORTER_KEYS.JIRA;
|
||||
};
|
||||
|
||||
export const transformIssue = (
|
||||
resourceId: string,
|
||||
projectId: string,
|
||||
ctx: TTransformationContext,
|
||||
issue: IJiraIssue,
|
||||
resourceUrl: string,
|
||||
stateMap: IStateConfig[],
|
||||
priorityMap: IPriorityConfig[]
|
||||
): Partial<PlaneIssue> => {
|
||||
const { resourceId, projectId, source } = ctx;
|
||||
const targetState = getTargetState(stateMap, issue.fields.status);
|
||||
const targetPriority = getTargetPriority(priorityMap, issue.fields.priority);
|
||||
const attachments = getTargetAttachments(resourceId, projectId, issue.fields.attachment);
|
||||
@@ -66,7 +72,7 @@ export const transformIssue = (
|
||||
assignees: issue.fields.assignee?.name ? [issue.fields.assignee.name] : [],
|
||||
links,
|
||||
external_id: `${projectId}_${resourceId}_${issue.id}`,
|
||||
external_source: E_IMPORTER_KEYS.JIRA_SERVER,
|
||||
external_source: source,
|
||||
created_by: issue.fields.creator?.name,
|
||||
name: issue.fields.summary ?? "Untitled",
|
||||
description_html: description,
|
||||
@@ -84,13 +90,13 @@ export const transformIssue = (
|
||||
};
|
||||
|
||||
export const transformIssueV2 = (
|
||||
resourceId: string,
|
||||
projectId: string,
|
||||
ctx: TTransformationContext,
|
||||
issue: IJiraIssue,
|
||||
resourceUrl: string,
|
||||
stateMap: IStateConfig[],
|
||||
priorityMap: IPriorityConfig[]
|
||||
): Partial<PlaneIssue> => {
|
||||
const { resourceId, projectId, source } = ctx;
|
||||
const targetState = getTargetState(stateMap, issue.fields.status);
|
||||
const targetPriority = getTargetPriority(priorityMap, issue.fields.priority);
|
||||
const attachments = getTargetAttachments(resourceId, projectId, issue.fields.attachment);
|
||||
@@ -111,11 +117,15 @@ export const transformIssueV2 = (
|
||||
issue.fields.labels.push("JIRA IMPORTED");
|
||||
|
||||
return {
|
||||
assignees: issue.fields.assignee?.emailAddress ? [issue.fields.assignee.emailAddress] : [],
|
||||
assignees: issue.fields.assignee?.emailAddress
|
||||
? [issue.fields.assignee.emailAddress]
|
||||
: issue.fields.assignee?.displayName
|
||||
? [issue.fields.assignee.displayName]
|
||||
: [],
|
||||
links,
|
||||
external_id: `${projectId}_${resourceId}_${issue.id}`,
|
||||
external_source: E_IMPORTER_KEYS.JIRA_SERVER,
|
||||
created_by: issue.fields.creator?.emailAddress,
|
||||
external_source: source,
|
||||
created_by: issue.fields.creator?.emailAddress || issue.fields.creator?.displayName,
|
||||
name: issue.fields.summary ?? "Untitled",
|
||||
description_html: description,
|
||||
target_date: issue.fields.duedate,
|
||||
@@ -136,20 +146,19 @@ export const transformLabel = (label: string): Partial<ExIssueLabel> => ({
|
||||
color: getRandomColor(),
|
||||
});
|
||||
|
||||
export const transformComment = (
|
||||
resourceId: string,
|
||||
projectId: string,
|
||||
comment: JiraComment
|
||||
): Partial<ExIssueComment> => ({
|
||||
external_id: `${projectId}_${resourceId}_${comment.id}`,
|
||||
external_source: E_IMPORTER_KEYS.JIRA_SERVER,
|
||||
created_at: getFormattedDate(comment.created),
|
||||
created_by: comment.author?.emailAddress,
|
||||
// @ts-expect-error
|
||||
comment_html: comment.body ?? "<p></p>",
|
||||
actor: comment.author?.emailAddress,
|
||||
issue: `${projectId}_${resourceId}_${comment.issue_id}`,
|
||||
});
|
||||
export const transformComment = (ctx: TTransformationContext, comment: JiraComment): Partial<ExIssueComment> => {
|
||||
const { resourceId, projectId, source } = ctx;
|
||||
return {
|
||||
external_id: `${projectId}_${resourceId}_${comment.id}`,
|
||||
external_source: source,
|
||||
created_at: comment.created,
|
||||
created_by: comment.author?.emailAddress || comment.author?.displayName,
|
||||
// @ts-expect-error
|
||||
comment_html: comment.renderedBody ? comment.renderedBody : (comment.body ?? "<p></p>"),
|
||||
actor: comment.author?.emailAddress || comment.author?.displayName,
|
||||
issue: `${projectId}_${resourceId}_${comment.issue_id}`,
|
||||
};
|
||||
};
|
||||
|
||||
export const transformUser = (user: ImportedJiraUser): Partial<PlaneUser> => {
|
||||
const [first_name, last_name] = user.full_name.split(" ");
|
||||
@@ -165,42 +174,46 @@ export const transformUser = (user: ImportedJiraUser): Partial<PlaneUser> => {
|
||||
};
|
||||
};
|
||||
|
||||
export const transformSprint = (resourceId: string, projectId: string, sprint: JiraSprint): Partial<ExCycle> => ({
|
||||
external_id: `${projectId}_${resourceId}_${sprint.sprint.id.toString()}`,
|
||||
external_source: E_IMPORTER_KEYS.JIRA_SERVER,
|
||||
name: sprint.sprint.name,
|
||||
start_date: getFormattedDate(sprint.sprint.startDate),
|
||||
end_date: getFormattedDate(sprint.sprint.endDate),
|
||||
created_at: getFormattedDate(sprint.sprint.createdDate),
|
||||
issues: sprint.issues.map((issue) => `${projectId}_${resourceId}_${issue.id}`),
|
||||
});
|
||||
export const transformSprint = (ctx: TTransformationContext, sprint: JiraSprint): Partial<ExCycle> => {
|
||||
const { resourceId, projectId, source } = ctx;
|
||||
return {
|
||||
external_id: `${projectId}_${resourceId}_${sprint.sprint.id.toString()}`,
|
||||
external_source: source,
|
||||
name: sprint.sprint.name,
|
||||
start_date: getFormattedDate(sprint.sprint.startDate),
|
||||
end_date: getFormattedDate(sprint.sprint.endDate),
|
||||
created_at: getFormattedDate(sprint.sprint.createdDate),
|
||||
issues: sprint.issues.map((issue) => `${projectId}_${resourceId}_${issue.id}`),
|
||||
};
|
||||
};
|
||||
|
||||
export const transformComponent = (
|
||||
resourceId: string,
|
||||
projectId: string,
|
||||
component: JiraComponent
|
||||
): Partial<ExModule> => ({
|
||||
external_id: `${projectId}_${resourceId}_${component.component.id}`,
|
||||
external_source: E_IMPORTER_KEYS.JIRA_SERVER,
|
||||
name: component.component.name,
|
||||
issues: component.issues.map((issue) => `${projectId}_${resourceId}_${issue.id}`),
|
||||
});
|
||||
export const transformComponent = (ctx: TTransformationContext, component: JiraComponent): Partial<ExModule> => {
|
||||
const { resourceId, projectId, source } = ctx;
|
||||
return {
|
||||
external_id: `${projectId}_${resourceId}_${component.component.id}`,
|
||||
external_source: source,
|
||||
name: component.component.name,
|
||||
issues: component.issues.map((issue) => `${projectId}_${resourceId}_${issue.id}`),
|
||||
};
|
||||
};
|
||||
|
||||
export const transformComponentV2 = (
|
||||
resourceId: string,
|
||||
projectId: string,
|
||||
ctx: TTransformationContext,
|
||||
component: ComponentWithIssueCount
|
||||
): Partial<ExModule> => ({
|
||||
external_id: `${projectId}_${resourceId}_${component.id}`,
|
||||
external_source: E_IMPORTER_KEYS.JIRA_SERVER,
|
||||
name: component.name,
|
||||
});
|
||||
): Partial<ExModule> => {
|
||||
const { resourceId, projectId, source } = ctx;
|
||||
return {
|
||||
external_id: `${projectId}_${resourceId}_${component.id}`,
|
||||
external_source: source,
|
||||
name: component.name,
|
||||
};
|
||||
};
|
||||
|
||||
export const transformIssueType = (
|
||||
resourceId: string,
|
||||
projectId: string,
|
||||
ctx: TTransformationContext,
|
||||
issueType: JiraIssueTypeDetails
|
||||
): Partial<ExIssueType> => {
|
||||
const { resourceId, projectId, source } = ctx;
|
||||
const isEpic = issueType.name?.toLowerCase().includes("epic");
|
||||
|
||||
return {
|
||||
@@ -209,15 +222,15 @@ export const transformIssueType = (
|
||||
is_active: true,
|
||||
is_epic: isEpic,
|
||||
external_id: `${projectId}_${resourceId}_${issueType.id}`,
|
||||
external_source: E_IMPORTER_KEYS.JIRA_SERVER,
|
||||
external_source: source,
|
||||
};
|
||||
};
|
||||
|
||||
export const transformIssueFields = (
|
||||
resourceId: string,
|
||||
projectId: string,
|
||||
ctx: TTransformationContext,
|
||||
issueField: JiraIssueField
|
||||
): Partial<ExIssueProperty> | undefined => {
|
||||
const { resourceId, projectId, source } = ctx;
|
||||
if (
|
||||
!issueField.schema ||
|
||||
!issueField.scope?.type ||
|
||||
@@ -231,7 +244,7 @@ export const transformIssueFields = (
|
||||
|
||||
return {
|
||||
external_id: `${projectId}_${resourceId}_${fieldId}`,
|
||||
external_source: E_IMPORTER_KEYS.JIRA_SERVER,
|
||||
external_source: source,
|
||||
display_name: issueField.name,
|
||||
type_id: issueField.scope?.type ? `${projectId}_${resourceId}_${issueField.scope?.type}` : undefined,
|
||||
is_required: false,
|
||||
@@ -241,26 +254,28 @@ export const transformIssueFields = (
|
||||
};
|
||||
|
||||
export const transformIssueFieldOptions = (
|
||||
resourceId: string,
|
||||
projectId: string,
|
||||
ctx: TTransformationContext,
|
||||
issueFieldOption: JiraIssueFieldOptions
|
||||
): Partial<ExIssuePropertyOption> => ({
|
||||
external_id: `${projectId}_${resourceId}_${issueFieldOption.id}`,
|
||||
external_source: E_IMPORTER_KEYS.JIRA_SERVER,
|
||||
name: issueFieldOption.value,
|
||||
is_active: issueFieldOption.disabled ? false : true,
|
||||
property_id: `${projectId}_${resourceId}_${issueFieldOption.fieldId}`,
|
||||
});
|
||||
): Partial<ExIssuePropertyOption> => {
|
||||
const { resourceId, projectId, source } = ctx;
|
||||
return {
|
||||
external_id: `${projectId}_${resourceId}_${issueFieldOption.id}`,
|
||||
external_source: source,
|
||||
name: issueFieldOption.value,
|
||||
is_active: issueFieldOption.disabled ? false : true,
|
||||
property_id: `${projectId}_${resourceId}_${issueFieldOption.fieldId}`,
|
||||
};
|
||||
};
|
||||
|
||||
export const transformIssuePropertyValues = (
|
||||
resourceId: string,
|
||||
projectId: string,
|
||||
ctx: TTransformationContext,
|
||||
issue: IJiraIssue,
|
||||
// eslint-disable-next-line no-undef
|
||||
planeIssueProperties: Map<string, Partial<ExIssueProperty>>, // TODO: replace Map with Record<string, Partial<ExIssueProperty>> in the future
|
||||
// eslint-disable-next-line no-undef
|
||||
jiraCustomFieldMap: Map<string, string> // TODO: replace Map with Record<string, string> in the future
|
||||
): TPropertyValuesPayload => {
|
||||
const { resourceId, projectId } = ctx;
|
||||
// Get all custom fields that are present in the issue and are also present in the plane issue properties
|
||||
const customFieldKeysToTransform = Object.keys(issue.fields).filter(
|
||||
(key) => key.startsWith("customfield_") && planeIssueProperties.has(key)
|
||||
@@ -283,18 +298,18 @@ export const transformIssuePropertyValues = (
|
||||
};
|
||||
|
||||
export const transformDefaultPropertyValues = (
|
||||
resourceId: string,
|
||||
projectId: string,
|
||||
ctx: TTransformationContext,
|
||||
issue: IJiraIssue,
|
||||
issueTypeId: string
|
||||
): TPropertyValuesPayload => {
|
||||
const { resourceId, projectId, source } = ctx;
|
||||
const propertyValuesPayload: TPropertyValuesPayload = {};
|
||||
|
||||
// Fix Versions
|
||||
if (issue.fields.fixVersions && Array.isArray(issue.fields.fixVersions)) {
|
||||
const fixVersionExternalId = `${resourceId}-${projectId}-${issueTypeId}-fix-version`;
|
||||
propertyValuesPayload[fixVersionExternalId] = issue.fields.fixVersions.map((version: any) => ({
|
||||
external_source: "JIRA_SERVER",
|
||||
external_source: source,
|
||||
external_id: version.id ? String(version.id) : undefined,
|
||||
value: version.name || "",
|
||||
}));
|
||||
@@ -304,7 +319,7 @@ export const transformDefaultPropertyValues = (
|
||||
if (issue.fields.versions && Array.isArray(issue.fields.versions)) {
|
||||
const affectedVersionExternalId = `${resourceId}-${projectId}-${issueTypeId}-affected-version`;
|
||||
propertyValuesPayload[affectedVersionExternalId] = issue.fields.versions.map((version: any) => ({
|
||||
external_source: "JIRA_SERVER",
|
||||
external_source: source,
|
||||
external_id: version.id ? String(version.id) : undefined,
|
||||
value: version.name || "",
|
||||
}));
|
||||
@@ -315,8 +330,8 @@ export const transformDefaultPropertyValues = (
|
||||
const reporterExternalId = `${resourceId}-${projectId}-${issueTypeId}-reporter`;
|
||||
propertyValuesPayload[reporterExternalId] = [
|
||||
{
|
||||
external_source: "JIRA_SERVER",
|
||||
external_id: issue.fields.reporter.emailAddress || issue.fields.reporter.name,
|
||||
external_source: source,
|
||||
external_id: issue.fields.reporter.emailAddress || issue.fields.reporter.displayName || "",
|
||||
value: issue.fields.reporter.emailAddress || issue.fields.reporter.displayName || "",
|
||||
},
|
||||
];
|
||||
|
||||
@@ -191,11 +191,11 @@ export const getPropertyValues = (
|
||||
break;
|
||||
case "com.atlassian.jira.plugin.system.customfieldtypes:userpicker":
|
||||
// Handle userpicker
|
||||
if (value.emailAddress) {
|
||||
if (value.emailAddress || value.displayName) {
|
||||
propertyValues.push({
|
||||
...commonPropertyProp,
|
||||
external_id: value.emailAddress,
|
||||
value: value.emailAddress,
|
||||
external_id: value.emailAddress ?? value.displayName,
|
||||
value: value.emailAddress ?? value.displayName,
|
||||
});
|
||||
}
|
||||
break;
|
||||
@@ -277,11 +277,11 @@ export const getPropertyValues = (
|
||||
// Handle multiuserpicker
|
||||
if (Array.isArray(value)) {
|
||||
value.forEach((val) => {
|
||||
if (val.emailAddress) {
|
||||
if (val.emailAddress || val.displayName) {
|
||||
propertyValues.push({
|
||||
...commonPropertyProp,
|
||||
external_id: val.emailAddress,
|
||||
value: val.emailAddress,
|
||||
external_id: val.emailAddress ?? val.displayName,
|
||||
value: val.emailAddress ?? val.displayName,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
// services
|
||||
import axios, { AxiosError } from "axios";
|
||||
import type { AxiosError } from "axios";
|
||||
import axios from "axios";
|
||||
import type { Paginated } from "jira.js";
|
||||
import { Board as BoardClient } from "jira.js/out/agile";
|
||||
import { Version2Client } from "jira.js/out/version2";
|
||||
@@ -7,29 +8,45 @@ import type {
|
||||
CustomFieldContextOption,
|
||||
FieldDetails,
|
||||
Issue,
|
||||
IssueTypeDetails,
|
||||
JiraStatus,
|
||||
Project,
|
||||
IssueTypeDetails,
|
||||
} from "jira.js/out/version2/models";
|
||||
import type { JiraCustomFieldWithCtx } from "@/jira-server/types/custom-fields";
|
||||
import type { JiraApiUser, JiraProps } from "..";
|
||||
import { fetchPaginatedData } from "..";
|
||||
import { fetchPaginatedData, EJiraAuthenticationType } from "..";
|
||||
|
||||
export class JiraV2Service {
|
||||
private jiraClient: Version2Client;
|
||||
private hostname: string;
|
||||
private patToken: string;
|
||||
private email: string;
|
||||
private authenticationType: EJiraAuthenticationType = EJiraAuthenticationType.PERSONAL_ACCESS_TOKEN;
|
||||
|
||||
constructor(props: JiraProps) {
|
||||
this.hostname = props.hostname;
|
||||
this.patToken = props.patToken;
|
||||
this.email = props.email;
|
||||
this.authenticationType = props.authenticationType;
|
||||
|
||||
this.jiraClient = new Version2Client({
|
||||
host: props.hostname,
|
||||
authentication: {
|
||||
personalAccessToken: props.patToken,
|
||||
},
|
||||
});
|
||||
if (this.authenticationType === EJiraAuthenticationType.BASIC) {
|
||||
this.jiraClient = new Version2Client({
|
||||
host: props.hostname,
|
||||
authentication: {
|
||||
basic: {
|
||||
email: this.email,
|
||||
apiToken: props.patToken,
|
||||
},
|
||||
},
|
||||
});
|
||||
} else {
|
||||
this.jiraClient = new Version2Client({
|
||||
host: props.hostname,
|
||||
authentication: {
|
||||
personalAccessToken: props.patToken,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
this.jiraClient.handleFailedResponse = async (request) => {
|
||||
const error = request as AxiosError;
|
||||
@@ -70,6 +87,14 @@ export class JiraV2Service {
|
||||
})) as JiraApiUser[];
|
||||
}
|
||||
|
||||
async getIssueWorklogs(issueId: string, startAt: number, maxResults: number) {
|
||||
return this.jiraClient.issueWorklogs.getIssueWorklog({
|
||||
issueIdOrKey: issueId,
|
||||
startAt: startAt,
|
||||
maxResults: maxResults,
|
||||
});
|
||||
}
|
||||
|
||||
// Verified
|
||||
async getNumberOfIssues(projectKey: string) {
|
||||
const issues = await this.jiraClient.issueSearch.searchForIssuesUsingJql({
|
||||
@@ -239,15 +264,11 @@ export class JiraV2Service {
|
||||
// Verified
|
||||
async getPaginatedIssueTypes(projectId: string, startAt?: number, maxResults?: number) {
|
||||
try {
|
||||
return axios
|
||||
.get(
|
||||
`${this.hostname}/rest/api/2/issuetype/page?projectIds=${projectId}&startAt=${startAt}&maxResults=${maxResults}`,
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.patToken}`,
|
||||
},
|
||||
}
|
||||
)
|
||||
return await this.jiraClient
|
||||
.sendRequestFullResponse<Paginated<IssueTypeDetails>>({
|
||||
method: "GET",
|
||||
url: `${this.hostname}/rest/api/2/issuetype/page?projectIds=${projectId}&startAt=${startAt}&maxResults=${maxResults}`,
|
||||
})
|
||||
.then((res) => res.data as Paginated<IssueTypeDetails>);
|
||||
} catch (e) {
|
||||
console.error("error getProjectIssueTypes", e);
|
||||
@@ -264,15 +285,13 @@ export class JiraV2Service {
|
||||
|
||||
async getCustomFieldsWithContext(projectId?: string) {
|
||||
try {
|
||||
return await fetchPaginatedData<JiraCustomFieldWithCtx>((startAt) =>
|
||||
axios
|
||||
.get(`${this.hostname}/rest/api/2/customFields?startAt=${startAt}&projectIds=${projectId ?? ""}`, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.patToken}`,
|
||||
},
|
||||
})
|
||||
.then((res) => res.data as Paginated<JiraCustomFieldWithCtx>)
|
||||
);
|
||||
return await fetchPaginatedData<JiraCustomFieldWithCtx>(async (startAt) => {
|
||||
const result = await this.jiraClient.sendRequestFullResponse<Paginated<JiraCustomFieldWithCtx>>({
|
||||
method: "GET",
|
||||
url: `${this.hostname}/rest/api/2/customFields?startAt=${startAt}&projectIds=${projectId ?? ""}`,
|
||||
});
|
||||
return result.data;
|
||||
});
|
||||
} catch (e) {
|
||||
console.error("error getCustomFieldsWithContext", e);
|
||||
throw e;
|
||||
|
||||
@@ -15,8 +15,15 @@ import type { ExProject, ExState } from "@plane/sdk";
|
||||
export type JiraProps = {
|
||||
hostname: string;
|
||||
patToken: string;
|
||||
email: string;
|
||||
authenticationType: EJiraAuthenticationType;
|
||||
};
|
||||
|
||||
export enum EJiraAuthenticationType {
|
||||
BASIC = "basic",
|
||||
PERSONAL_ACCESS_TOKEN = "personalAccessToken",
|
||||
}
|
||||
|
||||
export type ImportedJiraUser = {
|
||||
user_id: string;
|
||||
user_name: string;
|
||||
@@ -33,6 +40,7 @@ export type JiraApiUser = {
|
||||
key: string;
|
||||
name: string;
|
||||
emailAddress?: string;
|
||||
accountType?: string;
|
||||
avatarUrls: {
|
||||
"16x16": string;
|
||||
"24x24": string;
|
||||
|
||||
@@ -64,6 +64,37 @@ export async function pullIssues(client: JiraService, projectKey: string, from?:
|
||||
return issues;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull issues using nextPageToken pagination (Jira Cloud Enhanced Search API)
|
||||
* Note: Enhanced Search API does not return total count, only nextPageToken for pagination
|
||||
*/
|
||||
export async function pullIssuesV2(
|
||||
ctx: {
|
||||
client: JiraService;
|
||||
nextPageToken?: string;
|
||||
maxResults?: number;
|
||||
},
|
||||
projectKey: string,
|
||||
// We are using this property for the pagination context
|
||||
total = 0,
|
||||
from?: Date
|
||||
): Promise<{
|
||||
items: IJiraIssue[];
|
||||
hasMore: boolean;
|
||||
total?: number;
|
||||
nextPageToken?: string;
|
||||
}> {
|
||||
const { client, nextPageToken } = ctx;
|
||||
const result = await client.getProjectIssues(projectKey, nextPageToken, from ? formatDateStringForHHMM(from) : "");
|
||||
|
||||
return {
|
||||
items: result.issues || [],
|
||||
hasMore: !!result.nextPageToken,
|
||||
total: total,
|
||||
nextPageToken: result.nextPageToken,
|
||||
};
|
||||
}
|
||||
|
||||
export async function pullComments(issues: IJiraIssue[], client: JiraService): Promise<any[]> {
|
||||
const comments: JiraComment[] = [];
|
||||
|
||||
@@ -168,10 +199,11 @@ export const pullIssueFields = async (
|
||||
const customFields: JiraIssueField[] = [];
|
||||
try {
|
||||
// initialize fields
|
||||
const fields: FieldDetails[] = await client.getCustomFields();
|
||||
const allFields: FieldDetails[] = await client.getCustomFields();
|
||||
const filteredFields = allFields.filter((field) => field.custom);
|
||||
|
||||
// get all field contexts
|
||||
for (const field of fields) {
|
||||
for (const field of filteredFields) {
|
||||
// skip if field has no id
|
||||
if (!field.id) continue;
|
||||
|
||||
@@ -226,7 +258,11 @@ export const pullIssueFields = async (
|
||||
(startAt) => client.getIssueFieldOptions(field.id as string, Number(issueTypeContext.contextId), startAt),
|
||||
(values: CustomFieldContextOption[]) => {
|
||||
values.map((value) => {
|
||||
if (field.id) fieldOptions.push({ ...value, fieldId: field.id });
|
||||
if (field.id)
|
||||
fieldOptions.push({
|
||||
...value,
|
||||
fieldId: field.id.includes("customfield_") ? field.id.split("_").pop()! : field.id,
|
||||
});
|
||||
});
|
||||
},
|
||||
"values"
|
||||
|
||||
@@ -67,10 +67,16 @@ export class AssetService extends APIService {
|
||||
external_source?: string;
|
||||
}
|
||||
): Promise<string> {
|
||||
let fileType = file.type;
|
||||
|
||||
// If the file type includes a semicolon, we need to remove the charset
|
||||
if (fileType.includes(";")) {
|
||||
fileType = fileType.split(";")[0];
|
||||
}
|
||||
// First get the presigned URL
|
||||
const uploadResponse = await this.createAsset(workspaceSlug, {
|
||||
name,
|
||||
type: file.type,
|
||||
type: fileType,
|
||||
size,
|
||||
project_id: options?.project_id,
|
||||
external_id: options?.external_id,
|
||||
|
||||
Reference in New Issue
Block a user