Rework logic function module (#17588)
core-modules/logic-function/
├── logic-function.module.ts
├── logic-function-executor/
│ ├── logic-function-executor.module.ts
│ ├── commands/
│ │ └── add-packages.command.ts
│ ├── constants/
│ │ └── logic-function-executor.constants.ts
│ ├── factories/
│ │ └── logic-function-module.factory.ts
│ ├── interfaces/
│ │ └── logic-function-executor.interface.ts
│ └── services/
│ └── logic-function-executor.service.ts
├── logic-function-build/
│ ├── logic-function-build.module.ts
│ ├── services/
│ │ └── logic-function-build.service.ts
│ └── utils/
│ └── get-logic-function-base-folder-path.util.ts
├── logic-function-drivers/
│ ├── logic-function-drivers.module.ts
│ ├── constants/
│ │ └── ...
│ ├── drivers/
│ │ ├── disabled.driver.ts
│ │ ├── lambda.driver.ts
│ │ └── local.driver.ts
│ ├── interfaces/
│ │ └── logic-function-executor-driver.interface.ts
│ ├── layers/
│ │ └── ...
│ └── utils/
│ └── ...
├── logic-function-layer/
│ ├── logic-function-layer.module.ts
│ └── services/
│ └── logic-function-layer.service.ts
└── logic-function-trigger/
├── logic-function-trigger.module.ts
├── jobs/
│ └── logic-function-trigger.job.ts
└── triggers/
├── cron/
├── database-event/
└── route/
├── exceptions/
├── services/
│ └── route-trigger.service.ts
└── utils/
This commit is contained in:
+9
@@ -0,0 +1,9 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { LogicFunctionBuildService } from 'src/engine/core-modules/logic-function/logic-function-build/services/logic-function-build.service';
|
||||
|
||||
@Module({
|
||||
providers: [LogicFunctionBuildService],
|
||||
exports: [LogicFunctionBuildService],
|
||||
})
|
||||
export class LogicFunctionBuildModule {}
|
||||
+120
@@ -0,0 +1,120 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import fs from 'fs/promises';
|
||||
import { dirname, join } from 'path';
|
||||
|
||||
import { build } from 'esbuild';
|
||||
import { FileFolder } from 'twenty-shared/types';
|
||||
|
||||
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
|
||||
import { LambdaBuildDirectoryManager } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/lambda-build-directory-manager';
|
||||
import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
|
||||
import {
|
||||
getLogicFunctionBaseFolderPath,
|
||||
getRelativePathFromBase,
|
||||
} from 'src/engine/core-modules/logic-function/logic-function-build/utils/get-logic-function-base-folder-path.util';
|
||||
|
||||
export type FunctionBuildParams = {
|
||||
flatLogicFunction: FlatLogicFunction;
|
||||
applicationUniversalIdentifier: string;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class LogicFunctionBuildService {
|
||||
constructor(private readonly fileStorageService: FileStorageService) {}
|
||||
|
||||
async isBuilt({
|
||||
flatLogicFunction,
|
||||
applicationUniversalIdentifier,
|
||||
}: FunctionBuildParams): Promise<boolean> {
|
||||
return await this.fileStorageService.checkFileExists_v2({
|
||||
workspaceId: flatLogicFunction.workspaceId,
|
||||
applicationUniversalIdentifier,
|
||||
fileFolder: FileFolder.BuiltLogicFunction,
|
||||
resourcePath: flatLogicFunction.builtHandlerPath,
|
||||
});
|
||||
}
|
||||
|
||||
async buildAndUpload({
|
||||
flatLogicFunction,
|
||||
applicationUniversalIdentifier,
|
||||
}: FunctionBuildParams): Promise<void> {
|
||||
const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
|
||||
|
||||
try {
|
||||
const { sourceTemporaryDir } = await lambdaBuildDirectoryManager.init();
|
||||
|
||||
const baseFolderPath = getLogicFunctionBaseFolderPath(
|
||||
flatLogicFunction.sourceHandlerPath,
|
||||
);
|
||||
|
||||
await this.fileStorageService.downloadFolder_v2({
|
||||
workspaceId: flatLogicFunction.workspaceId,
|
||||
applicationUniversalIdentifier,
|
||||
fileFolder: FileFolder.Source,
|
||||
resourcePath: baseFolderPath,
|
||||
localPath: sourceTemporaryDir,
|
||||
});
|
||||
|
||||
const relativeSourcePath = getRelativePathFromBase(
|
||||
flatLogicFunction.sourceHandlerPath,
|
||||
baseFolderPath,
|
||||
);
|
||||
const relativeBuiltPath = getRelativePathFromBase(
|
||||
flatLogicFunction.builtHandlerPath,
|
||||
baseFolderPath,
|
||||
);
|
||||
|
||||
const builtBundleFilePath = await this.buildInMemory({
|
||||
sourceTemporaryDir,
|
||||
sourceHandlerPath: relativeSourcePath,
|
||||
builtHandlerPath: relativeBuiltPath,
|
||||
});
|
||||
|
||||
const builtFile = await fs.readFile(builtBundleFilePath, 'utf-8');
|
||||
|
||||
await this.fileStorageService.writeFile_v2({
|
||||
workspaceId: flatLogicFunction.workspaceId,
|
||||
applicationUniversalIdentifier,
|
||||
fileFolder: FileFolder.BuiltLogicFunction,
|
||||
resourcePath: flatLogicFunction.builtHandlerPath,
|
||||
sourceFile: builtFile,
|
||||
mimeType: 'application/javascript',
|
||||
settings: {
|
||||
isTemporaryFile: false,
|
||||
toDelete: false,
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
await lambdaBuildDirectoryManager.clean();
|
||||
}
|
||||
}
|
||||
|
||||
private async buildInMemory({
|
||||
sourceTemporaryDir,
|
||||
sourceHandlerPath,
|
||||
builtHandlerPath,
|
||||
}: {
|
||||
sourceTemporaryDir: string;
|
||||
sourceHandlerPath: string;
|
||||
builtHandlerPath: string;
|
||||
}): Promise<string> {
|
||||
const entryFilePath = join(sourceTemporaryDir, sourceHandlerPath);
|
||||
const builtBundleFilePath = join(sourceTemporaryDir, builtHandlerPath);
|
||||
|
||||
await fs.mkdir(dirname(builtBundleFilePath), { recursive: true });
|
||||
|
||||
await build({
|
||||
entryPoints: [entryFilePath],
|
||||
outfile: builtBundleFilePath,
|
||||
platform: 'node',
|
||||
format: 'esm',
|
||||
target: 'es2017',
|
||||
bundle: true,
|
||||
sourcemap: true,
|
||||
packages: 'external',
|
||||
});
|
||||
|
||||
return builtBundleFilePath;
|
||||
}
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
import { dirname } from 'path';
|
||||
|
||||
export const getLogicFunctionBaseFolderPath = (handlerPath: string): string => {
|
||||
return dirname(dirname(handlerPath));
|
||||
};
|
||||
|
||||
export const getRelativePathFromBase = (
|
||||
handlerPath: string,
|
||||
baseFolderPath: string,
|
||||
): string => {
|
||||
return handlerPath.replace(`${baseFolderPath}/`, '');
|
||||
};
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
import { promises as fs } from 'fs';
|
||||
import { randomBytes } from 'crypto';
|
||||
|
||||
export const handler = async (event) => {
|
||||
const randomId = randomBytes(16).toString('hex');
|
||||
|
||||
const mainPath = `/tmp/${randomId}.mjs`;
|
||||
|
||||
// eslint-disable-next-line no-undef
|
||||
const oldProcessEnv = { ...process.env };
|
||||
|
||||
try {
|
||||
const { code, params, env, handlerName } = event;
|
||||
|
||||
await fs.writeFile(mainPath, code, 'utf8');
|
||||
|
||||
// eslint-disable-next-line no-undef
|
||||
process.env = { ...process.env, ...(env ?? {}) };
|
||||
|
||||
const mainFile = await import(mainPath);
|
||||
|
||||
return await mainFile[handlerName](params);
|
||||
} finally {
|
||||
await fs.rm(mainPath, { force: true });
|
||||
// eslint-disable-next-line no-undef
|
||||
process.env = oldProcessEnv;
|
||||
}
|
||||
};
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
|
||||
export const LOGIC_FUNCTION_EXECUTOR_TMPDIR_FOLDER = join(
|
||||
tmpdir(),
|
||||
'logic-function-executor-tmpdir',
|
||||
);
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
export const SEED_PROJECT_INPUT_SCHEMA = {
|
||||
a: null,
|
||||
b: null,
|
||||
};
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
export const main = async (params: {
|
||||
a: string;
|
||||
b: number;
|
||||
}): Promise<object> => {
|
||||
const { a, b } = params;
|
||||
|
||||
// Rename the parameters and code below with your own logic
|
||||
// This is just an example
|
||||
const message = `Hello, input: ${a} and ${b}`;
|
||||
|
||||
return { message };
|
||||
};
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
import {
|
||||
type LogicFunctionExecutorDriver,
|
||||
type LogicFunctionExecuteResult,
|
||||
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-executor-driver.interface';
|
||||
|
||||
import {
|
||||
LogicFunctionException,
|
||||
LogicFunctionExceptionCode,
|
||||
} from 'src/engine/metadata-modules/logic-function/logic-function.exception';
|
||||
|
||||
export class DisabledDriver implements LogicFunctionExecutorDriver {
|
||||
async delete(): Promise<void> {
|
||||
// No-op when disabled
|
||||
}
|
||||
|
||||
async execute(): Promise<LogicFunctionExecuteResult> {
|
||||
throw new LogicFunctionException(
|
||||
'Logic function execution is disabled. Set LOGIC_FUNCTION_TYPE to LOCAL or LAMBDA to enable.',
|
||||
LogicFunctionExceptionCode.LOGIC_FUNCTION_DISABLED,
|
||||
);
|
||||
}
|
||||
}
|
||||
+389
@@ -0,0 +1,389 @@
|
||||
import * as fs from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
|
||||
import {
|
||||
CreateFunctionCommand,
|
||||
type CreateFunctionCommandInput,
|
||||
DeleteFunctionCommand,
|
||||
GetFunctionCommand,
|
||||
InvokeCommand,
|
||||
type InvokeCommandInput,
|
||||
Lambda,
|
||||
type LambdaClientConfig,
|
||||
ListLayerVersionsCommand,
|
||||
type ListLayerVersionsCommandInput,
|
||||
LogType,
|
||||
PublishLayerVersionCommand,
|
||||
type PublishLayerVersionCommandInput,
|
||||
ResourceNotFoundException,
|
||||
waitUntilFunctionUpdatedV2,
|
||||
} from '@aws-sdk/client-lambda';
|
||||
import { AssumeRoleCommand, STSClient } from '@aws-sdk/client-sts';
|
||||
import { FileFolder } from 'twenty-shared/types';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
|
||||
import {
|
||||
type LogicFunctionExecuteParams,
|
||||
type LogicFunctionExecuteResult,
|
||||
type LogicFunctionExecutorDriver,
|
||||
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-executor-driver.interface';
|
||||
|
||||
import { type FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
|
||||
import { copyAndBuildDependencies } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/copy-and-build-dependencies';
|
||||
import { copyExecutor } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/copy-executor';
|
||||
import { createZipFile } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/create-zip-file';
|
||||
import {
|
||||
LambdaBuildDirectoryManager,
|
||||
NODE_LAYER_SUBFOLDER,
|
||||
} from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/lambda-build-directory-manager';
|
||||
import { type FlatLogicFunctionLayer } from 'src/engine/metadata-modules/logic-function-layer/types/flat-logic-function-layer.type';
|
||||
import { LogicFunctionExecutionStatus } from 'src/engine/metadata-modules/logic-function/dtos/logic-function-execution-result.dto';
|
||||
import { LogicFunctionRuntime } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
|
||||
import {
|
||||
LogicFunctionException,
|
||||
LogicFunctionExceptionCode,
|
||||
} from 'src/engine/metadata-modules/logic-function/logic-function.exception';
|
||||
import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
|
||||
import { streamToBuffer } from 'src/utils/stream-to-buffer';
|
||||
|
||||
const UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS = 60;
|
||||
const CREDENTIALS_DURATION_IN_SECONDS = 60 * 60; // 1h
|
||||
|
||||
type LambdaDriverExecutorPayload = {
|
||||
code: string;
|
||||
params: object;
|
||||
env: Record<string, string>;
|
||||
handlerName: string;
|
||||
};
|
||||
|
||||
export interface LambdaDriverOptions extends LambdaClientConfig {
|
||||
fileStorageService: FileStorageService;
|
||||
region: string;
|
||||
lambdaRole: string;
|
||||
subhostingRole?: string;
|
||||
}
|
||||
|
||||
export class LambdaDriver implements LogicFunctionExecutorDriver {
|
||||
private lambdaClient: Lambda | undefined;
|
||||
private credentialsExpiry: Date | null = null;
|
||||
private readonly options: LambdaDriverOptions;
|
||||
private readonly fileStorageService: FileStorageService;
|
||||
|
||||
constructor(options: LambdaDriverOptions) {
|
||||
this.options = options;
|
||||
this.lambdaClient = undefined;
|
||||
this.fileStorageService = options.fileStorageService;
|
||||
}
|
||||
|
||||
private async getLambdaClient() {
|
||||
if (
|
||||
!isDefined(this.lambdaClient) ||
|
||||
(isDefined(this.options.subhostingRole) &&
|
||||
isDefined(this.credentialsExpiry) &&
|
||||
new Date() >= this.credentialsExpiry)
|
||||
) {
|
||||
this.lambdaClient = new Lambda({
|
||||
...this.options,
|
||||
...(isDefined(this.options.subhostingRole) && {
|
||||
credentials: await this.getAssumeRoleCredentials(),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
return this.lambdaClient;
|
||||
}
|
||||
|
||||
private async getAssumeRoleCredentials() {
|
||||
const stsClient = new STSClient({ region: this.options.region });
|
||||
|
||||
this.credentialsExpiry = new Date(
|
||||
Date.now() + (CREDENTIALS_DURATION_IN_SECONDS - 60 * 5) * 1000,
|
||||
);
|
||||
|
||||
const assumeRoleCommand = new AssumeRoleCommand({
|
||||
RoleArn: this.options.subhostingRole,
|
||||
RoleSessionName: 'LambdaSession',
|
||||
DurationSeconds: CREDENTIALS_DURATION_IN_SECONDS,
|
||||
});
|
||||
|
||||
const { Credentials } = await stsClient.send(assumeRoleCommand);
|
||||
|
||||
if (
|
||||
!isDefined(Credentials) ||
|
||||
!isDefined(Credentials.AccessKeyId) ||
|
||||
!isDefined(Credentials.SecretAccessKey) ||
|
||||
!isDefined(Credentials.SessionToken)
|
||||
) {
|
||||
throw new Error('Failed to assume role');
|
||||
}
|
||||
|
||||
return {
|
||||
accessKeyId: Credentials.AccessKeyId,
|
||||
secretAccessKey: Credentials.SecretAccessKey,
|
||||
sessionToken: Credentials.SessionToken,
|
||||
};
|
||||
}
|
||||
|
||||
private async waitFunctionUpdates(
|
||||
flatLogicFunction: FlatLogicFunction,
|
||||
maxWaitTime: number = UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS,
|
||||
) {
|
||||
const waitParams = {
|
||||
FunctionName: flatLogicFunction.id,
|
||||
};
|
||||
|
||||
await waitUntilFunctionUpdatedV2(
|
||||
{ client: await this.getLambdaClient(), maxWaitTime },
|
||||
waitParams,
|
||||
);
|
||||
}
|
||||
|
||||
private getLayerName(flatLogicFunctionLayer: FlatLogicFunctionLayer) {
|
||||
return flatLogicFunctionLayer.checksum;
|
||||
}
|
||||
|
||||
private async createLayerIfNotExists(
|
||||
flatLogicFunctionLayer: FlatLogicFunctionLayer,
|
||||
): Promise<string> {
|
||||
const layerName = this.getLayerName(flatLogicFunctionLayer);
|
||||
|
||||
const listLayerParams: ListLayerVersionsCommandInput = {
|
||||
LayerName: layerName,
|
||||
MaxItems: 1,
|
||||
};
|
||||
|
||||
const listLayerCommand = new ListLayerVersionsCommand(listLayerParams);
|
||||
|
||||
const listLayerResult = await (
|
||||
await this.getLambdaClient()
|
||||
).send(listLayerCommand);
|
||||
|
||||
if (isDefined(listLayerResult.LayerVersions?.[0]?.LayerVersionArn)) {
|
||||
return listLayerResult.LayerVersions[0].LayerVersionArn;
|
||||
}
|
||||
|
||||
const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
|
||||
const { sourceTemporaryDir, lambdaZipPath } =
|
||||
await lambdaBuildDirectoryManager.init();
|
||||
|
||||
const nodeDependenciesFolder = join(
|
||||
sourceTemporaryDir,
|
||||
NODE_LAYER_SUBFOLDER,
|
||||
);
|
||||
|
||||
await copyAndBuildDependencies(
|
||||
nodeDependenciesFolder,
|
||||
flatLogicFunctionLayer,
|
||||
);
|
||||
|
||||
await createZipFile(sourceTemporaryDir, lambdaZipPath);
|
||||
|
||||
const params: PublishLayerVersionCommandInput = {
|
||||
LayerName: layerName,
|
||||
Content: {
|
||||
ZipFile: await fs.readFile(lambdaZipPath),
|
||||
},
|
||||
CompatibleRuntimes: [
|
||||
LogicFunctionRuntime.NODE18,
|
||||
LogicFunctionRuntime.NODE22,
|
||||
],
|
||||
};
|
||||
|
||||
const command = new PublishLayerVersionCommand(params);
|
||||
|
||||
const result = await (await this.getLambdaClient()).send(command);
|
||||
|
||||
await lambdaBuildDirectoryManager.clean();
|
||||
|
||||
if (!isDefined(result.LayerVersionArn)) {
|
||||
throw new Error('new layer version arn if undefined');
|
||||
}
|
||||
|
||||
return result.LayerVersionArn;
|
||||
}
|
||||
|
||||
private async getLambdaExecutor(flatLogicFunction: FlatLogicFunction) {
|
||||
try {
|
||||
const getFunctionCommand: GetFunctionCommand = new GetFunctionCommand({
|
||||
FunctionName: flatLogicFunction.id,
|
||||
});
|
||||
|
||||
return await (await this.getLambdaClient()).send(getFunctionCommand);
|
||||
} catch (error) {
|
||||
if (!(error instanceof ResourceNotFoundException)) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async delete(flatLogicFunction: FlatLogicFunction) {
|
||||
const lambdaExecutor = await this.getLambdaExecutor(flatLogicFunction);
|
||||
|
||||
if (isDefined(lambdaExecutor)) {
|
||||
const deleteFunctionCommand = new DeleteFunctionCommand({
|
||||
FunctionName: flatLogicFunction.id,
|
||||
});
|
||||
|
||||
await (await this.getLambdaClient()).send(deleteFunctionCommand);
|
||||
}
|
||||
}
|
||||
|
||||
private async isAlreadyBuilt(
|
||||
flatLogicFunction: FlatLogicFunction,
|
||||
flatLogicFunctionLayer: FlatLogicFunctionLayer,
|
||||
) {
|
||||
const lambdaExecutor = await this.getLambdaExecutor(flatLogicFunction);
|
||||
|
||||
if (!isDefined(lambdaExecutor)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const layers = lambdaExecutor.Configuration?.Layers;
|
||||
|
||||
if (!isDefined(layers) || layers.length !== 1) {
|
||||
await this.delete(flatLogicFunction);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
const layerName = this.getLayerName(flatLogicFunctionLayer);
|
||||
|
||||
if (layers[0].Arn?.includes(layerName)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
await this.delete(flatLogicFunction);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private async build(
|
||||
flatLogicFunction: FlatLogicFunction,
|
||||
flatLogicFunctionLayer: FlatLogicFunctionLayer,
|
||||
) {
|
||||
if (await this.isAlreadyBuilt(flatLogicFunction, flatLogicFunctionLayer)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const layerArn = await this.createLayerIfNotExists(flatLogicFunctionLayer);
|
||||
|
||||
const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
|
||||
|
||||
const { sourceTemporaryDir, lambdaZipPath } =
|
||||
await lambdaBuildDirectoryManager.init();
|
||||
|
||||
await copyExecutor(sourceTemporaryDir);
|
||||
|
||||
await createZipFile(sourceTemporaryDir, lambdaZipPath);
|
||||
|
||||
const params: CreateFunctionCommandInput = {
|
||||
Code: {
|
||||
ZipFile: await fs.readFile(lambdaZipPath),
|
||||
},
|
||||
FunctionName: flatLogicFunction.id,
|
||||
Layers: [layerArn],
|
||||
Handler: 'index.handler',
|
||||
Role: this.options.lambdaRole,
|
||||
Runtime: flatLogicFunction.runtime,
|
||||
Timeout: 900, // timeout is handled by the logic function service
|
||||
};
|
||||
|
||||
const command = new CreateFunctionCommand(params);
|
||||
|
||||
await (await this.getLambdaClient()).send(command);
|
||||
|
||||
await lambdaBuildDirectoryManager.clean();
|
||||
}
|
||||
|
||||
private extractLogs(logString: string): string {
|
||||
const formattedLogString = Buffer.from(logString, 'base64')
|
||||
.toString('utf8')
|
||||
.split('\t')
|
||||
.join(' ');
|
||||
|
||||
return formattedLogString
|
||||
.replace(/^(START|END|REPORT).*\n?/gm, '')
|
||||
.replace(
|
||||
/^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z) [a-f0-9-]+ INFO /gm,
|
||||
'$1 INFO ',
|
||||
)
|
||||
.trim();
|
||||
}
|
||||
|
||||
async execute({
|
||||
flatLogicFunction,
|
||||
flatLogicFunctionLayer,
|
||||
applicationUniversalIdentifier,
|
||||
payload,
|
||||
env,
|
||||
}: LogicFunctionExecuteParams): Promise<LogicFunctionExecuteResult> {
|
||||
await this.build(flatLogicFunction, flatLogicFunctionLayer);
|
||||
|
||||
await this.waitFunctionUpdates(flatLogicFunction);
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
const compiledCode = (
|
||||
await streamToBuffer(
|
||||
await this.fileStorageService.readFile_v2({
|
||||
workspaceId: flatLogicFunction.workspaceId,
|
||||
applicationUniversalIdentifier,
|
||||
fileFolder: FileFolder.BuiltLogicFunction,
|
||||
resourcePath: flatLogicFunction.builtHandlerPath,
|
||||
}),
|
||||
)
|
||||
).toString('utf-8');
|
||||
|
||||
const executorPayload: LambdaDriverExecutorPayload = {
|
||||
params: payload,
|
||||
code: compiledCode,
|
||||
env: env ?? {},
|
||||
handlerName: flatLogicFunction.handlerName,
|
||||
};
|
||||
|
||||
const params: InvokeCommandInput = {
|
||||
FunctionName: flatLogicFunction.id,
|
||||
Payload: JSON.stringify(executorPayload),
|
||||
LogType: LogType.Tail,
|
||||
};
|
||||
|
||||
const command = new InvokeCommand(params);
|
||||
|
||||
try {
|
||||
const result = await (await this.getLambdaClient()).send(command);
|
||||
|
||||
const parsedResult = result.Payload
|
||||
? JSON.parse(result.Payload.transformToString())
|
||||
: {};
|
||||
|
||||
const logs = result.LogResult ? this.extractLogs(result.LogResult) : '';
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
if (result.FunctionError) {
|
||||
return {
|
||||
data: null,
|
||||
duration,
|
||||
status: LogicFunctionExecutionStatus.ERROR,
|
||||
error: parsedResult,
|
||||
logs,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
data: parsedResult,
|
||||
logs,
|
||||
duration,
|
||||
status: LogicFunctionExecutionStatus.SUCCESS,
|
||||
};
|
||||
} catch (error) {
|
||||
if (error instanceof ResourceNotFoundException) {
|
||||
throw new LogicFunctionException(
|
||||
`Function '${flatLogicFunction.id}' does not exist`,
|
||||
LogicFunctionExceptionCode.LOGIC_FUNCTION_NOT_FOUND,
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
+360
@@ -0,0 +1,360 @@
|
||||
import { promises as fs } from 'fs';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { join } from 'path';
|
||||
|
||||
import { FileFolder } from 'twenty-shared/types';
|
||||
|
||||
import {
|
||||
type LogicFunctionExecutorDriver,
|
||||
type LogicFunctionExecuteParams,
|
||||
type LogicFunctionExecuteResult,
|
||||
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-executor-driver.interface';
|
||||
|
||||
import { type FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
|
||||
import { LOGIC_FUNCTION_EXECUTOR_TMPDIR_FOLDER } from 'src/engine/core-modules/logic-function/logic-function-drivers/constants/logic-function-executor-tmpdir-folder';
|
||||
import { copyAndBuildDependencies } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/copy-and-build-dependencies';
|
||||
import { ConsoleListener } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/intercept-console';
|
||||
import { LambdaBuildDirectoryManager } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/lambda-build-directory-manager';
|
||||
import { type FlatLogicFunctionLayer } from 'src/engine/metadata-modules/logic-function-layer/types/flat-logic-function-layer.type';
|
||||
import { LogicFunctionExecutionStatus } from 'src/engine/metadata-modules/logic-function/dtos/logic-function-execution-result.dto';
|
||||
import {
|
||||
getLogicFunctionBaseFolderPath,
|
||||
getRelativePathFromBase,
|
||||
} from 'src/engine/core-modules/logic-function/logic-function-build/utils/get-logic-function-base-folder-path.util';
|
||||
|
||||
export interface LocalDriverOptions {
|
||||
fileStorageService: FileStorageService;
|
||||
}
|
||||
|
||||
export class LocalDriver implements LogicFunctionExecutorDriver {
|
||||
private readonly fileStorageService: FileStorageService;
|
||||
|
||||
constructor(options: LocalDriverOptions) {
|
||||
this.fileStorageService = options.fileStorageService;
|
||||
}
|
||||
|
||||
private getInMemoryLayerFolderPath = (
|
||||
flatLogicFunctionLayer: FlatLogicFunctionLayer,
|
||||
) => {
|
||||
return join(
|
||||
LOGIC_FUNCTION_EXECUTOR_TMPDIR_FOLDER,
|
||||
flatLogicFunctionLayer.checksum,
|
||||
);
|
||||
};
|
||||
|
||||
private async createLayerIfNotExists(
|
||||
flatLogicFunctionLayer: FlatLogicFunctionLayer,
|
||||
) {
|
||||
const inMemoryLayerFolderPath = this.getInMemoryLayerFolderPath(
|
||||
flatLogicFunctionLayer,
|
||||
);
|
||||
|
||||
try {
|
||||
await fs.access(inMemoryLayerFolderPath);
|
||||
} catch {
|
||||
await copyAndBuildDependencies(
|
||||
inMemoryLayerFolderPath,
|
||||
flatLogicFunctionLayer,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async delete() {}
|
||||
|
||||
private async build(flatLogicFunctionLayer: FlatLogicFunctionLayer) {
|
||||
await this.createLayerIfNotExists(flatLogicFunctionLayer);
|
||||
}
|
||||
|
||||
async execute({
|
||||
flatLogicFunction,
|
||||
flatLogicFunctionLayer,
|
||||
applicationUniversalIdentifier,
|
||||
payload,
|
||||
env,
|
||||
}: LogicFunctionExecuteParams): Promise<LogicFunctionExecuteResult> {
|
||||
await this.build(flatLogicFunctionLayer);
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
|
||||
|
||||
try {
|
||||
const { sourceTemporaryDir } = await lambdaBuildDirectoryManager.init();
|
||||
|
||||
const baseFolderPath = getLogicFunctionBaseFolderPath(
|
||||
flatLogicFunction.builtHandlerPath,
|
||||
);
|
||||
|
||||
await this.fileStorageService.downloadFolder_v2({
|
||||
workspaceId: flatLogicFunction.workspaceId,
|
||||
applicationUniversalIdentifier,
|
||||
fileFolder: FileFolder.BuiltLogicFunction,
|
||||
resourcePath: baseFolderPath,
|
||||
localPath: sourceTemporaryDir,
|
||||
});
|
||||
|
||||
try {
|
||||
await fs.symlink(
|
||||
join(
|
||||
this.getInMemoryLayerFolderPath(flatLogicFunctionLayer),
|
||||
'node_modules',
|
||||
),
|
||||
join(sourceTemporaryDir, 'node_modules'),
|
||||
'dir',
|
||||
);
|
||||
} catch (err) {
|
||||
if (err.code !== 'EEXIST') {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
let logs = '';
|
||||
|
||||
const consoleListener = new ConsoleListener();
|
||||
|
||||
consoleListener.intercept((type, args) => {
|
||||
const formattedArgs = args.map((arg) => {
|
||||
if (typeof arg === 'object' && arg !== null) {
|
||||
const seen = new WeakSet();
|
||||
|
||||
return JSON.stringify(
|
||||
arg,
|
||||
(_key, value) => {
|
||||
if (typeof value === 'object' && value !== null) {
|
||||
if (seen.has(value)) {
|
||||
return '[Circular]'; // Handle circular references
|
||||
}
|
||||
seen.add(value);
|
||||
}
|
||||
|
||||
return value;
|
||||
},
|
||||
2,
|
||||
);
|
||||
}
|
||||
|
||||
return arg;
|
||||
});
|
||||
|
||||
const formattedType = type === 'log' ? 'info' : type;
|
||||
|
||||
logs += `${new Date().toISOString()} ${formattedType.toUpperCase()} ${formattedArgs.join(' ')}\n`;
|
||||
});
|
||||
|
||||
try {
|
||||
const relativeBuiltPath = getRelativePathFromBase(
|
||||
flatLogicFunction.builtHandlerPath,
|
||||
baseFolderPath,
|
||||
);
|
||||
const builtBundleFilePath = join(sourceTemporaryDir, relativeBuiltPath);
|
||||
|
||||
const runnerPath = await this.writeBootstrapRunner({
|
||||
dir: sourceTemporaryDir,
|
||||
builtFileAbsPath: builtBundleFilePath,
|
||||
handlerName: flatLogicFunction.handlerName,
|
||||
});
|
||||
|
||||
const { ok, result, error, stack, stdout, stderr } =
|
||||
await this.runChildWithEnv({
|
||||
runnerPath,
|
||||
env: env ?? {},
|
||||
payload,
|
||||
timeoutMs: 900_000, // timeout is handled by the logic function service
|
||||
});
|
||||
|
||||
if (stdout)
|
||||
logs +=
|
||||
stdout
|
||||
.split('\n')
|
||||
.filter(Boolean)
|
||||
.map((l) => `${new Date().toISOString()} INFO ${l}`)
|
||||
.join('\n') + '\n';
|
||||
if (stderr)
|
||||
logs +=
|
||||
stderr
|
||||
.split('\n')
|
||||
.filter(Boolean)
|
||||
.map((l) => `${new Date().toISOString()} ERROR ${l}`)
|
||||
.join('\n') + '\n';
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
if (ok) {
|
||||
return {
|
||||
data: (result ?? null) as object | null,
|
||||
logs,
|
||||
duration,
|
||||
status: LogicFunctionExecutionStatus.SUCCESS,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
data: null,
|
||||
logs,
|
||||
duration,
|
||||
error: {
|
||||
errorType: 'UnhandledError',
|
||||
errorMessage: error || 'Unknown error',
|
||||
stackTrace: stack ? String(stack).split('\n') : [],
|
||||
},
|
||||
status: LogicFunctionExecutionStatus.ERROR,
|
||||
};
|
||||
} finally {
|
||||
consoleListener.release();
|
||||
}
|
||||
} finally {
|
||||
await lambdaBuildDirectoryManager.clean();
|
||||
}
|
||||
}
|
||||
|
||||
async writeBootstrapRunner({
|
||||
dir,
|
||||
builtFileAbsPath,
|
||||
handlerName,
|
||||
}: {
|
||||
dir: string;
|
||||
builtFileAbsPath: string;
|
||||
handlerName: string;
|
||||
}) {
|
||||
const runnerPath = join(dir, '__runner.cjs');
|
||||
const code = `
|
||||
// Auto-generated. Do not edit.
|
||||
const { pathToFileURL } = require('node:url');
|
||||
|
||||
(async () => {
|
||||
try {
|
||||
const builtUrl = pathToFileURL(${JSON.stringify(builtFileAbsPath)});
|
||||
const mod = await import(builtUrl.href);
|
||||
if (typeof mod.${handlerName} !== 'function') {
|
||||
throw new Error('Export "${handlerName}" not found in function bundle');
|
||||
}
|
||||
|
||||
let payload = undefined;
|
||||
if (process.send) {
|
||||
process.on('message', async (msg) => {
|
||||
if (!msg || msg.type !== 'run') return;
|
||||
try {
|
||||
const out = await mod.${handlerName}(msg.payload);
|
||||
process.send && process.send({ ok: true, result: out });
|
||||
process.exit(0);
|
||||
} catch (err) {
|
||||
process.send && process.send({ ok: false, error: String(err), stack: err?.stack });
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
// Fallback: read payload from argv[2] (JSON) and print to stdout
|
||||
const json = process.argv[2];
|
||||
payload = json ? JSON.parse(json) : undefined;
|
||||
const out = await mod.${handlerName}(payload);
|
||||
console.log(JSON.stringify({ ok: true, result: out }));
|
||||
process.exit(0);
|
||||
}
|
||||
} catch (err) {
|
||||
const msg = String(err);
|
||||
if (process.send) {
|
||||
process.send({ ok: false, error: msg, stack: err?.stack });
|
||||
} else {
|
||||
console.error(msg);
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
})();
|
||||
`;
|
||||
|
||||
await fs.writeFile(runnerPath, code, 'utf8');
|
||||
|
||||
return runnerPath;
|
||||
}
|
||||
|
||||
runChildWithEnv(options: {
|
||||
runnerPath: string;
|
||||
env: Record<string, string>;
|
||||
payload: unknown;
|
||||
timeoutMs: number;
|
||||
}) {
|
||||
const { runnerPath, env, payload, timeoutMs } = options;
|
||||
|
||||
return new Promise<{
|
||||
ok: boolean;
|
||||
result?: unknown;
|
||||
error?: string;
|
||||
stack?: string;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
}>((resolve) => {
|
||||
// Strip NODE_OPTIONS to prevent tsx loader from being inherited
|
||||
const { NODE_OPTIONS: _n1, ...cleanProcessEnv } = process.env;
|
||||
const { NODE_OPTIONS: _n2, ...cleanUserEnv } = env;
|
||||
|
||||
const child = spawn(process.execPath, [runnerPath], {
|
||||
env: { ...cleanProcessEnv, ...cleanUserEnv },
|
||||
stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
|
||||
});
|
||||
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
let settled = false;
|
||||
|
||||
child.stdout?.on('data', (d) => (stdout += String(d)));
|
||||
child.stderr?.on('data', (d) => (stderr += String(d)));
|
||||
|
||||
child.on(
|
||||
'message',
|
||||
(
|
||||
msg:
|
||||
| {
|
||||
ok: true;
|
||||
result?: unknown;
|
||||
stdout?: string;
|
||||
stderr?: string;
|
||||
}
|
||||
| {
|
||||
ok: false;
|
||||
error: string;
|
||||
stack?: string;
|
||||
stdout?: string;
|
||||
stderr?: string;
|
||||
},
|
||||
) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
resolve({ ...msg, stdout, stderr });
|
||||
},
|
||||
);
|
||||
|
||||
child.on('exit', (code) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (code === 0) {
|
||||
// Fallback path if no IPC (shouldn't happen with our stdio)
|
||||
resolve({ ok: true, stdout, stderr });
|
||||
} else {
|
||||
resolve({
|
||||
ok: false,
|
||||
error: `Exited with code ${code}`,
|
||||
stdout,
|
||||
stderr,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
const t = setTimeout(() => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
child.kill('SIGKILL');
|
||||
resolve({
|
||||
ok: false,
|
||||
error: `Timed out after ${timeoutMs}ms`,
|
||||
stdout,
|
||||
stderr,
|
||||
});
|
||||
}, timeoutMs);
|
||||
|
||||
// Kick it off
|
||||
child.send?.({ type: 'run', payload });
|
||||
|
||||
child.on('close', () => clearTimeout(t));
|
||||
});
|
||||
}
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
import { type FlatLogicFunctionLayer } from 'src/engine/metadata-modules/logic-function-layer/types/flat-logic-function-layer.type';
|
||||
import { type LogicFunctionExecutionStatus } from 'src/engine/metadata-modules/logic-function/dtos/logic-function-execution-result.dto';
|
||||
import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
|
||||
|
||||
export type LogicFunctionExecuteError = {
|
||||
errorType: string;
|
||||
errorMessage: string;
|
||||
stackTrace: string | string[];
|
||||
};
|
||||
|
||||
export type LogicFunctionExecuteResult = {
|
||||
data: object | null;
|
||||
duration: number;
|
||||
logs: string;
|
||||
status: LogicFunctionExecutionStatus;
|
||||
error?: LogicFunctionExecuteError;
|
||||
};
|
||||
|
||||
export type LogicFunctionExecuteParams = {
|
||||
flatLogicFunction: FlatLogicFunction;
|
||||
flatLogicFunctionLayer: FlatLogicFunctionLayer;
|
||||
applicationUniversalIdentifier: string;
|
||||
payload: object;
|
||||
env?: Record<string, string>;
|
||||
};
|
||||
|
||||
export interface LogicFunctionExecutorDriver {
|
||||
delete(flatLogicFunction: FlatLogicFunction): Promise<void>;
|
||||
execute(
|
||||
params: LogicFunctionExecuteParams,
|
||||
): Promise<LogicFunctionExecuteResult>;
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
{
|
||||
"dependencies": {
|
||||
"@types/bcrypt": "^5.0.2",
|
||||
"@types/deep-equal": "^1.0.4",
|
||||
"@types/lodash.camelcase": "^4.3.9",
|
||||
"@types/lodash.compact": "^3.0.9",
|
||||
"@types/lodash.groupby": "^4.6.9",
|
||||
"@types/lodash.identity": "^3.0.9",
|
||||
"@types/lodash.isempty": "^4.4.9",
|
||||
"@types/lodash.isequal": "^4.5.8",
|
||||
"@types/lodash.isobject": "^3.0.9",
|
||||
"@types/lodash.kebabcase": "^4.1.9",
|
||||
"@types/lodash.mapvalues": "^4.6.9",
|
||||
"@types/lodash.omit": "^4.5.9",
|
||||
"@types/lodash.pickby": "^4.6.9",
|
||||
"@types/lodash.snakecase": "^4.1.9",
|
||||
"@types/lodash.upperfirst": "^4.3.9",
|
||||
"@types/uuid": "^10.0.0",
|
||||
"archiver": "^7.0.1",
|
||||
"axios": "^1.12.0",
|
||||
"bcrypt": "^5.1.1",
|
||||
"body-parser": "^1.20.4",
|
||||
"deep-equal": "^2.2.3",
|
||||
"jsonwebtoken": "^9.0.2",
|
||||
"lodash.camelcase": "^4.3.0",
|
||||
"lodash.chunk": "^4.2.0",
|
||||
"lodash.compact": "^3.0.1",
|
||||
"lodash.groupby": "^4.6.0",
|
||||
"lodash.identity": "^3.0.0",
|
||||
"lodash.isempty": "^4.4.0",
|
||||
"lodash.isequal": "^4.5.0",
|
||||
"lodash.isobject": "^3.0.2",
|
||||
"lodash.kebabcase": "^4.1.1",
|
||||
"lodash.mapvalues": "^4.6.0",
|
||||
"lodash.merge": "^4.6.2",
|
||||
"lodash.omit": "^4.5.0",
|
||||
"lodash.pickby": "^4.6.0",
|
||||
"lodash.snakecase": "^4.1.1",
|
||||
"lodash.upperfirst": "^4.3.1",
|
||||
"nodemailer": "^7.0.11",
|
||||
"sharp": "^0.33.5",
|
||||
"uuid": "^10.0.0",
|
||||
"winston": "^3.14.2"
|
||||
}
|
||||
}
|
||||
+3374
File diff suppressed because it is too large
Load Diff
Vendored
Executable
+942
File diff suppressed because one or more lines are too long
+5
@@ -0,0 +1,5 @@
|
||||
enableInlineHunks: true
|
||||
|
||||
nodeLinker: node-modules
|
||||
|
||||
yarnPath: .yarn/releases/yarn-4.9.2.cjs
|
||||
+1
@@ -0,0 +1 @@
|
||||
export const LAST_LAYER_VERSION = 1;
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
import { type DynamicModule, Module } from '@nestjs/common';
|
||||
|
||||
import {
|
||||
LogicFunctionExecutorDriverType,
|
||||
type LogicFunctionExecutorModuleAsyncOptions,
|
||||
} from 'src/engine/core-modules/logic-function/logic-function-executor/interfaces/logic-function-executor.interface';
|
||||
|
||||
import { DisabledDriver } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/disabled.driver';
|
||||
import { LambdaDriver } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver';
|
||||
import { LocalDriver } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/local.driver';
|
||||
import { LOGIC_FUNCTION_EXECUTOR_DRIVER } from 'src/engine/core-modules/logic-function/logic-function-executor/constants/logic-function-executor.constants';
|
||||
|
||||
@Module({})
|
||||
export class LogicFunctionDriversModule {
|
||||
static forRootAsync(
|
||||
options: LogicFunctionExecutorModuleAsyncOptions,
|
||||
): DynamicModule {
|
||||
const provider = {
|
||||
provide: LOGIC_FUNCTION_EXECUTOR_DRIVER,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
useFactory: async (...args: any[]) => {
|
||||
const config = await options.useFactory(...args);
|
||||
|
||||
switch (config?.type) {
|
||||
case LogicFunctionExecutorDriverType.DISABLED:
|
||||
return new DisabledDriver();
|
||||
case LogicFunctionExecutorDriverType.LOCAL:
|
||||
return new LocalDriver(config.options);
|
||||
case LogicFunctionExecutorDriverType.LAMBDA:
|
||||
return new LambdaDriver(config.options);
|
||||
default: {
|
||||
const unknownConfig = config as { type?: string };
|
||||
|
||||
throw new Error(
|
||||
`Unknown logic function executor driver type: ${unknownConfig?.type}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
},
|
||||
inject: options.inject || [],
|
||||
};
|
||||
|
||||
return {
|
||||
module: LogicFunctionDriversModule,
|
||||
imports: options.imports || [],
|
||||
providers: [provider],
|
||||
exports: [LOGIC_FUNCTION_EXECUTOR_DRIVER],
|
||||
};
|
||||
}
|
||||
}
|
||||
+120
@@ -0,0 +1,120 @@
|
||||
import { type FlatApplicationVariable } from 'src/engine/core-modules/applicationVariable/types/flat-application-variable.type';
|
||||
import { type SecretEncryptionService } from 'src/engine/core-modules/secret-encryption/secret-encryption.service';
|
||||
import { buildEnvVar } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/build-env-var';
|
||||
|
||||
describe('buildEnvVar', () => {
|
||||
const mockSecretEncryptionService = {
|
||||
encrypt: jest.fn((value: string) => `encrypted_${value}`),
|
||||
decrypt: jest.fn((value: string) => value.replace('encrypted_', '')),
|
||||
} as unknown as SecretEncryptionService;
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('should return empty object for empty array', () => {
|
||||
const result = buildEnvVar([], mockSecretEncryptionService);
|
||||
|
||||
expect(result).toEqual({});
|
||||
});
|
||||
|
||||
it('should handle mixed secret and non-secret variables', () => {
|
||||
const flatVariables: FlatApplicationVariable[] = [
|
||||
{
|
||||
id: '1',
|
||||
key: 'PUBLIC_URL',
|
||||
value: 'https://example.com',
|
||||
description: 'Public URL',
|
||||
isSecret: false,
|
||||
applicationId: 'app-1',
|
||||
createdAt: '2024-01-01T00:00:00.000Z',
|
||||
updatedAt: '2024-01-01T00:00:00.000Z',
|
||||
},
|
||||
{
|
||||
id: '2',
|
||||
key: 'API_SECRET',
|
||||
value: 'encrypted_secret-123',
|
||||
description: 'API secret',
|
||||
isSecret: true,
|
||||
applicationId: 'app-1',
|
||||
createdAt: '2024-01-01T00:00:00.000Z',
|
||||
updatedAt: '2024-01-01T00:00:00.000Z',
|
||||
},
|
||||
{
|
||||
id: '3',
|
||||
key: 'DEBUG',
|
||||
value: 'true',
|
||||
description: 'Debug flag',
|
||||
isSecret: false,
|
||||
applicationId: 'app-1',
|
||||
createdAt: '2024-01-01T00:00:00.000Z',
|
||||
updatedAt: '2024-01-01T00:00:00.000Z',
|
||||
},
|
||||
];
|
||||
|
||||
const result = buildEnvVar(flatVariables, mockSecretEncryptionService);
|
||||
|
||||
expect(result).toEqual({
|
||||
PUBLIC_URL: 'https://example.com',
|
||||
API_SECRET: 'secret-123',
|
||||
DEBUG: 'true',
|
||||
});
|
||||
expect(mockSecretEncryptionService.decrypt).toHaveBeenCalledTimes(1);
|
||||
expect(mockSecretEncryptionService.decrypt).toHaveBeenCalledWith(
|
||||
'encrypted_secret-123',
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle null or undefined values', () => {
|
||||
const flatVariables: FlatApplicationVariable[] = [
|
||||
{
|
||||
id: '1',
|
||||
key: 'NULL_VALUE',
|
||||
value: null as unknown as string,
|
||||
description: '',
|
||||
isSecret: false,
|
||||
applicationId: 'app-1',
|
||||
createdAt: '2024-01-01T00:00:00.000Z',
|
||||
updatedAt: '2024-01-01T00:00:00.000Z',
|
||||
},
|
||||
{
|
||||
id: '2',
|
||||
key: 'UNDEFINED_VALUE',
|
||||
value: undefined as unknown as string,
|
||||
description: '',
|
||||
isSecret: false,
|
||||
applicationId: 'app-1',
|
||||
createdAt: '2024-01-01T00:00:00.000Z',
|
||||
updatedAt: '2024-01-01T00:00:00.000Z',
|
||||
},
|
||||
];
|
||||
|
||||
const result = buildEnvVar(flatVariables, mockSecretEncryptionService);
|
||||
|
||||
expect(result).toEqual({
|
||||
NULL_VALUE: '',
|
||||
UNDEFINED_VALUE: '',
|
||||
});
|
||||
});
|
||||
|
||||
it('should convert non-string values to strings', () => {
|
||||
const flatVariables: FlatApplicationVariable[] = [
|
||||
{
|
||||
id: '1',
|
||||
key: 'NUMBER_VALUE',
|
||||
value: 123 as unknown as string,
|
||||
description: '',
|
||||
isSecret: false,
|
||||
applicationId: 'app-1',
|
||||
createdAt: '2024-01-01T00:00:00.000Z',
|
||||
updatedAt: '2024-01-01T00:00:00.000Z',
|
||||
},
|
||||
];
|
||||
|
||||
const result = buildEnvVar(flatVariables, mockSecretEncryptionService);
|
||||
|
||||
expect(result).toEqual({
|
||||
NUMBER_VALUE: '123',
|
||||
});
|
||||
});
|
||||
});
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
import { type FlatApplicationVariable } from 'src/engine/core-modules/applicationVariable/types/flat-application-variable.type';
|
||||
import { type SecretEncryptionService } from 'src/engine/core-modules/secret-encryption/secret-encryption.service';
|
||||
|
||||
export const buildEnvVar = (
|
||||
flatApplicationVariables: FlatApplicationVariable[],
|
||||
secretEncryptionService: SecretEncryptionService,
|
||||
): Record<string, string> => {
|
||||
return flatApplicationVariables.reduce<Record<string, string>>(
|
||||
(acc, flatApplicationVariable) => {
|
||||
const value = String(flatApplicationVariable.value ?? '');
|
||||
|
||||
acc[flatApplicationVariable.key] = flatApplicationVariable.isSecret
|
||||
? secretEncryptionService.decrypt(value)
|
||||
: value;
|
||||
|
||||
return acc;
|
||||
},
|
||||
{},
|
||||
);
|
||||
};
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
import { execFile } from 'child_process';
|
||||
import { promises as fs, statSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { promisify } from 'util';
|
||||
|
||||
import { getLayerDependenciesDirName } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/get-layer-dependencies-dir-name';
|
||||
import { type FlatLogicFunctionLayer } from 'src/engine/metadata-modules/logic-function-layer/types/flat-logic-function-layer.type';
|
||||
|
||||
const execFilePromise = promisify(execFile);
|
||||
|
||||
export const copyAndBuildDependencies = async (
|
||||
buildDirectory: string,
|
||||
flatLogicFunctionLayer: FlatLogicFunctionLayer,
|
||||
) => {
|
||||
await fs.mkdir(buildDirectory, {
|
||||
recursive: true,
|
||||
});
|
||||
|
||||
const packageJson = flatLogicFunctionLayer.packageJson;
|
||||
|
||||
const yarnLock = flatLogicFunctionLayer.yarnLock;
|
||||
|
||||
await fs.writeFile(
|
||||
join(buildDirectory, 'package.json'),
|
||||
JSON.stringify(packageJson, null, 2),
|
||||
'utf8',
|
||||
);
|
||||
|
||||
await fs.writeFile(join(buildDirectory, 'yarn.lock'), yarnLock, 'utf8');
|
||||
|
||||
await fs.cp(getLayerDependenciesDirName('engine'), buildDirectory, {
|
||||
recursive: true,
|
||||
});
|
||||
|
||||
const localYarnPath = join(buildDirectory, '.yarn/releases/yarn-4.9.2.cjs');
|
||||
|
||||
// Strip NODE_OPTIONS to prevent tsx loader from interfering with yarn
|
||||
const { NODE_OPTIONS: _nodeOptions, ...cleanEnv } = process.env;
|
||||
|
||||
try {
|
||||
await execFilePromise(process.execPath, [localYarnPath], {
|
||||
cwd: buildDirectory,
|
||||
env: cleanEnv,
|
||||
});
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} catch (error: any) {
|
||||
const errorMessage =
|
||||
[error?.stdout, error?.stderr].filter(Boolean).join('\n') ||
|
||||
'Failed to install logic function executor dependencies';
|
||||
|
||||
throw new Error(errorMessage);
|
||||
}
|
||||
const objects = await fs.readdir(buildDirectory);
|
||||
|
||||
await Promise.all(
|
||||
objects
|
||||
.filter((object) => object !== 'node_modules')
|
||||
.map((object) => {
|
||||
const fullPath = join(buildDirectory, object);
|
||||
|
||||
return statSync(fullPath).isDirectory()
|
||||
? fs.rm(fullPath, { recursive: true, force: true })
|
||||
: fs.rm(fullPath);
|
||||
}),
|
||||
);
|
||||
};
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
import { promises as fs } from 'fs';
|
||||
|
||||
import { getExecutorFilePath } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/get-executor-file-path';
|
||||
|
||||
export const copyExecutor = async (buildDirectory: string) => {
|
||||
await fs.mkdir(buildDirectory, {
|
||||
recursive: true,
|
||||
});
|
||||
await fs.cp(getExecutorFilePath(), buildDirectory, {
|
||||
recursive: true,
|
||||
});
|
||||
};
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
import fs from 'fs';
|
||||
import { pipeline } from 'stream/promises';
|
||||
|
||||
import archiver from 'archiver';
|
||||
|
||||
export const createZipFile = async (
|
||||
sourceDir: string,
|
||||
outPath: string,
|
||||
): Promise<void> => {
|
||||
const output = fs.createWriteStream(outPath);
|
||||
const archive = archiver('zip', {
|
||||
zlib: { level: 9 }, // Compression level
|
||||
});
|
||||
|
||||
const p = pipeline(archive, output);
|
||||
|
||||
archive.directory(sourceDir, false);
|
||||
archive.finalize();
|
||||
|
||||
return p;
|
||||
};
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
import path from 'path';
|
||||
|
||||
import { ASSET_PATH } from 'src/constants/assets-path';
|
||||
|
||||
export const getExecutorFilePath = (): string => {
|
||||
const baseTypescriptProjectPath = path.join(
|
||||
ASSET_PATH,
|
||||
`engine/core-modules/logic-function/logic-function-drivers/constants/executor`,
|
||||
);
|
||||
|
||||
return path.resolve(__dirname, baseTypescriptProjectPath);
|
||||
};
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
import fs from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
|
||||
import { type PackageJson } from 'twenty-shared/application';
|
||||
|
||||
import { getLayerDependenciesDirName } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/get-layer-dependencies-dir-name';
|
||||
import { LAST_LAYER_VERSION } from 'src/engine/core-modules/logic-function/logic-function-drivers/layers/last-layer-version';
|
||||
|
||||
export type LayerDependencies = {
|
||||
packageJson: PackageJson;
|
||||
yarnLock: string;
|
||||
};
|
||||
|
||||
export const getLastCommonLayerDependencies = async (
|
||||
layerVersion = LAST_LAYER_VERSION,
|
||||
): Promise<LayerDependencies> => {
|
||||
const lastVersionLayerDirName = getLayerDependenciesDirName(layerVersion);
|
||||
const [packageJson, yarnLock] = await Promise.all([
|
||||
fs.readFile(join(lastVersionLayerDirName, 'package.json'), 'utf8'),
|
||||
fs.readFile(join(lastVersionLayerDirName, 'yarn.lock'), 'utf8'),
|
||||
]);
|
||||
|
||||
return { packageJson: JSON.parse(packageJson), yarnLock };
|
||||
};
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
import path from 'path';
|
||||
|
||||
import { ASSET_PATH } from 'src/constants/assets-path';
|
||||
|
||||
export const getLayerDependenciesDirName = (
|
||||
version: 'engine' | number,
|
||||
): string => {
|
||||
const baseTypescriptProjectPath = path.join(
|
||||
ASSET_PATH,
|
||||
`engine/core-modules/logic-function/logic-function-drivers/layers/${version}`,
|
||||
);
|
||||
|
||||
return path.resolve(__dirname, baseTypescriptProjectPath);
|
||||
};
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
import fs from 'fs/promises';
|
||||
import path, { join } from 'path';
|
||||
|
||||
import { ASSET_PATH } from 'src/constants/assets-path';
|
||||
|
||||
type File = { name: string; path: string; content: Buffer };
|
||||
|
||||
const getAllFiles = async (
|
||||
rootDir: string,
|
||||
dir: string = rootDir,
|
||||
files: File[] = [],
|
||||
): Promise<File[]> => {
|
||||
const dirEntries = await fs.readdir(dir, { withFileTypes: true });
|
||||
|
||||
for (const entry of dirEntries) {
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
files.push(...(await getAllFiles(rootDir, fullPath, files)));
|
||||
} else {
|
||||
files.push({
|
||||
path: path.relative(rootDir, dir),
|
||||
name: entry.name,
|
||||
content: await fs.readFile(fullPath),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return files;
|
||||
};
|
||||
|
||||
export const getSeedProjectFiles = (async () => {
|
||||
const seedProjectPath = join(
|
||||
ASSET_PATH,
|
||||
`engine/core-modules/logic-function/logic-function-drivers/constants/seed-project`,
|
||||
);
|
||||
|
||||
return await getAllFiles(seedProjectPath);
|
||||
})();
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
/* eslint-disable no-console */
|
||||
export class ConsoleListener {
|
||||
private readonly originalConsole;
|
||||
|
||||
constructor() {
|
||||
this.originalConsole = {
|
||||
log: console.log,
|
||||
error: console.error,
|
||||
warn: console.warn,
|
||||
info: console.info,
|
||||
debug: console.debug,
|
||||
};
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
intercept(callback: (type: string, message: any[]) => void) {
|
||||
Object.keys(this.originalConsole).forEach((method) => {
|
||||
// @ts-expect-error legacy noImplicitAny
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
console[method] = (...args: any[]) => {
|
||||
callback(method, args);
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
release() {
|
||||
Object.keys(this.originalConsole).forEach((method) => {
|
||||
// @ts-expect-error legacy noImplicitAny
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
console[method] = (...args: any[]) => {
|
||||
// @ts-expect-error legacy noImplicitAny
|
||||
this.originalConsole[method](...args);
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
import { join } from 'path';
|
||||
import * as fs from 'fs/promises';
|
||||
|
||||
import { v4 } from 'uuid';
|
||||
|
||||
import { LOGIC_FUNCTION_EXECUTOR_TMPDIR_FOLDER } from 'src/engine/core-modules/logic-function/logic-function-drivers/constants/logic-function-executor-tmpdir-folder';
|
||||
|
||||
export const NODE_LAYER_SUBFOLDER = 'nodejs';
|
||||
|
||||
const TEMPORARY_LAMBDA_FOLDER = 'lambda-build';
|
||||
const LAMBDA_ZIP_FILE_NAME = 'lambda.zip';
|
||||
|
||||
export class LambdaBuildDirectoryManager {
|
||||
private temporaryDir = join(
|
||||
LOGIC_FUNCTION_EXECUTOR_TMPDIR_FOLDER,
|
||||
`${TEMPORARY_LAMBDA_FOLDER}-${v4()}`,
|
||||
);
|
||||
|
||||
async init() {
|
||||
const sourceTemporaryDir = join(this.temporaryDir);
|
||||
const lambdaZipPath = join(this.temporaryDir, LAMBDA_ZIP_FILE_NAME);
|
||||
|
||||
await fs.mkdir(sourceTemporaryDir, { recursive: true });
|
||||
|
||||
return {
|
||||
sourceTemporaryDir,
|
||||
lambdaZipPath,
|
||||
};
|
||||
}
|
||||
|
||||
async clean() {
|
||||
await fs.rm(this.temporaryDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
+135
@@ -0,0 +1,135 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
|
||||
import { execFile } from 'child_process';
|
||||
import * as fs from 'fs/promises';
|
||||
import { resolve } from 'path';
|
||||
import { promisify } from 'util';
|
||||
|
||||
import { Command, CommandRunner, Option } from 'nest-commander';
|
||||
|
||||
const execFilePromise = promisify(execFile);
|
||||
|
||||
@Command({
|
||||
name: 'logic-function-executor:add-packages',
|
||||
description:
|
||||
'Create a new logic function executor layer version and install packages in it',
|
||||
})
|
||||
export class AddPackagesCommand extends CommandRunner {
|
||||
private readonly logger = new Logger(AddPackagesCommand.name);
|
||||
|
||||
@Option({
|
||||
flags: '-p, --packages <packages>',
|
||||
description: 'comma separated packages (eg: axios,uuid@9.0.1)',
|
||||
required: true,
|
||||
})
|
||||
parsePackages(val: string): string[] {
|
||||
return val.split(',');
|
||||
}
|
||||
|
||||
async run(
|
||||
_passedParams: string[],
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
options: Record<string, any>,
|
||||
): Promise<void> {
|
||||
this.logger.log('---------------------------------------');
|
||||
this.logger.warn('This command should be run locally only');
|
||||
this.logger.log('');
|
||||
|
||||
const layersFolder = this.getAbsoluteFilePath(
|
||||
`src/engine/core-modules/logic-function/logic-function-drivers/layers`,
|
||||
);
|
||||
|
||||
const currentVersion = await this.getLastLayerVersion();
|
||||
const newVersion = currentVersion + 1;
|
||||
|
||||
const currentVersionFolder = `${layersFolder}/${currentVersion}`;
|
||||
const newVersionFolder = `${layersFolder}/${newVersion}`;
|
||||
|
||||
await fs.cp(currentVersionFolder, newVersionFolder, { recursive: true });
|
||||
|
||||
// Install each package
|
||||
this.logger.log('Installing packages');
|
||||
await this.installPackages(options.packages, newVersionFolder);
|
||||
|
||||
this.logger.log('Cleaning');
|
||||
await this.cleanPackageInstallation(newVersionFolder);
|
||||
|
||||
this.logger.log('Updating last layer version');
|
||||
await this.updateLastLayerVersion(newVersion);
|
||||
|
||||
this.logger.log('Add changes to git');
|
||||
await this.addToGit(layersFolder);
|
||||
|
||||
this.logger.log('');
|
||||
this.logger.log(
|
||||
`New packages '${options.packages.join("', '")}' installed in new layer version '${newVersion}' `,
|
||||
);
|
||||
this.logger.log('Please commit your changes');
|
||||
this.logger.log('---------------------------------------');
|
||||
}
|
||||
|
||||
private getAbsoluteFilePath(path: string) {
|
||||
const rootPath = process.cwd();
|
||||
|
||||
return resolve(rootPath, path);
|
||||
}
|
||||
|
||||
private async addToGit(folderPath: string) {
|
||||
await execFilePromise('git', ['add', folderPath]);
|
||||
}
|
||||
|
||||
private async cleanPackageInstallation(folderPath: string) {
|
||||
await fs.rm(folderPath + '/node_modules', {
|
||||
recursive: true,
|
||||
force: true,
|
||||
});
|
||||
await fs.rm(folderPath + '/.yarn', {
|
||||
recursive: true,
|
||||
force: true,
|
||||
});
|
||||
}
|
||||
|
||||
private async installPackages(packages: string[], folderPath: string) {
|
||||
if (packages?.length) {
|
||||
for (const packageName of packages) {
|
||||
this.logger.log(`- adding '${packageName}'...`);
|
||||
try {
|
||||
await execFilePromise('yarn', ['add', packageName], {
|
||||
cwd: folderPath,
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Failed to install ${packageName}: ${(error as Error).message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async getLastLayerVersion() {
|
||||
const filePath = this.getAbsoluteFilePath(
|
||||
'src/engine/core-modules/logic-function/logic-function-drivers/layers/last-layer-version.ts',
|
||||
);
|
||||
|
||||
const content = await fs.readFile(filePath, 'utf8');
|
||||
const match = content.match(/export const LAST_LAYER_VERSION = (\d+);/);
|
||||
|
||||
if (!match) {
|
||||
throw new Error('LAST_LAYER_VERSION not found');
|
||||
}
|
||||
|
||||
return parseInt(match[1], 10);
|
||||
}
|
||||
|
||||
private async updateLastLayerVersion(newVersion: number) {
|
||||
const filePath = this.getAbsoluteFilePath(
|
||||
'src/engine/core-modules/logic-function/logic-function-drivers/layers/last-layer-version.ts',
|
||||
);
|
||||
|
||||
await fs.writeFile(
|
||||
filePath,
|
||||
`export const LAST_LAYER_VERSION = ${newVersion};\n`,
|
||||
'utf8',
|
||||
);
|
||||
}
|
||||
}
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
export const LOGIC_FUNCTION_EXECUTOR_DRIVER = Symbol(
|
||||
'LOGIC_FUNCTION_EXECUTOR_DRIVER',
|
||||
);
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
import { fromNodeProviderChain } from '@aws-sdk/credential-providers';
|
||||
|
||||
import {
|
||||
LogicFunctionExecutorDriverType,
|
||||
type LogicFunctionExecutorModuleOptions,
|
||||
} from 'src/engine/core-modules/logic-function/logic-function-executor/interfaces/logic-function-executor.interface';
|
||||
|
||||
import { type FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
|
||||
import { type TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
|
||||
|
||||
export const logicFunctionModuleFactory = async (
|
||||
twentyConfigService: TwentyConfigService,
|
||||
fileStorageService: FileStorageService,
|
||||
): Promise<LogicFunctionExecutorModuleOptions> => {
|
||||
const driverType = twentyConfigService.get('LOGIC_FUNCTION_TYPE');
|
||||
const options = { fileStorageService };
|
||||
|
||||
switch (driverType) {
|
||||
case LogicFunctionExecutorDriverType.DISABLED: {
|
||||
return {
|
||||
type: LogicFunctionExecutorDriverType.DISABLED,
|
||||
};
|
||||
}
|
||||
case LogicFunctionExecutorDriverType.LOCAL: {
|
||||
return {
|
||||
type: LogicFunctionExecutorDriverType.LOCAL,
|
||||
options,
|
||||
};
|
||||
}
|
||||
case LogicFunctionExecutorDriverType.LAMBDA: {
|
||||
const region = twentyConfigService.get('LOGIC_FUNCTION_LAMBDA_REGION');
|
||||
const accessKeyId = twentyConfigService.get(
|
||||
'LOGIC_FUNCTION_LAMBDA_ACCESS_KEY_ID',
|
||||
);
|
||||
const secretAccessKey = twentyConfigService.get(
|
||||
'LOGIC_FUNCTION_LAMBDA_SECRET_ACCESS_KEY',
|
||||
);
|
||||
const lambdaRole = twentyConfigService.get('LOGIC_FUNCTION_LAMBDA_ROLE');
|
||||
|
||||
const subhostingRole = twentyConfigService.get(
|
||||
'LOGIC_FUNCTION_LAMBDA_SUBHOSTING_ROLE',
|
||||
);
|
||||
|
||||
return {
|
||||
type: LogicFunctionExecutorDriverType.LAMBDA,
|
||||
options: {
|
||||
...options,
|
||||
credentials: accessKeyId
|
||||
? {
|
||||
accessKeyId,
|
||||
secretAccessKey,
|
||||
}
|
||||
: fromNodeProviderChain({
|
||||
clientConfig: { region },
|
||||
}),
|
||||
region,
|
||||
lambdaRole,
|
||||
subhostingRole,
|
||||
},
|
||||
};
|
||||
}
|
||||
default:
|
||||
throw new Error(
|
||||
`Invalid logic function executor driver type (${driverType}), check your .env file`,
|
||||
);
|
||||
}
|
||||
};
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
import { type FactoryProvider, type ModuleMetadata } from '@nestjs/common';
|
||||
|
||||
import { type LambdaDriverOptions } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver';
|
||||
import { type LocalDriverOptions } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/local.driver';
|
||||
|
||||
export enum LogicFunctionExecutorDriverType {
|
||||
DISABLED = 'DISABLED',
|
||||
LAMBDA = 'LAMBDA',
|
||||
LOCAL = 'LOCAL',
|
||||
}
|
||||
|
||||
export interface DisabledDriverFactoryOptions {
|
||||
type: LogicFunctionExecutorDriverType.DISABLED;
|
||||
}
|
||||
|
||||
export interface LocalDriverFactoryOptions {
|
||||
type: LogicFunctionExecutorDriverType.LOCAL;
|
||||
options: LocalDriverOptions;
|
||||
}
|
||||
|
||||
export interface LambdaDriverFactoryOptions {
|
||||
type: LogicFunctionExecutorDriverType.LAMBDA;
|
||||
options: LambdaDriverOptions;
|
||||
}
|
||||
|
||||
export type LogicFunctionExecutorModuleOptions =
|
||||
| DisabledDriverFactoryOptions
|
||||
| LocalDriverFactoryOptions
|
||||
| LambdaDriverFactoryOptions;
|
||||
|
||||
export type LogicFunctionExecutorModuleAsyncOptions = {
|
||||
useFactory: (
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
...args: any[]
|
||||
) =>
|
||||
| LogicFunctionExecutorModuleOptions
|
||||
| Promise<LogicFunctionExecutorModuleOptions>;
|
||||
} & Pick<ModuleMetadata, 'imports'> &
|
||||
Pick<FactoryProvider, 'inject'>;
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { AuditModule } from 'src/engine/core-modules/audit/audit.module';
|
||||
import { TokenModule } from 'src/engine/core-modules/auth/token/token.module';
|
||||
import { LogicFunctionBuildModule } from 'src/engine/core-modules/logic-function/logic-function-build/logic-function-build.module';
|
||||
import { AddPackagesCommand } from 'src/engine/core-modules/logic-function/logic-function-executor/commands/add-packages.command';
|
||||
import { LogicFunctionExecutorService } from 'src/engine/core-modules/logic-function/logic-function-executor/services/logic-function-executor.service';
|
||||
import { SecretEncryptionModule } from 'src/engine/core-modules/secret-encryption/secret-encryption.module';
|
||||
import { ThrottlerModule } from 'src/engine/core-modules/throttler/throttler.module';
|
||||
import { SubscriptionsModule } from 'src/engine/subscriptions/subscriptions.module';
|
||||
import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
ThrottlerModule,
|
||||
AuditModule,
|
||||
TokenModule,
|
||||
SecretEncryptionModule,
|
||||
SubscriptionsModule,
|
||||
WorkspaceCacheModule,
|
||||
LogicFunctionBuildModule,
|
||||
],
|
||||
providers: [LogicFunctionExecutorService, AddPackagesCommand],
|
||||
exports: [LogicFunctionExecutorService],
|
||||
})
|
||||
export class LogicFunctionExecutorModule {}
|
||||
+261
@@ -0,0 +1,261 @@
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
|
||||
import {
|
||||
DEFAULT_API_KEY_NAME,
|
||||
DEFAULT_API_URL_NAME,
|
||||
} from 'twenty-shared/application';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
|
||||
import {
|
||||
LogicFunctionExecutorDriver,
|
||||
type LogicFunctionExecuteParams,
|
||||
type LogicFunctionExecuteResult,
|
||||
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-executor-driver.interface';
|
||||
|
||||
import { AuditService } from 'src/engine/core-modules/audit/services/audit.service';
|
||||
import { LOGIC_FUNCTION_EXECUTED_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/logic-function/logic-function-executed';
|
||||
import { ApplicationTokenService } from 'src/engine/core-modules/auth/token/services/application-token.service';
|
||||
import { LogicFunctionBuildService } from 'src/engine/core-modules/logic-function/logic-function-build/services/logic-function-build.service';
|
||||
import { buildEnvVar } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/build-env-var';
|
||||
import { LOGIC_FUNCTION_EXECUTOR_DRIVER } from 'src/engine/core-modules/logic-function/logic-function-executor/constants/logic-function-executor.constants';
|
||||
import { SecretEncryptionService } from 'src/engine/core-modules/secret-encryption/secret-encryption.service';
|
||||
import { ThrottlerService } from 'src/engine/core-modules/throttler/throttler.service';
|
||||
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
|
||||
import { findFlatEntityByIdInFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-id-in-flat-entity-maps.util';
|
||||
import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
|
||||
import { SubscriptionChannel } from 'src/engine/subscriptions/enums/subscription-channel.enum';
|
||||
import { SubscriptionService } from 'src/engine/subscriptions/subscription.service';
|
||||
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
|
||||
import { cleanServerUrl } from 'src/utils/clean-server-url';
|
||||
|
||||
const MIN_TOKEN_EXPIRATION_IN_SECONDS = 5;
|
||||
|
||||
export class LogicFunctionExecutionException extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public readonly code: LogicFunctionExecutionExceptionCode,
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'LogicFunctionExecutionException';
|
||||
}
|
||||
}
|
||||
|
||||
export enum LogicFunctionExecutionExceptionCode {
|
||||
LOGIC_FUNCTION_NOT_FOUND = 'LOGIC_FUNCTION_NOT_FOUND',
|
||||
RATE_LIMIT_EXCEEDED = 'RATE_LIMIT_EXCEEDED',
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class LogicFunctionExecutorService
|
||||
implements LogicFunctionExecutorDriver
|
||||
{
|
||||
constructor(
|
||||
@Inject(LOGIC_FUNCTION_EXECUTOR_DRIVER)
|
||||
private driver: LogicFunctionExecutorDriver,
|
||||
private readonly throttlerService: ThrottlerService,
|
||||
private readonly twentyConfigService: TwentyConfigService,
|
||||
private readonly workspaceCacheService: WorkspaceCacheService,
|
||||
private readonly applicationTokenService: ApplicationTokenService,
|
||||
private readonly secretEncryptionService: SecretEncryptionService,
|
||||
private readonly functionBuildService: LogicFunctionBuildService,
|
||||
private readonly subscriptionService: SubscriptionService,
|
||||
private readonly auditService: AuditService,
|
||||
) {}
|
||||
|
||||
async delete(flatLogicFunction: FlatLogicFunction): Promise<void> {
|
||||
return this.driver.delete(flatLogicFunction);
|
||||
}
|
||||
|
||||
async execute(
|
||||
params: LogicFunctionExecuteParams,
|
||||
): Promise<LogicFunctionExecuteResult> {
|
||||
return this.driver.execute(params);
|
||||
}
|
||||
|
||||
async executeOneLogicFunction({
|
||||
id,
|
||||
workspaceId,
|
||||
payload,
|
||||
}: {
|
||||
id: string;
|
||||
workspaceId: string;
|
||||
payload: object;
|
||||
}): Promise<LogicFunctionExecuteResult> {
|
||||
await this.throttleExecution(workspaceId);
|
||||
|
||||
const {
|
||||
flatLogicFunctionMaps,
|
||||
flatApplicationMaps,
|
||||
applicationVariableMaps,
|
||||
logicFunctionLayerMaps,
|
||||
} = await this.workspaceCacheService.getOrRecompute(workspaceId, [
|
||||
'flatLogicFunctionMaps',
|
||||
'flatApplicationMaps',
|
||||
'applicationVariableMaps',
|
||||
'logicFunctionLayerMaps',
|
||||
]);
|
||||
|
||||
const flatLogicFunction = findFlatEntityByIdInFlatEntityMaps({
|
||||
flatEntityId: id,
|
||||
flatEntityMaps: flatLogicFunctionMaps,
|
||||
});
|
||||
|
||||
if (
|
||||
!isDefined(flatLogicFunction) ||
|
||||
isDefined(flatLogicFunction.deletedAt)
|
||||
) {
|
||||
throw new LogicFunctionExecutionException(
|
||||
`Logic function with id ${id} not found`,
|
||||
LogicFunctionExecutionExceptionCode.LOGIC_FUNCTION_NOT_FOUND,
|
||||
);
|
||||
}
|
||||
|
||||
const flatLogicFunctionLayer =
|
||||
logicFunctionLayerMaps.byId[flatLogicFunction.logicFunctionLayerId];
|
||||
|
||||
if (!isDefined(flatLogicFunctionLayer)) {
|
||||
throw new LogicFunctionExecutionException(
|
||||
`Logic function layer with id ${flatLogicFunction.logicFunctionLayerId} not found`,
|
||||
LogicFunctionExecutionExceptionCode.LOGIC_FUNCTION_NOT_FOUND,
|
||||
);
|
||||
}
|
||||
|
||||
const applicationAccessToken = isDefined(flatLogicFunction.applicationId)
|
||||
? await this.applicationTokenService.generateApplicationToken({
|
||||
workspaceId,
|
||||
applicationId: flatLogicFunction.applicationId,
|
||||
expiresInSeconds: Math.max(
|
||||
flatLogicFunction.timeoutSeconds,
|
||||
MIN_TOKEN_EXPIRATION_IN_SECONDS,
|
||||
),
|
||||
})
|
||||
: undefined;
|
||||
|
||||
const baseUrl = cleanServerUrl(this.twentyConfigService.get('SERVER_URL'));
|
||||
|
||||
const flatApplicationVariables = isDefined(flatLogicFunction.applicationId)
|
||||
? (applicationVariableMaps.byApplicationId[
|
||||
flatLogicFunction.applicationId
|
||||
] ?? [])
|
||||
: [];
|
||||
|
||||
const envVariables = {
|
||||
...(isDefined(baseUrl)
|
||||
? {
|
||||
[DEFAULT_API_URL_NAME]: baseUrl,
|
||||
}
|
||||
: {}),
|
||||
...(isDefined(applicationAccessToken)
|
||||
? {
|
||||
[DEFAULT_API_KEY_NAME]: applicationAccessToken.token,
|
||||
}
|
||||
: {}),
|
||||
...buildEnvVar(flatApplicationVariables, this.secretEncryptionService),
|
||||
};
|
||||
|
||||
const applicationUniversalIdentifier = isDefined(
|
||||
flatLogicFunction.applicationId,
|
||||
)
|
||||
? flatApplicationMaps.byId[flatLogicFunction.applicationId]
|
||||
?.universalIdentifier
|
||||
: undefined;
|
||||
|
||||
if (!isDefined(applicationUniversalIdentifier)) {
|
||||
throw new LogicFunctionExecutionException(
|
||||
`Application universal identifier not found for logic function ${flatLogicFunction.id}`,
|
||||
LogicFunctionExecutionExceptionCode.LOGIC_FUNCTION_NOT_FOUND,
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
!(await this.functionBuildService.isBuilt({
|
||||
flatLogicFunction,
|
||||
applicationUniversalIdentifier,
|
||||
}))
|
||||
) {
|
||||
await this.functionBuildService.buildAndUpload({
|
||||
flatLogicFunction,
|
||||
applicationUniversalIdentifier,
|
||||
});
|
||||
}
|
||||
|
||||
const resultLogicFunction = await this.callWithTimeout({
|
||||
callback: () =>
|
||||
this.execute({
|
||||
flatLogicFunction,
|
||||
flatLogicFunctionLayer,
|
||||
applicationUniversalIdentifier,
|
||||
payload,
|
||||
env: envVariables,
|
||||
}),
|
||||
timeoutMs: flatLogicFunction.timeoutSeconds * 1000,
|
||||
});
|
||||
|
||||
if (this.twentyConfigService.get('LOGIC_FUNCTION_LOGS_ENABLED')) {
|
||||
/* eslint-disable no-console */
|
||||
console.log(resultLogicFunction.logs);
|
||||
}
|
||||
|
||||
await this.subscriptionService.publish({
|
||||
channel: SubscriptionChannel.LOGIC_FUNCTION_LOGS_CHANNEL,
|
||||
workspaceId,
|
||||
payload: {
|
||||
logicFunctionLogs: {
|
||||
logs: resultLogicFunction.logs,
|
||||
id: flatLogicFunction.id,
|
||||
name: flatLogicFunction.name,
|
||||
universalIdentifier: flatLogicFunction.universalIdentifier,
|
||||
applicationId: flatLogicFunction.applicationId,
|
||||
applicationUniversalIdentifier,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
this.auditService
|
||||
.createContext({
|
||||
workspaceId,
|
||||
})
|
||||
.insertWorkspaceEvent(LOGIC_FUNCTION_EXECUTED_EVENT, {
|
||||
duration: resultLogicFunction.duration,
|
||||
status: resultLogicFunction.status,
|
||||
...(resultLogicFunction.error && {
|
||||
errorType: resultLogicFunction.error.errorType,
|
||||
}),
|
||||
functionId: flatLogicFunction.id,
|
||||
functionName: flatLogicFunction.name,
|
||||
});
|
||||
|
||||
return resultLogicFunction;
|
||||
}
|
||||
|
||||
private async throttleExecution(workspaceId: string) {
|
||||
try {
|
||||
await this.throttlerService.tokenBucketThrottleOrThrow(
|
||||
`${workspaceId}-logic-function-execution`,
|
||||
1,
|
||||
this.twentyConfigService.get('LOGIC_FUNCTION_EXEC_THROTTLE_LIMIT'),
|
||||
this.twentyConfigService.get('LOGIC_FUNCTION_EXEC_THROTTLE_TTL'),
|
||||
);
|
||||
} catch {
|
||||
throw new LogicFunctionExecutionException(
|
||||
'Logic function execution rate limit exceeded',
|
||||
LogicFunctionExecutionExceptionCode.RATE_LIMIT_EXCEEDED,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async callWithTimeout<T>({
|
||||
callback,
|
||||
timeoutMs,
|
||||
}: {
|
||||
callback: () => Promise<T>;
|
||||
timeoutMs: number;
|
||||
}): Promise<T> {
|
||||
return Promise.race([
|
||||
callback(),
|
||||
new Promise<T>((_, reject) =>
|
||||
setTimeout(() => reject(new Error('Execution timed out')), timeoutMs),
|
||||
),
|
||||
]);
|
||||
}
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { LogicFunctionLayerService } from 'src/engine/core-modules/logic-function/logic-function-layer/services/logic-function-layer.service';
|
||||
import { LogicFunctionLayerEntity } from 'src/engine/metadata-modules/logic-function-layer/logic-function-layer.entity';
|
||||
import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([LogicFunctionLayerEntity]),
|
||||
WorkspaceCacheModule,
|
||||
],
|
||||
providers: [LogicFunctionLayerService],
|
||||
exports: [LogicFunctionLayerService],
|
||||
})
|
||||
export class CoreLogicFunctionLayerModule {}
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { Repository } from 'typeorm';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
|
||||
import type { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity';
|
||||
|
||||
import { LogicFunctionLayerEntity } from 'src/engine/metadata-modules/logic-function-layer/logic-function-layer.entity';
|
||||
import { CreateLogicFunctionLayerInput } from 'src/engine/metadata-modules/logic-function-layer/dtos/create-logic-function-layer.input';
|
||||
import { getLastCommonLayerDependencies } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/get-last-common-layer-dependencies';
|
||||
import { logicFunctionCreateHash } from 'src/engine/metadata-modules/logic-function/utils/logic-function-create-hash.utils';
|
||||
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
|
||||
|
||||
@Injectable()
|
||||
export class LogicFunctionLayerService {
|
||||
constructor(
|
||||
@InjectRepository(LogicFunctionLayerEntity)
|
||||
private readonly logicFunctionLayerRepository: Repository<LogicFunctionLayerEntity>,
|
||||
private readonly workspaceCacheService: WorkspaceCacheService,
|
||||
) {}
|
||||
|
||||
async create(
|
||||
{ packageJson, yarnLock }: CreateLogicFunctionLayerInput,
|
||||
workspaceId: string,
|
||||
) {
|
||||
const checksum = logicFunctionCreateHash(yarnLock);
|
||||
|
||||
const logicFunctionLayer = this.logicFunctionLayerRepository.create({
|
||||
packageJson,
|
||||
yarnLock,
|
||||
checksum,
|
||||
workspaceId,
|
||||
});
|
||||
|
||||
const savedLayer =
|
||||
await this.logicFunctionLayerRepository.save(logicFunctionLayer);
|
||||
|
||||
await this.workspaceCacheService.invalidateAndRecompute(workspaceId, [
|
||||
'logicFunctionLayerMaps',
|
||||
]);
|
||||
|
||||
return savedLayer;
|
||||
}
|
||||
|
||||
async update(
|
||||
id: string,
|
||||
data: QueryDeepPartialEntity<LogicFunctionLayerEntity>,
|
||||
workspaceId: string,
|
||||
) {
|
||||
const checksum = data.yarnLock
|
||||
? logicFunctionCreateHash(data.yarnLock as string)
|
||||
: undefined;
|
||||
|
||||
const updateData = { ...data, ...(checksum && { checksum }) };
|
||||
|
||||
const result = await this.logicFunctionLayerRepository.update(
|
||||
id,
|
||||
updateData,
|
||||
);
|
||||
|
||||
await this.workspaceCacheService.invalidateAndRecompute(workspaceId, [
|
||||
'logicFunctionLayerMaps',
|
||||
]);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async createCommonLayerIfNotExist(workspaceId: string) {
|
||||
const { packageJson, yarnLock } = await getLastCommonLayerDependencies();
|
||||
const checksum = logicFunctionCreateHash(yarnLock);
|
||||
const commonLayer = await this.logicFunctionLayerRepository.findOne({
|
||||
where: {
|
||||
checksum,
|
||||
workspaceId,
|
||||
},
|
||||
});
|
||||
|
||||
if (isDefined(commonLayer)) {
|
||||
return commonLayer;
|
||||
}
|
||||
|
||||
return this.create({ packageJson, yarnLock }, workspaceId);
|
||||
}
|
||||
}
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
import { Scope } from '@nestjs/common';
|
||||
|
||||
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
|
||||
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
|
||||
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
|
||||
import { LogicFunctionExecutorService } from 'src/engine/core-modules/logic-function/logic-function-executor/services/logic-function-executor.service';
|
||||
|
||||
export type LogicFunctionTriggerJobData = {
|
||||
logicFunctionId: string;
|
||||
workspaceId: string;
|
||||
payload?: object;
|
||||
};
|
||||
|
||||
@Processor({
|
||||
queueName: MessageQueue.logicFunctionQueue,
|
||||
scope: Scope.REQUEST,
|
||||
})
|
||||
export class LogicFunctionTriggerJob {
|
||||
constructor(
|
||||
private readonly logicFunctionExecutorService: LogicFunctionExecutorService,
|
||||
) {}
|
||||
|
||||
@Process(LogicFunctionTriggerJob.name)
|
||||
async handle(logicFunctionPayloads: LogicFunctionTriggerJobData[]) {
|
||||
await Promise.all(
|
||||
logicFunctionPayloads.map(
|
||||
async (logicFunctionPayload) =>
|
||||
await this.logicFunctionExecutorService.executeOneLogicFunction({
|
||||
id: logicFunctionPayload.logicFunctionId,
|
||||
workspaceId: logicFunctionPayload.workspaceId,
|
||||
payload: logicFunctionPayload.payload ?? {},
|
||||
}),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { TokenModule } from 'src/engine/core-modules/auth/token/token.module';
|
||||
import { WorkspaceDomainsModule } from 'src/engine/core-modules/domain/workspace-domains/workspace-domains.module';
|
||||
import { LogicFunctionTriggerJob } from 'src/engine/core-modules/logic-function/logic-function-trigger/jobs/logic-function-trigger.job';
|
||||
import { CronTriggerCronCommand } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.command';
|
||||
import { CronTriggerCronJob } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job';
|
||||
import { CallDatabaseEventTriggerJobsJob } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job';
|
||||
import { RouteTriggerService } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/route/route-trigger.service';
|
||||
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import { LogicFunctionEntity } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([LogicFunctionEntity, WorkspaceEntity]),
|
||||
TokenModule,
|
||||
WorkspaceDomainsModule,
|
||||
],
|
||||
providers: [
|
||||
LogicFunctionTriggerJob,
|
||||
CronTriggerCronJob,
|
||||
CronTriggerCronCommand,
|
||||
CallDatabaseEventTriggerJobsJob,
|
||||
RouteTriggerService,
|
||||
],
|
||||
exports: [CronTriggerCronCommand, RouteTriggerService],
|
||||
})
|
||||
export class LogicFunctionTriggerModule {}
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
import { Command, CommandRunner } from 'nest-commander';
|
||||
|
||||
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
|
||||
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
|
||||
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
|
||||
import {
|
||||
CRON_TRIGGER_CRON_PATTERN,
|
||||
CronTriggerCronJob,
|
||||
} from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job';
|
||||
|
||||
@Command({
|
||||
name: 'cron:trigger:start-cron-trigger',
|
||||
description: 'Starts a cron job to trigger cron triggered logic functions',
|
||||
})
|
||||
export class CronTriggerCronCommand extends CommandRunner {
|
||||
constructor(
|
||||
@InjectMessageQueue(MessageQueue.cronQueue)
|
||||
private readonly messageQueueService: MessageQueueService,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
async run(): Promise<void> {
|
||||
await this.messageQueueService.addCron<undefined>({
|
||||
jobName: CronTriggerCronJob.name,
|
||||
data: undefined,
|
||||
options: {
|
||||
repeat: {
|
||||
pattern: CRON_TRIGGER_CRON_PATTERN,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
import { WorkspaceActivationStatus } from 'twenty-shared/workspace';
|
||||
import { IsNull, Not, Repository } from 'typeorm';
|
||||
|
||||
import { SentryCronMonitor } from 'src/engine/core-modules/cron/sentry-cron-monitor.decorator';
|
||||
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
|
||||
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
|
||||
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
|
||||
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
|
||||
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
|
||||
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
|
||||
import {
|
||||
LogicFunctionTriggerJob,
|
||||
LogicFunctionTriggerJobData,
|
||||
} from 'src/engine/core-modules/logic-function/logic-function-trigger/jobs/logic-function-trigger.job';
|
||||
import { LogicFunctionEntity } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
|
||||
import { shouldRunNow } from 'src/utils/should-run-now.utils';
|
||||
|
||||
export const CRON_TRIGGER_CRON_PATTERN = '* * * * *';
|
||||
|
||||
@Processor(MessageQueue.cronQueue)
|
||||
export class CronTriggerCronJob {
|
||||
constructor(
|
||||
@InjectMessageQueue(MessageQueue.logicFunctionQueue)
|
||||
private readonly messageQueueService: MessageQueueService,
|
||||
@InjectRepository(WorkspaceEntity)
|
||||
private readonly workspaceRepository: Repository<WorkspaceEntity>,
|
||||
@InjectRepository(LogicFunctionEntity)
|
||||
private readonly logicFunctionRepository: Repository<LogicFunctionEntity>,
|
||||
) {}
|
||||
|
||||
@Process(CronTriggerCronJob.name)
|
||||
@SentryCronMonitor(CronTriggerCronJob.name, CRON_TRIGGER_CRON_PATTERN)
|
||||
async handle() {
|
||||
const activeWorkspaces = await this.workspaceRepository.find({
|
||||
where: {
|
||||
activationStatus: WorkspaceActivationStatus.ACTIVE,
|
||||
},
|
||||
select: ['id'],
|
||||
});
|
||||
|
||||
const now = new Date();
|
||||
|
||||
for (const activeWorkspace of activeWorkspaces) {
|
||||
const logicFunctionsWithCronTrigger =
|
||||
await this.logicFunctionRepository.find({
|
||||
where: {
|
||||
workspaceId: activeWorkspace.id,
|
||||
cronTriggerSettings: Not(IsNull()),
|
||||
},
|
||||
select: ['id', 'cronTriggerSettings', 'workspaceId'],
|
||||
});
|
||||
|
||||
for (const logicFunction of logicFunctionsWithCronTrigger) {
|
||||
const cronSettings = logicFunction.cronTriggerSettings;
|
||||
|
||||
if (!isDefined(cronSettings?.pattern)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!shouldRunNow(cronSettings.pattern, now)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
await this.messageQueueService.add<LogicFunctionTriggerJobData[]>(
|
||||
LogicFunctionTriggerJob.name,
|
||||
[
|
||||
{
|
||||
logicFunctionId: logicFunction.id,
|
||||
workspaceId: logicFunction.workspaceId,
|
||||
payload: {},
|
||||
},
|
||||
],
|
||||
{ retryLimit: 3 },
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import chunk from 'lodash.chunk';
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
import { IsNull, Not, Repository } from 'typeorm';
|
||||
|
||||
import type { ObjectRecordEvent } from 'twenty-shared/database-events';
|
||||
|
||||
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
|
||||
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
|
||||
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
|
||||
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
|
||||
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
|
||||
import { transformEventBatchToEventPayloads } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/transform-event-batch-to-event-payloads';
|
||||
import {
|
||||
LogicFunctionTriggerJob,
|
||||
LogicFunctionTriggerJobData,
|
||||
} from 'src/engine/core-modules/logic-function/logic-function-trigger/jobs/logic-function-trigger.job';
|
||||
import { LogicFunctionEntity } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
|
||||
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
|
||||
|
||||
const DATABASE_EVENT_JOBS_CHUNK_SIZE = 20;
|
||||
|
||||
@Processor(MessageQueue.triggerQueue)
|
||||
export class CallDatabaseEventTriggerJobsJob {
|
||||
constructor(
|
||||
@InjectMessageQueue(MessageQueue.logicFunctionQueue)
|
||||
private readonly messageQueueService: MessageQueueService,
|
||||
@InjectRepository(LogicFunctionEntity)
|
||||
private readonly logicFunctionRepository: Repository<LogicFunctionEntity>,
|
||||
) {}
|
||||
|
||||
@Process(CallDatabaseEventTriggerJobsJob.name)
|
||||
async handle(workspaceEventBatch: WorkspaceEventBatch<ObjectRecordEvent>) {
|
||||
const logicFunctionsWithDatabaseEventTrigger =
|
||||
await this.logicFunctionRepository.find({
|
||||
where: {
|
||||
workspaceId: workspaceEventBatch.workspaceId,
|
||||
databaseEventTriggerSettings: Not(IsNull()),
|
||||
},
|
||||
select: ['id', 'databaseEventTriggerSettings', 'workspaceId'],
|
||||
});
|
||||
|
||||
const logicFunctionsToTrigger =
|
||||
logicFunctionsWithDatabaseEventTrigger.filter((logicFunction) =>
|
||||
this.shouldTriggerJob({
|
||||
workspaceEventBatch,
|
||||
eventName: isDefined(logicFunction.databaseEventTriggerSettings)
|
||||
? logicFunction.databaseEventTriggerSettings.eventName
|
||||
: '',
|
||||
}),
|
||||
);
|
||||
|
||||
const logicFunctionPayloads = transformEventBatchToEventPayloads({
|
||||
logicFunctions: logicFunctionsToTrigger,
|
||||
workspaceEventBatch,
|
||||
});
|
||||
|
||||
if (logicFunctionPayloads.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const logicFunctionPayloadsChunks = chunk(
|
||||
logicFunctionPayloads,
|
||||
DATABASE_EVENT_JOBS_CHUNK_SIZE,
|
||||
);
|
||||
|
||||
for (const logicFunctionPayloadsChunk of logicFunctionPayloadsChunks) {
|
||||
await this.messageQueueService.add<LogicFunctionTriggerJobData[]>(
|
||||
LogicFunctionTriggerJob.name,
|
||||
logicFunctionPayloadsChunk,
|
||||
{ retryLimit: 3 },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private shouldTriggerJob({
|
||||
workspaceEventBatch,
|
||||
eventName,
|
||||
}: {
|
||||
workspaceEventBatch: WorkspaceEventBatch<ObjectRecordEvent>;
|
||||
eventName: string;
|
||||
}) {
|
||||
const [nameSingular, operation] = workspaceEventBatch.name.split('.');
|
||||
|
||||
const validEventNames = [
|
||||
`${nameSingular}.${operation}`,
|
||||
`*.${operation}`,
|
||||
`${nameSingular}.*`,
|
||||
'*.*',
|
||||
];
|
||||
|
||||
return validEventNames.includes(eventName);
|
||||
}
|
||||
}
|
||||
+362
@@ -0,0 +1,362 @@
|
||||
import type { ObjectRecordEvent } from 'twenty-shared/database-events';
|
||||
|
||||
import { transformEventBatchToEventPayloads } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/transform-event-batch-to-event-payloads';
|
||||
import { getFlatObjectMetadataMock } from 'src/engine/metadata-modules/flat-object-metadata/__mocks__/get-flat-object-metadata.mock';
|
||||
import { type LogicFunctionEntity } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
|
||||
import type { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
|
||||
|
||||
const createMockLogicFunction = (
|
||||
overrides: Partial<LogicFunctionEntity> = {},
|
||||
): LogicFunctionEntity =>
|
||||
({
|
||||
id: 'function-1',
|
||||
workspaceId: 'workspace-1',
|
||||
databaseEventTriggerSettings: {
|
||||
eventName: 'company.updated',
|
||||
},
|
||||
...overrides,
|
||||
}) as LogicFunctionEntity;
|
||||
|
||||
const createMockEvent = (
|
||||
overrides: Partial<ObjectRecordEvent> = {},
|
||||
): ObjectRecordEvent =>
|
||||
({
|
||||
recordId: 'record-1',
|
||||
properties: {
|
||||
after: {},
|
||||
},
|
||||
...overrides,
|
||||
}) as ObjectRecordEvent;
|
||||
|
||||
const createMockWorkspaceEventBatch = (
|
||||
overrides: Partial<WorkspaceEventBatch<ObjectRecordEvent>> = {},
|
||||
): WorkspaceEventBatch<ObjectRecordEvent> => ({
|
||||
name: 'company.updated',
|
||||
workspaceId: 'workspace-1',
|
||||
objectMetadata: getFlatObjectMetadataMock({
|
||||
universalIdentifier: 'company-uuid',
|
||||
nameSingular: 'company',
|
||||
}),
|
||||
events: [createMockEvent()],
|
||||
...overrides,
|
||||
});
|
||||
|
||||
describe('transformEventBatchToEventPayloads', () => {
|
||||
describe('basic transformation', () => {
|
||||
it('should transform a single event batch with a single logic function', () => {
|
||||
const workspaceEventBatch = createMockWorkspaceEventBatch();
|
||||
const logicFunctions = [createMockLogicFunction()];
|
||||
|
||||
const result = transformEventBatchToEventPayloads({
|
||||
workspaceEventBatch,
|
||||
logicFunctions,
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0]).toEqual({
|
||||
logicFunctionId: 'function-1',
|
||||
workspaceId: 'workspace-1',
|
||||
payload: expect.objectContaining({
|
||||
name: 'company.updated',
|
||||
workspaceId: 'workspace-1',
|
||||
recordId: 'record-1',
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
it('should create multiple payloads for multiple events in a batch', () => {
|
||||
const workspaceEventBatch = createMockWorkspaceEventBatch({
|
||||
events: [
|
||||
createMockEvent({ recordId: 'record-1' }),
|
||||
createMockEvent({ recordId: 'record-2' }),
|
||||
createMockEvent({ recordId: 'record-3' }),
|
||||
],
|
||||
});
|
||||
const logicFunctions = [createMockLogicFunction()];
|
||||
|
||||
const result = transformEventBatchToEventPayloads({
|
||||
workspaceEventBatch,
|
||||
logicFunctions,
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(3);
|
||||
expect(
|
||||
result.map((r) => (r.payload as ObjectRecordEvent).recordId),
|
||||
).toEqual(['record-1', 'record-2', 'record-3']);
|
||||
});
|
||||
|
||||
it('should create payloads for each logic function', () => {
|
||||
const workspaceEventBatch = createMockWorkspaceEventBatch();
|
||||
const logicFunctions = [
|
||||
createMockLogicFunction({
|
||||
id: 'function-1',
|
||||
}),
|
||||
createMockLogicFunction({
|
||||
id: 'function-2',
|
||||
}),
|
||||
];
|
||||
|
||||
const result = transformEventBatchToEventPayloads({
|
||||
workspaceEventBatch,
|
||||
logicFunctions,
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result.map((r) => r.logicFunctionId)).toEqual([
|
||||
'function-1',
|
||||
'function-2',
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('updatedFields filtering', () => {
|
||||
it('should include all events when updatedFields is undefined', () => {
|
||||
const workspaceEventBatch = createMockWorkspaceEventBatch({
|
||||
name: 'company.updated',
|
||||
events: [
|
||||
createMockEvent({
|
||||
recordId: 'record-1',
|
||||
properties: { after: {}, updatedFields: ['name'] },
|
||||
}),
|
||||
createMockEvent({
|
||||
recordId: 'record-2',
|
||||
properties: { after: {}, updatedFields: ['address'] },
|
||||
}),
|
||||
],
|
||||
});
|
||||
const logicFunctions = [
|
||||
createMockLogicFunction({
|
||||
databaseEventTriggerSettings: { eventName: 'company.updated' },
|
||||
}),
|
||||
];
|
||||
|
||||
const result = transformEventBatchToEventPayloads({
|
||||
workspaceEventBatch,
|
||||
logicFunctions,
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('should include all events when updatedFields is empty array', () => {
|
||||
const workspaceEventBatch = createMockWorkspaceEventBatch({
|
||||
name: 'company.updated',
|
||||
events: [
|
||||
createMockEvent({
|
||||
recordId: 'record-1',
|
||||
properties: { after: {}, updatedFields: ['name'] },
|
||||
}),
|
||||
createMockEvent({
|
||||
recordId: 'record-2',
|
||||
properties: { after: {}, updatedFields: ['address'] },
|
||||
}),
|
||||
],
|
||||
});
|
||||
const logicFunctions = [
|
||||
createMockLogicFunction({
|
||||
databaseEventTriggerSettings: {
|
||||
eventName: 'company.updated',
|
||||
updatedFields: [],
|
||||
},
|
||||
}),
|
||||
];
|
||||
|
||||
const result = transformEventBatchToEventPayloads({
|
||||
workspaceEventBatch,
|
||||
logicFunctions,
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('should filter events to only those matching updatedFields', () => {
|
||||
const workspaceEventBatch = createMockWorkspaceEventBatch({
|
||||
name: 'company.updated',
|
||||
events: [
|
||||
createMockEvent({
|
||||
recordId: 'record-1',
|
||||
properties: { after: {}, updatedFields: ['name'] },
|
||||
}),
|
||||
createMockEvent({
|
||||
recordId: 'record-2',
|
||||
properties: { after: {}, updatedFields: ['address'] },
|
||||
}),
|
||||
createMockEvent({
|
||||
recordId: 'record-3',
|
||||
properties: { after: {}, updatedFields: ['name', 'description'] },
|
||||
}),
|
||||
],
|
||||
});
|
||||
const logicFunctions = [
|
||||
createMockLogicFunction({
|
||||
databaseEventTriggerSettings: {
|
||||
eventName: 'company.updated',
|
||||
updatedFields: ['name'],
|
||||
},
|
||||
}),
|
||||
];
|
||||
|
||||
const result = transformEventBatchToEventPayloads({
|
||||
workspaceEventBatch,
|
||||
logicFunctions,
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
expect(
|
||||
result.map((r) => (r.payload as ObjectRecordEvent).recordId),
|
||||
).toEqual(['record-1', 'record-3']);
|
||||
});
|
||||
|
||||
it('should filter events matching any of the specified updatedFields', () => {
|
||||
const workspaceEventBatch = createMockWorkspaceEventBatch({
|
||||
name: 'company.updated',
|
||||
events: [
|
||||
createMockEvent({
|
||||
recordId: 'record-1',
|
||||
properties: { after: {}, updatedFields: ['name'] },
|
||||
}),
|
||||
createMockEvent({
|
||||
recordId: 'record-2',
|
||||
properties: { after: {}, updatedFields: ['address'] },
|
||||
}),
|
||||
createMockEvent({
|
||||
recordId: 'record-3',
|
||||
properties: { after: {}, updatedFields: ['phone'] },
|
||||
}),
|
||||
],
|
||||
});
|
||||
const logicFunctions = [
|
||||
createMockLogicFunction({
|
||||
databaseEventTriggerSettings: {
|
||||
eventName: 'company.updated',
|
||||
updatedFields: ['name', 'address'],
|
||||
},
|
||||
}),
|
||||
];
|
||||
|
||||
const result = transformEventBatchToEventPayloads({
|
||||
workspaceEventBatch,
|
||||
logicFunctions,
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
expect(
|
||||
result.map((r) => (r.payload as ObjectRecordEvent).recordId),
|
||||
).toEqual(['record-1', 'record-2']);
|
||||
});
|
||||
|
||||
it('should return no events when none match the updatedFields filter', () => {
|
||||
const workspaceEventBatch = createMockWorkspaceEventBatch({
|
||||
name: 'company.updated',
|
||||
events: [
|
||||
createMockEvent({
|
||||
recordId: 'record-1',
|
||||
properties: { after: {}, updatedFields: ['name'] },
|
||||
}),
|
||||
createMockEvent({
|
||||
recordId: 'record-2',
|
||||
properties: { after: {}, updatedFields: ['address'] },
|
||||
}),
|
||||
],
|
||||
});
|
||||
const logicFunctions = [
|
||||
createMockLogicFunction({
|
||||
databaseEventTriggerSettings: {
|
||||
eventName: 'company.updated',
|
||||
updatedFields: ['phone'],
|
||||
},
|
||||
}),
|
||||
];
|
||||
|
||||
const result = transformEventBatchToEventPayloads({
|
||||
workspaceEventBatch,
|
||||
logicFunctions,
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should handle different updatedFields filters per logic function', () => {
|
||||
const workspaceEventBatch = createMockWorkspaceEventBatch({
|
||||
name: 'company.updated',
|
||||
events: [
|
||||
createMockEvent({
|
||||
recordId: 'record-1',
|
||||
properties: { after: {}, updatedFields: ['name'] },
|
||||
}),
|
||||
createMockEvent({
|
||||
recordId: 'record-2',
|
||||
properties: { after: {}, updatedFields: ['address'] },
|
||||
}),
|
||||
],
|
||||
});
|
||||
const logicFunctions = [
|
||||
createMockLogicFunction({
|
||||
id: 'function-1',
|
||||
databaseEventTriggerSettings: {
|
||||
eventName: 'company.updated',
|
||||
updatedFields: ['name'],
|
||||
},
|
||||
}),
|
||||
createMockLogicFunction({
|
||||
id: 'function-2',
|
||||
databaseEventTriggerSettings: {
|
||||
eventName: 'company.updated',
|
||||
updatedFields: ['address'],
|
||||
},
|
||||
}),
|
||||
];
|
||||
|
||||
const result = transformEventBatchToEventPayloads({
|
||||
workspaceEventBatch,
|
||||
logicFunctions,
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
|
||||
const function1Payloads = result.filter(
|
||||
(r) => r.logicFunctionId === 'function-1',
|
||||
);
|
||||
const function2Payloads = result.filter(
|
||||
(r) => r.logicFunctionId === 'function-2',
|
||||
);
|
||||
|
||||
expect(function1Payloads).toHaveLength(1);
|
||||
expect((function1Payloads[0].payload as ObjectRecordEvent).recordId).toBe(
|
||||
'record-1',
|
||||
);
|
||||
|
||||
expect(function2Payloads).toHaveLength(1);
|
||||
expect((function2Payloads[0].payload as ObjectRecordEvent).recordId).toBe(
|
||||
'record-2',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('edge cases', () => {
|
||||
it('should return empty array when no logic functions provided', () => {
|
||||
const workspaceEventBatch = createMockWorkspaceEventBatch();
|
||||
|
||||
const result = transformEventBatchToEventPayloads({
|
||||
workspaceEventBatch,
|
||||
logicFunctions: [],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should return empty array when no events in batch', () => {
|
||||
const workspaceEventBatch = createMockWorkspaceEventBatch({
|
||||
events: [],
|
||||
});
|
||||
const logicFunctions = [createMockLogicFunction()];
|
||||
|
||||
const result = transformEventBatchToEventPayloads({
|
||||
workspaceEventBatch,
|
||||
logicFunctions,
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
import { isDefined } from 'twenty-shared/utils';
|
||||
|
||||
import type {
|
||||
DatabaseEventPayload,
|
||||
ObjectRecordEvent,
|
||||
} from 'twenty-shared/database-events';
|
||||
|
||||
import { type LogicFunctionTriggerJobData } from 'src/engine/core-modules/logic-function/logic-function-trigger/jobs/logic-function-trigger.job';
|
||||
import { type LogicFunctionEntity } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
|
||||
import type { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
|
||||
|
||||
export const transformEventBatchToEventPayloads = ({
|
||||
workspaceEventBatch,
|
||||
logicFunctions,
|
||||
}: {
|
||||
workspaceEventBatch: WorkspaceEventBatch<ObjectRecordEvent>;
|
||||
logicFunctions: LogicFunctionEntity[];
|
||||
}): LogicFunctionTriggerJobData[] => {
|
||||
const result: LogicFunctionTriggerJobData[] = [];
|
||||
const { events, ...batchEventInfo } = workspaceEventBatch;
|
||||
const [, operation] = workspaceEventBatch.name.split('.');
|
||||
|
||||
for (const logicFunction of logicFunctions) {
|
||||
const triggerUpdatedFields =
|
||||
logicFunction.databaseEventTriggerSettings?.updatedFields;
|
||||
|
||||
const filteredEvents = filterEventsByUpdatedFields({
|
||||
events,
|
||||
operation,
|
||||
triggerUpdatedFields,
|
||||
});
|
||||
|
||||
for (const event of filteredEvents) {
|
||||
const payload: DatabaseEventPayload = { ...batchEventInfo, ...event };
|
||||
|
||||
result.push({
|
||||
logicFunctionId: logicFunction.id,
|
||||
workspaceId: logicFunction.workspaceId,
|
||||
payload,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
};
|
||||
|
||||
const filterEventsByUpdatedFields = ({
|
||||
events,
|
||||
operation,
|
||||
triggerUpdatedFields,
|
||||
}: {
|
||||
events: ObjectRecordEvent[];
|
||||
operation: string;
|
||||
triggerUpdatedFields?: string[];
|
||||
}): ObjectRecordEvent[] => {
|
||||
if (
|
||||
operation !== 'updated' ||
|
||||
!isDefined(triggerUpdatedFields) ||
|
||||
triggerUpdatedFields.length === 0
|
||||
) {
|
||||
return events;
|
||||
}
|
||||
|
||||
return events.filter((event) => {
|
||||
const eventUpdatedFields = (
|
||||
event.properties as { updatedFields?: string[] }
|
||||
)?.updatedFields;
|
||||
|
||||
if (!isDefined(eventUpdatedFields) || eventUpdatedFields.length === 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return eventUpdatedFields.some((fieldName: string) =>
|
||||
triggerUpdatedFields.includes(fieldName),
|
||||
);
|
||||
});
|
||||
};
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
import {
|
||||
type ArgumentsHost,
|
||||
Catch,
|
||||
type ExceptionFilter,
|
||||
} from '@nestjs/common';
|
||||
|
||||
import type { Response } from 'express';
|
||||
|
||||
import {
|
||||
RouteTriggerException,
|
||||
RouteTriggerExceptionCode,
|
||||
} from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/route/exceptions/route-trigger.exception';
|
||||
import type { CustomException } from 'src/utils/custom-exception';
|
||||
import { HttpExceptionHandlerService } from 'src/engine/core-modules/exception-handler/http-exception-handler.service';
|
||||
|
||||
@Catch(RouteTriggerException)
|
||||
export class RouteTriggerRestApiExceptionFilter implements ExceptionFilter {
|
||||
constructor(
|
||||
private readonly httpExceptionHandlerService: HttpExceptionHandlerService,
|
||||
) {}
|
||||
|
||||
catch(exception: RouteTriggerException, host: ArgumentsHost) {
|
||||
const ctx = host.switchToHttp();
|
||||
const response = ctx.getResponse<Response>();
|
||||
|
||||
switch (exception.code) {
|
||||
case RouteTriggerExceptionCode.WORKSPACE_NOT_FOUND:
|
||||
case RouteTriggerExceptionCode.ROUTE_NOT_FOUND:
|
||||
case RouteTriggerExceptionCode.TRIGGER_NOT_FOUND:
|
||||
case RouteTriggerExceptionCode.LOGIC_FUNCTION_NOT_FOUND:
|
||||
return this.httpExceptionHandlerService.handleError(
|
||||
exception as CustomException,
|
||||
response,
|
||||
404,
|
||||
);
|
||||
case RouteTriggerExceptionCode.FORBIDDEN_EXCEPTION:
|
||||
return this.httpExceptionHandlerService.handleError(
|
||||
exception as CustomException,
|
||||
response,
|
||||
403,
|
||||
);
|
||||
case RouteTriggerExceptionCode.LOGIC_FUNCTION_EXECUTION_ERROR:
|
||||
return this.httpExceptionHandlerService.handleError(
|
||||
exception as CustomException,
|
||||
response,
|
||||
500,
|
||||
);
|
||||
case RouteTriggerExceptionCode.ROUTE_ALREADY_EXIST:
|
||||
case RouteTriggerExceptionCode.ROUTE_PATH_ALREADY_EXIST:
|
||||
default: {
|
||||
return this.httpExceptionHandlerService.handleError(
|
||||
exception as CustomException,
|
||||
response,
|
||||
400,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
import { type MessageDescriptor } from '@lingui/core';
|
||||
import { msg } from '@lingui/core/macro';
|
||||
import { assertUnreachable } from 'twenty-shared/utils';
|
||||
|
||||
import { CustomException } from 'src/utils/custom-exception';
|
||||
|
||||
export enum RouteTriggerExceptionCode {
|
||||
WORKSPACE_NOT_FOUND = 'WORKSPACE_NOT_FOUND',
|
||||
ROUTE_NOT_FOUND = 'ROUTE_NOT_FOUND',
|
||||
TRIGGER_NOT_FOUND = 'TRIGGER_NOT_FOUND',
|
||||
LOGIC_FUNCTION_NOT_FOUND = 'LOGIC_FUNCTION_NOT_FOUND',
|
||||
ROUTE_ALREADY_EXIST = 'ROUTE_ALREADY_EXIST',
|
||||
ROUTE_PATH_ALREADY_EXIST = 'ROUTE_PATH_ALREADY_EXIST',
|
||||
FORBIDDEN_EXCEPTION = 'FORBIDDEN_EXCEPTION',
|
||||
LOGIC_FUNCTION_EXECUTION_ERROR = 'LOGIC_FUNCTION_EXECUTION_ERROR',
|
||||
}
|
||||
|
||||
const getRouteTriggerExceptionUserFriendlyMessage = (
|
||||
code: RouteTriggerExceptionCode,
|
||||
) => {
|
||||
switch (code) {
|
||||
case RouteTriggerExceptionCode.WORKSPACE_NOT_FOUND:
|
||||
return msg`Workspace not found.`;
|
||||
case RouteTriggerExceptionCode.ROUTE_NOT_FOUND:
|
||||
return msg`Route not found.`;
|
||||
case RouteTriggerExceptionCode.TRIGGER_NOT_FOUND:
|
||||
return msg`Trigger not found.`;
|
||||
case RouteTriggerExceptionCode.LOGIC_FUNCTION_NOT_FOUND:
|
||||
return msg`Logic function not found.`;
|
||||
case RouteTriggerExceptionCode.ROUTE_ALREADY_EXIST:
|
||||
return msg`Route already exists.`;
|
||||
case RouteTriggerExceptionCode.ROUTE_PATH_ALREADY_EXIST:
|
||||
return msg`Route path already exists.`;
|
||||
case RouteTriggerExceptionCode.FORBIDDEN_EXCEPTION:
|
||||
return msg`You do not have permission to perform this action.`;
|
||||
case RouteTriggerExceptionCode.LOGIC_FUNCTION_EXECUTION_ERROR:
|
||||
return msg`Logic function execution failed.`;
|
||||
default:
|
||||
assertUnreachable(code);
|
||||
}
|
||||
};
|
||||
|
||||
export class RouteTriggerException extends CustomException<RouteTriggerExceptionCode> {
|
||||
constructor(
|
||||
message: string,
|
||||
code: RouteTriggerExceptionCode,
|
||||
{ userFriendlyMessage }: { userFriendlyMessage?: MessageDescriptor } = {},
|
||||
) {
|
||||
super(message, code, {
|
||||
userFriendlyMessage:
|
||||
userFriendlyMessage ??
|
||||
getRouteTriggerExceptionUserFriendlyMessage(code),
|
||||
});
|
||||
}
|
||||
}
|
||||
+169
@@ -0,0 +1,169 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { Request } from 'express';
|
||||
import { match } from 'path-to-regexp';
|
||||
import { assertIsDefinedOrThrow, isDefined } from 'twenty-shared/utils';
|
||||
import { IsNull, Not, Repository } from 'typeorm';
|
||||
import { HTTPMethod } from 'twenty-shared/types';
|
||||
|
||||
import { AccessTokenService } from 'src/engine/core-modules/auth/token/services/access-token.service';
|
||||
import { WorkspaceDomainsService } from 'src/engine/core-modules/domain/workspace-domains/services/workspace-domains.service';
|
||||
import { LogicFunctionExecutorService } from 'src/engine/core-modules/logic-function/logic-function-executor/services/logic-function-executor.service';
|
||||
import {
|
||||
RouteTriggerException,
|
||||
RouteTriggerExceptionCode,
|
||||
} from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/route/exceptions/route-trigger.exception';
|
||||
import { buildLogicFunctionEvent } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/route/utils/build-logic-function-event.util';
|
||||
import { LogicFunctionEntity } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
|
||||
|
||||
@Injectable()
|
||||
export class RouteTriggerService {
|
||||
constructor(
|
||||
private readonly accessTokenService: AccessTokenService,
|
||||
private readonly logicFunctionExecutorService: LogicFunctionExecutorService,
|
||||
private readonly workspaceDomainsService: WorkspaceDomainsService,
|
||||
@InjectRepository(LogicFunctionEntity)
|
||||
private readonly logicFunctionRepository: Repository<LogicFunctionEntity>,
|
||||
) {}
|
||||
|
||||
private async getLogicFunctionWithPathParamsOrFail({
|
||||
request,
|
||||
httpMethod,
|
||||
}: {
|
||||
request: Request;
|
||||
httpMethod: HTTPMethod;
|
||||
}): Promise<{
|
||||
logicFunction: LogicFunctionEntity;
|
||||
pathParams: Partial<Record<string, string | string[]>>;
|
||||
}> {
|
||||
const host = `${request.protocol}://${request.get('host')}`;
|
||||
|
||||
const workspace =
|
||||
await this.workspaceDomainsService.getWorkspaceByOriginOrDefaultWorkspace(
|
||||
host,
|
||||
);
|
||||
|
||||
assertIsDefinedOrThrow(
|
||||
workspace,
|
||||
new RouteTriggerException(
|
||||
'Workspace not found',
|
||||
RouteTriggerExceptionCode.WORKSPACE_NOT_FOUND,
|
||||
),
|
||||
);
|
||||
|
||||
const logicFunctionsWithHttpRouteTrigger =
|
||||
await this.logicFunctionRepository.find({
|
||||
where: {
|
||||
workspaceId: workspace.id,
|
||||
httpRouteTriggerSettings: Not(IsNull()),
|
||||
},
|
||||
});
|
||||
|
||||
const requestPath = request.path.replace(/^\/s\//, '/');
|
||||
|
||||
for (const logicFunction of logicFunctionsWithHttpRouteTrigger) {
|
||||
const httpRouteSettings = logicFunction.httpRouteTriggerSettings;
|
||||
|
||||
if (
|
||||
!isDefined(httpRouteSettings) ||
|
||||
httpRouteSettings.httpMethod !== httpMethod
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const routeMatcher = match(httpRouteSettings.path, {
|
||||
decode: decodeURIComponent,
|
||||
});
|
||||
const routeMatched = routeMatcher(requestPath);
|
||||
|
||||
if (routeMatched) {
|
||||
return {
|
||||
logicFunction,
|
||||
pathParams: routeMatched.params,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
throw new RouteTriggerException(
|
||||
'No Route trigger found',
|
||||
RouteTriggerExceptionCode.TRIGGER_NOT_FOUND,
|
||||
);
|
||||
}
|
||||
|
||||
private async validateWorkspaceFromRequest({
|
||||
request,
|
||||
workspaceId,
|
||||
}: {
|
||||
request: Request;
|
||||
workspaceId: string;
|
||||
}) {
|
||||
const authContext =
|
||||
await this.accessTokenService.validateTokenByRequest(request);
|
||||
|
||||
if (!isDefined(authContext.workspace)) {
|
||||
throw new RouteTriggerException(
|
||||
'Workspace not found',
|
||||
RouteTriggerExceptionCode.WORKSPACE_NOT_FOUND,
|
||||
);
|
||||
}
|
||||
|
||||
if (authContext.workspace.id !== workspaceId) {
|
||||
throw new RouteTriggerException(
|
||||
'You are not authorized',
|
||||
RouteTriggerExceptionCode.FORBIDDEN_EXCEPTION,
|
||||
);
|
||||
}
|
||||
|
||||
return authContext;
|
||||
}
|
||||
|
||||
async handle({
|
||||
request,
|
||||
httpMethod,
|
||||
}: {
|
||||
request: Request;
|
||||
httpMethod: HTTPMethod;
|
||||
}) {
|
||||
const { logicFunction, pathParams } =
|
||||
await this.getLogicFunctionWithPathParamsOrFail({
|
||||
request,
|
||||
httpMethod,
|
||||
});
|
||||
|
||||
const httpRouteSettings = logicFunction.httpRouteTriggerSettings;
|
||||
|
||||
if (httpRouteSettings?.isAuthRequired) {
|
||||
await this.validateWorkspaceFromRequest({
|
||||
request,
|
||||
workspaceId: logicFunction.workspaceId,
|
||||
});
|
||||
}
|
||||
|
||||
const event = buildLogicFunctionEvent({
|
||||
request,
|
||||
pathParameters: pathParams,
|
||||
forwardedRequestHeaders: httpRouteSettings?.forwardedRequestHeaders ?? [],
|
||||
});
|
||||
|
||||
const result =
|
||||
await this.logicFunctionExecutorService.executeOneLogicFunction({
|
||||
id: logicFunction.id,
|
||||
workspaceId: logicFunction.workspaceId,
|
||||
payload: event,
|
||||
});
|
||||
|
||||
if (!isDefined(result)) {
|
||||
return result;
|
||||
}
|
||||
|
||||
if (result.error) {
|
||||
throw new RouteTriggerException(
|
||||
result.error.errorMessage,
|
||||
RouteTriggerExceptionCode.LOGIC_FUNCTION_EXECUTION_ERROR,
|
||||
);
|
||||
}
|
||||
|
||||
return result.data;
|
||||
}
|
||||
}
|
||||
+438
@@ -0,0 +1,438 @@
|
||||
import { type Request } from 'express';
|
||||
|
||||
import {
|
||||
buildLogicFunctionEvent,
|
||||
extractBody,
|
||||
filterRequestHeaders,
|
||||
normalizePathParameters,
|
||||
normalizeQueryStringParameters,
|
||||
} from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/route/utils/build-logic-function-event.util';
|
||||
|
||||
describe('filterRequestHeaders', () => {
|
||||
it('should filter headers based on allowed names', () => {
|
||||
const requestHeaders = {
|
||||
'content-type': 'application/json',
|
||||
authorization: 'Bearer token123',
|
||||
'x-custom-header': 'custom-value',
|
||||
'user-agent': 'test-agent',
|
||||
};
|
||||
const forwardedRequestHeaders = ['content-type', 'authorization'];
|
||||
|
||||
const result = filterRequestHeaders({
|
||||
requestHeaders,
|
||||
forwardedRequestHeaders,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
'content-type': 'application/json',
|
||||
authorization: 'Bearer token123',
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle case-insensitive header names', () => {
|
||||
const requestHeaders = {
|
||||
'content-type': 'application/json',
|
||||
authorization: 'Bearer token123',
|
||||
};
|
||||
const forwardedRequestHeaders = ['Content-Type', 'AUTHORIZATION'];
|
||||
|
||||
const result = filterRequestHeaders({
|
||||
requestHeaders,
|
||||
forwardedRequestHeaders,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
'content-type': 'application/json',
|
||||
authorization: 'Bearer token123',
|
||||
});
|
||||
});
|
||||
|
||||
it('should return empty object when no headers match', () => {
|
||||
const requestHeaders = {
|
||||
'content-type': 'application/json',
|
||||
};
|
||||
const forwardedRequestHeaders = ['x-custom-header'];
|
||||
|
||||
const result = filterRequestHeaders({
|
||||
requestHeaders,
|
||||
forwardedRequestHeaders,
|
||||
});
|
||||
|
||||
expect(result).toEqual({});
|
||||
});
|
||||
|
||||
it('should return empty object when forwardedRequestHeaders is empty', () => {
|
||||
const requestHeaders = {
|
||||
'content-type': 'application/json',
|
||||
};
|
||||
|
||||
const result = filterRequestHeaders({
|
||||
requestHeaders,
|
||||
forwardedRequestHeaders: [],
|
||||
});
|
||||
|
||||
expect(result).toEqual({});
|
||||
});
|
||||
|
||||
it('should convert array header values to comma-separated string', () => {
|
||||
const requestHeaders = {
|
||||
'x-custom-array-header': ['value1', 'value2', 'value3'],
|
||||
};
|
||||
const forwardedRequestHeaders = ['x-custom-array-header'];
|
||||
|
||||
const result = filterRequestHeaders({
|
||||
requestHeaders,
|
||||
forwardedRequestHeaders,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
'x-custom-array-header': 'value1, value2, value3',
|
||||
});
|
||||
});
|
||||
|
||||
it('should skip undefined header values', () => {
|
||||
const requestHeaders = {
|
||||
'content-type': 'application/json',
|
||||
'x-missing': undefined,
|
||||
};
|
||||
const forwardedRequestHeaders = ['content-type', 'x-missing'];
|
||||
|
||||
const result = filterRequestHeaders({
|
||||
requestHeaders,
|
||||
forwardedRequestHeaders,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
'content-type': 'application/json',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractBody', () => {
|
||||
it('should return null for undefined body', () => {
|
||||
const request = { body: undefined } as Request;
|
||||
|
||||
const result = extractBody(request);
|
||||
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it('should return null for null body', () => {
|
||||
const request = { body: null } as unknown as Request;
|
||||
|
||||
const result = extractBody(request);
|
||||
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it('should parse string body as JSON', () => {
|
||||
const request = { body: '{"key":"value"}' } as unknown as Request;
|
||||
|
||||
const result = extractBody(request);
|
||||
|
||||
expect(result).toEqual({ key: 'value' });
|
||||
});
|
||||
|
||||
it('should wrap non-JSON string body in raw property', () => {
|
||||
const request = { body: 'plain text body' } as unknown as Request;
|
||||
|
||||
const result = extractBody(request);
|
||||
|
||||
expect(result).toEqual({ raw: 'plain text body' });
|
||||
});
|
||||
|
||||
it('should return object body as-is (parsed JSON)', () => {
|
||||
const request = {
|
||||
body: { key: 'value', nested: { foo: 'bar' } },
|
||||
} as Request;
|
||||
|
||||
const result = extractBody(request);
|
||||
|
||||
expect(result).toEqual({ key: 'value', nested: { foo: 'bar' } });
|
||||
});
|
||||
|
||||
it('should parse Buffer body as JSON', () => {
|
||||
const request = {
|
||||
body: Buffer.from('{"buffered":"json"}'),
|
||||
} as unknown as Request;
|
||||
|
||||
const result = extractBody(request);
|
||||
|
||||
expect(result).toEqual({ buffered: 'json' });
|
||||
});
|
||||
|
||||
it('should wrap non-JSON Buffer body in raw property', () => {
|
||||
const request = {
|
||||
body: Buffer.from('buffer content'),
|
||||
} as unknown as Request;
|
||||
|
||||
const result = extractBody(request);
|
||||
|
||||
expect(result).toEqual({ raw: 'buffer content' });
|
||||
});
|
||||
|
||||
it('should handle empty object body', () => {
|
||||
const request = { body: {} } as Request;
|
||||
|
||||
const result = extractBody(request);
|
||||
|
||||
expect(result).toEqual({});
|
||||
});
|
||||
|
||||
it('should handle array body', () => {
|
||||
const request = { body: [1, 2, 3] } as unknown as Request;
|
||||
|
||||
const result = extractBody(request);
|
||||
|
||||
expect(result).toEqual([1, 2, 3]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('normalizeQueryStringParameters', () => {
|
||||
it('should handle simple string parameters', () => {
|
||||
const query = { page: '1', limit: '10' };
|
||||
|
||||
const result = normalizeQueryStringParameters(query);
|
||||
|
||||
expect(result).toEqual({ page: '1', limit: '10' });
|
||||
});
|
||||
|
||||
it('should join array parameters with commas', () => {
|
||||
const query = { ids: ['1', '2', '3'] };
|
||||
|
||||
const result = normalizeQueryStringParameters(query);
|
||||
|
||||
expect(result).toEqual({ ids: '1,2,3' });
|
||||
});
|
||||
|
||||
it('should skip undefined parameters', () => {
|
||||
const query = { page: '1', missing: undefined };
|
||||
|
||||
const result = normalizeQueryStringParameters(query);
|
||||
|
||||
expect(result).toEqual({ page: '1' });
|
||||
});
|
||||
|
||||
it('should handle empty query object', () => {
|
||||
const query = {};
|
||||
|
||||
const result = normalizeQueryStringParameters(query);
|
||||
|
||||
expect(result).toEqual({});
|
||||
});
|
||||
|
||||
it('should stringify nested objects', () => {
|
||||
const query = { filter: { name: 'test' } as unknown as string };
|
||||
|
||||
const result = normalizeQueryStringParameters(query);
|
||||
|
||||
expect(result).toEqual({ filter: '{"name":"test"}' });
|
||||
});
|
||||
|
||||
it('should filter non-string values from arrays and join with commas', () => {
|
||||
const query = { ids: ['1', undefined as unknown as string, '2'] };
|
||||
|
||||
const result = normalizeQueryStringParameters(query);
|
||||
|
||||
expect(result).toEqual({ ids: '1,2' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('normalizePathParameters', () => {
|
||||
it('should handle simple string parameters', () => {
|
||||
const pathParams = { id: '123', slug: 'test' };
|
||||
|
||||
const result = normalizePathParameters(pathParams);
|
||||
|
||||
expect(result).toEqual({ id: '123', slug: 'test' });
|
||||
});
|
||||
|
||||
it('should join array parameters with commas', () => {
|
||||
const pathParams = { ids: ['1', '2', '3'] };
|
||||
|
||||
const result = normalizePathParameters(pathParams);
|
||||
|
||||
expect(result).toEqual({ ids: '1,2,3' });
|
||||
});
|
||||
|
||||
it('should skip undefined parameters', () => {
|
||||
const pathParams = { id: '123', missing: undefined };
|
||||
|
||||
const result = normalizePathParameters(pathParams);
|
||||
|
||||
expect(result).toEqual({ id: '123' });
|
||||
});
|
||||
|
||||
it('should handle empty object', () => {
|
||||
const pathParams = {};
|
||||
|
||||
const result = normalizePathParameters(pathParams);
|
||||
|
||||
expect(result).toEqual({});
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildLogicFunctionEvent', () => {
|
||||
const createMockRequest = (overrides: Partial<Request> = {}): Request =>
|
||||
({
|
||||
headers: {},
|
||||
query: {},
|
||||
body: undefined,
|
||||
method: 'GET',
|
||||
path: '/test',
|
||||
...overrides,
|
||||
}) as Request;
|
||||
|
||||
it('should build a complete event from Express request', () => {
|
||||
const request = createMockRequest({
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
authorization: 'Bearer token',
|
||||
'user-agent': 'test',
|
||||
},
|
||||
query: { page: '1' },
|
||||
body: { data: 'test' },
|
||||
method: 'POST',
|
||||
path: '/s/users/123',
|
||||
});
|
||||
|
||||
const result = buildLogicFunctionEvent({
|
||||
request,
|
||||
pathParameters: { id: '123' },
|
||||
forwardedRequestHeaders: ['content-type', 'authorization'],
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
authorization: 'Bearer token',
|
||||
},
|
||||
queryStringParameters: { page: '1' },
|
||||
pathParameters: { id: '123' },
|
||||
body: { data: 'test' },
|
||||
isBase64Encoded: false,
|
||||
requestContext: {
|
||||
http: {
|
||||
method: 'POST',
|
||||
path: '/s/users/123',
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should preserve the request path as-is', () => {
|
||||
const request = createMockRequest({
|
||||
path: '/s/api/users',
|
||||
});
|
||||
|
||||
const result = buildLogicFunctionEvent({
|
||||
request,
|
||||
pathParameters: {},
|
||||
forwardedRequestHeaders: [],
|
||||
});
|
||||
|
||||
expect(result.requestContext.http.path).toBe('/s/api/users');
|
||||
});
|
||||
|
||||
it('should preserve path without prefix', () => {
|
||||
const request = createMockRequest({
|
||||
path: '/api/users',
|
||||
});
|
||||
|
||||
const result = buildLogicFunctionEvent({
|
||||
request,
|
||||
pathParameters: {},
|
||||
forwardedRequestHeaders: [],
|
||||
});
|
||||
|
||||
expect(result.requestContext.http.path).toBe('/api/users');
|
||||
});
|
||||
|
||||
it('should handle GET request with no body', () => {
|
||||
const request = createMockRequest({
|
||||
method: 'GET',
|
||||
query: { search: 'test' },
|
||||
body: undefined,
|
||||
});
|
||||
|
||||
const result = buildLogicFunctionEvent({
|
||||
request,
|
||||
pathParameters: {},
|
||||
forwardedRequestHeaders: [],
|
||||
});
|
||||
|
||||
expect(result.body).toBeNull();
|
||||
expect(result.queryStringParameters).toEqual({ search: 'test' });
|
||||
});
|
||||
|
||||
it('should handle DELETE request with path parameters', () => {
|
||||
const request = createMockRequest({
|
||||
method: 'DELETE',
|
||||
path: '/s/users/456',
|
||||
});
|
||||
|
||||
const result = buildLogicFunctionEvent({
|
||||
request,
|
||||
pathParameters: { userId: '456' },
|
||||
forwardedRequestHeaders: [],
|
||||
});
|
||||
|
||||
expect(result.requestContext.http.method).toBe('DELETE');
|
||||
expect(result.pathParameters).toEqual({ userId: '456' });
|
||||
});
|
||||
|
||||
it('should filter only allowed headers', () => {
|
||||
const request = createMockRequest({
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
authorization: 'Bearer secret',
|
||||
'x-api-key': 'key123',
|
||||
cookie: 'session=abc',
|
||||
},
|
||||
});
|
||||
|
||||
const result = buildLogicFunctionEvent({
|
||||
request,
|
||||
pathParameters: {},
|
||||
forwardedRequestHeaders: ['x-api-key'],
|
||||
});
|
||||
|
||||
expect(result.headers).toEqual({
|
||||
'x-api-key': 'key123',
|
||||
});
|
||||
expect(result.headers['authorization']).toBeUndefined();
|
||||
expect(result.headers['cookie']).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should set isBase64Encoded to false', () => {
|
||||
const request = createMockRequest();
|
||||
|
||||
const result = buildLogicFunctionEvent({
|
||||
request,
|
||||
pathParameters: {},
|
||||
forwardedRequestHeaders: [],
|
||||
});
|
||||
|
||||
expect(result.isBase64Encoded).toBe(false);
|
||||
});
|
||||
|
||||
it('should handle complex path parameters', () => {
|
||||
const request = createMockRequest({
|
||||
path: '/s/organizations/org1/users/user1/posts',
|
||||
});
|
||||
|
||||
const result = buildLogicFunctionEvent({
|
||||
request,
|
||||
pathParameters: {
|
||||
orgId: 'org1',
|
||||
userId: 'user1',
|
||||
},
|
||||
forwardedRequestHeaders: [],
|
||||
});
|
||||
|
||||
expect(result.pathParameters).toEqual({
|
||||
orgId: 'org1',
|
||||
userId: 'user1',
|
||||
});
|
||||
});
|
||||
});
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
import { type Request } from 'express';
|
||||
import { type LogicFunctionEvent } from 'twenty-shared/types';
|
||||
|
||||
/**
|
||||
* Filters HTTP headers from Express request based on allowed header names
|
||||
* Header names are case-insensitive as per HTTP specification
|
||||
*/
|
||||
export const filterRequestHeaders = ({
|
||||
requestHeaders,
|
||||
forwardedRequestHeaders,
|
||||
}: {
|
||||
requestHeaders: Request['headers'];
|
||||
forwardedRequestHeaders: string[];
|
||||
}): Record<string, string | undefined> => {
|
||||
const lowercaseForwardedHeaders = forwardedRequestHeaders.map((h) =>
|
||||
h.toLowerCase(),
|
||||
);
|
||||
|
||||
const filteredHeaders: Record<string, string | undefined> = {};
|
||||
|
||||
for (const headerName of lowercaseForwardedHeaders) {
|
||||
const headerValue = requestHeaders[headerName];
|
||||
|
||||
if (headerValue !== undefined) {
|
||||
filteredHeaders[headerName] = Array.isArray(headerValue)
|
||||
? headerValue.join(', ')
|
||||
: headerValue;
|
||||
}
|
||||
}
|
||||
|
||||
return filteredHeaders;
|
||||
};
|
||||
|
||||
/**
|
||||
* Extracts the body from Express request as an object
|
||||
* Express body-parser middleware parses JSON bodies automatically
|
||||
* Returns null if body is empty/undefined
|
||||
*/
|
||||
export const extractBody = (request: Request): object | null => {
|
||||
if (request.body === undefined || request.body === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (typeof request.body === 'object' && !Buffer.isBuffer(request.body)) {
|
||||
return request.body;
|
||||
}
|
||||
|
||||
if (typeof request.body === 'string') {
|
||||
try {
|
||||
return JSON.parse(request.body);
|
||||
} catch {
|
||||
return { raw: request.body };
|
||||
}
|
||||
}
|
||||
|
||||
if (Buffer.isBuffer(request.body)) {
|
||||
try {
|
||||
return JSON.parse(request.body.toString('utf-8'));
|
||||
} catch {
|
||||
return { raw: request.body.toString('utf-8') };
|
||||
}
|
||||
}
|
||||
|
||||
return { raw: String(request.body) };
|
||||
};
|
||||
|
||||
/**
|
||||
* Converts Express query parameters to a normalized string format
|
||||
* Arrays are joined with commas (e.g., ['1', '2', '3'] → '1,2,3')
|
||||
*/
|
||||
export const normalizeQueryStringParameters = (
|
||||
query: Request['query'],
|
||||
): Record<string, string | undefined> => {
|
||||
const normalized: Record<string, string | undefined> = {};
|
||||
|
||||
for (const [key, value] of Object.entries(query)) {
|
||||
if (value === undefined) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
const stringValues = value.filter(
|
||||
(v): v is string => typeof v === 'string',
|
||||
);
|
||||
|
||||
normalized[key] = stringValues.join(',');
|
||||
} else if (typeof value === 'string') {
|
||||
normalized[key] = value;
|
||||
} else if (typeof value === 'object') {
|
||||
normalized[key] = JSON.stringify(value);
|
||||
}
|
||||
}
|
||||
|
||||
return normalized;
|
||||
};
|
||||
|
||||
/**
|
||||
* Normalizes path parameters to string format
|
||||
* Arrays are joined with commas (e.g., ['1', '2', '3'] → '1,2,3')
|
||||
*/
|
||||
export const normalizePathParameters = (
|
||||
pathParams: Record<string, string | string[] | undefined>,
|
||||
): Record<string, string | undefined> => {
|
||||
const normalized: Record<string, string | undefined> = {};
|
||||
|
||||
for (const [key, value] of Object.entries(pathParams)) {
|
||||
if (value === undefined) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
normalized[key] = value.join(',');
|
||||
} else {
|
||||
normalized[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
return normalized;
|
||||
};
|
||||
|
||||
/**
|
||||
* Builds an AWS HTTP API v2 compatible event from an Express request
|
||||
* @see https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-develop-integrations-lambda.html
|
||||
*/
|
||||
export const buildLogicFunctionEvent = ({
|
||||
request,
|
||||
pathParameters,
|
||||
forwardedRequestHeaders,
|
||||
}: {
|
||||
request: Request;
|
||||
pathParameters: Record<string, string | string[] | undefined>;
|
||||
forwardedRequestHeaders: string[];
|
||||
}): LogicFunctionEvent => {
|
||||
return {
|
||||
headers: filterRequestHeaders({
|
||||
requestHeaders: request.headers,
|
||||
forwardedRequestHeaders,
|
||||
}),
|
||||
queryStringParameters: normalizeQueryStringParameters(request.query),
|
||||
pathParameters: normalizePathParameters(pathParameters),
|
||||
body: extractBody(request),
|
||||
isBase64Encoded: false,
|
||||
requestContext: {
|
||||
http: {
|
||||
method: request.method,
|
||||
path: request.path,
|
||||
},
|
||||
},
|
||||
};
|
||||
};
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
import { type DynamicModule, Global, Module } from '@nestjs/common';
|
||||
|
||||
import { type LogicFunctionExecutorModuleAsyncOptions } from 'src/engine/core-modules/logic-function/logic-function-executor/interfaces/logic-function-executor.interface';
|
||||
|
||||
import { LogicFunctionBuildModule } from 'src/engine/core-modules/logic-function/logic-function-build/logic-function-build.module';
|
||||
import { LogicFunctionDriversModule } from 'src/engine/core-modules/logic-function/logic-function-drivers/logic-function-drivers.module';
|
||||
import { LogicFunctionExecutorModule } from 'src/engine/core-modules/logic-function/logic-function-executor/logic-function-executor.module';
|
||||
import { CoreLogicFunctionLayerModule } from 'src/engine/core-modules/logic-function/logic-function-layer/logic-function-layer.module';
|
||||
import { LogicFunctionTriggerModule } from 'src/engine/core-modules/logic-function/logic-function-trigger/logic-function-trigger.module';
|
||||
|
||||
@Global()
|
||||
@Module({})
|
||||
export class LogicFunctionModule {
|
||||
static forRootAsync(
|
||||
options: LogicFunctionExecutorModuleAsyncOptions,
|
||||
): DynamicModule {
|
||||
return {
|
||||
module: LogicFunctionModule,
|
||||
imports: [
|
||||
LogicFunctionDriversModule.forRootAsync(options),
|
||||
LogicFunctionExecutorModule,
|
||||
LogicFunctionBuildModule,
|
||||
CoreLogicFunctionLayerModule,
|
||||
LogicFunctionTriggerModule,
|
||||
],
|
||||
exports: [
|
||||
LogicFunctionDriversModule,
|
||||
LogicFunctionExecutorModule,
|
||||
LogicFunctionBuildModule,
|
||||
CoreLogicFunctionLayerModule,
|
||||
LogicFunctionTriggerModule,
|
||||
],
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user