Rework logic function module (#17588)

core-modules/logic-function/
├── logic-function.module.ts
├── logic-function-executor/
│   ├── logic-function-executor.module.ts
│   ├── commands/
│   │   └── add-packages.command.ts
│   ├── constants/
│   │   └── logic-function-executor.constants.ts
│   ├── factories/
│   │   └── logic-function-module.factory.ts
│   ├── interfaces/
│   │   └── logic-function-executor.interface.ts
│   └── services/
│       └── logic-function-executor.service.ts
├── logic-function-build/
│   ├── logic-function-build.module.ts
│   ├── services/
│   │   └── logic-function-build.service.ts
│   └── utils/
│       └── get-logic-function-base-folder-path.util.ts
├── logic-function-drivers/
│   ├── logic-function-drivers.module.ts
│   ├── constants/
│   │   └── ...
│   ├── drivers/
│   │   ├── disabled.driver.ts
│   │   ├── lambda.driver.ts
│   │   └── local.driver.ts
│   ├── interfaces/
│   │   └── logic-function-executor-driver.interface.ts
│   ├── layers/
│   │   └── ...
│   └── utils/
│       └── ...
├── logic-function-layer/
│   ├── logic-function-layer.module.ts
│   └── services/
│       └── logic-function-layer.service.ts
└── logic-function-trigger/
    ├── logic-function-trigger.module.ts
    ├── jobs/
    │   └── logic-function-trigger.job.ts
    └── triggers/
        ├── cron/
        ├── database-event/
        └── route/
            ├── exceptions/
            ├── services/
            │   └── route-trigger.service.ts
            └── utils/
This commit is contained in:
Charles Bochet
2026-01-30 19:28:20 +01:00
committed by GitHub
parent cb5e5e4622
commit 4a770eafa1
73 changed files with 497 additions and 409 deletions
@@ -0,0 +1,9 @@
import { Module } from '@nestjs/common';
import { LogicFunctionBuildService } from 'src/engine/core-modules/logic-function/logic-function-build/services/logic-function-build.service';
@Module({
providers: [LogicFunctionBuildService],
exports: [LogicFunctionBuildService],
})
export class LogicFunctionBuildModule {}
@@ -0,0 +1,120 @@
import { Injectable } from '@nestjs/common';
import fs from 'fs/promises';
import { dirname, join } from 'path';
import { build } from 'esbuild';
import { FileFolder } from 'twenty-shared/types';
import { FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { LambdaBuildDirectoryManager } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/lambda-build-directory-manager';
import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
import {
getLogicFunctionBaseFolderPath,
getRelativePathFromBase,
} from 'src/engine/core-modules/logic-function/logic-function-build/utils/get-logic-function-base-folder-path.util';
export type FunctionBuildParams = {
flatLogicFunction: FlatLogicFunction;
applicationUniversalIdentifier: string;
};
@Injectable()
export class LogicFunctionBuildService {
constructor(private readonly fileStorageService: FileStorageService) {}
async isBuilt({
flatLogicFunction,
applicationUniversalIdentifier,
}: FunctionBuildParams): Promise<boolean> {
return await this.fileStorageService.checkFileExists_v2({
workspaceId: flatLogicFunction.workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.BuiltLogicFunction,
resourcePath: flatLogicFunction.builtHandlerPath,
});
}
async buildAndUpload({
flatLogicFunction,
applicationUniversalIdentifier,
}: FunctionBuildParams): Promise<void> {
const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
try {
const { sourceTemporaryDir } = await lambdaBuildDirectoryManager.init();
const baseFolderPath = getLogicFunctionBaseFolderPath(
flatLogicFunction.sourceHandlerPath,
);
await this.fileStorageService.downloadFolder_v2({
workspaceId: flatLogicFunction.workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.Source,
resourcePath: baseFolderPath,
localPath: sourceTemporaryDir,
});
const relativeSourcePath = getRelativePathFromBase(
flatLogicFunction.sourceHandlerPath,
baseFolderPath,
);
const relativeBuiltPath = getRelativePathFromBase(
flatLogicFunction.builtHandlerPath,
baseFolderPath,
);
const builtBundleFilePath = await this.buildInMemory({
sourceTemporaryDir,
sourceHandlerPath: relativeSourcePath,
builtHandlerPath: relativeBuiltPath,
});
const builtFile = await fs.readFile(builtBundleFilePath, 'utf-8');
await this.fileStorageService.writeFile_v2({
workspaceId: flatLogicFunction.workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.BuiltLogicFunction,
resourcePath: flatLogicFunction.builtHandlerPath,
sourceFile: builtFile,
mimeType: 'application/javascript',
settings: {
isTemporaryFile: false,
toDelete: false,
},
});
} finally {
await lambdaBuildDirectoryManager.clean();
}
}
private async buildInMemory({
sourceTemporaryDir,
sourceHandlerPath,
builtHandlerPath,
}: {
sourceTemporaryDir: string;
sourceHandlerPath: string;
builtHandlerPath: string;
}): Promise<string> {
const entryFilePath = join(sourceTemporaryDir, sourceHandlerPath);
const builtBundleFilePath = join(sourceTemporaryDir, builtHandlerPath);
await fs.mkdir(dirname(builtBundleFilePath), { recursive: true });
await build({
entryPoints: [entryFilePath],
outfile: builtBundleFilePath,
platform: 'node',
format: 'esm',
target: 'es2017',
bundle: true,
sourcemap: true,
packages: 'external',
});
return builtBundleFilePath;
}
}
@@ -0,0 +1,12 @@
import { dirname } from 'path';
export const getLogicFunctionBaseFolderPath = (handlerPath: string): string => {
return dirname(dirname(handlerPath));
};
export const getRelativePathFromBase = (
handlerPath: string,
baseFolderPath: string,
): string => {
return handlerPath.replace(`${baseFolderPath}/`, '');
};
@@ -0,0 +1,28 @@
import { promises as fs } from 'fs';
import { randomBytes } from 'crypto';
export const handler = async (event) => {
const randomId = randomBytes(16).toString('hex');
const mainPath = `/tmp/${randomId}.mjs`;
// eslint-disable-next-line no-undef
const oldProcessEnv = { ...process.env };
try {
const { code, params, env, handlerName } = event;
await fs.writeFile(mainPath, code, 'utf8');
// eslint-disable-next-line no-undef
process.env = { ...process.env, ...(env ?? {}) };
const mainFile = await import(mainPath);
return await mainFile[handlerName](params);
} finally {
await fs.rm(mainPath, { force: true });
// eslint-disable-next-line no-undef
process.env = oldProcessEnv;
}
};
@@ -0,0 +1,7 @@
import { join } from 'path';
import { tmpdir } from 'os';
export const LOGIC_FUNCTION_EXECUTOR_TMPDIR_FOLDER = join(
tmpdir(),
'logic-function-executor-tmpdir',
);
@@ -0,0 +1,4 @@
export const SEED_PROJECT_INPUT_SCHEMA = {
a: null,
b: null,
};
@@ -0,0 +1,12 @@
export const main = async (params: {
a: string;
b: number;
}): Promise<object> => {
const { a, b } = params;
// Rename the parameters and code below with your own logic
// This is just an example
const message = `Hello, input: ${a} and ${b}`;
return { message };
};
@@ -0,0 +1,22 @@
import {
type LogicFunctionExecutorDriver,
type LogicFunctionExecuteResult,
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-executor-driver.interface';
import {
LogicFunctionException,
LogicFunctionExceptionCode,
} from 'src/engine/metadata-modules/logic-function/logic-function.exception';
export class DisabledDriver implements LogicFunctionExecutorDriver {
async delete(): Promise<void> {
// No-op when disabled
}
async execute(): Promise<LogicFunctionExecuteResult> {
throw new LogicFunctionException(
'Logic function execution is disabled. Set LOGIC_FUNCTION_TYPE to LOCAL or LAMBDA to enable.',
LogicFunctionExceptionCode.LOGIC_FUNCTION_DISABLED,
);
}
}
@@ -0,0 +1,389 @@
import * as fs from 'fs/promises';
import { join } from 'path';
import {
CreateFunctionCommand,
type CreateFunctionCommandInput,
DeleteFunctionCommand,
GetFunctionCommand,
InvokeCommand,
type InvokeCommandInput,
Lambda,
type LambdaClientConfig,
ListLayerVersionsCommand,
type ListLayerVersionsCommandInput,
LogType,
PublishLayerVersionCommand,
type PublishLayerVersionCommandInput,
ResourceNotFoundException,
waitUntilFunctionUpdatedV2,
} from '@aws-sdk/client-lambda';
import { AssumeRoleCommand, STSClient } from '@aws-sdk/client-sts';
import { FileFolder } from 'twenty-shared/types';
import { isDefined } from 'twenty-shared/utils';
import {
type LogicFunctionExecuteParams,
type LogicFunctionExecuteResult,
type LogicFunctionExecutorDriver,
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-executor-driver.interface';
import { type FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { copyAndBuildDependencies } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/copy-and-build-dependencies';
import { copyExecutor } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/copy-executor';
import { createZipFile } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/create-zip-file';
import {
LambdaBuildDirectoryManager,
NODE_LAYER_SUBFOLDER,
} from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/lambda-build-directory-manager';
import { type FlatLogicFunctionLayer } from 'src/engine/metadata-modules/logic-function-layer/types/flat-logic-function-layer.type';
import { LogicFunctionExecutionStatus } from 'src/engine/metadata-modules/logic-function/dtos/logic-function-execution-result.dto';
import { LogicFunctionRuntime } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import {
LogicFunctionException,
LogicFunctionExceptionCode,
} from 'src/engine/metadata-modules/logic-function/logic-function.exception';
import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
import { streamToBuffer } from 'src/utils/stream-to-buffer';
const UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS = 60;
const CREDENTIALS_DURATION_IN_SECONDS = 60 * 60; // 1h
type LambdaDriverExecutorPayload = {
code: string;
params: object;
env: Record<string, string>;
handlerName: string;
};
export interface LambdaDriverOptions extends LambdaClientConfig {
fileStorageService: FileStorageService;
region: string;
lambdaRole: string;
subhostingRole?: string;
}
export class LambdaDriver implements LogicFunctionExecutorDriver {
private lambdaClient: Lambda | undefined;
private credentialsExpiry: Date | null = null;
private readonly options: LambdaDriverOptions;
private readonly fileStorageService: FileStorageService;
constructor(options: LambdaDriverOptions) {
this.options = options;
this.lambdaClient = undefined;
this.fileStorageService = options.fileStorageService;
}
private async getLambdaClient() {
if (
!isDefined(this.lambdaClient) ||
(isDefined(this.options.subhostingRole) &&
isDefined(this.credentialsExpiry) &&
new Date() >= this.credentialsExpiry)
) {
this.lambdaClient = new Lambda({
...this.options,
...(isDefined(this.options.subhostingRole) && {
credentials: await this.getAssumeRoleCredentials(),
}),
});
}
return this.lambdaClient;
}
private async getAssumeRoleCredentials() {
const stsClient = new STSClient({ region: this.options.region });
this.credentialsExpiry = new Date(
Date.now() + (CREDENTIALS_DURATION_IN_SECONDS - 60 * 5) * 1000,
);
const assumeRoleCommand = new AssumeRoleCommand({
RoleArn: this.options.subhostingRole,
RoleSessionName: 'LambdaSession',
DurationSeconds: CREDENTIALS_DURATION_IN_SECONDS,
});
const { Credentials } = await stsClient.send(assumeRoleCommand);
if (
!isDefined(Credentials) ||
!isDefined(Credentials.AccessKeyId) ||
!isDefined(Credentials.SecretAccessKey) ||
!isDefined(Credentials.SessionToken)
) {
throw new Error('Failed to assume role');
}
return {
accessKeyId: Credentials.AccessKeyId,
secretAccessKey: Credentials.SecretAccessKey,
sessionToken: Credentials.SessionToken,
};
}
private async waitFunctionUpdates(
flatLogicFunction: FlatLogicFunction,
maxWaitTime: number = UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS,
) {
const waitParams = {
FunctionName: flatLogicFunction.id,
};
await waitUntilFunctionUpdatedV2(
{ client: await this.getLambdaClient(), maxWaitTime },
waitParams,
);
}
private getLayerName(flatLogicFunctionLayer: FlatLogicFunctionLayer) {
return flatLogicFunctionLayer.checksum;
}
private async createLayerIfNotExists(
flatLogicFunctionLayer: FlatLogicFunctionLayer,
): Promise<string> {
const layerName = this.getLayerName(flatLogicFunctionLayer);
const listLayerParams: ListLayerVersionsCommandInput = {
LayerName: layerName,
MaxItems: 1,
};
const listLayerCommand = new ListLayerVersionsCommand(listLayerParams);
const listLayerResult = await (
await this.getLambdaClient()
).send(listLayerCommand);
if (isDefined(listLayerResult.LayerVersions?.[0]?.LayerVersionArn)) {
return listLayerResult.LayerVersions[0].LayerVersionArn;
}
const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
const { sourceTemporaryDir, lambdaZipPath } =
await lambdaBuildDirectoryManager.init();
const nodeDependenciesFolder = join(
sourceTemporaryDir,
NODE_LAYER_SUBFOLDER,
);
await copyAndBuildDependencies(
nodeDependenciesFolder,
flatLogicFunctionLayer,
);
await createZipFile(sourceTemporaryDir, lambdaZipPath);
const params: PublishLayerVersionCommandInput = {
LayerName: layerName,
Content: {
ZipFile: await fs.readFile(lambdaZipPath),
},
CompatibleRuntimes: [
LogicFunctionRuntime.NODE18,
LogicFunctionRuntime.NODE22,
],
};
const command = new PublishLayerVersionCommand(params);
const result = await (await this.getLambdaClient()).send(command);
await lambdaBuildDirectoryManager.clean();
if (!isDefined(result.LayerVersionArn)) {
throw new Error('new layer version arn if undefined');
}
return result.LayerVersionArn;
}
private async getLambdaExecutor(flatLogicFunction: FlatLogicFunction) {
try {
const getFunctionCommand: GetFunctionCommand = new GetFunctionCommand({
FunctionName: flatLogicFunction.id,
});
return await (await this.getLambdaClient()).send(getFunctionCommand);
} catch (error) {
if (!(error instanceof ResourceNotFoundException)) {
throw error;
}
}
}
async delete(flatLogicFunction: FlatLogicFunction) {
const lambdaExecutor = await this.getLambdaExecutor(flatLogicFunction);
if (isDefined(lambdaExecutor)) {
const deleteFunctionCommand = new DeleteFunctionCommand({
FunctionName: flatLogicFunction.id,
});
await (await this.getLambdaClient()).send(deleteFunctionCommand);
}
}
private async isAlreadyBuilt(
flatLogicFunction: FlatLogicFunction,
flatLogicFunctionLayer: FlatLogicFunctionLayer,
) {
const lambdaExecutor = await this.getLambdaExecutor(flatLogicFunction);
if (!isDefined(lambdaExecutor)) {
return false;
}
const layers = lambdaExecutor.Configuration?.Layers;
if (!isDefined(layers) || layers.length !== 1) {
await this.delete(flatLogicFunction);
return false;
}
const layerName = this.getLayerName(flatLogicFunctionLayer);
if (layers[0].Arn?.includes(layerName)) {
return true;
}
await this.delete(flatLogicFunction);
return false;
}
private async build(
flatLogicFunction: FlatLogicFunction,
flatLogicFunctionLayer: FlatLogicFunctionLayer,
) {
if (await this.isAlreadyBuilt(flatLogicFunction, flatLogicFunctionLayer)) {
return;
}
const layerArn = await this.createLayerIfNotExists(flatLogicFunctionLayer);
const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
const { sourceTemporaryDir, lambdaZipPath } =
await lambdaBuildDirectoryManager.init();
await copyExecutor(sourceTemporaryDir);
await createZipFile(sourceTemporaryDir, lambdaZipPath);
const params: CreateFunctionCommandInput = {
Code: {
ZipFile: await fs.readFile(lambdaZipPath),
},
FunctionName: flatLogicFunction.id,
Layers: [layerArn],
Handler: 'index.handler',
Role: this.options.lambdaRole,
Runtime: flatLogicFunction.runtime,
Timeout: 900, // timeout is handled by the logic function service
};
const command = new CreateFunctionCommand(params);
await (await this.getLambdaClient()).send(command);
await lambdaBuildDirectoryManager.clean();
}
private extractLogs(logString: string): string {
const formattedLogString = Buffer.from(logString, 'base64')
.toString('utf8')
.split('\t')
.join(' ');
return formattedLogString
.replace(/^(START|END|REPORT).*\n?/gm, '')
.replace(
/^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z) [a-f0-9-]+ INFO /gm,
'$1 INFO ',
)
.trim();
}
async execute({
flatLogicFunction,
flatLogicFunctionLayer,
applicationUniversalIdentifier,
payload,
env,
}: LogicFunctionExecuteParams): Promise<LogicFunctionExecuteResult> {
await this.build(flatLogicFunction, flatLogicFunctionLayer);
await this.waitFunctionUpdates(flatLogicFunction);
const startTime = Date.now();
const compiledCode = (
await streamToBuffer(
await this.fileStorageService.readFile_v2({
workspaceId: flatLogicFunction.workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.BuiltLogicFunction,
resourcePath: flatLogicFunction.builtHandlerPath,
}),
)
).toString('utf-8');
const executorPayload: LambdaDriverExecutorPayload = {
params: payload,
code: compiledCode,
env: env ?? {},
handlerName: flatLogicFunction.handlerName,
};
const params: InvokeCommandInput = {
FunctionName: flatLogicFunction.id,
Payload: JSON.stringify(executorPayload),
LogType: LogType.Tail,
};
const command = new InvokeCommand(params);
try {
const result = await (await this.getLambdaClient()).send(command);
const parsedResult = result.Payload
? JSON.parse(result.Payload.transformToString())
: {};
const logs = result.LogResult ? this.extractLogs(result.LogResult) : '';
const duration = Date.now() - startTime;
if (result.FunctionError) {
return {
data: null,
duration,
status: LogicFunctionExecutionStatus.ERROR,
error: parsedResult,
logs,
};
}
return {
data: parsedResult,
logs,
duration,
status: LogicFunctionExecutionStatus.SUCCESS,
};
} catch (error) {
if (error instanceof ResourceNotFoundException) {
throw new LogicFunctionException(
`Function '${flatLogicFunction.id}' does not exist`,
LogicFunctionExceptionCode.LOGIC_FUNCTION_NOT_FOUND,
);
}
throw error;
}
}
}
@@ -0,0 +1,360 @@
import { promises as fs } from 'fs';
import { spawn } from 'node:child_process';
import { join } from 'path';
import { FileFolder } from 'twenty-shared/types';
import {
type LogicFunctionExecutorDriver,
type LogicFunctionExecuteParams,
type LogicFunctionExecuteResult,
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-executor-driver.interface';
import { type FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { LOGIC_FUNCTION_EXECUTOR_TMPDIR_FOLDER } from 'src/engine/core-modules/logic-function/logic-function-drivers/constants/logic-function-executor-tmpdir-folder';
import { copyAndBuildDependencies } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/copy-and-build-dependencies';
import { ConsoleListener } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/intercept-console';
import { LambdaBuildDirectoryManager } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/lambda-build-directory-manager';
import { type FlatLogicFunctionLayer } from 'src/engine/metadata-modules/logic-function-layer/types/flat-logic-function-layer.type';
import { LogicFunctionExecutionStatus } from 'src/engine/metadata-modules/logic-function/dtos/logic-function-execution-result.dto';
import {
getLogicFunctionBaseFolderPath,
getRelativePathFromBase,
} from 'src/engine/core-modules/logic-function/logic-function-build/utils/get-logic-function-base-folder-path.util';
export interface LocalDriverOptions {
fileStorageService: FileStorageService;
}
export class LocalDriver implements LogicFunctionExecutorDriver {
private readonly fileStorageService: FileStorageService;
constructor(options: LocalDriverOptions) {
this.fileStorageService = options.fileStorageService;
}
private getInMemoryLayerFolderPath = (
flatLogicFunctionLayer: FlatLogicFunctionLayer,
) => {
return join(
LOGIC_FUNCTION_EXECUTOR_TMPDIR_FOLDER,
flatLogicFunctionLayer.checksum,
);
};
private async createLayerIfNotExists(
flatLogicFunctionLayer: FlatLogicFunctionLayer,
) {
const inMemoryLayerFolderPath = this.getInMemoryLayerFolderPath(
flatLogicFunctionLayer,
);
try {
await fs.access(inMemoryLayerFolderPath);
} catch {
await copyAndBuildDependencies(
inMemoryLayerFolderPath,
flatLogicFunctionLayer,
);
}
}
async delete() {}
private async build(flatLogicFunctionLayer: FlatLogicFunctionLayer) {
await this.createLayerIfNotExists(flatLogicFunctionLayer);
}
async execute({
flatLogicFunction,
flatLogicFunctionLayer,
applicationUniversalIdentifier,
payload,
env,
}: LogicFunctionExecuteParams): Promise<LogicFunctionExecuteResult> {
await this.build(flatLogicFunctionLayer);
const startTime = Date.now();
const lambdaBuildDirectoryManager = new LambdaBuildDirectoryManager();
try {
const { sourceTemporaryDir } = await lambdaBuildDirectoryManager.init();
const baseFolderPath = getLogicFunctionBaseFolderPath(
flatLogicFunction.builtHandlerPath,
);
await this.fileStorageService.downloadFolder_v2({
workspaceId: flatLogicFunction.workspaceId,
applicationUniversalIdentifier,
fileFolder: FileFolder.BuiltLogicFunction,
resourcePath: baseFolderPath,
localPath: sourceTemporaryDir,
});
try {
await fs.symlink(
join(
this.getInMemoryLayerFolderPath(flatLogicFunctionLayer),
'node_modules',
),
join(sourceTemporaryDir, 'node_modules'),
'dir',
);
} catch (err) {
if (err.code !== 'EEXIST') {
throw err;
}
}
let logs = '';
const consoleListener = new ConsoleListener();
consoleListener.intercept((type, args) => {
const formattedArgs = args.map((arg) => {
if (typeof arg === 'object' && arg !== null) {
const seen = new WeakSet();
return JSON.stringify(
arg,
(_key, value) => {
if (typeof value === 'object' && value !== null) {
if (seen.has(value)) {
return '[Circular]'; // Handle circular references
}
seen.add(value);
}
return value;
},
2,
);
}
return arg;
});
const formattedType = type === 'log' ? 'info' : type;
logs += `${new Date().toISOString()} ${formattedType.toUpperCase()} ${formattedArgs.join(' ')}\n`;
});
try {
const relativeBuiltPath = getRelativePathFromBase(
flatLogicFunction.builtHandlerPath,
baseFolderPath,
);
const builtBundleFilePath = join(sourceTemporaryDir, relativeBuiltPath);
const runnerPath = await this.writeBootstrapRunner({
dir: sourceTemporaryDir,
builtFileAbsPath: builtBundleFilePath,
handlerName: flatLogicFunction.handlerName,
});
const { ok, result, error, stack, stdout, stderr } =
await this.runChildWithEnv({
runnerPath,
env: env ?? {},
payload,
timeoutMs: 900_000, // timeout is handled by the logic function service
});
if (stdout)
logs +=
stdout
.split('\n')
.filter(Boolean)
.map((l) => `${new Date().toISOString()} INFO ${l}`)
.join('\n') + '\n';
if (stderr)
logs +=
stderr
.split('\n')
.filter(Boolean)
.map((l) => `${new Date().toISOString()} ERROR ${l}`)
.join('\n') + '\n';
const duration = Date.now() - startTime;
if (ok) {
return {
data: (result ?? null) as object | null,
logs,
duration,
status: LogicFunctionExecutionStatus.SUCCESS,
};
}
return {
data: null,
logs,
duration,
error: {
errorType: 'UnhandledError',
errorMessage: error || 'Unknown error',
stackTrace: stack ? String(stack).split('\n') : [],
},
status: LogicFunctionExecutionStatus.ERROR,
};
} finally {
consoleListener.release();
}
} finally {
await lambdaBuildDirectoryManager.clean();
}
}
async writeBootstrapRunner({
dir,
builtFileAbsPath,
handlerName,
}: {
dir: string;
builtFileAbsPath: string;
handlerName: string;
}) {
const runnerPath = join(dir, '__runner.cjs');
const code = `
// Auto-generated. Do not edit.
const { pathToFileURL } = require('node:url');
(async () => {
try {
const builtUrl = pathToFileURL(${JSON.stringify(builtFileAbsPath)});
const mod = await import(builtUrl.href);
if (typeof mod.${handlerName} !== 'function') {
throw new Error('Export "${handlerName}" not found in function bundle');
}
let payload = undefined;
if (process.send) {
process.on('message', async (msg) => {
if (!msg || msg.type !== 'run') return;
try {
const out = await mod.${handlerName}(msg.payload);
process.send && process.send({ ok: true, result: out });
process.exit(0);
} catch (err) {
process.send && process.send({ ok: false, error: String(err), stack: err?.stack });
process.exit(1);
}
});
} else {
// Fallback: read payload from argv[2] (JSON) and print to stdout
const json = process.argv[2];
payload = json ? JSON.parse(json) : undefined;
const out = await mod.${handlerName}(payload);
console.log(JSON.stringify({ ok: true, result: out }));
process.exit(0);
}
} catch (err) {
const msg = String(err);
if (process.send) {
process.send({ ok: false, error: msg, stack: err?.stack });
} else {
console.error(msg);
}
process.exit(1);
}
})();
`;
await fs.writeFile(runnerPath, code, 'utf8');
return runnerPath;
}
runChildWithEnv(options: {
runnerPath: string;
env: Record<string, string>;
payload: unknown;
timeoutMs: number;
}) {
const { runnerPath, env, payload, timeoutMs } = options;
return new Promise<{
ok: boolean;
result?: unknown;
error?: string;
stack?: string;
stdout: string;
stderr: string;
}>((resolve) => {
// Strip NODE_OPTIONS to prevent tsx loader from being inherited
const { NODE_OPTIONS: _n1, ...cleanProcessEnv } = process.env;
const { NODE_OPTIONS: _n2, ...cleanUserEnv } = env;
const child = spawn(process.execPath, [runnerPath], {
env: { ...cleanProcessEnv, ...cleanUserEnv },
stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
});
let stdout = '';
let stderr = '';
let settled = false;
child.stdout?.on('data', (d) => (stdout += String(d)));
child.stderr?.on('data', (d) => (stderr += String(d)));
child.on(
'message',
(
msg:
| {
ok: true;
result?: unknown;
stdout?: string;
stderr?: string;
}
| {
ok: false;
error: string;
stack?: string;
stdout?: string;
stderr?: string;
},
) => {
if (settled) return;
settled = true;
resolve({ ...msg, stdout, stderr });
},
);
child.on('exit', (code) => {
if (settled) return;
settled = true;
if (code === 0) {
// Fallback path if no IPC (shouldn't happen with our stdio)
resolve({ ok: true, stdout, stderr });
} else {
resolve({
ok: false,
error: `Exited with code ${code}`,
stdout,
stderr,
});
}
});
const t = setTimeout(() => {
if (settled) return;
settled = true;
child.kill('SIGKILL');
resolve({
ok: false,
error: `Timed out after ${timeoutMs}ms`,
stdout,
stderr,
});
}, timeoutMs);
// Kick it off
child.send?.({ type: 'run', payload });
child.on('close', () => clearTimeout(t));
});
}
}
@@ -0,0 +1,32 @@
import { type FlatLogicFunctionLayer } from 'src/engine/metadata-modules/logic-function-layer/types/flat-logic-function-layer.type';
import { type LogicFunctionExecutionStatus } from 'src/engine/metadata-modules/logic-function/dtos/logic-function-execution-result.dto';
import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
export type LogicFunctionExecuteError = {
errorType: string;
errorMessage: string;
stackTrace: string | string[];
};
export type LogicFunctionExecuteResult = {
data: object | null;
duration: number;
logs: string;
status: LogicFunctionExecutionStatus;
error?: LogicFunctionExecuteError;
};
export type LogicFunctionExecuteParams = {
flatLogicFunction: FlatLogicFunction;
flatLogicFunctionLayer: FlatLogicFunctionLayer;
applicationUniversalIdentifier: string;
payload: object;
env?: Record<string, string>;
};
export interface LogicFunctionExecutorDriver {
delete(flatLogicFunction: FlatLogicFunction): Promise<void>;
execute(
params: LogicFunctionExecuteParams,
): Promise<LogicFunctionExecuteResult>;
}
@@ -0,0 +1,45 @@
{
"dependencies": {
"@types/bcrypt": "^5.0.2",
"@types/deep-equal": "^1.0.4",
"@types/lodash.camelcase": "^4.3.9",
"@types/lodash.compact": "^3.0.9",
"@types/lodash.groupby": "^4.6.9",
"@types/lodash.identity": "^3.0.9",
"@types/lodash.isempty": "^4.4.9",
"@types/lodash.isequal": "^4.5.8",
"@types/lodash.isobject": "^3.0.9",
"@types/lodash.kebabcase": "^4.1.9",
"@types/lodash.mapvalues": "^4.6.9",
"@types/lodash.omit": "^4.5.9",
"@types/lodash.pickby": "^4.6.9",
"@types/lodash.snakecase": "^4.1.9",
"@types/lodash.upperfirst": "^4.3.9",
"@types/uuid": "^10.0.0",
"archiver": "^7.0.1",
"axios": "^1.12.0",
"bcrypt": "^5.1.1",
"body-parser": "^1.20.4",
"deep-equal": "^2.2.3",
"jsonwebtoken": "^9.0.2",
"lodash.camelcase": "^4.3.0",
"lodash.chunk": "^4.2.0",
"lodash.compact": "^3.0.1",
"lodash.groupby": "^4.6.0",
"lodash.identity": "^3.0.0",
"lodash.isempty": "^4.4.0",
"lodash.isequal": "^4.5.0",
"lodash.isobject": "^3.0.2",
"lodash.kebabcase": "^4.1.1",
"lodash.mapvalues": "^4.6.0",
"lodash.merge": "^4.6.2",
"lodash.omit": "^4.5.0",
"lodash.pickby": "^4.6.0",
"lodash.snakecase": "^4.1.1",
"lodash.upperfirst": "^4.3.1",
"nodemailer": "^7.0.11",
"sharp": "^0.33.5",
"uuid": "^10.0.0",
"winston": "^3.14.2"
}
}
File diff suppressed because one or more lines are too long
@@ -0,0 +1,5 @@
enableInlineHunks: true
nodeLinker: node-modules
yarnPath: .yarn/releases/yarn-4.9.2.cjs
@@ -0,0 +1 @@
export const LAST_LAYER_VERSION = 1;
@@ -0,0 +1,50 @@
import { type DynamicModule, Module } from '@nestjs/common';
import {
LogicFunctionExecutorDriverType,
type LogicFunctionExecutorModuleAsyncOptions,
} from 'src/engine/core-modules/logic-function/logic-function-executor/interfaces/logic-function-executor.interface';
import { DisabledDriver } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/disabled.driver';
import { LambdaDriver } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver';
import { LocalDriver } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/local.driver';
import { LOGIC_FUNCTION_EXECUTOR_DRIVER } from 'src/engine/core-modules/logic-function/logic-function-executor/constants/logic-function-executor.constants';
@Module({})
export class LogicFunctionDriversModule {
static forRootAsync(
options: LogicFunctionExecutorModuleAsyncOptions,
): DynamicModule {
const provider = {
provide: LOGIC_FUNCTION_EXECUTOR_DRIVER,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
useFactory: async (...args: any[]) => {
const config = await options.useFactory(...args);
switch (config?.type) {
case LogicFunctionExecutorDriverType.DISABLED:
return new DisabledDriver();
case LogicFunctionExecutorDriverType.LOCAL:
return new LocalDriver(config.options);
case LogicFunctionExecutorDriverType.LAMBDA:
return new LambdaDriver(config.options);
default: {
const unknownConfig = config as { type?: string };
throw new Error(
`Unknown logic function executor driver type: ${unknownConfig?.type}`,
);
}
}
},
inject: options.inject || [],
};
return {
module: LogicFunctionDriversModule,
imports: options.imports || [],
providers: [provider],
exports: [LOGIC_FUNCTION_EXECUTOR_DRIVER],
};
}
}
@@ -0,0 +1,120 @@
import { type FlatApplicationVariable } from 'src/engine/core-modules/applicationVariable/types/flat-application-variable.type';
import { type SecretEncryptionService } from 'src/engine/core-modules/secret-encryption/secret-encryption.service';
import { buildEnvVar } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/build-env-var';
describe('buildEnvVar', () => {
const mockSecretEncryptionService = {
encrypt: jest.fn((value: string) => `encrypted_${value}`),
decrypt: jest.fn((value: string) => value.replace('encrypted_', '')),
} as unknown as SecretEncryptionService;
beforeEach(() => {
jest.clearAllMocks();
});
it('should return empty object for empty array', () => {
const result = buildEnvVar([], mockSecretEncryptionService);
expect(result).toEqual({});
});
it('should handle mixed secret and non-secret variables', () => {
const flatVariables: FlatApplicationVariable[] = [
{
id: '1',
key: 'PUBLIC_URL',
value: 'https://example.com',
description: 'Public URL',
isSecret: false,
applicationId: 'app-1',
createdAt: '2024-01-01T00:00:00.000Z',
updatedAt: '2024-01-01T00:00:00.000Z',
},
{
id: '2',
key: 'API_SECRET',
value: 'encrypted_secret-123',
description: 'API secret',
isSecret: true,
applicationId: 'app-1',
createdAt: '2024-01-01T00:00:00.000Z',
updatedAt: '2024-01-01T00:00:00.000Z',
},
{
id: '3',
key: 'DEBUG',
value: 'true',
description: 'Debug flag',
isSecret: false,
applicationId: 'app-1',
createdAt: '2024-01-01T00:00:00.000Z',
updatedAt: '2024-01-01T00:00:00.000Z',
},
];
const result = buildEnvVar(flatVariables, mockSecretEncryptionService);
expect(result).toEqual({
PUBLIC_URL: 'https://example.com',
API_SECRET: 'secret-123',
DEBUG: 'true',
});
expect(mockSecretEncryptionService.decrypt).toHaveBeenCalledTimes(1);
expect(mockSecretEncryptionService.decrypt).toHaveBeenCalledWith(
'encrypted_secret-123',
);
});
it('should handle null or undefined values', () => {
const flatVariables: FlatApplicationVariable[] = [
{
id: '1',
key: 'NULL_VALUE',
value: null as unknown as string,
description: '',
isSecret: false,
applicationId: 'app-1',
createdAt: '2024-01-01T00:00:00.000Z',
updatedAt: '2024-01-01T00:00:00.000Z',
},
{
id: '2',
key: 'UNDEFINED_VALUE',
value: undefined as unknown as string,
description: '',
isSecret: false,
applicationId: 'app-1',
createdAt: '2024-01-01T00:00:00.000Z',
updatedAt: '2024-01-01T00:00:00.000Z',
},
];
const result = buildEnvVar(flatVariables, mockSecretEncryptionService);
expect(result).toEqual({
NULL_VALUE: '',
UNDEFINED_VALUE: '',
});
});
it('should convert non-string values to strings', () => {
const flatVariables: FlatApplicationVariable[] = [
{
id: '1',
key: 'NUMBER_VALUE',
value: 123 as unknown as string,
description: '',
isSecret: false,
applicationId: 'app-1',
createdAt: '2024-01-01T00:00:00.000Z',
updatedAt: '2024-01-01T00:00:00.000Z',
},
];
const result = buildEnvVar(flatVariables, mockSecretEncryptionService);
expect(result).toEqual({
NUMBER_VALUE: '123',
});
});
});
@@ -0,0 +1,20 @@
import { type FlatApplicationVariable } from 'src/engine/core-modules/applicationVariable/types/flat-application-variable.type';
import { type SecretEncryptionService } from 'src/engine/core-modules/secret-encryption/secret-encryption.service';
export const buildEnvVar = (
flatApplicationVariables: FlatApplicationVariable[],
secretEncryptionService: SecretEncryptionService,
): Record<string, string> => {
return flatApplicationVariables.reduce<Record<string, string>>(
(acc, flatApplicationVariable) => {
const value = String(flatApplicationVariable.value ?? '');
acc[flatApplicationVariable.key] = flatApplicationVariable.isSecret
? secretEncryptionService.decrypt(value)
: value;
return acc;
},
{},
);
};
@@ -0,0 +1,66 @@
import { execFile } from 'child_process';
import { promises as fs, statSync } from 'fs';
import { join } from 'path';
import { promisify } from 'util';
import { getLayerDependenciesDirName } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/get-layer-dependencies-dir-name';
import { type FlatLogicFunctionLayer } from 'src/engine/metadata-modules/logic-function-layer/types/flat-logic-function-layer.type';
const execFilePromise = promisify(execFile);
export const copyAndBuildDependencies = async (
buildDirectory: string,
flatLogicFunctionLayer: FlatLogicFunctionLayer,
) => {
await fs.mkdir(buildDirectory, {
recursive: true,
});
const packageJson = flatLogicFunctionLayer.packageJson;
const yarnLock = flatLogicFunctionLayer.yarnLock;
await fs.writeFile(
join(buildDirectory, 'package.json'),
JSON.stringify(packageJson, null, 2),
'utf8',
);
await fs.writeFile(join(buildDirectory, 'yarn.lock'), yarnLock, 'utf8');
await fs.cp(getLayerDependenciesDirName('engine'), buildDirectory, {
recursive: true,
});
const localYarnPath = join(buildDirectory, '.yarn/releases/yarn-4.9.2.cjs');
// Strip NODE_OPTIONS to prevent tsx loader from interfering with yarn
const { NODE_OPTIONS: _nodeOptions, ...cleanEnv } = process.env;
try {
await execFilePromise(process.execPath, [localYarnPath], {
cwd: buildDirectory,
env: cleanEnv,
});
// eslint-disable-next-line @typescript-eslint/no-explicit-any
} catch (error: any) {
const errorMessage =
[error?.stdout, error?.stderr].filter(Boolean).join('\n') ||
'Failed to install logic function executor dependencies';
throw new Error(errorMessage);
}
const objects = await fs.readdir(buildDirectory);
await Promise.all(
objects
.filter((object) => object !== 'node_modules')
.map((object) => {
const fullPath = join(buildDirectory, object);
return statSync(fullPath).isDirectory()
? fs.rm(fullPath, { recursive: true, force: true })
: fs.rm(fullPath);
}),
);
};
@@ -0,0 +1,12 @@
import { promises as fs } from 'fs';
import { getExecutorFilePath } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/get-executor-file-path';
export const copyExecutor = async (buildDirectory: string) => {
await fs.mkdir(buildDirectory, {
recursive: true,
});
await fs.cp(getExecutorFilePath(), buildDirectory, {
recursive: true,
});
};
@@ -0,0 +1,21 @@
import fs from 'fs';
import { pipeline } from 'stream/promises';
import archiver from 'archiver';
export const createZipFile = async (
sourceDir: string,
outPath: string,
): Promise<void> => {
const output = fs.createWriteStream(outPath);
const archive = archiver('zip', {
zlib: { level: 9 }, // Compression level
});
const p = pipeline(archive, output);
archive.directory(sourceDir, false);
archive.finalize();
return p;
};
@@ -0,0 +1,12 @@
import path from 'path';
import { ASSET_PATH } from 'src/constants/assets-path';
export const getExecutorFilePath = (): string => {
const baseTypescriptProjectPath = path.join(
ASSET_PATH,
`engine/core-modules/logic-function/logic-function-drivers/constants/executor`,
);
return path.resolve(__dirname, baseTypescriptProjectPath);
};
@@ -0,0 +1,24 @@
import fs from 'fs/promises';
import { join } from 'path';
import { type PackageJson } from 'twenty-shared/application';
import { getLayerDependenciesDirName } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/get-layer-dependencies-dir-name';
import { LAST_LAYER_VERSION } from 'src/engine/core-modules/logic-function/logic-function-drivers/layers/last-layer-version';
export type LayerDependencies = {
packageJson: PackageJson;
yarnLock: string;
};
export const getLastCommonLayerDependencies = async (
layerVersion = LAST_LAYER_VERSION,
): Promise<LayerDependencies> => {
const lastVersionLayerDirName = getLayerDependenciesDirName(layerVersion);
const [packageJson, yarnLock] = await Promise.all([
fs.readFile(join(lastVersionLayerDirName, 'package.json'), 'utf8'),
fs.readFile(join(lastVersionLayerDirName, 'yarn.lock'), 'utf8'),
]);
return { packageJson: JSON.parse(packageJson), yarnLock };
};
@@ -0,0 +1,14 @@
import path from 'path';
import { ASSET_PATH } from 'src/constants/assets-path';
export const getLayerDependenciesDirName = (
version: 'engine' | number,
): string => {
const baseTypescriptProjectPath = path.join(
ASSET_PATH,
`engine/core-modules/logic-function/logic-function-drivers/layers/${version}`,
);
return path.resolve(__dirname, baseTypescriptProjectPath);
};
@@ -0,0 +1,39 @@
import fs from 'fs/promises';
import path, { join } from 'path';
import { ASSET_PATH } from 'src/constants/assets-path';
type File = { name: string; path: string; content: Buffer };
const getAllFiles = async (
rootDir: string,
dir: string = rootDir,
files: File[] = [],
): Promise<File[]> => {
const dirEntries = await fs.readdir(dir, { withFileTypes: true });
for (const entry of dirEntries) {
const fullPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
files.push(...(await getAllFiles(rootDir, fullPath, files)));
} else {
files.push({
path: path.relative(rootDir, dir),
name: entry.name,
content: await fs.readFile(fullPath),
});
}
}
return files;
};
export const getSeedProjectFiles = (async () => {
const seedProjectPath = join(
ASSET_PATH,
`engine/core-modules/logic-function/logic-function-drivers/constants/seed-project`,
);
return await getAllFiles(seedProjectPath);
})();
@@ -0,0 +1,36 @@
/* eslint-disable no-console */
export class ConsoleListener {
private readonly originalConsole;
constructor() {
this.originalConsole = {
log: console.log,
error: console.error,
warn: console.warn,
info: console.info,
debug: console.debug,
};
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
intercept(callback: (type: string, message: any[]) => void) {
Object.keys(this.originalConsole).forEach((method) => {
// @ts-expect-error legacy noImplicitAny
// eslint-disable-next-line @typescript-eslint/no-explicit-any
console[method] = (...args: any[]) => {
callback(method, args);
};
});
}
release() {
Object.keys(this.originalConsole).forEach((method) => {
// @ts-expect-error legacy noImplicitAny
// eslint-disable-next-line @typescript-eslint/no-explicit-any
console[method] = (...args: any[]) => {
// @ts-expect-error legacy noImplicitAny
this.originalConsole[method](...args);
};
});
}
}
@@ -0,0 +1,34 @@
import { join } from 'path';
import * as fs from 'fs/promises';
import { v4 } from 'uuid';
import { LOGIC_FUNCTION_EXECUTOR_TMPDIR_FOLDER } from 'src/engine/core-modules/logic-function/logic-function-drivers/constants/logic-function-executor-tmpdir-folder';
export const NODE_LAYER_SUBFOLDER = 'nodejs';
const TEMPORARY_LAMBDA_FOLDER = 'lambda-build';
const LAMBDA_ZIP_FILE_NAME = 'lambda.zip';
export class LambdaBuildDirectoryManager {
private temporaryDir = join(
LOGIC_FUNCTION_EXECUTOR_TMPDIR_FOLDER,
`${TEMPORARY_LAMBDA_FOLDER}-${v4()}`,
);
async init() {
const sourceTemporaryDir = join(this.temporaryDir);
const lambdaZipPath = join(this.temporaryDir, LAMBDA_ZIP_FILE_NAME);
await fs.mkdir(sourceTemporaryDir, { recursive: true });
return {
sourceTemporaryDir,
lambdaZipPath,
};
}
async clean() {
await fs.rm(this.temporaryDir, { recursive: true, force: true });
}
}
@@ -0,0 +1,135 @@
import { Logger } from '@nestjs/common';
import { execFile } from 'child_process';
import * as fs from 'fs/promises';
import { resolve } from 'path';
import { promisify } from 'util';
import { Command, CommandRunner, Option } from 'nest-commander';
const execFilePromise = promisify(execFile);
@Command({
name: 'logic-function-executor:add-packages',
description:
'Create a new logic function executor layer version and install packages in it',
})
export class AddPackagesCommand extends CommandRunner {
private readonly logger = new Logger(AddPackagesCommand.name);
@Option({
flags: '-p, --packages <packages>',
description: 'comma separated packages (eg: axios,uuid@9.0.1)',
required: true,
})
parsePackages(val: string): string[] {
return val.split(',');
}
async run(
_passedParams: string[],
// eslint-disable-next-line @typescript-eslint/no-explicit-any
options: Record<string, any>,
): Promise<void> {
this.logger.log('---------------------------------------');
this.logger.warn('This command should be run locally only');
this.logger.log('');
const layersFolder = this.getAbsoluteFilePath(
`src/engine/core-modules/logic-function/logic-function-drivers/layers`,
);
const currentVersion = await this.getLastLayerVersion();
const newVersion = currentVersion + 1;
const currentVersionFolder = `${layersFolder}/${currentVersion}`;
const newVersionFolder = `${layersFolder}/${newVersion}`;
await fs.cp(currentVersionFolder, newVersionFolder, { recursive: true });
// Install each package
this.logger.log('Installing packages');
await this.installPackages(options.packages, newVersionFolder);
this.logger.log('Cleaning');
await this.cleanPackageInstallation(newVersionFolder);
this.logger.log('Updating last layer version');
await this.updateLastLayerVersion(newVersion);
this.logger.log('Add changes to git');
await this.addToGit(layersFolder);
this.logger.log('');
this.logger.log(
`New packages '${options.packages.join("', '")}' installed in new layer version '${newVersion}' `,
);
this.logger.log('Please commit your changes');
this.logger.log('---------------------------------------');
}
private getAbsoluteFilePath(path: string) {
const rootPath = process.cwd();
return resolve(rootPath, path);
}
private async addToGit(folderPath: string) {
await execFilePromise('git', ['add', folderPath]);
}
private async cleanPackageInstallation(folderPath: string) {
await fs.rm(folderPath + '/node_modules', {
recursive: true,
force: true,
});
await fs.rm(folderPath + '/.yarn', {
recursive: true,
force: true,
});
}
private async installPackages(packages: string[], folderPath: string) {
if (packages?.length) {
for (const packageName of packages) {
this.logger.log(`- adding '${packageName}'...`);
try {
await execFilePromise('yarn', ['add', packageName], {
cwd: folderPath,
});
} catch (error) {
this.logger.error(
`Failed to install ${packageName}: ${(error as Error).message}`,
);
}
}
}
}
private async getLastLayerVersion() {
const filePath = this.getAbsoluteFilePath(
'src/engine/core-modules/logic-function/logic-function-drivers/layers/last-layer-version.ts',
);
const content = await fs.readFile(filePath, 'utf8');
const match = content.match(/export const LAST_LAYER_VERSION = (\d+);/);
if (!match) {
throw new Error('LAST_LAYER_VERSION not found');
}
return parseInt(match[1], 10);
}
private async updateLastLayerVersion(newVersion: number) {
const filePath = this.getAbsoluteFilePath(
'src/engine/core-modules/logic-function/logic-function-drivers/layers/last-layer-version.ts',
);
await fs.writeFile(
filePath,
`export const LAST_LAYER_VERSION = ${newVersion};\n`,
'utf8',
);
}
}
@@ -0,0 +1,3 @@
export const LOGIC_FUNCTION_EXECUTOR_DRIVER = Symbol(
'LOGIC_FUNCTION_EXECUTOR_DRIVER',
);
@@ -0,0 +1,67 @@
import { fromNodeProviderChain } from '@aws-sdk/credential-providers';
import {
LogicFunctionExecutorDriverType,
type LogicFunctionExecutorModuleOptions,
} from 'src/engine/core-modules/logic-function/logic-function-executor/interfaces/logic-function-executor.interface';
import { type FileStorageService } from 'src/engine/core-modules/file-storage/file-storage.service';
import { type TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
export const logicFunctionModuleFactory = async (
twentyConfigService: TwentyConfigService,
fileStorageService: FileStorageService,
): Promise<LogicFunctionExecutorModuleOptions> => {
const driverType = twentyConfigService.get('LOGIC_FUNCTION_TYPE');
const options = { fileStorageService };
switch (driverType) {
case LogicFunctionExecutorDriverType.DISABLED: {
return {
type: LogicFunctionExecutorDriverType.DISABLED,
};
}
case LogicFunctionExecutorDriverType.LOCAL: {
return {
type: LogicFunctionExecutorDriverType.LOCAL,
options,
};
}
case LogicFunctionExecutorDriverType.LAMBDA: {
const region = twentyConfigService.get('LOGIC_FUNCTION_LAMBDA_REGION');
const accessKeyId = twentyConfigService.get(
'LOGIC_FUNCTION_LAMBDA_ACCESS_KEY_ID',
);
const secretAccessKey = twentyConfigService.get(
'LOGIC_FUNCTION_LAMBDA_SECRET_ACCESS_KEY',
);
const lambdaRole = twentyConfigService.get('LOGIC_FUNCTION_LAMBDA_ROLE');
const subhostingRole = twentyConfigService.get(
'LOGIC_FUNCTION_LAMBDA_SUBHOSTING_ROLE',
);
return {
type: LogicFunctionExecutorDriverType.LAMBDA,
options: {
...options,
credentials: accessKeyId
? {
accessKeyId,
secretAccessKey,
}
: fromNodeProviderChain({
clientConfig: { region },
}),
region,
lambdaRole,
subhostingRole,
},
};
}
default:
throw new Error(
`Invalid logic function executor driver type (${driverType}), check your .env file`,
);
}
};
@@ -0,0 +1,39 @@
import { type FactoryProvider, type ModuleMetadata } from '@nestjs/common';
import { type LambdaDriverOptions } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda.driver';
import { type LocalDriverOptions } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/local.driver';
export enum LogicFunctionExecutorDriverType {
DISABLED = 'DISABLED',
LAMBDA = 'LAMBDA',
LOCAL = 'LOCAL',
}
export interface DisabledDriverFactoryOptions {
type: LogicFunctionExecutorDriverType.DISABLED;
}
export interface LocalDriverFactoryOptions {
type: LogicFunctionExecutorDriverType.LOCAL;
options: LocalDriverOptions;
}
export interface LambdaDriverFactoryOptions {
type: LogicFunctionExecutorDriverType.LAMBDA;
options: LambdaDriverOptions;
}
export type LogicFunctionExecutorModuleOptions =
| DisabledDriverFactoryOptions
| LocalDriverFactoryOptions
| LambdaDriverFactoryOptions;
export type LogicFunctionExecutorModuleAsyncOptions = {
useFactory: (
// eslint-disable-next-line @typescript-eslint/no-explicit-any
...args: any[]
) =>
| LogicFunctionExecutorModuleOptions
| Promise<LogicFunctionExecutorModuleOptions>;
} & Pick<ModuleMetadata, 'imports'> &
Pick<FactoryProvider, 'inject'>;
@@ -0,0 +1,26 @@
import { Module } from '@nestjs/common';
import { AuditModule } from 'src/engine/core-modules/audit/audit.module';
import { TokenModule } from 'src/engine/core-modules/auth/token/token.module';
import { LogicFunctionBuildModule } from 'src/engine/core-modules/logic-function/logic-function-build/logic-function-build.module';
import { AddPackagesCommand } from 'src/engine/core-modules/logic-function/logic-function-executor/commands/add-packages.command';
import { LogicFunctionExecutorService } from 'src/engine/core-modules/logic-function/logic-function-executor/services/logic-function-executor.service';
import { SecretEncryptionModule } from 'src/engine/core-modules/secret-encryption/secret-encryption.module';
import { ThrottlerModule } from 'src/engine/core-modules/throttler/throttler.module';
import { SubscriptionsModule } from 'src/engine/subscriptions/subscriptions.module';
import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';
@Module({
imports: [
ThrottlerModule,
AuditModule,
TokenModule,
SecretEncryptionModule,
SubscriptionsModule,
WorkspaceCacheModule,
LogicFunctionBuildModule,
],
providers: [LogicFunctionExecutorService, AddPackagesCommand],
exports: [LogicFunctionExecutorService],
})
export class LogicFunctionExecutorModule {}
@@ -0,0 +1,261 @@
import { Inject, Injectable } from '@nestjs/common';
import {
DEFAULT_API_KEY_NAME,
DEFAULT_API_URL_NAME,
} from 'twenty-shared/application';
import { isDefined } from 'twenty-shared/utils';
import {
LogicFunctionExecutorDriver,
type LogicFunctionExecuteParams,
type LogicFunctionExecuteResult,
} from 'src/engine/core-modules/logic-function/logic-function-drivers/interfaces/logic-function-executor-driver.interface';
import { AuditService } from 'src/engine/core-modules/audit/services/audit.service';
import { LOGIC_FUNCTION_EXECUTED_EVENT } from 'src/engine/core-modules/audit/utils/events/workspace-event/logic-function/logic-function-executed';
import { ApplicationTokenService } from 'src/engine/core-modules/auth/token/services/application-token.service';
import { LogicFunctionBuildService } from 'src/engine/core-modules/logic-function/logic-function-build/services/logic-function-build.service';
import { buildEnvVar } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/build-env-var';
import { LOGIC_FUNCTION_EXECUTOR_DRIVER } from 'src/engine/core-modules/logic-function/logic-function-executor/constants/logic-function-executor.constants';
import { SecretEncryptionService } from 'src/engine/core-modules/secret-encryption/secret-encryption.service';
import { ThrottlerService } from 'src/engine/core-modules/throttler/throttler.service';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
import { findFlatEntityByIdInFlatEntityMaps } from 'src/engine/metadata-modules/flat-entity/utils/find-flat-entity-by-id-in-flat-entity-maps.util';
import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-function/types/flat-logic-function.type';
import { SubscriptionChannel } from 'src/engine/subscriptions/enums/subscription-channel.enum';
import { SubscriptionService } from 'src/engine/subscriptions/subscription.service';
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
import { cleanServerUrl } from 'src/utils/clean-server-url';
const MIN_TOKEN_EXPIRATION_IN_SECONDS = 5;
export class LogicFunctionExecutionException extends Error {
constructor(
message: string,
public readonly code: LogicFunctionExecutionExceptionCode,
) {
super(message);
this.name = 'LogicFunctionExecutionException';
}
}
export enum LogicFunctionExecutionExceptionCode {
LOGIC_FUNCTION_NOT_FOUND = 'LOGIC_FUNCTION_NOT_FOUND',
RATE_LIMIT_EXCEEDED = 'RATE_LIMIT_EXCEEDED',
}
@Injectable()
export class LogicFunctionExecutorService
implements LogicFunctionExecutorDriver
{
constructor(
@Inject(LOGIC_FUNCTION_EXECUTOR_DRIVER)
private driver: LogicFunctionExecutorDriver,
private readonly throttlerService: ThrottlerService,
private readonly twentyConfigService: TwentyConfigService,
private readonly workspaceCacheService: WorkspaceCacheService,
private readonly applicationTokenService: ApplicationTokenService,
private readonly secretEncryptionService: SecretEncryptionService,
private readonly functionBuildService: LogicFunctionBuildService,
private readonly subscriptionService: SubscriptionService,
private readonly auditService: AuditService,
) {}
async delete(flatLogicFunction: FlatLogicFunction): Promise<void> {
return this.driver.delete(flatLogicFunction);
}
async execute(
params: LogicFunctionExecuteParams,
): Promise<LogicFunctionExecuteResult> {
return this.driver.execute(params);
}
async executeOneLogicFunction({
id,
workspaceId,
payload,
}: {
id: string;
workspaceId: string;
payload: object;
}): Promise<LogicFunctionExecuteResult> {
await this.throttleExecution(workspaceId);
const {
flatLogicFunctionMaps,
flatApplicationMaps,
applicationVariableMaps,
logicFunctionLayerMaps,
} = await this.workspaceCacheService.getOrRecompute(workspaceId, [
'flatLogicFunctionMaps',
'flatApplicationMaps',
'applicationVariableMaps',
'logicFunctionLayerMaps',
]);
const flatLogicFunction = findFlatEntityByIdInFlatEntityMaps({
flatEntityId: id,
flatEntityMaps: flatLogicFunctionMaps,
});
if (
!isDefined(flatLogicFunction) ||
isDefined(flatLogicFunction.deletedAt)
) {
throw new LogicFunctionExecutionException(
`Logic function with id ${id} not found`,
LogicFunctionExecutionExceptionCode.LOGIC_FUNCTION_NOT_FOUND,
);
}
const flatLogicFunctionLayer =
logicFunctionLayerMaps.byId[flatLogicFunction.logicFunctionLayerId];
if (!isDefined(flatLogicFunctionLayer)) {
throw new LogicFunctionExecutionException(
`Logic function layer with id ${flatLogicFunction.logicFunctionLayerId} not found`,
LogicFunctionExecutionExceptionCode.LOGIC_FUNCTION_NOT_FOUND,
);
}
const applicationAccessToken = isDefined(flatLogicFunction.applicationId)
? await this.applicationTokenService.generateApplicationToken({
workspaceId,
applicationId: flatLogicFunction.applicationId,
expiresInSeconds: Math.max(
flatLogicFunction.timeoutSeconds,
MIN_TOKEN_EXPIRATION_IN_SECONDS,
),
})
: undefined;
const baseUrl = cleanServerUrl(this.twentyConfigService.get('SERVER_URL'));
const flatApplicationVariables = isDefined(flatLogicFunction.applicationId)
? (applicationVariableMaps.byApplicationId[
flatLogicFunction.applicationId
] ?? [])
: [];
const envVariables = {
...(isDefined(baseUrl)
? {
[DEFAULT_API_URL_NAME]: baseUrl,
}
: {}),
...(isDefined(applicationAccessToken)
? {
[DEFAULT_API_KEY_NAME]: applicationAccessToken.token,
}
: {}),
...buildEnvVar(flatApplicationVariables, this.secretEncryptionService),
};
const applicationUniversalIdentifier = isDefined(
flatLogicFunction.applicationId,
)
? flatApplicationMaps.byId[flatLogicFunction.applicationId]
?.universalIdentifier
: undefined;
if (!isDefined(applicationUniversalIdentifier)) {
throw new LogicFunctionExecutionException(
`Application universal identifier not found for logic function ${flatLogicFunction.id}`,
LogicFunctionExecutionExceptionCode.LOGIC_FUNCTION_NOT_FOUND,
);
}
if (
!(await this.functionBuildService.isBuilt({
flatLogicFunction,
applicationUniversalIdentifier,
}))
) {
await this.functionBuildService.buildAndUpload({
flatLogicFunction,
applicationUniversalIdentifier,
});
}
const resultLogicFunction = await this.callWithTimeout({
callback: () =>
this.execute({
flatLogicFunction,
flatLogicFunctionLayer,
applicationUniversalIdentifier,
payload,
env: envVariables,
}),
timeoutMs: flatLogicFunction.timeoutSeconds * 1000,
});
if (this.twentyConfigService.get('LOGIC_FUNCTION_LOGS_ENABLED')) {
/* eslint-disable no-console */
console.log(resultLogicFunction.logs);
}
await this.subscriptionService.publish({
channel: SubscriptionChannel.LOGIC_FUNCTION_LOGS_CHANNEL,
workspaceId,
payload: {
logicFunctionLogs: {
logs: resultLogicFunction.logs,
id: flatLogicFunction.id,
name: flatLogicFunction.name,
universalIdentifier: flatLogicFunction.universalIdentifier,
applicationId: flatLogicFunction.applicationId,
applicationUniversalIdentifier,
},
},
});
this.auditService
.createContext({
workspaceId,
})
.insertWorkspaceEvent(LOGIC_FUNCTION_EXECUTED_EVENT, {
duration: resultLogicFunction.duration,
status: resultLogicFunction.status,
...(resultLogicFunction.error && {
errorType: resultLogicFunction.error.errorType,
}),
functionId: flatLogicFunction.id,
functionName: flatLogicFunction.name,
});
return resultLogicFunction;
}
private async throttleExecution(workspaceId: string) {
try {
await this.throttlerService.tokenBucketThrottleOrThrow(
`${workspaceId}-logic-function-execution`,
1,
this.twentyConfigService.get('LOGIC_FUNCTION_EXEC_THROTTLE_LIMIT'),
this.twentyConfigService.get('LOGIC_FUNCTION_EXEC_THROTTLE_TTL'),
);
} catch {
throw new LogicFunctionExecutionException(
'Logic function execution rate limit exceeded',
LogicFunctionExecutionExceptionCode.RATE_LIMIT_EXCEEDED,
);
}
}
private async callWithTimeout<T>({
callback,
timeoutMs,
}: {
callback: () => Promise<T>;
timeoutMs: number;
}): Promise<T> {
return Promise.race([
callback(),
new Promise<T>((_, reject) =>
setTimeout(() => reject(new Error('Execution timed out')), timeoutMs),
),
]);
}
}
@@ -0,0 +1,16 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { LogicFunctionLayerService } from 'src/engine/core-modules/logic-function/logic-function-layer/services/logic-function-layer.service';
import { LogicFunctionLayerEntity } from 'src/engine/metadata-modules/logic-function-layer/logic-function-layer.entity';
import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache.module';
@Module({
imports: [
TypeOrmModule.forFeature([LogicFunctionLayerEntity]),
WorkspaceCacheModule,
],
providers: [LogicFunctionLayerService],
exports: [LogicFunctionLayerService],
})
export class CoreLogicFunctionLayerModule {}
@@ -0,0 +1,85 @@
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { isDefined } from 'twenty-shared/utils';
import type { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity';
import { LogicFunctionLayerEntity } from 'src/engine/metadata-modules/logic-function-layer/logic-function-layer.entity';
import { CreateLogicFunctionLayerInput } from 'src/engine/metadata-modules/logic-function-layer/dtos/create-logic-function-layer.input';
import { getLastCommonLayerDependencies } from 'src/engine/core-modules/logic-function/logic-function-drivers/utils/get-last-common-layer-dependencies';
import { logicFunctionCreateHash } from 'src/engine/metadata-modules/logic-function/utils/logic-function-create-hash.utils';
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
@Injectable()
export class LogicFunctionLayerService {
constructor(
@InjectRepository(LogicFunctionLayerEntity)
private readonly logicFunctionLayerRepository: Repository<LogicFunctionLayerEntity>,
private readonly workspaceCacheService: WorkspaceCacheService,
) {}
async create(
{ packageJson, yarnLock }: CreateLogicFunctionLayerInput,
workspaceId: string,
) {
const checksum = logicFunctionCreateHash(yarnLock);
const logicFunctionLayer = this.logicFunctionLayerRepository.create({
packageJson,
yarnLock,
checksum,
workspaceId,
});
const savedLayer =
await this.logicFunctionLayerRepository.save(logicFunctionLayer);
await this.workspaceCacheService.invalidateAndRecompute(workspaceId, [
'logicFunctionLayerMaps',
]);
return savedLayer;
}
async update(
id: string,
data: QueryDeepPartialEntity<LogicFunctionLayerEntity>,
workspaceId: string,
) {
const checksum = data.yarnLock
? logicFunctionCreateHash(data.yarnLock as string)
: undefined;
const updateData = { ...data, ...(checksum && { checksum }) };
const result = await this.logicFunctionLayerRepository.update(
id,
updateData,
);
await this.workspaceCacheService.invalidateAndRecompute(workspaceId, [
'logicFunctionLayerMaps',
]);
return result;
}
async createCommonLayerIfNotExist(workspaceId: string) {
const { packageJson, yarnLock } = await getLastCommonLayerDependencies();
const checksum = logicFunctionCreateHash(yarnLock);
const commonLayer = await this.logicFunctionLayerRepository.findOne({
where: {
checksum,
workspaceId,
},
});
if (isDefined(commonLayer)) {
return commonLayer;
}
return this.create({ packageJson, yarnLock }, workspaceId);
}
}
@@ -0,0 +1,36 @@
import { Scope } from '@nestjs/common';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { LogicFunctionExecutorService } from 'src/engine/core-modules/logic-function/logic-function-executor/services/logic-function-executor.service';
export type LogicFunctionTriggerJobData = {
logicFunctionId: string;
workspaceId: string;
payload?: object;
};
@Processor({
queueName: MessageQueue.logicFunctionQueue,
scope: Scope.REQUEST,
})
export class LogicFunctionTriggerJob {
constructor(
private readonly logicFunctionExecutorService: LogicFunctionExecutorService,
) {}
@Process(LogicFunctionTriggerJob.name)
async handle(logicFunctionPayloads: LogicFunctionTriggerJobData[]) {
await Promise.all(
logicFunctionPayloads.map(
async (logicFunctionPayload) =>
await this.logicFunctionExecutorService.executeOneLogicFunction({
id: logicFunctionPayload.logicFunctionId,
workspaceId: logicFunctionPayload.workspaceId,
payload: logicFunctionPayload.payload ?? {},
}),
),
);
}
}
@@ -0,0 +1,29 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { TokenModule } from 'src/engine/core-modules/auth/token/token.module';
import { WorkspaceDomainsModule } from 'src/engine/core-modules/domain/workspace-domains/workspace-domains.module';
import { LogicFunctionTriggerJob } from 'src/engine/core-modules/logic-function/logic-function-trigger/jobs/logic-function-trigger.job';
import { CronTriggerCronCommand } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.command';
import { CronTriggerCronJob } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job';
import { CallDatabaseEventTriggerJobsJob } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job';
import { RouteTriggerService } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/route/route-trigger.service';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import { LogicFunctionEntity } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
@Module({
imports: [
TypeOrmModule.forFeature([LogicFunctionEntity, WorkspaceEntity]),
TokenModule,
WorkspaceDomainsModule,
],
providers: [
LogicFunctionTriggerJob,
CronTriggerCronJob,
CronTriggerCronCommand,
CallDatabaseEventTriggerJobsJob,
RouteTriggerService,
],
exports: [CronTriggerCronCommand, RouteTriggerService],
})
export class LogicFunctionTriggerModule {}
@@ -0,0 +1,34 @@
import { Command, CommandRunner } from 'nest-commander';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import {
CRON_TRIGGER_CRON_PATTERN,
CronTriggerCronJob,
} from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job';
@Command({
name: 'cron:trigger:start-cron-trigger',
description: 'Starts a cron job to trigger cron triggered logic functions',
})
export class CronTriggerCronCommand extends CommandRunner {
constructor(
@InjectMessageQueue(MessageQueue.cronQueue)
private readonly messageQueueService: MessageQueueService,
) {
super();
}
async run(): Promise<void> {
await this.messageQueueService.addCron<undefined>({
jobName: CronTriggerCronJob.name,
data: undefined,
options: {
repeat: {
pattern: CRON_TRIGGER_CRON_PATTERN,
},
},
});
}
}
@@ -0,0 +1,81 @@
import { InjectRepository } from '@nestjs/typeorm';
import { isDefined } from 'twenty-shared/utils';
import { WorkspaceActivationStatus } from 'twenty-shared/workspace';
import { IsNull, Not, Repository } from 'typeorm';
import { SentryCronMonitor } from 'src/engine/core-modules/cron/sentry-cron-monitor.decorator';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import {
LogicFunctionTriggerJob,
LogicFunctionTriggerJobData,
} from 'src/engine/core-modules/logic-function/logic-function-trigger/jobs/logic-function-trigger.job';
import { LogicFunctionEntity } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import { shouldRunNow } from 'src/utils/should-run-now.utils';
export const CRON_TRIGGER_CRON_PATTERN = '* * * * *';
@Processor(MessageQueue.cronQueue)
export class CronTriggerCronJob {
constructor(
@InjectMessageQueue(MessageQueue.logicFunctionQueue)
private readonly messageQueueService: MessageQueueService,
@InjectRepository(WorkspaceEntity)
private readonly workspaceRepository: Repository<WorkspaceEntity>,
@InjectRepository(LogicFunctionEntity)
private readonly logicFunctionRepository: Repository<LogicFunctionEntity>,
) {}
@Process(CronTriggerCronJob.name)
@SentryCronMonitor(CronTriggerCronJob.name, CRON_TRIGGER_CRON_PATTERN)
async handle() {
const activeWorkspaces = await this.workspaceRepository.find({
where: {
activationStatus: WorkspaceActivationStatus.ACTIVE,
},
select: ['id'],
});
const now = new Date();
for (const activeWorkspace of activeWorkspaces) {
const logicFunctionsWithCronTrigger =
await this.logicFunctionRepository.find({
where: {
workspaceId: activeWorkspace.id,
cronTriggerSettings: Not(IsNull()),
},
select: ['id', 'cronTriggerSettings', 'workspaceId'],
});
for (const logicFunction of logicFunctionsWithCronTrigger) {
const cronSettings = logicFunction.cronTriggerSettings;
if (!isDefined(cronSettings?.pattern)) {
continue;
}
if (!shouldRunNow(cronSettings.pattern, now)) {
continue;
}
await this.messageQueueService.add<LogicFunctionTriggerJobData[]>(
LogicFunctionTriggerJob.name,
[
{
logicFunctionId: logicFunction.id,
workspaceId: logicFunction.workspaceId,
payload: {},
},
],
{ retryLimit: 3 },
);
}
}
}
}
@@ -0,0 +1,95 @@
import { InjectRepository } from '@nestjs/typeorm';
import chunk from 'lodash.chunk';
import { isDefined } from 'twenty-shared/utils';
import { IsNull, Not, Repository } from 'typeorm';
import type { ObjectRecordEvent } from 'twenty-shared/database-events';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { transformEventBatchToEventPayloads } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/transform-event-batch-to-event-payloads';
import {
LogicFunctionTriggerJob,
LogicFunctionTriggerJobData,
} from 'src/engine/core-modules/logic-function/logic-function-trigger/jobs/logic-function-trigger.job';
import { LogicFunctionEntity } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
const DATABASE_EVENT_JOBS_CHUNK_SIZE = 20;
@Processor(MessageQueue.triggerQueue)
export class CallDatabaseEventTriggerJobsJob {
constructor(
@InjectMessageQueue(MessageQueue.logicFunctionQueue)
private readonly messageQueueService: MessageQueueService,
@InjectRepository(LogicFunctionEntity)
private readonly logicFunctionRepository: Repository<LogicFunctionEntity>,
) {}
@Process(CallDatabaseEventTriggerJobsJob.name)
async handle(workspaceEventBatch: WorkspaceEventBatch<ObjectRecordEvent>) {
const logicFunctionsWithDatabaseEventTrigger =
await this.logicFunctionRepository.find({
where: {
workspaceId: workspaceEventBatch.workspaceId,
databaseEventTriggerSettings: Not(IsNull()),
},
select: ['id', 'databaseEventTriggerSettings', 'workspaceId'],
});
const logicFunctionsToTrigger =
logicFunctionsWithDatabaseEventTrigger.filter((logicFunction) =>
this.shouldTriggerJob({
workspaceEventBatch,
eventName: isDefined(logicFunction.databaseEventTriggerSettings)
? logicFunction.databaseEventTriggerSettings.eventName
: '',
}),
);
const logicFunctionPayloads = transformEventBatchToEventPayloads({
logicFunctions: logicFunctionsToTrigger,
workspaceEventBatch,
});
if (logicFunctionPayloads.length === 0) {
return;
}
const logicFunctionPayloadsChunks = chunk(
logicFunctionPayloads,
DATABASE_EVENT_JOBS_CHUNK_SIZE,
);
for (const logicFunctionPayloadsChunk of logicFunctionPayloadsChunks) {
await this.messageQueueService.add<LogicFunctionTriggerJobData[]>(
LogicFunctionTriggerJob.name,
logicFunctionPayloadsChunk,
{ retryLimit: 3 },
);
}
}
private shouldTriggerJob({
workspaceEventBatch,
eventName,
}: {
workspaceEventBatch: WorkspaceEventBatch<ObjectRecordEvent>;
eventName: string;
}) {
const [nameSingular, operation] = workspaceEventBatch.name.split('.');
const validEventNames = [
`${nameSingular}.${operation}`,
`*.${operation}`,
`${nameSingular}.*`,
'*.*',
];
return validEventNames.includes(eventName);
}
}
@@ -0,0 +1,362 @@
import type { ObjectRecordEvent } from 'twenty-shared/database-events';
import { transformEventBatchToEventPayloads } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/transform-event-batch-to-event-payloads';
import { getFlatObjectMetadataMock } from 'src/engine/metadata-modules/flat-object-metadata/__mocks__/get-flat-object-metadata.mock';
import { type LogicFunctionEntity } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import type { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
const createMockLogicFunction = (
overrides: Partial<LogicFunctionEntity> = {},
): LogicFunctionEntity =>
({
id: 'function-1',
workspaceId: 'workspace-1',
databaseEventTriggerSettings: {
eventName: 'company.updated',
},
...overrides,
}) as LogicFunctionEntity;
const createMockEvent = (
overrides: Partial<ObjectRecordEvent> = {},
): ObjectRecordEvent =>
({
recordId: 'record-1',
properties: {
after: {},
},
...overrides,
}) as ObjectRecordEvent;
const createMockWorkspaceEventBatch = (
overrides: Partial<WorkspaceEventBatch<ObjectRecordEvent>> = {},
): WorkspaceEventBatch<ObjectRecordEvent> => ({
name: 'company.updated',
workspaceId: 'workspace-1',
objectMetadata: getFlatObjectMetadataMock({
universalIdentifier: 'company-uuid',
nameSingular: 'company',
}),
events: [createMockEvent()],
...overrides,
});
describe('transformEventBatchToEventPayloads', () => {
describe('basic transformation', () => {
it('should transform a single event batch with a single logic function', () => {
const workspaceEventBatch = createMockWorkspaceEventBatch();
const logicFunctions = [createMockLogicFunction()];
const result = transformEventBatchToEventPayloads({
workspaceEventBatch,
logicFunctions,
});
expect(result).toHaveLength(1);
expect(result[0]).toEqual({
logicFunctionId: 'function-1',
workspaceId: 'workspace-1',
payload: expect.objectContaining({
name: 'company.updated',
workspaceId: 'workspace-1',
recordId: 'record-1',
}),
});
});
it('should create multiple payloads for multiple events in a batch', () => {
const workspaceEventBatch = createMockWorkspaceEventBatch({
events: [
createMockEvent({ recordId: 'record-1' }),
createMockEvent({ recordId: 'record-2' }),
createMockEvent({ recordId: 'record-3' }),
],
});
const logicFunctions = [createMockLogicFunction()];
const result = transformEventBatchToEventPayloads({
workspaceEventBatch,
logicFunctions,
});
expect(result).toHaveLength(3);
expect(
result.map((r) => (r.payload as ObjectRecordEvent).recordId),
).toEqual(['record-1', 'record-2', 'record-3']);
});
it('should create payloads for each logic function', () => {
const workspaceEventBatch = createMockWorkspaceEventBatch();
const logicFunctions = [
createMockLogicFunction({
id: 'function-1',
}),
createMockLogicFunction({
id: 'function-2',
}),
];
const result = transformEventBatchToEventPayloads({
workspaceEventBatch,
logicFunctions,
});
expect(result).toHaveLength(2);
expect(result.map((r) => r.logicFunctionId)).toEqual([
'function-1',
'function-2',
]);
});
});
describe('updatedFields filtering', () => {
it('should include all events when updatedFields is undefined', () => {
const workspaceEventBatch = createMockWorkspaceEventBatch({
name: 'company.updated',
events: [
createMockEvent({
recordId: 'record-1',
properties: { after: {}, updatedFields: ['name'] },
}),
createMockEvent({
recordId: 'record-2',
properties: { after: {}, updatedFields: ['address'] },
}),
],
});
const logicFunctions = [
createMockLogicFunction({
databaseEventTriggerSettings: { eventName: 'company.updated' },
}),
];
const result = transformEventBatchToEventPayloads({
workspaceEventBatch,
logicFunctions,
});
expect(result).toHaveLength(2);
});
it('should include all events when updatedFields is empty array', () => {
const workspaceEventBatch = createMockWorkspaceEventBatch({
name: 'company.updated',
events: [
createMockEvent({
recordId: 'record-1',
properties: { after: {}, updatedFields: ['name'] },
}),
createMockEvent({
recordId: 'record-2',
properties: { after: {}, updatedFields: ['address'] },
}),
],
});
const logicFunctions = [
createMockLogicFunction({
databaseEventTriggerSettings: {
eventName: 'company.updated',
updatedFields: [],
},
}),
];
const result = transformEventBatchToEventPayloads({
workspaceEventBatch,
logicFunctions,
});
expect(result).toHaveLength(2);
});
it('should filter events to only those matching updatedFields', () => {
const workspaceEventBatch = createMockWorkspaceEventBatch({
name: 'company.updated',
events: [
createMockEvent({
recordId: 'record-1',
properties: { after: {}, updatedFields: ['name'] },
}),
createMockEvent({
recordId: 'record-2',
properties: { after: {}, updatedFields: ['address'] },
}),
createMockEvent({
recordId: 'record-3',
properties: { after: {}, updatedFields: ['name', 'description'] },
}),
],
});
const logicFunctions = [
createMockLogicFunction({
databaseEventTriggerSettings: {
eventName: 'company.updated',
updatedFields: ['name'],
},
}),
];
const result = transformEventBatchToEventPayloads({
workspaceEventBatch,
logicFunctions,
});
expect(result).toHaveLength(2);
expect(
result.map((r) => (r.payload as ObjectRecordEvent).recordId),
).toEqual(['record-1', 'record-3']);
});
it('should filter events matching any of the specified updatedFields', () => {
const workspaceEventBatch = createMockWorkspaceEventBatch({
name: 'company.updated',
events: [
createMockEvent({
recordId: 'record-1',
properties: { after: {}, updatedFields: ['name'] },
}),
createMockEvent({
recordId: 'record-2',
properties: { after: {}, updatedFields: ['address'] },
}),
createMockEvent({
recordId: 'record-3',
properties: { after: {}, updatedFields: ['phone'] },
}),
],
});
const logicFunctions = [
createMockLogicFunction({
databaseEventTriggerSettings: {
eventName: 'company.updated',
updatedFields: ['name', 'address'],
},
}),
];
const result = transformEventBatchToEventPayloads({
workspaceEventBatch,
logicFunctions,
});
expect(result).toHaveLength(2);
expect(
result.map((r) => (r.payload as ObjectRecordEvent).recordId),
).toEqual(['record-1', 'record-2']);
});
it('should return no events when none match the updatedFields filter', () => {
const workspaceEventBatch = createMockWorkspaceEventBatch({
name: 'company.updated',
events: [
createMockEvent({
recordId: 'record-1',
properties: { after: {}, updatedFields: ['name'] },
}),
createMockEvent({
recordId: 'record-2',
properties: { after: {}, updatedFields: ['address'] },
}),
],
});
const logicFunctions = [
createMockLogicFunction({
databaseEventTriggerSettings: {
eventName: 'company.updated',
updatedFields: ['phone'],
},
}),
];
const result = transformEventBatchToEventPayloads({
workspaceEventBatch,
logicFunctions,
});
expect(result).toHaveLength(0);
});
it('should handle different updatedFields filters per logic function', () => {
const workspaceEventBatch = createMockWorkspaceEventBatch({
name: 'company.updated',
events: [
createMockEvent({
recordId: 'record-1',
properties: { after: {}, updatedFields: ['name'] },
}),
createMockEvent({
recordId: 'record-2',
properties: { after: {}, updatedFields: ['address'] },
}),
],
});
const logicFunctions = [
createMockLogicFunction({
id: 'function-1',
databaseEventTriggerSettings: {
eventName: 'company.updated',
updatedFields: ['name'],
},
}),
createMockLogicFunction({
id: 'function-2',
databaseEventTriggerSettings: {
eventName: 'company.updated',
updatedFields: ['address'],
},
}),
];
const result = transformEventBatchToEventPayloads({
workspaceEventBatch,
logicFunctions,
});
expect(result).toHaveLength(2);
const function1Payloads = result.filter(
(r) => r.logicFunctionId === 'function-1',
);
const function2Payloads = result.filter(
(r) => r.logicFunctionId === 'function-2',
);
expect(function1Payloads).toHaveLength(1);
expect((function1Payloads[0].payload as ObjectRecordEvent).recordId).toBe(
'record-1',
);
expect(function2Payloads).toHaveLength(1);
expect((function2Payloads[0].payload as ObjectRecordEvent).recordId).toBe(
'record-2',
);
});
});
describe('edge cases', () => {
it('should return empty array when no logic functions provided', () => {
const workspaceEventBatch = createMockWorkspaceEventBatch();
const result = transformEventBatchToEventPayloads({
workspaceEventBatch,
logicFunctions: [],
});
expect(result).toHaveLength(0);
});
it('should return empty array when no events in batch', () => {
const workspaceEventBatch = createMockWorkspaceEventBatch({
events: [],
});
const logicFunctions = [createMockLogicFunction()];
const result = transformEventBatchToEventPayloads({
workspaceEventBatch,
logicFunctions,
});
expect(result).toHaveLength(0);
});
});
});
@@ -0,0 +1,77 @@
import { isDefined } from 'twenty-shared/utils';
import type {
DatabaseEventPayload,
ObjectRecordEvent,
} from 'twenty-shared/database-events';
import { type LogicFunctionTriggerJobData } from 'src/engine/core-modules/logic-function/logic-function-trigger/jobs/logic-function-trigger.job';
import { type LogicFunctionEntity } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
import type { WorkspaceEventBatch } from 'src/engine/workspace-event-emitter/types/workspace-event-batch.type';
export const transformEventBatchToEventPayloads = ({
workspaceEventBatch,
logicFunctions,
}: {
workspaceEventBatch: WorkspaceEventBatch<ObjectRecordEvent>;
logicFunctions: LogicFunctionEntity[];
}): LogicFunctionTriggerJobData[] => {
const result: LogicFunctionTriggerJobData[] = [];
const { events, ...batchEventInfo } = workspaceEventBatch;
const [, operation] = workspaceEventBatch.name.split('.');
for (const logicFunction of logicFunctions) {
const triggerUpdatedFields =
logicFunction.databaseEventTriggerSettings?.updatedFields;
const filteredEvents = filterEventsByUpdatedFields({
events,
operation,
triggerUpdatedFields,
});
for (const event of filteredEvents) {
const payload: DatabaseEventPayload = { ...batchEventInfo, ...event };
result.push({
logicFunctionId: logicFunction.id,
workspaceId: logicFunction.workspaceId,
payload,
});
}
}
return result;
};
const filterEventsByUpdatedFields = ({
events,
operation,
triggerUpdatedFields,
}: {
events: ObjectRecordEvent[];
operation: string;
triggerUpdatedFields?: string[];
}): ObjectRecordEvent[] => {
if (
operation !== 'updated' ||
!isDefined(triggerUpdatedFields) ||
triggerUpdatedFields.length === 0
) {
return events;
}
return events.filter((event) => {
const eventUpdatedFields = (
event.properties as { updatedFields?: string[] }
)?.updatedFields;
if (!isDefined(eventUpdatedFields) || eventUpdatedFields.length === 0) {
return false;
}
return eventUpdatedFields.some((fieldName: string) =>
triggerUpdatedFields.includes(fieldName),
);
});
};
@@ -0,0 +1,59 @@
import {
type ArgumentsHost,
Catch,
type ExceptionFilter,
} from '@nestjs/common';
import type { Response } from 'express';
import {
RouteTriggerException,
RouteTriggerExceptionCode,
} from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/route/exceptions/route-trigger.exception';
import type { CustomException } from 'src/utils/custom-exception';
import { HttpExceptionHandlerService } from 'src/engine/core-modules/exception-handler/http-exception-handler.service';
@Catch(RouteTriggerException)
export class RouteTriggerRestApiExceptionFilter implements ExceptionFilter {
constructor(
private readonly httpExceptionHandlerService: HttpExceptionHandlerService,
) {}
catch(exception: RouteTriggerException, host: ArgumentsHost) {
const ctx = host.switchToHttp();
const response = ctx.getResponse<Response>();
switch (exception.code) {
case RouteTriggerExceptionCode.WORKSPACE_NOT_FOUND:
case RouteTriggerExceptionCode.ROUTE_NOT_FOUND:
case RouteTriggerExceptionCode.TRIGGER_NOT_FOUND:
case RouteTriggerExceptionCode.LOGIC_FUNCTION_NOT_FOUND:
return this.httpExceptionHandlerService.handleError(
exception as CustomException,
response,
404,
);
case RouteTriggerExceptionCode.FORBIDDEN_EXCEPTION:
return this.httpExceptionHandlerService.handleError(
exception as CustomException,
response,
403,
);
case RouteTriggerExceptionCode.LOGIC_FUNCTION_EXECUTION_ERROR:
return this.httpExceptionHandlerService.handleError(
exception as CustomException,
response,
500,
);
case RouteTriggerExceptionCode.ROUTE_ALREADY_EXIST:
case RouteTriggerExceptionCode.ROUTE_PATH_ALREADY_EXIST:
default: {
return this.httpExceptionHandlerService.handleError(
exception as CustomException,
response,
400,
);
}
}
}
}
@@ -0,0 +1,55 @@
import { type MessageDescriptor } from '@lingui/core';
import { msg } from '@lingui/core/macro';
import { assertUnreachable } from 'twenty-shared/utils';
import { CustomException } from 'src/utils/custom-exception';
export enum RouteTriggerExceptionCode {
WORKSPACE_NOT_FOUND = 'WORKSPACE_NOT_FOUND',
ROUTE_NOT_FOUND = 'ROUTE_NOT_FOUND',
TRIGGER_NOT_FOUND = 'TRIGGER_NOT_FOUND',
LOGIC_FUNCTION_NOT_FOUND = 'LOGIC_FUNCTION_NOT_FOUND',
ROUTE_ALREADY_EXIST = 'ROUTE_ALREADY_EXIST',
ROUTE_PATH_ALREADY_EXIST = 'ROUTE_PATH_ALREADY_EXIST',
FORBIDDEN_EXCEPTION = 'FORBIDDEN_EXCEPTION',
LOGIC_FUNCTION_EXECUTION_ERROR = 'LOGIC_FUNCTION_EXECUTION_ERROR',
}
const getRouteTriggerExceptionUserFriendlyMessage = (
code: RouteTriggerExceptionCode,
) => {
switch (code) {
case RouteTriggerExceptionCode.WORKSPACE_NOT_FOUND:
return msg`Workspace not found.`;
case RouteTriggerExceptionCode.ROUTE_NOT_FOUND:
return msg`Route not found.`;
case RouteTriggerExceptionCode.TRIGGER_NOT_FOUND:
return msg`Trigger not found.`;
case RouteTriggerExceptionCode.LOGIC_FUNCTION_NOT_FOUND:
return msg`Logic function not found.`;
case RouteTriggerExceptionCode.ROUTE_ALREADY_EXIST:
return msg`Route already exists.`;
case RouteTriggerExceptionCode.ROUTE_PATH_ALREADY_EXIST:
return msg`Route path already exists.`;
case RouteTriggerExceptionCode.FORBIDDEN_EXCEPTION:
return msg`You do not have permission to perform this action.`;
case RouteTriggerExceptionCode.LOGIC_FUNCTION_EXECUTION_ERROR:
return msg`Logic function execution failed.`;
default:
assertUnreachable(code);
}
};
export class RouteTriggerException extends CustomException<RouteTriggerExceptionCode> {
constructor(
message: string,
code: RouteTriggerExceptionCode,
{ userFriendlyMessage }: { userFriendlyMessage?: MessageDescriptor } = {},
) {
super(message, code, {
userFriendlyMessage:
userFriendlyMessage ??
getRouteTriggerExceptionUserFriendlyMessage(code),
});
}
}
@@ -0,0 +1,169 @@
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Request } from 'express';
import { match } from 'path-to-regexp';
import { assertIsDefinedOrThrow, isDefined } from 'twenty-shared/utils';
import { IsNull, Not, Repository } from 'typeorm';
import { HTTPMethod } from 'twenty-shared/types';
import { AccessTokenService } from 'src/engine/core-modules/auth/token/services/access-token.service';
import { WorkspaceDomainsService } from 'src/engine/core-modules/domain/workspace-domains/services/workspace-domains.service';
import { LogicFunctionExecutorService } from 'src/engine/core-modules/logic-function/logic-function-executor/services/logic-function-executor.service';
import {
RouteTriggerException,
RouteTriggerExceptionCode,
} from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/route/exceptions/route-trigger.exception';
import { buildLogicFunctionEvent } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/route/utils/build-logic-function-event.util';
import { LogicFunctionEntity } from 'src/engine/metadata-modules/logic-function/logic-function.entity';
@Injectable()
export class RouteTriggerService {
constructor(
private readonly accessTokenService: AccessTokenService,
private readonly logicFunctionExecutorService: LogicFunctionExecutorService,
private readonly workspaceDomainsService: WorkspaceDomainsService,
@InjectRepository(LogicFunctionEntity)
private readonly logicFunctionRepository: Repository<LogicFunctionEntity>,
) {}
private async getLogicFunctionWithPathParamsOrFail({
request,
httpMethod,
}: {
request: Request;
httpMethod: HTTPMethod;
}): Promise<{
logicFunction: LogicFunctionEntity;
pathParams: Partial<Record<string, string | string[]>>;
}> {
const host = `${request.protocol}://${request.get('host')}`;
const workspace =
await this.workspaceDomainsService.getWorkspaceByOriginOrDefaultWorkspace(
host,
);
assertIsDefinedOrThrow(
workspace,
new RouteTriggerException(
'Workspace not found',
RouteTriggerExceptionCode.WORKSPACE_NOT_FOUND,
),
);
const logicFunctionsWithHttpRouteTrigger =
await this.logicFunctionRepository.find({
where: {
workspaceId: workspace.id,
httpRouteTriggerSettings: Not(IsNull()),
},
});
const requestPath = request.path.replace(/^\/s\//, '/');
for (const logicFunction of logicFunctionsWithHttpRouteTrigger) {
const httpRouteSettings = logicFunction.httpRouteTriggerSettings;
if (
!isDefined(httpRouteSettings) ||
httpRouteSettings.httpMethod !== httpMethod
) {
continue;
}
const routeMatcher = match(httpRouteSettings.path, {
decode: decodeURIComponent,
});
const routeMatched = routeMatcher(requestPath);
if (routeMatched) {
return {
logicFunction,
pathParams: routeMatched.params,
};
}
}
throw new RouteTriggerException(
'No Route trigger found',
RouteTriggerExceptionCode.TRIGGER_NOT_FOUND,
);
}
private async validateWorkspaceFromRequest({
request,
workspaceId,
}: {
request: Request;
workspaceId: string;
}) {
const authContext =
await this.accessTokenService.validateTokenByRequest(request);
if (!isDefined(authContext.workspace)) {
throw new RouteTriggerException(
'Workspace not found',
RouteTriggerExceptionCode.WORKSPACE_NOT_FOUND,
);
}
if (authContext.workspace.id !== workspaceId) {
throw new RouteTriggerException(
'You are not authorized',
RouteTriggerExceptionCode.FORBIDDEN_EXCEPTION,
);
}
return authContext;
}
async handle({
request,
httpMethod,
}: {
request: Request;
httpMethod: HTTPMethod;
}) {
const { logicFunction, pathParams } =
await this.getLogicFunctionWithPathParamsOrFail({
request,
httpMethod,
});
const httpRouteSettings = logicFunction.httpRouteTriggerSettings;
if (httpRouteSettings?.isAuthRequired) {
await this.validateWorkspaceFromRequest({
request,
workspaceId: logicFunction.workspaceId,
});
}
const event = buildLogicFunctionEvent({
request,
pathParameters: pathParams,
forwardedRequestHeaders: httpRouteSettings?.forwardedRequestHeaders ?? [],
});
const result =
await this.logicFunctionExecutorService.executeOneLogicFunction({
id: logicFunction.id,
workspaceId: logicFunction.workspaceId,
payload: event,
});
if (!isDefined(result)) {
return result;
}
if (result.error) {
throw new RouteTriggerException(
result.error.errorMessage,
RouteTriggerExceptionCode.LOGIC_FUNCTION_EXECUTION_ERROR,
);
}
return result.data;
}
}
@@ -0,0 +1,438 @@
import { type Request } from 'express';
import {
buildLogicFunctionEvent,
extractBody,
filterRequestHeaders,
normalizePathParameters,
normalizeQueryStringParameters,
} from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/route/utils/build-logic-function-event.util';
describe('filterRequestHeaders', () => {
it('should filter headers based on allowed names', () => {
const requestHeaders = {
'content-type': 'application/json',
authorization: 'Bearer token123',
'x-custom-header': 'custom-value',
'user-agent': 'test-agent',
};
const forwardedRequestHeaders = ['content-type', 'authorization'];
const result = filterRequestHeaders({
requestHeaders,
forwardedRequestHeaders,
});
expect(result).toEqual({
'content-type': 'application/json',
authorization: 'Bearer token123',
});
});
it('should handle case-insensitive header names', () => {
const requestHeaders = {
'content-type': 'application/json',
authorization: 'Bearer token123',
};
const forwardedRequestHeaders = ['Content-Type', 'AUTHORIZATION'];
const result = filterRequestHeaders({
requestHeaders,
forwardedRequestHeaders,
});
expect(result).toEqual({
'content-type': 'application/json',
authorization: 'Bearer token123',
});
});
it('should return empty object when no headers match', () => {
const requestHeaders = {
'content-type': 'application/json',
};
const forwardedRequestHeaders = ['x-custom-header'];
const result = filterRequestHeaders({
requestHeaders,
forwardedRequestHeaders,
});
expect(result).toEqual({});
});
it('should return empty object when forwardedRequestHeaders is empty', () => {
const requestHeaders = {
'content-type': 'application/json',
};
const result = filterRequestHeaders({
requestHeaders,
forwardedRequestHeaders: [],
});
expect(result).toEqual({});
});
it('should convert array header values to comma-separated string', () => {
const requestHeaders = {
'x-custom-array-header': ['value1', 'value2', 'value3'],
};
const forwardedRequestHeaders = ['x-custom-array-header'];
const result = filterRequestHeaders({
requestHeaders,
forwardedRequestHeaders,
});
expect(result).toEqual({
'x-custom-array-header': 'value1, value2, value3',
});
});
it('should skip undefined header values', () => {
const requestHeaders = {
'content-type': 'application/json',
'x-missing': undefined,
};
const forwardedRequestHeaders = ['content-type', 'x-missing'];
const result = filterRequestHeaders({
requestHeaders,
forwardedRequestHeaders,
});
expect(result).toEqual({
'content-type': 'application/json',
});
});
});
describe('extractBody', () => {
it('should return null for undefined body', () => {
const request = { body: undefined } as Request;
const result = extractBody(request);
expect(result).toBeNull();
});
it('should return null for null body', () => {
const request = { body: null } as unknown as Request;
const result = extractBody(request);
expect(result).toBeNull();
});
it('should parse string body as JSON', () => {
const request = { body: '{"key":"value"}' } as unknown as Request;
const result = extractBody(request);
expect(result).toEqual({ key: 'value' });
});
it('should wrap non-JSON string body in raw property', () => {
const request = { body: 'plain text body' } as unknown as Request;
const result = extractBody(request);
expect(result).toEqual({ raw: 'plain text body' });
});
it('should return object body as-is (parsed JSON)', () => {
const request = {
body: { key: 'value', nested: { foo: 'bar' } },
} as Request;
const result = extractBody(request);
expect(result).toEqual({ key: 'value', nested: { foo: 'bar' } });
});
it('should parse Buffer body as JSON', () => {
const request = {
body: Buffer.from('{"buffered":"json"}'),
} as unknown as Request;
const result = extractBody(request);
expect(result).toEqual({ buffered: 'json' });
});
it('should wrap non-JSON Buffer body in raw property', () => {
const request = {
body: Buffer.from('buffer content'),
} as unknown as Request;
const result = extractBody(request);
expect(result).toEqual({ raw: 'buffer content' });
});
it('should handle empty object body', () => {
const request = { body: {} } as Request;
const result = extractBody(request);
expect(result).toEqual({});
});
it('should handle array body', () => {
const request = { body: [1, 2, 3] } as unknown as Request;
const result = extractBody(request);
expect(result).toEqual([1, 2, 3]);
});
});
describe('normalizeQueryStringParameters', () => {
it('should handle simple string parameters', () => {
const query = { page: '1', limit: '10' };
const result = normalizeQueryStringParameters(query);
expect(result).toEqual({ page: '1', limit: '10' });
});
it('should join array parameters with commas', () => {
const query = { ids: ['1', '2', '3'] };
const result = normalizeQueryStringParameters(query);
expect(result).toEqual({ ids: '1,2,3' });
});
it('should skip undefined parameters', () => {
const query = { page: '1', missing: undefined };
const result = normalizeQueryStringParameters(query);
expect(result).toEqual({ page: '1' });
});
it('should handle empty query object', () => {
const query = {};
const result = normalizeQueryStringParameters(query);
expect(result).toEqual({});
});
it('should stringify nested objects', () => {
const query = { filter: { name: 'test' } as unknown as string };
const result = normalizeQueryStringParameters(query);
expect(result).toEqual({ filter: '{"name":"test"}' });
});
it('should filter non-string values from arrays and join with commas', () => {
const query = { ids: ['1', undefined as unknown as string, '2'] };
const result = normalizeQueryStringParameters(query);
expect(result).toEqual({ ids: '1,2' });
});
});
describe('normalizePathParameters', () => {
it('should handle simple string parameters', () => {
const pathParams = { id: '123', slug: 'test' };
const result = normalizePathParameters(pathParams);
expect(result).toEqual({ id: '123', slug: 'test' });
});
it('should join array parameters with commas', () => {
const pathParams = { ids: ['1', '2', '3'] };
const result = normalizePathParameters(pathParams);
expect(result).toEqual({ ids: '1,2,3' });
});
it('should skip undefined parameters', () => {
const pathParams = { id: '123', missing: undefined };
const result = normalizePathParameters(pathParams);
expect(result).toEqual({ id: '123' });
});
it('should handle empty object', () => {
const pathParams = {};
const result = normalizePathParameters(pathParams);
expect(result).toEqual({});
});
});
describe('buildLogicFunctionEvent', () => {
const createMockRequest = (overrides: Partial<Request> = {}): Request =>
({
headers: {},
query: {},
body: undefined,
method: 'GET',
path: '/test',
...overrides,
}) as Request;
it('should build a complete event from Express request', () => {
const request = createMockRequest({
headers: {
'content-type': 'application/json',
authorization: 'Bearer token',
'user-agent': 'test',
},
query: { page: '1' },
body: { data: 'test' },
method: 'POST',
path: '/s/users/123',
});
const result = buildLogicFunctionEvent({
request,
pathParameters: { id: '123' },
forwardedRequestHeaders: ['content-type', 'authorization'],
});
expect(result).toEqual({
headers: {
'content-type': 'application/json',
authorization: 'Bearer token',
},
queryStringParameters: { page: '1' },
pathParameters: { id: '123' },
body: { data: 'test' },
isBase64Encoded: false,
requestContext: {
http: {
method: 'POST',
path: '/s/users/123',
},
},
});
});
it('should preserve the request path as-is', () => {
const request = createMockRequest({
path: '/s/api/users',
});
const result = buildLogicFunctionEvent({
request,
pathParameters: {},
forwardedRequestHeaders: [],
});
expect(result.requestContext.http.path).toBe('/s/api/users');
});
it('should preserve path without prefix', () => {
const request = createMockRequest({
path: '/api/users',
});
const result = buildLogicFunctionEvent({
request,
pathParameters: {},
forwardedRequestHeaders: [],
});
expect(result.requestContext.http.path).toBe('/api/users');
});
it('should handle GET request with no body', () => {
const request = createMockRequest({
method: 'GET',
query: { search: 'test' },
body: undefined,
});
const result = buildLogicFunctionEvent({
request,
pathParameters: {},
forwardedRequestHeaders: [],
});
expect(result.body).toBeNull();
expect(result.queryStringParameters).toEqual({ search: 'test' });
});
it('should handle DELETE request with path parameters', () => {
const request = createMockRequest({
method: 'DELETE',
path: '/s/users/456',
});
const result = buildLogicFunctionEvent({
request,
pathParameters: { userId: '456' },
forwardedRequestHeaders: [],
});
expect(result.requestContext.http.method).toBe('DELETE');
expect(result.pathParameters).toEqual({ userId: '456' });
});
it('should filter only allowed headers', () => {
const request = createMockRequest({
headers: {
'content-type': 'application/json',
authorization: 'Bearer secret',
'x-api-key': 'key123',
cookie: 'session=abc',
},
});
const result = buildLogicFunctionEvent({
request,
pathParameters: {},
forwardedRequestHeaders: ['x-api-key'],
});
expect(result.headers).toEqual({
'x-api-key': 'key123',
});
expect(result.headers['authorization']).toBeUndefined();
expect(result.headers['cookie']).toBeUndefined();
});
it('should set isBase64Encoded to false', () => {
const request = createMockRequest();
const result = buildLogicFunctionEvent({
request,
pathParameters: {},
forwardedRequestHeaders: [],
});
expect(result.isBase64Encoded).toBe(false);
});
it('should handle complex path parameters', () => {
const request = createMockRequest({
path: '/s/organizations/org1/users/user1/posts',
});
const result = buildLogicFunctionEvent({
request,
pathParameters: {
orgId: 'org1',
userId: 'user1',
},
forwardedRequestHeaders: [],
});
expect(result.pathParameters).toEqual({
orgId: 'org1',
userId: 'user1',
});
});
});
@@ -0,0 +1,150 @@
import { type Request } from 'express';
import { type LogicFunctionEvent } from 'twenty-shared/types';
/**
* Filters HTTP headers from Express request based on allowed header names
* Header names are case-insensitive as per HTTP specification
*/
export const filterRequestHeaders = ({
requestHeaders,
forwardedRequestHeaders,
}: {
requestHeaders: Request['headers'];
forwardedRequestHeaders: string[];
}): Record<string, string | undefined> => {
const lowercaseForwardedHeaders = forwardedRequestHeaders.map((h) =>
h.toLowerCase(),
);
const filteredHeaders: Record<string, string | undefined> = {};
for (const headerName of lowercaseForwardedHeaders) {
const headerValue = requestHeaders[headerName];
if (headerValue !== undefined) {
filteredHeaders[headerName] = Array.isArray(headerValue)
? headerValue.join(', ')
: headerValue;
}
}
return filteredHeaders;
};
/**
* Extracts the body from Express request as an object
* Express body-parser middleware parses JSON bodies automatically
* Returns null if body is empty/undefined
*/
export const extractBody = (request: Request): object | null => {
if (request.body === undefined || request.body === null) {
return null;
}
if (typeof request.body === 'object' && !Buffer.isBuffer(request.body)) {
return request.body;
}
if (typeof request.body === 'string') {
try {
return JSON.parse(request.body);
} catch {
return { raw: request.body };
}
}
if (Buffer.isBuffer(request.body)) {
try {
return JSON.parse(request.body.toString('utf-8'));
} catch {
return { raw: request.body.toString('utf-8') };
}
}
return { raw: String(request.body) };
};
/**
* Converts Express query parameters to a normalized string format
* Arrays are joined with commas (e.g., ['1', '2', '3'] → '1,2,3')
*/
export const normalizeQueryStringParameters = (
query: Request['query'],
): Record<string, string | undefined> => {
const normalized: Record<string, string | undefined> = {};
for (const [key, value] of Object.entries(query)) {
if (value === undefined) {
continue;
}
if (Array.isArray(value)) {
const stringValues = value.filter(
(v): v is string => typeof v === 'string',
);
normalized[key] = stringValues.join(',');
} else if (typeof value === 'string') {
normalized[key] = value;
} else if (typeof value === 'object') {
normalized[key] = JSON.stringify(value);
}
}
return normalized;
};
/**
* Normalizes path parameters to string format
* Arrays are joined with commas (e.g., ['1', '2', '3'] → '1,2,3')
*/
export const normalizePathParameters = (
pathParams: Record<string, string | string[] | undefined>,
): Record<string, string | undefined> => {
const normalized: Record<string, string | undefined> = {};
for (const [key, value] of Object.entries(pathParams)) {
if (value === undefined) {
continue;
}
if (Array.isArray(value)) {
normalized[key] = value.join(',');
} else {
normalized[key] = value;
}
}
return normalized;
};
/**
* Builds an AWS HTTP API v2 compatible event from an Express request
* @see https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-develop-integrations-lambda.html
*/
export const buildLogicFunctionEvent = ({
request,
pathParameters,
forwardedRequestHeaders,
}: {
request: Request;
pathParameters: Record<string, string | string[] | undefined>;
forwardedRequestHeaders: string[];
}): LogicFunctionEvent => {
return {
headers: filterRequestHeaders({
requestHeaders: request.headers,
forwardedRequestHeaders,
}),
queryStringParameters: normalizeQueryStringParameters(request.query),
pathParameters: normalizePathParameters(pathParameters),
body: extractBody(request),
isBase64Encoded: false,
requestContext: {
http: {
method: request.method,
path: request.path,
},
},
};
};
@@ -0,0 +1,35 @@
import { type DynamicModule, Global, Module } from '@nestjs/common';
import { type LogicFunctionExecutorModuleAsyncOptions } from 'src/engine/core-modules/logic-function/logic-function-executor/interfaces/logic-function-executor.interface';
import { LogicFunctionBuildModule } from 'src/engine/core-modules/logic-function/logic-function-build/logic-function-build.module';
import { LogicFunctionDriversModule } from 'src/engine/core-modules/logic-function/logic-function-drivers/logic-function-drivers.module';
import { LogicFunctionExecutorModule } from 'src/engine/core-modules/logic-function/logic-function-executor/logic-function-executor.module';
import { CoreLogicFunctionLayerModule } from 'src/engine/core-modules/logic-function/logic-function-layer/logic-function-layer.module';
import { LogicFunctionTriggerModule } from 'src/engine/core-modules/logic-function/logic-function-trigger/logic-function-trigger.module';
@Global()
@Module({})
export class LogicFunctionModule {
static forRootAsync(
options: LogicFunctionExecutorModuleAsyncOptions,
): DynamicModule {
return {
module: LogicFunctionModule,
imports: [
LogicFunctionDriversModule.forRootAsync(options),
LogicFunctionExecutorModule,
LogicFunctionBuildModule,
CoreLogicFunctionLayerModule,
LogicFunctionTriggerModule,
],
exports: [
LogicFunctionDriversModule,
LogicFunctionExecutorModule,
LogicFunctionBuildModule,
CoreLogicFunctionLayerModule,
LogicFunctionTriggerModule,
],
};
}
}