feat: Add Agent Evaluation System and Refactor AI Modules (#16111)
## Summary This PR introduces a comprehensive agent evaluation system and refactors the AI module structure for better organization. ## Key Changes ### 🎯 Agent Evaluation System - Added **Agent Turn Evaluation** entities, DTOs, and database schema - New GraphQL mutations: `evaluateAgentTurn` and `runEvaluationInput` - Added `evaluationInputs` field to Agent entity for storing test inputs - New `AgentTurnGraderService` for automatic turn evaluation - Added evaluation UI with new **Evals** and **Logs** tabs in agent detail pages ### 🏗️ Entity & Module Refactoring - Renamed `AgentChatMessage` → `AgentMessage` for clarity - Consolidated chat entities: `AgentMessage`, `AgentTurn`, and `AgentChatThread` - Reorganized AI modules under `ai/` subdirectory structure - Updated imports across codebase to reflect new module paths ### 🤖 New Agents & Roles - Added **Dashboard Builder Agent** for dashboard creation and management - Added **Dashboard Manager Role** with appropriate permissions - Updated role permissions to be more granular (users vs agents vs API keys) ### 🔐 Permission System Updates - Added `HTTP_REQUEST_TOOL` permission flag - Updated Workflow Manager role permissions (restricted tool access) - Enhanced permission flag types to differentiate between user/agent/API key contexts - Added `isRelevantForAgents`, `isRelevantForApiKeys`, `isRelevantForUsers` to permission flags ### 📨 Message Role Enhancement - Added `system` role to `AgentMessageRole` enum (alongside user/assistant) - Updated message handling to support system prompts ### 🎨 UI/UX Improvements - New tabs in agent detail: **Evals** and **Logs** - Added turn detail page: `/ai/agents/:agentId/turns/:turnId` - Fixed text overflow in `SettingsListItemCardContent` - Updated role applicability labels ("Assignable to Workspace Members") ### 🛠️ Technical Improvements - Fixed Zod schema validation for UUID and Date fields (use string validators) - Updated `ToolRegistryService` to properly register HTTP tool with permission flag - Enhanced error handling in agent execution services - Updated database migrations for new entity schema ## Database Migrations - `1764210000000-add-system-role-to-agent-message.ts` - `1764220000000-add-evaluation-inputs-to-agent.ts` - `1764200000000-add-agent-turn-evaluation.ts` - `1764100000000-refactor-agent-chat-entities.ts` ## Testing - [ ] Agent evaluation flow tested - [ ] Dashboard Builder agent tested - [ ] Permission system validated - [ ] UI tabs and navigation tested - [ ] Database migrations run successfully ## Breaking Changes ⚠️ **Entity Rename**: `AgentChatMessage` renamed to `AgentMessage` - GraphQL queries need updating ## Related Issues <!-- Link any related issues here --> ## Screenshots <!-- Add screenshots if applicable -->
This commit is contained in:
Félix Malfait
+5
-10
@@ -1,26 +1,22 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
|
||||
import { AiBillingModule } from 'src/engine/metadata-modules/ai-billing/ai-billing.module';
|
||||
import { AiModelsModule } from 'src/engine/metadata-modules/ai-models/ai-models.module';
|
||||
import { AiToolsModule } from 'src/engine/metadata-modules/ai-tools/ai-tools.module';
|
||||
import { UserWorkspaceModule } from 'src/engine/core-modules/user-workspace/user-workspace.module';
|
||||
import { AgentEntity } from 'src/engine/metadata-modules/ai-agent/entities/agent.entity';
|
||||
import { RoleTargetsEntity } from 'src/engine/metadata-modules/role/role-targets.entity';
|
||||
import { AiAgentExecutionModule } from 'src/engine/metadata-modules/ai/ai-agent-execution/ai-agent-execution.module';
|
||||
import { AgentEntity } from 'src/engine/metadata-modules/ai/ai-agent/entities/agent.entity';
|
||||
import { AiBillingModule } from 'src/engine/metadata-modules/ai/ai-billing/ai-billing.module';
|
||||
import { UserRoleModule } from 'src/engine/metadata-modules/user-role/user-role.module';
|
||||
import { ScopedWorkspaceContextFactory } from 'src/engine/twenty-orm/factories/scoped-workspace-context.factory';
|
||||
import { WorkflowExecutionContextService } from 'src/modules/workflow/workflow-executor/services/workflow-execution-context.service';
|
||||
import { AiAgentExecutorService } from 'src/modules/workflow/workflow-executor/workflow-actions/ai-agent/services/ai-agent-executor.service';
|
||||
import { WorkflowRunModule } from 'src/modules/workflow/workflow-runner/workflow-run/workflow-run.module';
|
||||
|
||||
import { AiAgentWorkflowAction } from './ai-agent.workflow-action';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
AiAgentExecutionModule,
|
||||
AiBillingModule,
|
||||
AiModelsModule,
|
||||
AiToolsModule,
|
||||
TypeOrmModule.forFeature([AgentEntity, RoleTargetsEntity]),
|
||||
TypeOrmModule.forFeature([AgentEntity]),
|
||||
WorkflowRunModule,
|
||||
UserWorkspaceModule,
|
||||
UserRoleModule,
|
||||
@@ -29,7 +25,6 @@ import { AiAgentWorkflowAction } from './ai-agent.workflow-action';
|
||||
ScopedWorkspaceContextFactory,
|
||||
WorkflowExecutionContextService,
|
||||
AiAgentWorkflowAction,
|
||||
AiAgentExecutorService,
|
||||
],
|
||||
exports: [AiAgentWorkflowAction],
|
||||
})
|
||||
|
||||
+6
-6
@@ -6,12 +6,14 @@ import { Repository } from 'typeorm';
|
||||
|
||||
import { type WorkflowAction } from 'src/modules/workflow/workflow-executor/interfaces/workflow-action.interface';
|
||||
|
||||
import { AIBillingService } from 'src/engine/metadata-modules/ai-billing/services/ai-billing.service';
|
||||
import { AgentEntity } from 'src/engine/metadata-modules/ai-agent/entities/agent.entity';
|
||||
import { AgentAsyncExecutorService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service';
|
||||
import {
|
||||
AgentException,
|
||||
AgentExceptionCode,
|
||||
} from 'src/engine/metadata-modules/ai-agent/agent.exception';
|
||||
} from 'src/engine/metadata-modules/ai/ai-agent/agent.exception';
|
||||
import { AgentEntity } from 'src/engine/metadata-modules/ai/ai-agent/entities/agent.entity';
|
||||
import { AIBillingService } from 'src/engine/metadata-modules/ai/ai-billing/services/ai-billing.service';
|
||||
import { DEFAULT_SMART_MODEL } from 'src/engine/metadata-modules/ai/ai-models/constants/ai-models.const';
|
||||
import {
|
||||
WorkflowStepExecutorException,
|
||||
WorkflowStepExecutorExceptionCode,
|
||||
@@ -20,15 +22,13 @@ import { WorkflowExecutionContextService } from 'src/modules/workflow/workflow-e
|
||||
import { type WorkflowActionInput } from 'src/modules/workflow/workflow-executor/types/workflow-action-input';
|
||||
import { type WorkflowActionOutput } from 'src/modules/workflow/workflow-executor/types/workflow-action-output.type';
|
||||
import { findStepOrThrow } from 'src/modules/workflow/workflow-executor/utils/find-step-or-throw.util';
|
||||
import { AiAgentExecutorService } from 'src/modules/workflow/workflow-executor/workflow-actions/ai-agent/services/ai-agent-executor.service';
|
||||
import { DEFAULT_SMART_MODEL } from 'src/engine/metadata-modules/ai-models/constants/ai-models.const';
|
||||
|
||||
import { isWorkflowAiAgentAction } from './guards/is-workflow-ai-agent-action.guard';
|
||||
|
||||
@Injectable()
|
||||
export class AiAgentWorkflowAction implements WorkflowAction {
|
||||
constructor(
|
||||
private readonly aiAgentExecutionService: AiAgentExecutorService,
|
||||
private readonly aiAgentExecutionService: AgentAsyncExecutorService,
|
||||
private readonly aiBillingService: AIBillingService,
|
||||
private readonly workflowExecutionContextService: WorkflowExecutionContextService,
|
||||
@InjectRepository(AgentEntity)
|
||||
|
||||
-174
@@ -1,174 +0,0 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import {
|
||||
generateObject,
|
||||
generateText,
|
||||
jsonSchema,
|
||||
stepCountIs,
|
||||
ToolSet,
|
||||
} from 'ai';
|
||||
import { type ActorMetadata } from 'twenty-shared/types';
|
||||
import { Repository } from 'typeorm';
|
||||
|
||||
import { AI_TELEMETRY_CONFIG } from 'src/engine/metadata-modules/ai-models/constants/ai-telemetry.const';
|
||||
import { AiModelRegistryService } from 'src/engine/metadata-modules/ai-models/services/ai-model-registry.service';
|
||||
import { ToolAdapterService } from 'src/engine/metadata-modules/ai-tools/services/tool-adapter.service';
|
||||
import { ToolService } from 'src/engine/metadata-modules/ai-tools/services/tool.service';
|
||||
import { AgentExecutionResult } from 'src/engine/metadata-modules/ai-agent/services/agent-execution.service';
|
||||
import { AgentEntity } from 'src/engine/metadata-modules/ai-agent/entities/agent.entity';
|
||||
import {
|
||||
AgentException,
|
||||
AgentExceptionCode,
|
||||
} from 'src/engine/metadata-modules/ai-agent/agent.exception';
|
||||
import { AGENT_CONFIG } from 'src/engine/metadata-modules/ai-agent/constants/agent-config.const';
|
||||
import { AGENT_SYSTEM_PROMPTS } from 'src/engine/metadata-modules/ai-agent/constants/agent-system-prompts.const';
|
||||
import { RoleTargetsEntity } from 'src/engine/metadata-modules/role/role-targets.entity';
|
||||
import { type RolePermissionConfig } from 'src/engine/twenty-orm/types/role-permission-config';
|
||||
|
||||
@Injectable()
|
||||
export class AiAgentExecutorService {
|
||||
private readonly logger = new Logger(AiAgentExecutorService.name);
|
||||
constructor(
|
||||
private readonly aiModelRegistryService: AiModelRegistryService,
|
||||
private readonly toolAdapterService: ToolAdapterService,
|
||||
@InjectRepository(RoleTargetsEntity)
|
||||
private readonly roleTargetsRepository: Repository<RoleTargetsEntity>,
|
||||
private readonly toolService: ToolService,
|
||||
) {}
|
||||
|
||||
private async getTools(
|
||||
agentId: string,
|
||||
workspaceId: string,
|
||||
actorContext?: ActorMetadata,
|
||||
rolePermissionConfig?: RolePermissionConfig,
|
||||
): Promise<ToolSet> {
|
||||
const roleTarget = await this.roleTargetsRepository.findOne({
|
||||
where: {
|
||||
agentId: agentId,
|
||||
workspaceId,
|
||||
},
|
||||
select: ['roleId'],
|
||||
});
|
||||
|
||||
const agentRoleId = roleTarget?.roleId;
|
||||
|
||||
if (!rolePermissionConfig && !agentRoleId) {
|
||||
return await this.toolAdapterService.getTools();
|
||||
}
|
||||
|
||||
let effectiveRoleContext: RolePermissionConfig;
|
||||
|
||||
if (
|
||||
rolePermissionConfig &&
|
||||
('intersectionOf' in rolePermissionConfig ||
|
||||
'unionOf' in rolePermissionConfig)
|
||||
) {
|
||||
effectiveRoleContext = rolePermissionConfig;
|
||||
} else if (agentRoleId) {
|
||||
effectiveRoleContext = { unionOf: [agentRoleId] };
|
||||
} else {
|
||||
return await this.toolAdapterService.getTools();
|
||||
}
|
||||
|
||||
const actionTools = await this.toolAdapterService.getTools(
|
||||
effectiveRoleContext,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
const databaseTools = await this.toolService.listTools(
|
||||
effectiveRoleContext,
|
||||
workspaceId,
|
||||
actorContext,
|
||||
);
|
||||
|
||||
return {
|
||||
...databaseTools,
|
||||
...actionTools,
|
||||
};
|
||||
}
|
||||
|
||||
async executeAgent({
|
||||
agent,
|
||||
userPrompt,
|
||||
actorContext,
|
||||
rolePermissionConfig,
|
||||
}: {
|
||||
agent: AgentEntity | null;
|
||||
userPrompt: string;
|
||||
actorContext?: ActorMetadata;
|
||||
rolePermissionConfig?: RolePermissionConfig;
|
||||
}): Promise<AgentExecutionResult> {
|
||||
try {
|
||||
const registeredModel =
|
||||
await this.aiModelRegistryService.resolveModelForAgent(agent);
|
||||
|
||||
const tools = agent
|
||||
? await this.getTools(
|
||||
agent.id,
|
||||
agent.workspaceId,
|
||||
actorContext,
|
||||
rolePermissionConfig,
|
||||
)
|
||||
: {};
|
||||
|
||||
this.logger.log(`Generated ${Object.keys(tools).length} tools for agent`);
|
||||
|
||||
const textResponse = await generateText({
|
||||
system: `${AGENT_SYSTEM_PROMPTS.BASE}\n${AGENT_SYSTEM_PROMPTS.WORKFLOW_ADDITIONS}\n\n${agent ? agent.prompt : ''}`,
|
||||
tools,
|
||||
model: registeredModel.model,
|
||||
prompt: userPrompt,
|
||||
stopWhen: stepCountIs(AGENT_CONFIG.MAX_STEPS),
|
||||
experimental_telemetry: AI_TELEMETRY_CONFIG,
|
||||
});
|
||||
|
||||
const agentSchema =
|
||||
agent?.responseFormat?.type === 'json'
|
||||
? agent.responseFormat.schema
|
||||
: undefined;
|
||||
|
||||
if (!agentSchema) {
|
||||
return {
|
||||
result: { response: textResponse.text },
|
||||
usage: textResponse.usage,
|
||||
};
|
||||
}
|
||||
|
||||
const output = await generateObject({
|
||||
system: AGENT_SYSTEM_PROMPTS.OUTPUT_GENERATOR,
|
||||
model: registeredModel.model,
|
||||
prompt: `Based on the following execution results, generate the structured output according to the schema:
|
||||
|
||||
Execution Results: ${textResponse.text}
|
||||
|
||||
Please generate the structured output based on the execution results and context above.`,
|
||||
schema: jsonSchema(agentSchema),
|
||||
experimental_telemetry: AI_TELEMETRY_CONFIG,
|
||||
});
|
||||
|
||||
return {
|
||||
result: output.object as object,
|
||||
usage: {
|
||||
inputTokens:
|
||||
(textResponse.usage?.inputTokens ?? 0) +
|
||||
(output.usage?.inputTokens ?? 0),
|
||||
outputTokens:
|
||||
(textResponse.usage?.outputTokens ?? 0) +
|
||||
(output.usage?.outputTokens ?? 0),
|
||||
totalTokens:
|
||||
(textResponse.usage?.totalTokens ?? 0) +
|
||||
(output.usage?.totalTokens ?? 0),
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
if (error instanceof AgentException) {
|
||||
throw error;
|
||||
}
|
||||
throw new AgentException(
|
||||
error instanceof Error ? error.message : 'Agent execution failed',
|
||||
AgentExceptionCode.AGENT_EXECUTION_FAILED,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user