feat: add configurable response format for AI agents (text/JSON) (#15953)
## Summary
This PR adds configurable response format support for AI agents,
allowing them to return either plain text or structured JSON data based
on a defined schema.
## Key Features
### 1. Agent Response Format Configuration
- Added `AgentResponseFormat` type supporting:
- `text`: Returns plain text responses (default)
- `json`: Returns structured JSON based on defined schema
- New `AgentResponseSchema` type moved to `twenty-shared/ai` for sharing
between frontend/backend
### 2. Settings UI
- New `SettingsAgentResponseFormat` component for configuring response
format
- Visual schema builder for defining JSON output structure
- Real-time validation and preview
- Integrated into agent settings tab
### 3. Workflow Integration
- AI Agent workflow action automatically uses agent's configured
response format
- Output schema dynamically generated from agent's response format
- Workflow variable picker shows structured fields for JSON responses
- Backward compatible with existing text-only agents
### 4. Backend Implementation
- Added `convertAgentSchemaToZod` utility to validate JSON responses
- Agent executor service handles both text and JSON generation
- Automatic agent creation/cloning when adding AI agent steps to
workflows
- Unique agent naming with conflict resolution
### 5. Database Migration
- Migration `1763622159656-update-agent-response-format.ts`
- Sets default `responseFormat` to `{"type":"text"}` for existing agents
- Updated all standard agents with proper response format
## Changes by Module
### Frontend (`twenty-front`)
- 🆕 `AgentResponseFormat` type
- 🆕 `SettingsAgentResponseFormat` component
- ✏️ Updated `WorkflowEditActionAiAgent` to support response format
configuration
- 🗑️ Removed deprecated `useAiAgentOutputSchema` hook and
`AiAgentOutputSchema` type
### Backend (`twenty-server`)
- 🆕 `AgentResponseFormat` type in agent entity
- 🆕 `convertAgentSchemaToZod` utility for schema validation
- ✏️ Updated `AiAgentExecutorService` to handle both text and JSON
generation
- ✏️ Updated `WorkflowSchemaWorkspaceService` to generate output schema
from agent config
- ✏️ Enhanced `WorkflowVersionStepOperationsWorkspaceService` with agent
creation/cloning
- 🆕 Agent naming constants for conflict resolution
### Shared (`twenty-shared`)
- 🆕 `AgentResponseSchema` type
- 🆕 `ModelConfiguration` type moved to shared package
- Updated exports in `ai/index.ts`
## Code Quality
- Removed useless comments following code style guidelines
- All linter checks passed
- Type-safe implementation with proper TypeScript types
## Testing
- ✅ Database migration tested
- ✅ Agent creation/cloning in workflows verified
- ✅ Response format switching (text ↔ JSON) validated
- ✅ Backward compatibility with existing agents confirmed
## Migration Notes
- Existing agents will have `responseFormat: {type: 'text'}` set
automatically
- No breaking changes - all existing functionality preserved
- Agents can be updated to use JSON format through settings UI
This commit is contained in:
-1
@@ -80,7 +80,6 @@ export class AiAgentWorkflowAction implements WorkflowAction {
|
||||
const { result, usage } = await this.aiAgentExecutionService.executeAgent(
|
||||
{
|
||||
agent,
|
||||
schema: step.settings.outputSchema,
|
||||
userPrompt: resolveInput(prompt, context) as string,
|
||||
actorContext: executionContext.isActingOnBehalfOfUser
|
||||
? executionContext.initiator
|
||||
|
||||
+17
-9
@@ -1,9 +1,15 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
|
||||
import { generateObject, generateText, stepCountIs, ToolSet } from 'ai';
|
||||
import { Repository } from 'typeorm';
|
||||
import {
|
||||
generateObject,
|
||||
generateText,
|
||||
jsonSchema,
|
||||
stepCountIs,
|
||||
ToolSet,
|
||||
} from 'ai';
|
||||
import { type ActorMetadata } from 'twenty-shared/types';
|
||||
import { Repository } from 'typeorm';
|
||||
|
||||
import { AI_TELEMETRY_CONFIG } from 'src/engine/core-modules/ai/constants/ai-telemetry.const';
|
||||
import { AiModelRegistryService } from 'src/engine/core-modules/ai/services/ai-model-registry.service';
|
||||
@@ -17,10 +23,8 @@ import {
|
||||
} from 'src/engine/metadata-modules/agent/agent.exception';
|
||||
import { AGENT_CONFIG } from 'src/engine/metadata-modules/agent/constants/agent-config.const';
|
||||
import { AGENT_SYSTEM_PROMPTS } from 'src/engine/metadata-modules/agent/constants/agent-system-prompts.const';
|
||||
import { convertOutputSchemaToZod } from 'src/engine/metadata-modules/agent/utils/convert-output-schema-to-zod';
|
||||
import { RoleTargetsEntity } from 'src/engine/metadata-modules/role/role-targets.entity';
|
||||
import { type RolePermissionConfig } from 'src/engine/twenty-orm/types/role-permission-config';
|
||||
import { OutputSchema } from 'src/modules/workflow/workflow-builder/workflow-schema/types/output-schema.type';
|
||||
|
||||
@Injectable()
|
||||
export class AiAgentExecutorService {
|
||||
@@ -86,13 +90,11 @@ export class AiAgentExecutorService {
|
||||
|
||||
async executeAgent({
|
||||
agent,
|
||||
schema,
|
||||
userPrompt,
|
||||
actorContext,
|
||||
rolePermissionConfig,
|
||||
}: {
|
||||
agent: AgentEntity | null;
|
||||
schema: OutputSchema;
|
||||
userPrompt: string;
|
||||
actorContext?: ActorMetadata;
|
||||
rolePermissionConfig?: RolePermissionConfig;
|
||||
@@ -121,12 +123,18 @@ export class AiAgentExecutorService {
|
||||
experimental_telemetry: AI_TELEMETRY_CONFIG,
|
||||
});
|
||||
|
||||
if (Object.keys(schema).length === 0) {
|
||||
const agentSchema =
|
||||
agent?.responseFormat?.type === 'json'
|
||||
? agent.responseFormat.schema
|
||||
: undefined;
|
||||
|
||||
if (!agentSchema) {
|
||||
return {
|
||||
result: { response: textResponse.text },
|
||||
usage: textResponse.usage,
|
||||
};
|
||||
}
|
||||
|
||||
const output = await generateObject({
|
||||
system: AGENT_SYSTEM_PROMPTS.OUTPUT_GENERATOR,
|
||||
model: registeredModel.model,
|
||||
@@ -135,12 +143,12 @@ export class AiAgentExecutorService {
|
||||
Execution Results: ${textResponse.text}
|
||||
|
||||
Please generate the structured output based on the execution results and context above.`,
|
||||
schema: convertOutputSchemaToZod(schema),
|
||||
schema: jsonSchema(agentSchema),
|
||||
experimental_telemetry: AI_TELEMETRY_CONFIG,
|
||||
});
|
||||
|
||||
return {
|
||||
result: output.object,
|
||||
result: output.object as object,
|
||||
usage: {
|
||||
inputTokens:
|
||||
(textResponse.usage?.inputTokens ?? 0) +
|
||||
|
||||
Reference in New Issue
Block a user