feat: add function calling tools

- add custom intructions for each prompt that are silently passed to the agent. - add minor code fixes - add .env.template and change AZURE_AI_PROJECTS_CONNECTION_STRING env name to AI_FOUNDRY_PROJECT_CONNECTION_STRING as in .env

Author: manekinekko

.env.template

@@ -1,3 +1,3 @@
-AZURE_AI_PROJECTS_CONNECTION_STRING=
+AI_FOUNDRY_PROJECT_CONNECTION_STRING=
 AI_SEARCH_CONNECTION_STRING=
 AI_MODEL=gpt-4o

.gitignore

@@ -14,4 +14,6 @@ downloads/*.png
 # Ignore any Webpack generated files
 dist
 
-tsconfig.tsbuildinfo
+tsconfig.tsbuildinfo
+
+.DS_Store

package-lock.json

@@ -5,8 +5,16 @@ export const promptConfig: Record<string, PromptConfig> = {
  prompt: 'I need to solve the equation `3x + 11 = 14`. Can you help me?',
  emoji: '🧮'
  },
+ localCpusUsage: {
+ prompt: 'What is the average CPUs usage on my local machine?',
+ instructions: 'Always use the function tool to get the CPUs usage.',
+ emoji: '💾',
+ tool: 'function-tool',
+ executor: null, // set at runtime (see toolService.ts)
+ },
  codeGenerator: {
  prompt: 'Write a function that finds prime numbers',
+ instructions: 'Always use the code interpreter to generate code.',
  tool: 'code-interpreter',
  emoji: '💻'
  },
@@ -16,19 +24,22 @@ export const promptConfig: Record<string, PromptConfig> = {
  - Relationships between Price, Mileage, and Year. 
  - Sales by SalesPerson.
  - Sales by Make, Model, and Year for 2023.`,
+ instructions: 'Always use the code interpreter to generate visualizations.',
  tool: 'code-interpreter',
  filePath: './files/car_sales_data.csv',
  emoji: '📊'
  },
  hotelReviews: {
- prompt: 'Tell me about the hotel reviews in the HotelReviews_data.csv.',
+ prompt: 'Tell me about the hotel reviews in the hotel_reviews_data.csv.',
+ instructions: 'Always use the code interpreter to analyze the data.',
  tool: 'code-interpreter',
  fileId: '',
  filePath: './files/hotel_reviews_data.csv',
  emoji: '🏨'
  },
  insuranceCoverage: {
  prompt: 'What are my health insurance plan coverage types?',
+ instructions: 'Always use the AI Search tool to analyze the data.',
  tool: 'ai-search',
  emoji: '🏥'
  }

src/config/promptConfig.ts

@@ -18,7 +18,7 @@ export async function processSelectedPrompt(client: AIProjectsClient, selectedKe
 
  const agent = await client.agents.createAgent(model, {
  name: 'my-agent',
- instructions: 'You are a helpful agent',
+ instructions: `You are a helpful agent. ${selectedPromptConfig.instructions}`,
  temperature: 0.5,
  tools: selectedPromptConfig.tools,
  toolResources: selectedPromptConfig.toolResources
@@ -27,7 +27,7 @@ export async function processSelectedPrompt(client: AIProjectsClient, selectedKe
  const thread = await client.agents.createThread();
  await addMessageToThread(client, thread.id, selectedPromptConfig.prompt);
 
- const runId = await runAgent(client, thread, agent);
+ const runId = await runAgent(client, thread, agent, selectedPromptConfig);
  await printThreadMessages(selectedPromptConfig, client, thread.id);
  await getRunStats(runId, client, thread);
 

src/services/agentService.ts

@@ -1,6 +1,6 @@
 import fs from 'fs';
 import { AIProjectsClient, AgentThreadOutput, DoneEvent, ErrorEvent, MessageStreamEvent, RunStreamEvent, isOutputOfType } from '@azure/ai-projects';
-import type { AgentOutput, MessageDeltaChunk, MessageDeltaTextContent, MessageImageFileContentOutput, MessageTextContentOutput, OpenAIPageableListOfThreadMessageOutput, ThreadRunOutput } from '@azure/ai-projects';
+import type { AgentOutput, FunctionToolDefinitionOutput, MessageDeltaChunk, MessageDeltaTextContent, MessageImageFileContentOutput, MessageTextContentOutput, OpenAIPageableListOfThreadMessageOutput, SubmitToolOutputsActionOutput, ThreadRunOutput, ToolOutput } from '@azure/ai-projects';
 import { PromptConfig } from '../types.js';
 
 export async function addMessageToThread(client: AIProjectsClient, threadId: string, message: string) {
@@ -18,10 +18,18 @@ export async function printThreadMessages(selectedPromptConfig: PromptConfig, cl
  const messagesArray = messages.data;
  for (let i = messagesArray.length - 1; i >= 0; i--) {
  const m = messagesArray[i];
+ const content = m.content[0];
+
+ if (!content) {
+ // Skip if no content
+ continue;
+ }
+
  console.log(`Type: ${m.content[0].type}`);
  if (isOutputOfType<MessageTextContentOutput>(m.content[0], 'text')) {
  const textContent = m.content[0] as MessageTextContentOutput;
- console.log(`Text: ${textContent.text.value}`);
+ const role = m.role === 'user' ? 'User' : 'Agent';
+ console.log(`${role}: ${textContent.text.value}`);
  }
  }
 
@@ -77,7 +85,7 @@ export async function getRunStats(runId: string, client: AIProjectsClient, threa
  }
 }
 
-export async function runAgent(client: AIProjectsClient, thread: AgentThreadOutput, agent: AgentOutput): Promise<string> {
+export async function runAgent(client: AIProjectsClient, thread: AgentThreadOutput, agent: AgentOutput, promptConfig: PromptConfig): Promise<string> {
  const run = client.agents.createRun(thread.id, agent.id);
  const streamEventMessages = await run.stream();
  let runId = '';
@@ -104,8 +112,16 @@ export async function runAgent(client: AIProjectsClient, thread: AgentThreadOutp
  }
  break;
 
+ case RunStreamEvent.ThreadRunRequiresAction:
+ console.log("\nThread run requires action.");
+ const run = eventMessage.data as ThreadRunOutput;
+ if (run.requiredAction) {
+ await processRequiredAction(client, thread, run, promptConfig);
+ }
+ break;
+
  case RunStreamEvent.ThreadRunCompleted:
- console.log('\nThread run completed.');
+ console.log("\nThread run completed.");
  break;
 
  case ErrorEvent.Error:
@@ -119,4 +135,42 @@ export async function runAgent(client: AIProjectsClient, thread: AgentThreadOutp
  }
 
  return runId;
-}
+}
+
+async function processRequiredAction(
+ client: AIProjectsClient,
+ thread: AgentThreadOutput,
+ run: ThreadRunOutput,
+ promptConfig: PromptConfig
+) {
+ if (
+ run.requiredAction &&
+ isOutputOfType<SubmitToolOutputsActionOutput>(
+ run.requiredAction,
+ "submit_tool_outputs"
+ )
+ ) {
+ const submitToolOutputsActionOutput =
+ run.requiredAction as SubmitToolOutputsActionOutput;
+ const toolCalls = submitToolOutputsActionOutput.submitToolOutputs.toolCalls;
+ const toolResponses: ToolOutput[] = [];
+ for (const toolCall of toolCalls) {
+ if (isOutputOfType<FunctionToolDefinitionOutput>(toolCall, "function")) {
+ const toolResponse = promptConfig.executor?.invokeTool(toolCall) as ToolOutput;
+ console.log(
+ `💡 Function tool ${toolCall.id} - ${toolResponse.output}`
+ );
+ if (toolResponse) {
+ toolResponses.push(toolResponse);
+ }
+ }
+ }
+ if (toolResponses.length > 0) {
+ run = await client.agents.submitToolOutputsToRun(
+ thread.id,
+ run.id,
+ toolResponses
+ );
+ }
+ }
+}

src/services/threadService.ts

@@ -1,7 +1,15 @@
-import fs from 'fs';
-import { AIProjectsClient, ToolUtility } from '@azure/ai-projects';
-import { aiSearchConnectionString } from '../config/env.js';
-import { PromptConfig } from '../types.js';
+import fs from "fs";
+import { cpus } from "os";
+import {
+ AIProjectsClient,
+ FunctionToolDefinition,
+ FunctionToolDefinitionOutput,
+ RequiredToolCallOutput,
+ ToolOutput,
+ ToolUtility,
+} from "@azure/ai-projects";
+import { aiSearchConnectionString } from "../config/env.js";
+import { PromptConfig } from "../types.js";
 
 export async function createTools(selectedPromptConfig: PromptConfig, client: AIProjectsClient) {
  if (selectedPromptConfig.tool === 'code-interpreter') {
@@ -13,11 +21,18 @@ export async function createTools(selectedPromptConfig: PromptConfig, client: AI
  selectedPromptConfig.toolResources = codeInterpreterTool.resources;
  }
 
- if (selectedPromptConfig.aiSearch) {
- const azureAISearchTool = await createAISearchTool(client);
- selectedPromptConfig.tools = [azureAISearchTool.definition];
- selectedPromptConfig.toolResources = azureAISearchTool.resources;
- }
+ if (selectedPromptConfig.aiSearch) {
+ const azureAISearchTool = await createAISearchTool(client);
+ selectedPromptConfig.tools = [azureAISearchTool.definition];
+ selectedPromptConfig.toolResources = azureAISearchTool.resources;
+ }
+
+ if (selectedPromptConfig.tool === "function-tool") {
+ selectedPromptConfig.executor = FunctionToolExecutor;
+ selectedPromptConfig.tools = [
+ ...FunctionToolExecutor.getFunctionDefinitions(),
+ ];
+ }
 }
 
 export async function getCodeInterpreter(selectedPromptConfig: PromptConfig, client: AIProjectsClient) {
@@ -41,4 +56,70 @@ export async function createAISearchTool(client: AIProjectsClient) {
  aiSearchConnection.id,
  aiSearchConnection.name
  );
-}
+}
+
+class FunctionToolFactory {
+ static getCpuUsage() {
+ return `CPU Usage: ${cpus()[0].model} ${Math.floor( (cpus().reduce((acc, core) => acc + core.speed, 0)) / 1000)}%`;
+ }
+}
+
+export class FunctionToolExecutor {
+ static functionTools: {
+ func: Function;
+ definition: FunctionToolDefinition;
+ }[] = [
+ {
+ func: FunctionToolFactory.getCpuUsage,
+ ...ToolUtility.createFunctionTool({
+ name: "getCpuUsage",
+ description: "Gets the current CPU usage of the system.",
+ parameters: {},
+ }),
+ },
+ ];
+
+ public static invokeTool(
+ toolCall: RequiredToolCallOutput & FunctionToolDefinitionOutput
+ ): ToolOutput | undefined {
+ console.log(`💡 Function tool ${toolCall.id} - ${toolCall.function.name}`);
+ const args: string[] = [];
+ if (toolCall.function.parameters) {
+ try {
+ const params = JSON.parse(toolCall.function.parameters) as Record<
+ string,
+ string
+ >;
+ for (const key in params) {
+ if (Object.prototype.hasOwnProperty.call(params, key)) {
+ args.push(params[key] as string);
+ }
+ }
+ } catch (error) {
+ console.error(
+ `Failed to parse parameters: ${toolCall.function.parameters}`,
+ error
+ );
+ return undefined;
+ }
+ }
+ const result = this.functionTools
+ .find((tool) => tool.definition.function.name === toolCall.function.name)
+ ?.func(...args);
+
+ console.log(`💡 Function tool ${toolCall.id} - completed`);
+
+ return result
+ ? {
+ toolCallId: toolCall.id,
+ output: JSON.stringify(result),
+ }
+ : undefined;
+ }
+
+ public static getFunctionDefinitions(): FunctionToolDefinition[] {
+ return FunctionToolExecutor.functionTools.map((tool) => {
+ return tool.definition;
+ });
+ }
+}

src/services/toolService.ts

@@ -2,11 +2,13 @@ import type { ToolDefinition } from '@azure/ai-projects';
 
 export interface PromptConfig {
  prompt: string;
+ instructions?: string;
  emoji?: string;
- tool?: string;
+ tool?: "code-interpreter" | "function-tool" | "ai-search";
  filePath?: string;
  fileId?: string;
  aiSearch?: boolean;
+ executor?: any;
  tools?: ToolDefinition[];
  toolResources?: any;
 }