feat: 🎉 support streamChatCompletion

Author: qddegtya

examples/serve-astack/backend/src/routes/chat.ts

@@ -1,6 +1,6 @@
 import type { FastifyInstance } from 'fastify';
 import { classifyIntent, getStreamingAgentByIntent } from '../agents/index.js';
-import { createLLMClient, chatWithLLM } from '../services/llm.js';
+import { createLLMClient, chatWithLLMStreaming } from '../services/llm.js';
 
 // AI SDK 5.0 compatible types
 interface UIMessagePart {
@@ -58,34 +58,34 @@ export default async function chatRoutes(fastify: FastifyInstance) {
  .join(' '),
  }));
 
- const response = await chatWithLLM(llmClient, llmMessages);
-
  // Set proper headers for AI SDK Data Stream Protocol
  reply.type('text/plain; charset=utf-8');
  reply.header('X-Vercel-AI-Data-Stream', 'v1');
 
- // Send text chunks using AI SDK Data Stream Protocol format
- const textContent = response.content;
- const chunks = textContent.split('');
+ let completionTokens = 0;
 
- // Simulate streaming by sending character by character
- for (let i = 0; i < chunks.length; i++) {
- // Text Part: 0:string\n
- const chunk = `0:${JSON.stringify(chunks[i])}\n`;
- reply.raw.write(chunk);
+ try {
+ // Real streaming from LLM
+ for await (const chunk of chatWithLLMStreaming(llmClient, llmMessages)) {
+ // Send each chunk as it arrives from LLM
+ const textPart = `0:${JSON.stringify(chunk)}\n`;
+ reply.raw.write(textPart);
+ completionTokens += chunk.length;
+ }
 
- // Small delay to simulate streaming
- await new Promise(resolve => setTimeout(resolve, 50));
+ // Send completion using Finish Message Part
+ const finishPart = `d:${JSON.stringify({
+ finishReason: 'stop',
+ usage: { promptTokens: 0, completionTokens },
+ })}\n`;
+ reply.raw.write(finishPart);
+ reply.raw.end();
+ } catch (error) {
+ fastify.log.error(error, 'Error in LLM streaming');
+ const errorPart = `3:${JSON.stringify(error instanceof Error ? error.message : 'LLM streaming error')}\n`;
+ reply.raw.write(errorPart);
+ reply.raw.end();
  }
-
- // Send completion using Finish Message Part
- // d:{finishReason:'stop' | 'length' | 'content-filter' | 'tool-calls' | 'error' | 'other' | 'unknown';usage:{promptTokens:number; completionTokens:number;}}
- const finishPart = `d:${JSON.stringify({
- finishReason: 'stop',
- usage: { promptTokens: 0, completionTokens: chunks.length },
- })}\n`;
- reply.raw.write(finishPart);
- reply.raw.end();
  } else {
  // Handle agent-based processing with AI SDK Data Stream Protocol
  const streamingAgent = getStreamingAgentByIntent(intent);

examples/serve-astack/backend/src/services/llm.ts

@@ -50,3 +50,27 @@ export async function chatWithLLM(
  content: response.content,
  };
 }
+
+export async function* chatWithLLMStreaming(
+ client: ModelProvider,
+ messages: LLMMessage[]
+): AsyncGenerator<string> {
+ // 检查是否支持流式调用
+ if (!client.streamChatCompletion) {
+ throw new Error('ModelProvider does not support streaming');
+ }
+
+ // 使用流式调用
+ const response = await client.streamChatCompletion(
+ messages.map(msg => ({
+ role: msg.role,
+ content: msg.content,
+ }))
+ );
+
+ for await (const chunk of response) {
+ if (chunk.content) {
+ yield chunk.content;
+ }
+ }
+}

examples/serve-astack/frontend/package.json

@@ -10,25 +10,27 @@
  "lint": "eslint"
  },
  "dependencies": {
- "react": "19.1.0",
- "react-dom": "19.1.0",
- "next": "15.5.4",
  "@ai-sdk/react": "^1.0.18",
- "@radix-ui/react-slot": "^1.0.2",
  "@radix-ui/react-scroll-area": "^1.0.5",
+ "@radix-ui/react-slot": "^1.0.2",
  "class-variance-authority": "^0.7.0",
  "clsx": "^2.1.0",
- "lucide-react": "^0.263.1"
+ "lucide-react": "^0.263.1",
+ "next": "15.5.4",
+ "react": "19.1.0",
+ "react-dom": "19.1.0",
+ "shiki": "^3.13.0",
+ "streamdown": "^1.3.0"
  },
  "devDependencies": {
- "typescript": "^5",
+ "@eslint/eslintrc": "^3",
+ "@tailwindcss/postcss": "^4",
  "@types/node": "^20",
  "@types/react": "^19",
  "@types/react-dom": "^19",
- "@tailwindcss/postcss": "^4",
- "tailwindcss": "^4",
  "eslint": "^9",
  "eslint-config-next": "15.5.4",
- "@eslint/eslintrc": "^3"
+ "tailwindcss": "^4",
+ "typescript": "^5"
  }
 }

examples/serve-astack/frontend/src/app/globals.css

@@ -1,4 +1,5 @@
 @import "tailwindcss";
+@source "../node_modules/streamdown/dist/index.js";
 
 :root {
  --background: #ffffff;

examples/serve-astack/frontend/src/app/page.tsx

@@ -3,6 +3,7 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
 import { useChat } from '@ai-sdk/react';
+import { Streamdown } from 'streamdown';
 import {
  Send,
  Bot,
@@ -327,11 +328,8 @@ export default function ChatPage() {
  {message.parts.map((part, partIndex) => {
  if (part.type === 'text') {
  return (
- <div
- key={partIndex}
- className="leading-relaxed whitespace-pre-wrap"
- >
- {part.text}
+ <div key={partIndex} className="leading-relaxed">
+ <Streamdown>{part.text}</Streamdown>
  </div>
  );
  }

packages/components/src/agents/StreamingAgent.ts

@@ -215,9 +215,9 @@ export class StreamingAgent extends Component {
  };
 
  // 调用模型获取回复（与原 Agent 完全一致）
- const modelResponse = await model.chatCompletion(currentMessages, {
+ const modelResponse = (await model.chatCompletion(currentMessages, {
  temporaryTools: tools,
- });
+ })) as MessageWithToolCalls;
 
  if (verbose) {
  console.log(

packages/components/src/agents/index.ts

@@ -170,6 +170,17 @@ export interface ModelProvider {
  messages: Message[],
  options?: ModelProviderOptions
  ): Promise<MessageWithToolCalls>;
+
+ /**
+ * 流式调用模型生成回复
+ * @param messages 输入消息列表
+ * @param options 可选的模型调用选项
+ * @returns 流式回复消息的异步生成器
+ */
+ streamChatCompletion?(
+ messages: Message[],
+ options?: ModelProviderOptions
+ ): AsyncGenerator<Partial<MessageWithToolCalls>>;
 }
 
 /**

packages/integrations/src/model-provider/deepseek/index.ts

@@ -224,6 +224,124 @@ class Deepseek extends Component {
  return response.choices[0].message.content || '';
  }
 
+ /**
+ * 流式处理对话消息
+ * @param messages 对话消息数组
+ * @param options 可选的调用选项，包含临时工具列表
+ * @returns 生成的流式响应消息异步生成器
+ */
+ async *streamChatCompletion(
+ messages: Message[],
+ options?: { temporaryTools?: unknown[] }
+ ): AsyncGenerator<Partial<Message>> {
+ // 转换消息格式 (复用现有逻辑)
+ const formattedMessages: Array<OpenAI.Chat.ChatCompletionMessageParam> = [];
+
+ // 添加系统提示
+ if (this.systemPrompt) {
+ formattedMessages.push({
+ role: 'system',
+ content: this.systemPrompt,
+ });
+ }
+
+ // 添加用户提供的消息
+ formattedMessages.push(
+ ...messages.map(msg => {
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ const formattedMsg: any = {
+ role: msg.role,
+ content: msg.content,
+ };
+
+ if (msg.role === 'tool' && msg.tool_call_id) {
+ formattedMsg.tool_call_id = msg.tool_call_id;
+ }
+
+ return formattedMsg;
+ })
+ );
+
+ // 创建请求参数
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ const requestParams: any = {
+ model: this.model,
+ messages: formattedMessages,
+ temperature: this.temperature,
+ max_tokens: this.maxTokens,
+ top_p: this.topP,
+ stream: true, // 开启流式模式
+ };
+
+ // 决定使用哪些工具
+ const toolsToUse = options?.temporaryTools || this.tools;
+
+ // 如果有工具定义，添加到请求中
+ if (toolsToUse && toolsToUse.length > 0) {
+ const formattedTools = toolsToUse.map(tool => {
+ type ToolType = {
+ type?: string;
+ name?: string;
+ description?: string;
+ parameters?: Record<string, unknown>;
+ };
+
+ const typedTool = tool as ToolType;
+
+ if (typedTool.type === 'function') {
+ return tool;
+ }
+
+ return {
+ type: 'function',
+ function: {
+ name: typedTool.name,
+ description: typedTool.description,
+ parameters: typedTool.parameters || {},
+ },
+ };
+ });
+
+ requestParams.tools = formattedTools;
+ }
+
+ // 调用流式 API
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ const stream = (await this.client.chat.completions.create(requestParams)) as any;
+
+ // 处理流式响应
+ for await (const chunk of stream) {
+ const delta = chunk.choices[0]?.delta;
+
+ if (delta?.content) {
+ // 流式文本内容
+ yield {
+ role: 'assistant',
+ content: delta.content,
+ };
+ }
+
+ if (delta?.tool_calls) {
+ // 流式工具调用（如果需要）
+ yield {
+ role: 'assistant',
+ content: '',
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ tool_calls: delta.tool_calls.map((toolCall: any) => ({
+ id: toolCall.id || '',
+ function: {
+ name: toolCall.function?.name || '',
+ arguments: toolCall.function?.arguments || '',
+ },
+ type: 'function',
+ tool_name: toolCall.function?.name || '',
+ arguments: JSON.parse(toolCall.function?.arguments || '{}'),
+ })),
+ };
+ }
+ }
+ }
+
  /**
  * 处理对话消息
  * @param messages 对话消息数组