Update build_qwen_chat_input

Author: xusenlin

api/core/default.py

@@ -253,7 +253,7 @@ def build_chat_inputs(
  )
  elif check_is_qwen(self.model):
  inputs = build_qwen_chat_input(
- self.tokenizer, messages, self.context_len, max_new_tokens, functions, tools,
+ self.tokenizer, messages, functions=functions, tools=tools,
  )
  elif check_is_xverse(self.model):
  inputs = build_xverse_chat_input(

api/core/vllm_engine.py

@@ -85,10 +85,8 @@ def apply_chat_template(
  return build_qwen_chat_input(
  self.tokenizer,
  messages,
- self.max_model_len,
- max_tokens,
- functions,
- tools,
+ functions=functions,
+ tools=tools,
  )
  else:
  return self.prompt_adapter.apply_chat_template(messages)

api/generation/qwen.py

@@ -1,9 +1,7 @@
 import json
-import re
 from copy import deepcopy
 from typing import List, Union, Optional, Dict, Any, Tuple
 
-from fastapi import HTTPException
 from loguru import logger
 from openai.types.chat import (
  ChatCompletionMessageParam,
@@ -12,7 +10,6 @@
 )
 from transformers import PreTrainedTokenizer
 
-from api.generation.utils import parse_messages
 from api.utils.protocol import Role
 
 TOOL_DESC = """{name_for_model}: Call this tool to interact with the {name_for_human} API. What is the {name_for_human} API useful for? {description_for_model} Parameters: {parameters}"""
@@ -40,8 +37,7 @@
 def build_qwen_chat_input(
  tokenizer: PreTrainedTokenizer,
  messages: List[ChatCompletionMessageParam],
- context_len: int = 8192,
- max_new_tokens: int = 256,
+ max_window_size: int = 6144,
  functions: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
  tools: Optional[List[Dict[str, Any]]] = None,
 ) -> List[int]:
@@ -54,71 +50,65 @@ def build_qwen_chat_input(
  Args:
  tokenizer: The tokenizer used to encode the input tokens.
  messages: The list of chat messages.
- context_len: The maximum length of the context.
- max_new_tokens: The maximum number of new tokens to add.
+ max_window_size: The maximum length of the context.
  functions: Optional dictionary or list of dictionaries representing the functions.
  tools: Optional list of dictionaries representing the tools.
 
  Returns:
  The list of input tokens.
  """
- query, history = process_qwen_messages(messages, functions, tools)
+ query, history, system = process_qwen_messages(messages, functions, tools)
  if query is _TEXT_COMPLETION_CMD:
- return build_last_message_input(tokenizer, history)
-
- messages = []
- for q, r in history:
- messages.extend(
- [
- ChatCompletionUserMessageParam(role="user", content=q),
- ChatCompletionAssistantMessageParam(role="assistant", content=r)
- ]
- )
- messages.append(ChatCompletionUserMessageParam(role="user", content=query))
-
- max_input_tokens = context_len - max_new_tokens
- system, rounds = parse_messages(messages)
- system = f"You are a helpful assistant.{system}"
+ return build_last_message_input(tokenizer, history, system)
 
  im_start_tokens, im_end_tokens = [tokenizer.im_start_id], [tokenizer.im_end_id]
  nl_tokens = tokenizer.encode("\n")
 
- def _tokenize_str(role, content):
- return tokenizer.encode(
- role, allowed_special=set()
- ) + nl_tokens + tokenizer.encode(content, allowed_special=set())
+ if hasattr(tokenizer, "IMAGE_ST"):
+ def _tokenize_str(role, content):
+ return tokenizer.encode(
+ role, allowed_special=set(tokenizer.IMAGE_ST)
+ ) + nl_tokens + tokenizer.encode(content, allowed_special=set(tokenizer.IMAGE_ST))
+ else:
+ def _tokenize_str(role, content):
+ return tokenizer.encode(
+ role, allowed_special=set()
+ ) + nl_tokens + tokenizer.encode(content, allowed_special=set())
 
  system_tokens_part = _tokenize_str("system", system)
  system_tokens = im_start_tokens + system_tokens_part + im_end_tokens
- max_history_tokens = max_input_tokens - len(system_tokens)
 
- history_tokens = []
- for r in rounds[::-1]:
- round_tokens = []
- for message in r:
- if round_tokens:
- round_tokens += nl_tokens
+ context_tokens = []
+ for turn_query, turn_response in reversed(history):
+ query_tokens_part = _tokenize_str("user", turn_query)
+ query_tokens = im_start_tokens + query_tokens_part + im_end_tokens
 
- if message["role"] == Role.USER:
- content_tokens = im_start_tokens + _tokenize_str("user", message["content"]) + im_end_tokens
- else:
- content_tokens = im_start_tokens + _tokenize_str("assistant", message["content"]) + im_end_tokens
+ response_tokens_part = _tokenize_str("assistant", turn_response)
+ response_tokens = im_start_tokens + response_tokens_part + im_end_tokens
 
-  round_tokens.extend(content_tokens)
+ next_context_tokens = nl_tokens + query_tokens + nl_tokens + response_tokens
 
- if len(history_tokens) == 0 or len(history_tokens) + len(round_tokens) <= max_history_tokens:
- if history_tokens:
- history_tokens = nl_tokens + history_tokens
+ current_context_size = (
+ len(system_tokens) + len(next_context_tokens) + len(context_tokens)
+ )
+ if current_context_size < max_window_size:
+ context_tokens = next_context_tokens + context_tokens
+ else:
+ break
 
- history_tokens = round_tokens + history_tokens # concat left
- if len(history_tokens) < max_history_tokens:
- continue
- break
+ context_tokens = system_tokens + context_tokens
+ context_tokens += (
+ nl_tokens
+ + im_start_tokens
+ + _tokenize_str("user", query)
+ + im_end_tokens
+ + nl_tokens
+ + im_start_tokens
+ + tokenizer.encode("assistant")
+ + nl_tokens
+ )
 
- input_tokens = system_tokens + nl_tokens + history_tokens
- if messages[-1]["role"] != Role.ASSISTANT:
- input_tokens += nl_tokens + im_start_tokens + tokenizer.encode("assistant") + nl_tokens
- return input_tokens[-max_input_tokens:] # truncate left
+ return context_tokens
 
 
 def check_is_qwen(model) -> bool:
@@ -138,7 +128,7 @@ def process_qwen_messages(
  messages: List[ChatCompletionMessageParam],
  functions: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
  tools: Optional[List[Dict[str, Any]]] = None,
-) -> Tuple[str, List[List[str]]]:
+) -> Tuple[str, List[List[str]], str]:
  """
  Process the Qwen messages and generate a query and history.
 
@@ -148,21 +138,16 @@ def process_qwen_messages(
  tools (Optional[List[Dict[str, Any]]]): The tools to be used.
 
  Returns:
- Tuple[str, List[List[str]]]: The generated query and history.
+ Tuple[str, List[List[str]], str]: The generated query and history and system.
  """
  if all(m["role"] != Role.USER for m in messages):
- raise HTTPException(
- status_code=400,
- detail=f"Invalid request: Expecting at least one user message.",
- )
+ raise ValueError(f"Invalid messages: Expecting at least one user message.")
 
  messages = deepcopy(messages)
- default_system = "You are a helpful assistant."
- system = ""
  if messages[0]["role"] == Role.SYSTEM:
  system = messages.pop(0)["content"].lstrip("\n").rstrip()
-  if system == default_system:
-  system = ""
+ else:
+ system = "You are a helpful assistant."
 
  if tools:
  functions = [t["function"] for t in tools]
@@ -191,55 +176,37 @@ def process_qwen_messages(
 
  tools_text = "\n\n".join(tools_text)
  tools_name_text = ", ".join(tools_name_text)
- system += "\n\n" + REACT_INSTRUCTION.format(
+ instruction = REACT_INSTRUCTION.format(
  tools_text=tools_text,
  tools_name_text=tools_name_text,
- )
- system = system.lstrip("\n").rstrip()
+ ).lstrip('\n').rstrip()
+ else:
+ instruction = ""
 
- dummy_thought = {
- "en": "\nThought: I now know the final answer.\nFinal answer: ",
- "zh": "\nThought: 我会作答了。\nFinal answer: ",
- }
-
- _messages = messages
+ messages_with_fncall = messages
  messages = []
- for m_idx, m in enumerate(_messages):
+ for m_idx, m in enumerate(messages_with_fncall):
  role, content = m["role"], m["content"]
  func_call, tool_calls = m.get("function_call", None), m.get("tool_calls", None)
- if content:
- content = content.lstrip("\n").rstrip()
+
+ content = content or ''
+ content = content.lstrip('\n').rstrip()
+
  if role in [Role.FUNCTION, Role.TOOL]:
  if (len(messages) == 0) or (messages[-1]["role"] != Role.ASSISTANT):
- raise HTTPException(
- status_code=400,
- detail=f"Invalid request: Expecting role assistant before role function.",
- )
+ raise ValueError(f"Invalid messages: Expecting role assistant before role function.")
+
  messages[-1]["content"] += f"\nObservation: {content}"
- if m_idx == len(_messages) - 1:
+ if m_idx == len(messages_with_fncall) - 1:
  messages[-1]["content"] += "\nThought:"
+
  elif role == Role.ASSISTANT:
  if len(messages) == 0:
- raise HTTPException(
- status_code=400,
- detail=f"Invalid request: Expecting role user before role assistant.",
- )
- last_msg = messages[-1]["content"]
- last_msg_has_zh = len(re.findall(r"[\u4e00-\u9fff]+", last_msg)) > 0
+ raise ValueError(f"Invalid messages: Expecting role user before role assistant.")
 
  if func_call is None and tool_calls is None:
  if functions or tool_calls:
- content = dummy_thought["zh" if last_msg_has_zh else "en"] + content
- else:
- if func_call:
- f_name, f_args = func_call.get("name"), func_call.get("arguments")
- else:
- f_name, f_args = tool_calls[0]["function"]["name"], tool_calls[0]["function"]["arguments"]
- if not content:
- if last_msg_has_zh:
- content = f"Thought: 我可以使用 {f_name} API。"
- else:
- content = f"Thought: I can use {f_name}."
+ content = f"Thought: I now know the final answer.\nFinal Answer: {content}"
 
  if messages[-1]["role"] == Role.USER:
  messages.append(
@@ -252,46 +219,39 @@ def process_qwen_messages(
  ChatCompletionUserMessageParam(role="user", content=content.lstrip("\n").rstrip())
  )
  else:
- raise HTTPException(
- status_code=400, detail=f"Invalid request: Incorrect role {role}."
- )
+ raise ValueError(f"Invalid messages: Incorrect role {role}.")
 
  query = _TEXT_COMPLETION_CMD
  if messages[-1]["role"] == Role.USER:
  query = messages[-1]["content"]
  messages = messages[:-1]
 
  if len(messages) % 2 != 0:
- raise HTTPException(status_code=400, detail="Invalid request")
+ raise ValueError("Invalid messages")
 
  history = [] # [(Q1, A1), (Q2, A2), ..., (Q_last_turn, A_last_turn)]
  for i in range(0, len(messages), 2):
  if messages[i]["role"] == Role.USER and messages[i + 1]["role"] == Role.ASSISTANT:
  usr_msg = messages[i]["content"].lstrip("\n").rstrip()
  bot_msg = messages[i + 1]["content"].lstrip("\n").rstrip()
- if system and (i == len(messages) - 2):
- usr_msg = f"{system}\n\nQuestion: {usr_msg}"
- system = ""
- for t in dummy_thought.values():
- t = t.lstrip("\n")
- if bot_msg.startswith(t) and ("\nAction: " in bot_msg):
- bot_msg = bot_msg[len(t):]
+ if instruction and (i == len(messages) - 2):
+ usr_msg = f"{instruction}\n\nQuestion: {usr_msg}"
+ instruction = ''
  history.append([usr_msg, bot_msg])
  else:
- raise HTTPException(
- status_code=400,
- detail="Invalid request: Expecting exactly one user (or function) role before every assistant role.",
- )
- if system:
+ raise ValueError("Invalid messages: Expecting exactly one user (or function) role before every assistant role.")
+
+ if instruction:
  assert query is not _TEXT_COMPLETION_CMD
- query = f"{system}\n\nQuestion: {query}"
- return query, history
+ query = f"{instruction}\n\nQuestion: {query}"
+
+ return query, history, system
 
 
-def build_last_message_input(tokenizer: PreTrainedTokenizer, history: list):
+def build_last_message_input(tokenizer: PreTrainedTokenizer, history: List[List[str]], system: str):
  im_start = "<|im_start|>"
  im_end = "<|im_end|>"
- prompt = f"{im_start}system\nYou are a helpful assistant.{im_end}"
+ prompt = f"{im_start}system\n{system}{im_end}"
  for i, (query, response) in enumerate(history):
  query = query.lstrip("\n").rstrip()
  response = response.lstrip("\n").rstrip()