Add neuralchat server unit test cases (#276)

Author: lvliang-intel

.github/workflows/script/unitTest/run_unit_test_neuralchat.sh

@@ -28,6 +28,24 @@ function pytest() {
 
  itrex_path=$(python -c 'import intel_extension_for_transformers; import os; print(os.path.dirname(intel_extension_for_transformers.__file__))')
  find . -name "test*.py" | sed 's,\.\/,coverage run --source='"${itrex_path}"' --append ,g' | sed 's/$/ --verbose/' >run.sh
+ echo -e '
+# Kill the neuralchat server processes
+ports="7000 8000 9000"
+# Loop through each port and find associated PIDs
+for port in $ports; do
+ # Use lsof to find the processes associated with the port
+ pids=$(lsof -ti :$port)
+
+ if [ -n "$pids" ]; then
+ echo "Processes running on port $port: $pids"
+ # Terminate the processes gracefully with SIGTERM
+ kill $pids
+ echo "Terminated processes on port $port."
+ else
+ echo "No processes found on port $port."
+ fi
+done
+' >> run.sh
  coverage erase
 
  # run UT
@@ -52,6 +70,9 @@ function pytest() {
 
 function main() {
  bash /intel-extension-for-transformers/.github/workflows/script/unitTest/env_setup.sh
+ apt-get update
+ apt-get install ffmpeg -y
+ apt-get install lsof
  wget http://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2.19_amd64.deb
  dpkg -i libssl1.1_1.1.1f-1ubuntu2.19_amd64.deb
  python -m pip install --upgrade --force-reinstall torch

.github/workflows/unit-test-neuralchat.yml

@@ -24,27 +24,27 @@ env:
 
 jobs:
  unit-test:
- runs-on: [self-hosted, Linux, X64, itrex-node]
+ runs-on: [self-hosted, Linux, X64, neuralchat-node]
  strategy:
  matrix:
  include:
  - test_branch: ${{ github.ref }}
  test_name: "PR-test"
- - test_branch: "main"
- test_name: "baseline"
+ # - test_branch: "main"
+ #  test_name: "baseline"
  steps:
- - name: Docker Clean Up
+ - name: podman Clean Up
  run: |
- docker ps -a
- if [[ $(docker ps -a | grep -i '${{ env.CONTAINER_NAME }}'$) ]]; then
- docker start ${{ env.CONTAINER_NAME }}
+ podman ps -a
+ if [[ $(podman ps -a | grep -i '${{ env.CONTAINER_NAME }}'$) ]]; then
+ podman start ${{ env.CONTAINER_NAME }}
  echo "remove left files through container ..."
- docker exec ${{ env.CONTAINER_NAME }} bash -c "ls -a /intel-extension-for-transformers && rm -fr /intel-extension-for-transformers/* && rm -fr /intel-extension-for-transformers/.* || true"
+ podman exec ${{ env.CONTAINER_NAME }} bash -c "ls -a /intel-extension-for-transformers && rm -fr /intel-extension-for-transformers/* && rm -fr /intel-extension-for-transformers/.* || true"
  fi
- if [[ $(docker ps -a | grep -i '${{ env.EXTRA_CONTAINER_NAME }}'$) ]]; then
- docker start ${{ env.EXTRA_CONTAINER_NAME }}
+ if [[ $(podman ps -a | grep -i '${{ env.EXTRA_CONTAINER_NAME }}'$) ]]; then
+ podman start ${{ env.EXTRA_CONTAINER_NAME }}
  echo "remove left files through container ..."
- docker exec ${{ env.EXTRA_CONTAINER_NAME }} bash -c "ls -a /intel-extension-for-transformers && rm -fr /intel-extension-for-transformers/* && rm -fr /intel-extension-for-transformers/.* || true"
+ podman exec ${{ env.EXTRA_CONTAINER_NAME }} bash -c "ls -a /intel-extension-for-transformers && rm -fr /intel-extension-for-transformers/* && rm -fr /intel-extension-for-transformers/.* || true"
  fi
 
  - name: Checkout out Repo
@@ -54,28 +54,28 @@ jobs:
  ref: ${{ matrix.test_branch }}
  fetch-tags: true
 
- - name: Docker Build
+ - name: podman Build
  run: |
- docker build -f ${{ github.workspace }}/.github/workflows/docker/${{ env.DOCKER_FILE_NAME }}.dockerfile -t ${{ env.REPO_NAME }}:${{ env.REPO_TAG }} .
+ podman build -f ${{ github.workspace }}/.github/workflows/docker/${{ env.DOCKER_FILE_NAME }}.dockerfile -t ${{ env.REPO_NAME }}:${{ env.REPO_TAG }} .
 
- - name: Docker Run
+ - name: podman Run
  run: |
- if [[ $(docker ps -a | grep -i '${{ env.CONTAINER_NAME }}'$) ]]; then
- docker stop ${{ env.CONTAINER_NAME }}
- docker rm -vf ${{ env.CONTAINER_NAME }} || true
+ if [[ $(podman ps -a | grep -i '${{ env.CONTAINER_NAME }}'$) ]]; then
+ podman stop ${{ env.CONTAINER_NAME }}
+ podman rm -vf ${{ env.CONTAINER_NAME }} || true
  fi
- docker run -dit --disable-content-trust --privileged --name=${{ env.CONTAINER_NAME }} -v /dev/shm:/dev/shm \
+ podman run -dit --disable-content-trust --privileged --name=${{ env.CONTAINER_NAME }} -v /dev/shm:/dev/shm \
  -v ${{ github.workspace }}:/intel-extension-for-transformers \
  ${{ env.REPO_NAME }}:${{ env.REPO_TAG }}
 
  - name: Env build
  run: |
- docker exec ${{ env.CONTAINER_NAME }} \
+ podman exec ${{ env.CONTAINER_NAME }} \
  bash /intel-extension-for-transformers/.github/workflows/script/prepare_env.sh
 
  - name: Binary build
  run: |
- docker exec ${{ env.CONTAINER_NAME }} \
+ podman exec ${{ env.CONTAINER_NAME }} \
  bash -c "cd /intel-extension-for-transformers/.github/workflows/script \
  && bash install_binary.sh \
  && pip install intel_extension_for_pytorch wget sentencepiece \
@@ -91,14 +91,14 @@ jobs:
 
  - name: Run UT
  run: |
- docker exec ${{ env.CONTAINER_NAME }} \
+ podman exec ${{ env.CONTAINER_NAME }} \
  bash -c "cd /intel-extension-for-transformers/.github/workflows/script/unitTest \
  && bash run_unit_test_neuralchat.sh --test_name=${{ matrix.test_name }}"
 
  - name: Collect log
  if: ${{ !cancelled() }}
  run: |
- docker exec ${{ env.CONTAINER_NAME }} \
+ podman exec ${{ env.CONTAINER_NAME }} \
  bash -c "cd /intel-extension-for-transformers && \
  mv /log_dir . "
  

intel_extension_for_transformers/neural_chat/assets/speaker_embeddings/spk_embed_pat.pt renamed to intel_extension_for_transformers/neural_chat/assets/speaker_embeddings/spk_embed_male.pt

@@ -21,6 +21,7 @@
 from transformers import TrainingArguments, BitsAndBytesConfig
 from transformers.utils.versions import require_version
 from dataclasses import dataclass
+from .utils.common import get_device_type
 
 from .plugins import plugins
 
@@ -429,9 +430,15 @@ def __init__(self,
  self.model_name_or_path = model_name_or_path
  self.tokenizer_name_or_path = tokenizer_name_or_path
  self.hf_access_token = hf_access_token
- self.device = device
+ if device == "auto":
+ self.device = get_device_type()
+ else:
+ self.device = device
+
  self.plugins = plugins
- self.loading_config = loading_config if loading_config is not None else LoadingModelConfig()
+ self.loading_config = loading_config if loading_config is not None else \
+ LoadingModelConfig(cpu_jit=True if self.device == "cpu" else False, \
+ use_hpu_graphs = True if self.device == "hpu" else False)
  self.optimization_config = optimization_config if optimization_config is not None else AMPConfig()
  assert type(self.optimization_config) in [AMPConfig, WeightOnlyQuantizationConfig, BitsAndBytesConfig], \
  f"Expect optimization_config be an object of AMPConfig, WeightOnlyQuantizationConfig" + \

intel_extension_for_transformers/neural_chat/config.py

@@ -33,7 +33,7 @@ class TextToSpeech():
  """Convert text to speech with a driven speaker embedding
 
  1) Default voice (Original model + Proved good default speaker embedding from trained dataset)
- 2) Finetuned voice (Fine-tuned offline model of specific person, such as Pat's voice + corresponding embedding)
+ 2) Finetuned voice (Fine-tuned offline model of specific person's voice + corresponding embedding)
  3) Customized voice (Original model + User's customized input voice embedding)
  """
  def __init__(self, output_audio_path="./response.wav", voice="default", stream_mode=False, device="cpu", 
@@ -66,16 +66,16 @@ def __init__(self, output_audio_path="./response.wav", voice="default", stream_m
  self.default_speaker_embedding = torch.load(default_speaker_embedding_path)
 
  # preload the demo model in case of time-consuming runtime loading
- self.pat_model = None
- if os.path.exists("pat.pt"):
- self.pat_model = torch.load("pat.pt", map_location=device)
+ self.demo_model = None
+ if os.path.exists("demo_model.pt"):
+ self.demo_model = torch.load("demo_model.pt", map_location=device)
 
- self.pat_speaker_embeddings = None
- pat_speaker_embedding_path = os.path.join(script_dir, '../../../assets/speaker_embeddings/spk_embed_pat.pt')
+ self.male_speaker_embeddings = None
+ pat_speaker_embedding_path = os.path.join(script_dir, '../../../assets/speaker_embeddings/spk_embed_male.pt')
  if os.path.exists(pat_speaker_embedding_path):
- self.pat_speaker_embeddings = torch.load(pat_speaker_embedding_path)
- elif os.path.exists(os.path.join(asset_path, 'speaker_embeddings/spk_embed_pat.pt')):
- self.pat_speaker_embeddings = torch.load(os.path.join(asset_path, 'speaker_embeddings/spk_embed_pat.pt'))
+ self.male_speaker_embeddings = torch.load(pat_speaker_embedding_path)
+ elif os.path.exists(os.path.join(asset_path, 'speaker_embeddings/spk_embed_male.pt')):
+ self.male_speaker_embeddings = torch.load(os.path.join(asset_path, 'speaker_embeddings/spk_embed_male.pt'))
 
  self.cpu_pool = None
  if not torch.cuda.is_available():
@@ -148,7 +148,7 @@ def text2speech(self, text, output_audio_path, voice="default", do_batch_tts=Fal
  """Text to speech.
 
  text: the input text
- voice: default/pat/huma/tom/eric...
+ voice: default/male/female/...
  batch_length: the batch length for spliting long texts into batches to do text to speech
  """
  print(text)
@@ -164,15 +164,15 @@ def text2speech(self, text, output_audio_path, voice="default", do_batch_tts=Fal
  print(texts)
  model = self.original_model
  speaker_embeddings = self.default_speaker_embedding
- if voice == "pat":
- if self.pat_model == None:
+ if voice == "male":
+ if self.demo_model == None:
  print("Finetuned model is not found! Use the default one")
  else:
- model = self.pat_model
- if self.pat_speaker_embeddings == None:
- print("Pat's speaker embedding is not found! Use the default one")
+ model = self.demo_model
+ if self.male_speaker_embeddings == None:
+ print("Male speaker embedding is not found! Use the default one")
  else:
- speaker_embeddings = self.pat_speaker_embeddings
+ speaker_embeddings = self.male_speaker_embeddings
  elif voice != "default":
  speaker_embeddings = torch.load(self._lookup_voice_embedding(voice))
  all_speech = np.array([])

intel_extension_for_transformers/neural_chat/pipeline/plugins/audio/tts.py

@@ -23,23 +23,56 @@
 host: 0.0.0.0
 port: 8000
 
-# task choices = ['textchat', 'voicechat', 'retrieval', 'text2image', 'finetune']
-tasks_list: ['textchat', 'finetune', 'retrieval'] # text chatbot with document retrieval
+model_name_or_path: "meta-llama/Llama-2-7b-chat-hf"
+device: "auto"
+
+asr:
+ enable: true
+ args:
+ # support cpu, hpu, xpu, cuda
+ device: "cpu"
+ # support openai/whisper series
+ model_name_or_path: "openai/whisper-small"
+ # only can be set to true when the device is set to "cpu"
+ bf16: false
+
+tts:
+ enable: true
+ args:
+ device: "cpu"
+ voice: "default"
+ stream_mode: false
+ output_audio_path: "./output_audio.wav"
 
-# plugins choices = ['audio', 'retrieval', 'caching', 'memory_controller', 'intent_detection', 'safety_checker']
-plugins_list: ['audio', 'retrieval', 'caching']
+asr_chinese:
+ enable: false
 
-audio:
- audio_input: true
- audio_output: true
- language: "english"
+tts_chinese:
+ enable: false
+ args:
+ device: "cpu"
+ spk_id: 0
+ stream_mode: false
+ output_audio_path: "./output_audio.wav"
 
 retrieval:
- retrieval_type: "dense"
- retrieval_document_path: "../../assets/docs/"
+ enable: true
+ args:
+ retrieval_type: "dense"
+ input_path: "../../assets/docs/"
+ embedding_model: "hkunlp/instructor-large"
+ persist_dir: "./output"
+ max_length: 512
+ process: true
 
-caching:
- cache_chat_config_file: "../../plugins/caching/cache_config.yaml"
- cache_embedding_model_dir: "hkunlp/instructor-large"
+cache:
+ enable: true
+ args:
+ config_dir: "../../pipeline/plugins/caching/cache_config.yaml"
+ embedding_model_dir: "hkunlp/instructor-large"
 
-model_name: "meta-llama/Llama-2-7b-chat-hf"
+safety_checker:
+ enable: true
+
+# task choices = ['textchat', 'voicechat', 'retrieval', 'text2image', 'finetune']
+tasks_list: ['textchat', 'retrieval']

intel_extension_for_transformers/neural_chat/server/config/neuralchat.yaml

@@ -17,7 +17,6 @@
 
 import argparse
 import sys
-import os
 from typing import List
 
 
@@ -36,7 +35,7 @@
 from .restful.api import setup_router
 from ..config import PipelineConfig
 from ..chatbot import build_chatbot
-
+from ..plugins import plugins
 
 __all__ = ['NeuralChatServerExecutor']
 
@@ -97,31 +96,23 @@ def init(self, config):
  Returns:
  bool:
  """
- plugin_list = list(plugin for plugin in config.plugins_list)
- params = {}
- # Model configuration
- if config.model_name:
- params["model_name_or_path"] = config.model_name
- # Audio plugin configuration
- if "audio" in plugin_list:
- params["audio_input"] = config.audio.audio_input
- params["audio_output"] = config.audio.audio_output
- # Retrieval plugin configuration
- if "retrieval" in plugin_list:
- params["retrieval_type"] = config.retrieval.retrieval_type
- script_dir = os.path.dirname(os.path.abspath(__file__))
- retrieval_document_path = os.path.join(script_dir, config.retrieval.retrieval_document_path)
- params["retrieval_document_path"] = retrieval_document_path
- # Caching plugin configuration
- if "caching" in plugin_list:
- params["cache_chat_config_file"] = config.caching.cache_chat_config_file
- script_dir = os.path.dirname(os.path.abspath(__file__))
- retrieval_document_path = os.path.join(script_dir, config.caching.cache_embedding_model_dir)
- params["cache_embedding_model_dir"] = retrieval_document_path
- # Other plugins configurations
- for plugin in ["memory_controller", "intent_detection", "safety_checker"]:
- if plugin in config.plugins_list:
- params[plugin] = True
+ device = config.get("device", "auto")
+ model_name_or_path = config.get("model_name_or_path", "meta-llama/Llama-2-7b-hf")
+
+ # Update plugins based on YAML configuration
+ for plugin_name, plugin_config in plugins.items():
+ yaml_config = config.get(plugin_name, {})
+ if yaml_config.get("enable"):
+ plugin_config["enable"] = True
+ plugin_config["args"] = yaml_config.get("args", {})
+
+ # Create a dictionary of parameters for PipelineConfig
+ params = {
+ "model_name_or_path": model_name_or_path,
+ "device": device,
+ "plugins": plugins
+ }
+
  pipeline_config = PipelineConfig(**params)
  self.chatbot = build_chatbot(pipeline_config)
 
@@ -150,4 +141,7 @@ def __call__(self,
  config = get_config(config_file)
  if self.init(config):
  logging.basicConfig(filename=log_file, level=logging.INFO)
- uvicorn.run(app, host=config.host, port=config.port)
+ try:
+ uvicorn.run(app, host=config.host, port=config.port)
+ except Exception as e:
+ print(f"Error starting uvicorn: {str(e)}")

intel_extension_for_transformers/neural_chat/server/neuralchat_server.py

@@ -21,7 +21,6 @@
 from typing import Optional
 from fastapi import APIRouter
 from ...cli.log import logger
-from ...config import GenerationConfig
 from ...server.restful.openai_protocol import ChatCompletionRequest, ChatCompletionResponse
 
 
@@ -75,8 +74,7 @@ async def handle_chat_completion_request(self, request: ChatCompletionRequest) -
 
  try:
  logger.info(f"Predicting chat completion using prompt '{request.prompt}'")
- config = GenerationConfig(max_new_tokens=64)
- response = chatbot.predict(query=request.prompt, config=config)
+ response = chatbot.predict(query=request.prompt)
  except Exception as e:
  raise Exception(e)
  else:

intel_extension_for_transformers/neural_chat/server/restful/textchat_api.py

@@ -35,3 +35,4 @@ markdown
 rouge_score
 openpyxl
 numpy==1.23.5
+tiktoken==0.4.0