Fix IntelON notebook demo issues

intel_extension_for_transformers/llm/inference/inference.py

@@ -696,6 +696,10 @@ def generate_output():
  input_tokens = prepare_inputs(
  input_tokens, model.device if hasattr(model, 'device') else torch.device(device)
  )
+ else:
+ input_tokens = tokenizer.batch_encode_plus(
+ [prompt], return_tensors="pt", padding=True
+ )
  with context:
  generation_kwargs = dict(
  streamer=streamer,

intel_extension_for_transformers/neural_chat/chatbot.py

@@ -21,7 +21,7 @@
 from .config import PipelineConfig
 from .config import BaseFinetuningConfig
 from .config import DeviceOptions
-from .plugins import plugins, global_plugins
+from .plugins import plugins
 
 def build_chatbot(config: PipelineConfig=None):
  """Build the chatbot with a given configuration.
@@ -101,9 +101,6 @@ def build_chatbot(config: PipelineConfig=None):
  plugins[plugin_name]["instance"] = plugins[plugin_name]['class'](**plugin_value['args'])
  adapter.register_plugin_instance(plugin_name, plugins[plugin_name]["instance"])
 
- global_plugins.reset_plugins()
- plugins = global_plugins.plugins
-
  parameters = {}
  parameters["model_name"] = config.model_name_or_path
  if config.tokenizer_name_or_path:

intel_extension_for_transformers/neural_chat/docker/Dockerfile

@@ -70,7 +70,7 @@ RUN source activate && conda activate chatbot-finetuning && pip install oneccl_b
  git clone https://github.com/huggingface/peft.git && cd peft && python setup.py install && \
  cd /intel-extension-for-transformers && pip install -v . && \
  cd ./intel_extension_for_transformers/neural_chat/examples/instruction_tuning && pip install -r requirements.txt && \
- cd /intel-extension-for-transformers/intel-extension-for-transformers/intel_extension_for_transformers/neural_chat && pip install -r requirements.txt
+ cd /intel-extension-for-transformers/intel-extension-for-transformers/intel_extension_for_transformers/neural_chat && pip install -r requirements_cpu.txt
 
 # Enable passwordless ssh for mpirun^M
 RUN mkdir /var/run/sshd
@@ -120,7 +120,7 @@ RUN git clone --single-branch --branch=${ITREX_VER} ${REPO} intel-extension-for-
  cd /intel-extension-for-transformers/intel_extension_for_transformers/neural_chat/examples/instruction_tuning/ && \
  pip install -r requirements.txt && \
  cd /intel-extension-for-transformers/intel_extension_for_transformers/neural_chat/ && \
- pip install -r requirements.txt
+ pip install -r requirements_hpu.txt
 
 # Build ITREX
 RUN cd /intel-extension-for-transformers && pip install -v . && \

intel_extension_for_transformers/neural_chat/docker/inference/Dockerfile

@@ -72,7 +72,7 @@ RUN source activate && conda activate chatbot-demo && \
  pip install schema && \
  pip install datasets torch transformers sentencepiece peft evaluate nltk rouge_score && \
  cd /root/chatbot && git clone https://github.com/intel/intel-extension-for-transformers.git \
- && cd ./intel-extension-for-transformers/intel_extension_for_transformers/neural_chat/ && pip install -r requirements.txt
+ && cd ./intel-extension-for-transformers/intel_extension_for_transformers/neural_chat/ && pip install -r requirements_cpu.txt
 
 ENV KMP_BLOCKTIME=1
 ENV KMP_SETTINGS=1
@@ -111,7 +111,7 @@ RUN git clone https://github.com/huggingface/optimum-habana.git && \
 
 RUN git clone https://github.com/intel/intel-extension-for-transformers.git \
  && cd ./intel-extension-for-transformers/intel_extension_for_transformers/neural_chat/ \
- && pip install -r requirements.txt \
+ && pip install -r requirements_hpu.txt \
  && pip install transformers==4.32.0 \
  && pip install accelerate==0.22.0 \
  && pip uninstall -y intel_extension_for_pytorch

intel_extension_for_transformers/neural_chat/models/base_model.py

@@ -19,9 +19,9 @@
 from typing import List
 import os
 from fastchat.conversation import get_conv_template, Conversation
-from intel_extension_for_transformers.llm.inference import load_model, predict, predict_stream, MODELS
+from intel_extension_for_transformers.llm.inference import load_model, predict, predict_stream
 from ..config import GenerationConfig
-from ..plugins import is_plugin_enabled, get_plugin_instance, get_registered_plugins, get_plugin_arguments
+from ..plugins import is_plugin_enabled, get_plugin_instance, get_registered_plugins, plugins
 from ..utils.common import is_audio_file
 
 
@@ -204,6 +204,15 @@ def predict(self, query, config=None):
  if hasattr(plugin_instance, 'post_llm_inference_actions'):
  response = plugin_instance.post_llm_inference_actions(response)
 
+ # clear plugins config
+ for key in plugins:
+ plugins[key] = {
+ "enable": False,
+ "class": None,
+ "args": {},
+ "instance": None
+ }
+
  return response
 
  def chat_stream(self, query, config=None):

intel_extension_for_transformers/neural_chat/requirements.txt

@@ -35,3 +35,4 @@ rouge_score
 openpyxl
 numpy==1.23.5
 tiktoken==0.4.0
+lm_eval

intel_extension_for_transformers/neural_chat/requirements_cpu.txt

@@ -0,0 +1,38 @@
+transformers>=4.32.0
+peft
+fschat
+torch
+torchaudio
+intel_extension_for_pytorch
+num2words
+speechbrain
+paddlepaddle
+paddlespeech==1.4.1
+shortuuid
+gptcache
+evaluate
+pydub
+python-multipart
+PyPDF2
+langchain
+python-docx
+scikit-learn
+farm-haystack
+librosa
+beautifulsoup4
+InstructorEmbedding
+chromadb
+fastapi
+pydantic
+starlette
+yacs
+uvicorn
+optimum
+sentence_transformers
+unstructured
+markdown
+rouge_score
+openpyxl
+numpy==1.23.5
+tiktoken==0.4.0
+lm_eval

intel_extension_for_transformers/neural_chat/requirements_hpu.txt

@@ -0,0 +1,35 @@
+transformers>=4.32.0
+peft
+fschat
+num2words
+speechbrain
+paddlepaddle
+paddlespeech==1.4.1
+shortuuid
+gptcache
+evaluate
+pydub
+python-multipart
+PyPDF2
+langchain
+python-docx
+scikit-learn
+farm-haystack
+librosa
+beautifulsoup4
+InstructorEmbedding
+chromadb
+fastapi
+pydantic
+starlette
+yacs
+uvicorn
+optimum
+sentence_transformers
+unstructured
+markdown
+rouge_score
+openpyxl
+numpy==1.23.5
+tiktoken==0.4.0
+lm_eval

intel_extension_for_transformers/neural_chat/tests/api/test_chatbot_build_api.py

@@ -72,7 +72,7 @@ def test_build_chatbot_with_audio_plugin(self):
 
  self.assertIsNotNone(response)
  print("output audio path: ", response)
- self.assertTrue(os.path.exists(plugins.tts.args["output_audio_path"]))
+ self.assertTrue(os.path.exists("./output_audio.wav"))
 
  def test_build_chatbot_with_safety_checker_plugin(self):
  plugins.safety_checker.enable = True

intel_extension_for_transformers/neural_chat/tests/api/test_inference.py

@@ -64,5 +64,14 @@ def test_quantization(self):
  config = AMPConfig()
  optimize_model(model="facebook/opt-125m", config=config)
 
+ def test_text_chat_stream(self):
+ config = PipelineConfig(model_name_or_path="facebook/opt-125m")
+ chatbot = build_chatbot(config)
+ stream_text = ""
+ for text in chatbot.predict_stream("Tell me about Intel Xeon Scalable Processors."):
+ stream_text += text
+ print(text)
+ self.assertIsNotNone(stream_text)
+
 if __name__ == '__main__':
  unittest.main()

intel_extension_for_transformers/neural_chat/tests/requirements.txt

@@ -36,3 +36,4 @@ rouge_score
 openpyxl
 numpy==1.23.5
 tiktoken==0.4.0
+lm_eval