Update NeuralChat plugins API (#156)

* Update NeuralChat plugins API Signed-off-by: lvliang-intel <liang1.lv@intel.com> * update safety checker and intent detection plugin Signed-off-by: lvliang-intel <liang1.lv@intel.com> * add finetuning lm_eval metric (#147) * add lm_eval metric. * support summarization task evaluation. * update doc. Signed-off-by: Lv, Kaokao <kaokao.lv@intel.com> * update doc. Signed-off-by: Lv, Kaokao <kaokao.lv@intel.com> * Update README.md * Update README.md * add summarization function. * add evaluation desc. Signed-off-by: Lv, Kaokao <kaokao.lv@intel.com> --------- Signed-off-by: Lv, Kaokao <kaokao.lv@intel.com> Co-authored-by: Haihao Shen <haihao.shen@intel.com> * remove hard print (#158) Signed-off-by: Lv, Kaokao <kaokao.lv@intel.com> * Support PEFT model (#153) Signed-off-by: Mengni Wang <mengni.wang@intel.com> Co-authored-by: Haihao Shen <haihao.shen@intel.com> * update code for comments Signed-off-by: lvliang-intel <liang1.lv@intel.com> --------- Signed-off-by: lvliang-intel <liang1.lv@intel.com> Signed-off-by: Lv, Kaokao <kaokao.lv@intel.com> Signed-off-by: Mengni Wang <mengni.wang@intel.com> Co-authored-by: Haihao Shen <haihao.shen@intel.com> Co-authored-by: lkk <33276950+lkk12014402@users.noreply.github.com> Co-authored-by: Wang, Mengni <mengni.wang@intel.com>

Author: 4 people

neural_chat/__init__.py

@@ -24,4 +24,5 @@
 from .chatbot import optimize_model
 from .server.neuralchat_server import NeuralChatServerExecutor
 from .server.neuralchat_client import TextChatClientExecutor, VoiceChatClientExecutor, FinetuingClientExecutor
+from .plugins import plugins
 

neural_chat/chatbot.py

@@ -20,20 +20,22 @@
 from .config import PipelineConfig
 from .config import OptimizationConfig
 from .config import FinetuningConfig
+from .plugins import is_plugin_enabled, get_plugin_instance, get_registered_plugins
 from .pipeline.finetuning.finetuning import Finetuning
 from .pipeline.optimization.optimization import Optimization
-from .config import DeviceOptions, AudioLanguageOptions, RetrievalTypeOptions
+from .config import DeviceOptions
 from .models.base_model import get_model_adapter
 from .utils.common import get_device_type
 from .pipeline.plugins.caching.cache import init_similar_cache_from_config
 from .pipeline.plugins.audio.asr import AudioSpeechRecognition
 from .pipeline.plugins.audio.asr_chinese import ChineseAudioSpeechRecognition
 from .pipeline.plugins.audio.tts import TextToSpeech
-from .pipeline.plugins.audio.tts_chinese_tts import ChineseTextToSpeech
+from .pipeline.plugins.audio.tts_chinese import ChineseTextToSpeech
 from .pipeline.plugins.retrievers.indexing.document_parser import DocumentIndexing
 from .pipeline.plugins.retrievers.retriever.langchain import ChromaRetriever
 from .pipeline.plugins.retrievers.retriever import BM25Retriever
-from .pipeline.plugins.security.sensitive_checker import SensitiveChecker
+from .pipeline.plugins.security.safety_checker import SafetyChecker
+from .pipeline.plugins.intent_detector import IntentDetector
 from .models.llama_model import LlamaModel
 from .models.mpt_model import MptModel
 from .models.chatglm_model import ChatGlmModel
@@ -66,57 +68,12 @@ def build_chatbot(config: PipelineConfig=None):
  # get model adapter
  adapter = get_model_adapter(config.model_name_or_path)
 
- # construct document retrieval using retrieval plugin
- if config.retrieval:
- if config.retrieval_type not in [option.name.lower() for option in RetrievalTypeOptions]:
- valid_options = ", ".join([option.name.lower() for option in RetrievalTypeOptions])
- raise ValueError(f"Invalid retrieval type value '{config.retrieval_type}'. Must be one of {valid_options}")
- if not config.retrieval_document_path:
- raise ValueError("Must provide a retrieval document path")
- if not os.path.exists(config.retrieval_document_path):
- raise ValueError(f"The retrieval document path {config.retrieval_document_path} is not exist.")
- db = DocumentIndexing(config.retrieval_type).KB_construct(config.retrieval_document_path)
- if config.retrieval_type == "dense":
- retriever = ChromaRetriever(db).retriever
- else:
- retriever = BM25Retriever(document_store = db)
- adapter.register_retriever(retriever, config.retrieval_type)
-
- # construct audio plugin
- if config.audio_input or config.audio_output:
- if config.audio_lang not in [option.name.lower() for option in AudioLanguageOptions]:
- valid_options = ", ".join([option.name.lower() for option in AudioLanguageOptions])
- raise ValueError(f"Invalid audio language value '{config.audio_lang}'. Must be one of {valid_options}")
- if config.audio_input:
- if config.audio_lang == AudioLanguageOptions.CHINESE.name.lower():
- asr = ChineseAudioSpeechRecognition()
- else:
- asr = AudioSpeechRecognition()
- adapter.register_asr(asr)
- if config.audio_output:
- if config.audio_lang == AudioLanguageOptions.CHINESE.name.lower():
- tts = ChineseTextToSpeech()
- else:
- tts = TextToSpeech()
- adapter.register_tts(tts)
-
- # construct response caching
- if config.cache_chat:
- if not config.cache_chat_config_file:
- cache_chat_config_file = "./pipeline/plugins/caching/cache_config.yaml"
- else:
- cache_chat_config_file = config.cache_chat_config_file
- if not config.cache_embedding_model_dir:
- cache_embedding_model_dir = "hkunlp/instructor-large"
- else:
- cache_embedding_model_dir = config.cache_embedding_model_dir
- init_similar_cache_from_config(config_dir=cache_chat_config_file,
- embedding_model_dir=cache_embedding_model_dir)
-
- # construct safety checker
- if config.safety_checker:
- safety_checker = SensitiveChecker()
- adapter.register_safety_checker(safety_checker)
+ # register plugin instance in model adaptor
+ for plugin_name in get_registered_plugins():
+ if is_plugin_enabled(plugin_name):
+ plugin_instance = get_plugin_instance(plugin_name)
+ if plugin_instance:
+ adapter.register_plugin_instance(plugin_name, plugin_instance)
 
  parameters = {}
  parameters["model_name"] = config.model_name_or_path
@@ -133,6 +90,7 @@ def build_chatbot(config: PipelineConfig=None):
  parameters["dtype"] = config.optimization_config.amp_config.dtype
  parameters["optimization_config"] = config.optimization_config
  adapter.load_model(parameters)
+
  return adapter
 
 def finetune_model(config: FinetuningConfig):

neural_chat/cli/cli_commands.py

@@ -25,7 +25,7 @@
 from neural_chat.pipeline.plugins.audio.asr import AudioSpeechRecognition
 from neural_chat.pipeline.plugins.audio.asr_chinese import ChineseAudioSpeechRecognition
 from neural_chat.pipeline.plugins.audio.tts import TextToSpeech
-from neural_chat.pipeline.plugins.audio.tts_chinese_tts import ChineseTextToSpeech
+from neural_chat.pipeline.plugins.audio.tts_chinese import ChineseTextToSpeech
 
 __all__ = ['BaseCommand', 'HelpCommand', 'TextChatExecutor', 'VoiceChatExecutor', 'FinetuingExecutor']
 

neural_chat/config.py

@@ -22,6 +22,12 @@
 from transformers.utils.versions import require_version
 from dataclasses import dataclass
 
+from neural_chat.pipeline.plugins.audio.asr import AudioSpeechRecognition
+from neural_chat.pipeline.plugins.audio.asr_chinese import ChineseAudioSpeechRecognition
+from neural_chat.pipeline.plugins.audio.tts import TextToSpeech
+from neural_chat.pipeline.plugins.audio.tts_chinese import ChineseTextToSpeech
+from .plugins import plugins
+
 from enum import Enum, auto
 
 class DeviceOptions(Enum):
@@ -383,27 +389,25 @@ class IntentConfig:
  ipex_int8: bool = False
 
 
-@dataclass
 class PipelineConfig:
- model_name_or_path: str = "meta-llama/Llama-2-7b-hf"
- tokenizer_name_or_path: str = None
- device: str = "auto"
- retrieval: bool = False
- retrieval_type: str = "dense"
- retrieval_document_path: str = None
- retrieval_config: RetrieverConfig = RetrieverConfig()
- audio_input: bool = False
- audio_output: bool = False
- audio_lang: str = "english"
- txt2Image: bool = False
- cache_chat: bool = False
- cache_chat_config_file: str = None
- cache_embedding_model_dir: str = None
- intent_detection: bool = False
- intent_config: IntentConfig = IntentConfig()
- memory_controller: bool = False
- safety_checker: bool = False
- saftey_config: SafetyConfig = SafetyConfig()
- loading_config: LoadingModelConfig = LoadingModelConfig()
- optimization_config: OptimizationConfig = OptimizationConfig()
+ def __init__(self,
+ model_name_or_path="meta-llama/Llama-2-7b-hf",
+ tokenizer_name_or_path=None,
+ device="auto",
+ plugins=plugins,
+ loading_config=None,
+ optimization_config=None):
+ self.model_name_or_path = model_name_or_path
+ self.tokenizer_name_or_path = tokenizer_name_or_path
+ self.device = device
+ self.plugins = plugins
+ self.loading_config = loading_config if loading_config is not None else LoadingModelConfig()
+ self.optimization_config = optimization_config if optimization_config is not None else OptimizationConfig()
+ for plugin_name, plugin_value in self.plugins.items():
+ if plugin_value['enable']:
+ print(f"create {plugin_name} plugin instance...")
+ print(f"plugin parameters: ", plugin_value['args'])
+ plugins[plugin_name]["instance"] = plugin_value['class'](**plugin_value['args'])
+
+
 

neural_chat/models/base_model.py

@@ -15,12 +15,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from abc import ABC, abstractmethod
+from abc import ABC
 from typing import List
 import os
 from fastchat.conversation import get_conv_template, Conversation
-from neural_chat.pipeline.inference.inference import load_model, predict, predict_stream
+from neural_chat.pipeline.inference.inference import load_model, predict, predict_stream, MODELS
 from neural_chat.config import GenerationConfig
+from neural_chat.plugins import is_plugin_enabled, get_plugin_instance, get_registered_plugins, get_plugin_arguments
 from neural_chat.utils.common import is_audio_file
 from neural_chat.pipeline.plugins.prompts.prompt import generate_qa_prompt, generate_prompt
 
@@ -59,7 +60,7 @@ def construct_prompt(query, retriever, retrieval_type):
  return generate_qa_prompt(query, context)
  else:
  return generate_prompt(query)
- 
+
 
 class BaseModel(ABC):
  """
@@ -79,6 +80,7 @@ def __init__(self):
  self.retrieval_type = None
  self.safety_checker = None
  self.intent_detection = False
+ self.cache = None
 
  def match(self, model_path: str):
  """
@@ -148,34 +150,42 @@ def predict(self, query, config=None):
  if is_audio_file(query):
  if not os.path.exists(query):
  raise ValueError(f"The audio file path {query} is invalid.")
- if self.asr:
- query = self.asr.audio2text(query)
- else:
- raise ValueError(f"The query {query} is audio file but there is no ASR registered.")
+
+ # plugin pre actions
+ for plugin_name in get_registered_plugins():
+ if is_plugin_enabled(plugin_name):
+ plugin_instance = get_plugin_instance(plugin_name)
+ if plugin_instance:
+ if hasattr(plugin_instance, 'pre_llm_inference_actions'):
+ if plugin_name == "asr" and not is_audio_file(query):
+ continue
+ if plugin_name == "intent_detection":
+ response = plugin_instance.pre_llm_inference_actions(query,
+ MODELS[self.model_name]["model"], MODELS[self.model_name]["tokenizer"])
+ else:
+ response = plugin_instance.pre_llm_inference_actions(query)
+ if plugin_name == "safety_checker" and response:
+ return "Your query contains sensitive words, please try another query."
+ elif plugin_name == "intent_detection":
+ if 'qa' not in response.lower():
+ query = generate_prompt(query)
+ else:
+ query = generate_qa_prompt(query)
+ else:
+ query = response
  assert query is not None, "Query cannot be None."
 
- if self.intent_detection:
- intent = predict(**construct_parameters(query, self.model_name, config.intent_config))
- if 'qa' not in intent.lower():
- intent = "chitchat"
- query = generate_prompt(query)
- elif self.retriever:
- query = construct_prompt(query, self.retriever, self.retrieval_type)
- else:
- query = generate_qa_prompt(query)
- else:
- if self.retriever:
- query = construct_prompt(query, self.retriever, self.retrieval_type)
-
- if self.safety_checker:
- assert self.safety_checker.sensitive_check(query) is False, "The input query contains sensitive words." 
+ # LLM inference
  response = predict(**construct_parameters(query, self.model_name, config))
- if self.safety_checker:
- if self.safety_checker.sensitive_check(response):
- response = self.safety_checker.sensitive_filter(response)
- if self.tts:
- self.tts.text2speech(response, config.audio_output_path)
- response = config.audio_output_path
+
+ # plugin post actions
+ for plugin_name in get_registered_plugins():
+ if is_plugin_enabled(plugin_name):
+ plugin_instance = get_plugin_instance(plugin_name)
+ if plugin_instance:
+ if hasattr(plugin_instance, 'post_llm_inference_actions'):
+ response = plugin_instance.post_llm_inference_actions(response)
+
  return response
 
  def chat_stream(self, query, config=None):
@@ -210,43 +220,29 @@ def get_default_conv_template(self, model_path: str) -> Conversation:
  """
  return get_conv_template("one_shot")
 
- def register_tts(self, instance):
- """
- Register a text-to-speech (TTS) instance.
-
- Args:
- instance: An instance of a TTS module.
- """
- self.tts = instance
-
- def register_asr(self, instance):
- """
- Register an automatic speech recognition (ASR) instance.
-
- Args:
- instance: An instance of an ASR module.
- """
- self.asr = instance
-
- def register_safety_checker(self, instance):
- """
- Register a safety checker instance.
-
- Args:
- instance: An instance of a safety checker module.
- """
- self.safety_checker = instance
-
- def register_retriever(self, retriever, retrieval_type):
+ def register_plugin_instance(self, plugin_name, instance):
  """
- Register a database retriever.
+ Register a plugin instance.
 
  Args:
- instance: An instance of a retriever.
- retrieval_type: The type of the retrieval method.
+ instance: An instance of a plugin.
  """
- self.retriever = retriever
- self.retrieval_type = retrieval_type
+ if plugin_name == "tts":
+ self.tts = instance
+ if plugin_name == "tts_chinese":
+ self.tts_chinese = instance
+ if plugin_name == "asr":
+ self.asr = instance
+ if plugin_name == "asr_chinese":
+ self.asr_chinese = instance
+ if plugin_name == "retrieval":
+ self.retrieval = instance
+ if plugin_name == "cache":
+ self.cache = instance
+ if plugin_name == "intent_detection":
+ self.intent_detection = instance
+ if plugin_name == "safety_checker":
+ self.safety_checker = instance
 
 
 # A global registry for all model adapters
@@ -266,4 +262,4 @@ def get_model_adapter(model_name_path: str) -> BaseModel:
  if adapter.match(model_path_basename) and type(adapter) != BaseModel:
  return adapter
 
- raise ValueError(f"No valid model adapter for {model_name_path}")
+ raise ValueError(f"No valid model adapter for {model_name_path}")

neural_chat/pipeline/plugins/audio/__init__.py

@@ -1,4 +1,4 @@
 from .asr import AudioSpeechRecognition
 from .asr_chinese import ChineseAudioSpeechRecognition
 from .tts import TextToSpeech
-from .tts_chinese_tts import ChineseTextToSpeech
+from .tts_chinese import ChineseTextToSpeech

neural_chat/pipeline/plugins/audio/asr.py

@@ -22,8 +22,10 @@
 import contextlib
 from pydub import AudioSegment
 
+from neural_chat.plugins import register_plugin
 
-class AudioSpeechRecognition:
+@register_plugin('asr')
+class AudioSpeechRecognition():
  """Convert audio to text."""
  def __init__(self, model_name_or_path="openai/whisper-small", bf16=False, device="cpu"):
  self.device = device
@@ -58,4 +60,8 @@ def audio2text(self, audio_path):
  predicted_ids = self.model.generate(inputs)
  result = self.processor.tokenizer.batch_decode(predicted_ids, skip_special_tokens=True, normalize=True)[0]
  print(f"generated text in {time.time() - start} seconds, and the result is: {result}")
- return result
+ return result
+
+
+ def pre_llm_inference_actions(self, audio_path):
+ return self.audio2text(audio_path)

neural_chat/pipeline/plugins/audio/asr_chinese.py

@@ -17,8 +17,10 @@
 
 from paddlespeech.cli.asr.infer import ASRExecutor
 import time
+from neural_chat.plugins import register_plugin
 
-class ChineseAudioSpeechRecognition:
+@register_plugin('asr_chinese')
+class ChineseAudioSpeechRecognition():
  """Convert audio to text in Chinese."""
  def __init__(self):
  self.asr = ASRExecutor()
@@ -31,3 +33,6 @@ def audio2text(self, audio_path):
  start = time.time()
  result = self.asr(audio_file=audio_path)
  print(f"generated text in {time.time() - start} seconds, and the result is: {result}")
+
+ def pre_llm_inference_actions(self, audio_path):
+ return self.audio2text(audio_path)