[tokenizer] Fix AutoTokenizer

paddlenlp/transformers/auto/factory.py

@@ -79,15 +79,15 @@ def __getitem__(self, key):
  def _load_attr_from_module(self, model_type, attr):
  module_name = model_type_to_module_name(model_type)
  if module_name not in self._modules:
- if "Tokenizer" in model_type:
+ if any(["Tokenizer" in name for name in [model_type, attr]]):
  try:
  self._modules[module_name] = importlib.import_module(
  f".{module_name}.tokenizer", "paddlenlp.transformers"
  )
  except ImportError:
  pass
  if module_name not in self._modules:
- if "Config" in model_type:
+ if any(["Config" in name for name in [model_type, attr]]):
  try:
  self._modules[module_name] = importlib.import_module(
  f".{module_name}.configuration", "paddlenlp.transformers"

paddlenlp/transformers/auto/tokenizer.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import importlib
+import inspect
 import io
 import json
 import os
@@ -460,7 +461,7 @@
  return tokenizer_class_fast.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
  else:
  if tokenizer_class_py is not None:
- if isinstance(tokenizer_class_py, str):
+ if inspect.isclass(tokenizer_class_py):
  return tokenizer_class_py.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
  else:
  # Use the first tokenizer class in the list