upgrade huggingface use_auth_token to token (#943)

Author: lkk12014402

intel_extension_for_transformers/llm/finetuning/finetuning.py

@@ -144,7 +144,7 @@ def load_dataset(self, data_args, model_args, training_args):
  data_args.dataset_name,
  data_args.dataset_config_name,
  cache_dir=model_args.cache_dir,
- use_auth_token=True if model_args.use_auth_token else None,
+ token=model_args.token,
  streaming=data_args.streaming,
  )
 
@@ -154,15 +154,15 @@ def load_dataset(self, data_args, model_args, training_args):
  data_args.dataset_config_name,
  split=f"train[:{data_args.validation_split_percentage}%]",
  cache_dir=model_args.cache_dir,
- use_auth_token=True if model_args.use_auth_token else None,
+ token=model_args.token,
  streaming=data_args.streaming,
  )
  raw_datasets["train"] = load_dataset(
  data_args.dataset_name,
  data_args.dataset_config_name,
  split=f"train[{data_args.validation_split_percentage}%:]",
  cache_dir=model_args.cache_dir,
- use_auth_token=True if model_args.use_auth_token else None,
+ token=model_args.token,
  streaming=data_args.streaming,
  )
  else:
@@ -184,7 +184,7 @@ def load_dataset(self, data_args, model_args, training_args):
  extension,
  data_files=data_files,
  cache_dir=model_args.cache_dir,
- use_auth_token=True if model_args.use_auth_token else None,
+ token=model_args.token,
  **dataset_args,
  )
 
@@ -195,15 +195,15 @@ def load_dataset(self, data_args, model_args, training_args):
  data_files=data_files,
  split=f"train[:{data_args.validation_split_percentage}%]",
  cache_dir=model_args.cache_dir,
- use_auth_token=True if model_args.use_auth_token else None,
+ token=model_args.token,
  **dataset_args,
  )
  raw_datasets["train"] = load_dataset(
  extension,
  data_files=data_files,
  split=f"train[{data_args.validation_split_percentage}%:]",
  cache_dir=model_args.cache_dir,
- use_auth_token=True if model_args.use_auth_token else None,
+ token=model_args.token,
  **dataset_args,
  )
  return raw_datasets
@@ -212,7 +212,7 @@ def load_model_config(self, model_args):
  config_kwargs = {
  "cache_dir": model_args.cache_dir,
  "revision": model_args.model_revision,
- "use_auth_token": True if model_args.use_auth_token else None,
+ "token": model_args.token,
  "trust_remote_code": True if model_args.trust_remote_code else None,
  }
  if model_args.config_name:
@@ -230,7 +230,7 @@ def load_tokenizer(self, model_args):
  "cache_dir": model_args.cache_dir,
  "use_fast": model_args.use_fast_tokenizer,
  "revision": model_args.model_revision,
- "use_auth_token": True if model_args.use_auth_token else None,
+ "token": model_args.token,
  "trust_remote_code": model_args.trust_remote_code,
  }
  if model_args.tokenizer_name:
@@ -378,7 +378,7 @@ def finetune_clm(self, model_args, data_args, training_args, finetune_args, conf
  device_map=self.device_map,
  quantization_config=self.bitsandbytes_quant_config,
  revision=model_args.model_revision,
- use_auth_token=True if model_args.use_auth_token else None,
+ token=model_args.token,
  trust_remote_code=True if model_args.trust_remote_code else None,
  torch_dtype=model_dtype,
  low_cpu_mem_usage=low_cpu_mem_usage,
@@ -807,7 +807,7 @@ def preprocess_logits_for_metrics(logits, labels):
  device_map=self.device_map,
  quantization_config=self.bitsandbytes_quant_config,
  revision=model_args.model_revision,
- use_auth_token=True if model_args.use_auth_token else None,
+ token=model_args.token,
  torch_dtype=model_dtype,
  load_in_4bit=self.load_in_4bit,
  load_in_8bit=self.load_in_8bit,

intel_extension_for_transformers/neural_chat/config.py

@@ -92,11 +92,20 @@ class ModelArguments:
  "help": "The specific model version to use (can be a branch name, tag name or commit id)."
  },
  )
+ token: str = field(
+ default=None,
+ metadata={
+ "help": (
+ "The token to use as HTTP bearer authorization for remote files. If not specified, will use the token "
+ "generated when running `huggingface-cli login` (stored in `~/.huggingface`)."
+ )
+ },
+ )
  use_auth_token: bool = field(
  default=False,
  metadata={
- "help": "Will use the token generated when running `transformers-cli login` (necessary to use this script "
- "with private models)."
+ "help": "The `use_auth_token` argument is deprecated and will be removed in v4.34."
+ "Please use `token` instead."
  },
  )
  trust_remote_code: bool = field(