Override Transformers defaults by GGUF defaults (#42770)

* Override Transformers defaults by GGUF defaults In some models, GGUF uses default or fixed values different from this library. To integrate GGUF-based models without additional configuration, we need some kind of compatibility layer. This commit provides additional mapping to provide GGUF-specific default values to initialize parameters in this library. Currently, only fixed "norm_topk_prob" value of Qwen3 MoE (True) is defined because (a) it differs from the default value of this library (False) and (b) if this parameter is incorrectly set, it results in almost completely garbled output. Signed-off-by: Tsukasa OI <floss_llm@irq.a4lg.com> * Apply suggestions from code review Co-authored-by: Mohamed Mekkouri <93391238+MekkCyber@users.noreply.github.com> --------- Signed-off-by: Tsukasa OI <floss_llm@irq.a4lg.com> Co-authored-by: Mohamed Mekkouri <93391238+MekkCyber@users.noreply.github.com>

Author: a4lg

src/transformers/integrations/__init__.py

@@ -54,6 +54,7 @@
  "finegrained_fp8": ["FP8Linear", "replace_with_fp8_linear"],
  "fsdp": ["is_fsdp_enabled", "is_fsdp_managed_module"],
  "ggml": [
+ "GGUF_CONFIG_DEFAULTS_MAPPING",
  "GGUF_CONFIG_MAPPING",
  "GGUF_TOKENIZER_MAPPING",
  "_gguf_parse_value",
@@ -201,6 +202,7 @@
  from .finegrained_fp8 import FP8Linear, replace_with_fp8_linear
  from .fsdp import is_fsdp_enabled, is_fsdp_managed_module
  from .ggml import (
+ GGUF_CONFIG_DEFAULTS_MAPPING,
  GGUF_CONFIG_MAPPING,
  GGUF_TOKENIZER_MAPPING,
  _gguf_parse_value,

src/transformers/integrations/ggml.py

@@ -313,6 +313,16 @@
  },
 }
 
+# We only need to set here the parameters that default to different values between transformers and llamacpp.
+GGUF_CONFIG_DEFAULTS_MAPPING = {
+ "qwen3_moe": {
+ # NOTE: Qwen3MoeConfig defaults to false but llama.cpp needs this to be true.
+ # See: https://github.com/ggml-org/llama.cpp/blob/17f7f4baad8b3a716ee139da7bb56ae984e8c0fa/src/models/qwen3moe.cpp#L85-L96
+ # (the parameter right after LLM_FFN_SILU corresponds to norm_topk_prob)
+ "norm_topk_prob": True,
+ },
+}
+
 
 def _gguf_parse_value(_value, data_type):
  if not isinstance(data_type, list):

src/transformers/modeling_gguf_pytorch_utils.py

@@ -20,6 +20,7 @@
 from tqdm.auto import tqdm
 
 from .integrations import (
+ GGUF_CONFIG_DEFAULTS_MAPPING,
  GGUF_CONFIG_MAPPING,
  GGUF_TOKENIZER_MAPPING,
  _gguf_parse_value,
@@ -437,6 +438,13 @@ def load_gguf_checkpoint(gguf_checkpoint_path, return_tensors=False, model_to_lo
  all("output.weight" != tensor.name for tensor in reader.tensors) or architecture in exceptions
  )
 
+ # Set GGUF-specific default values
+ config_defaults = GGUF_CONFIG_DEFAULTS_MAPPING.get(
+ updated_architecture, GGUF_CONFIG_DEFAULTS_MAPPING.get(architecture) or {}
+ )
+ for key, value in config_defaults.items():
+ parsed_parameters["config"].setdefault(key, value)
+
  # List all key-value pairs in a columnized format
  for gguf_key, field in reader.fields.items():
  gguf_key = gguf_key.replace(architecture, updated_architecture)