[MoE] fix expert parallel (#9760)

* fix moe uc

Author: DesmonDay

llm/run_finetune.py

@@ -151,6 +151,9 @@ def main():
  quantization_config=quantization_config,
  )
 
+ if "Qwen2Moe" in str(model_config.architectures) and training_args.data_parallel_degree > 1:
+ training_args.use_expert_parallel = True
+
  LlmMetaConfig.set_llm_config(model_config, training_args)
  model_config.use_fast_layer_norm = model_args.use_fast_layer_norm
 

llm/run_pretrain.py

@@ -478,6 +478,9 @@ def main():
  except:
  print("Not register llama pp reshard information.")
 
+ if "Qwen2Moe" in str(config.architectures) and training_args.data_parallel_degree > 1:
+ training_args.use_expert_parallel = True
+
  if model_args.continue_training:
  # NOTE(gongenlei): new add
  if training_args.autotuner_benchmark:

paddlenlp/transformers/moe_layer.py

@@ -162,12 +162,14 @@ def __init__(
  self.moe_num_experts_per_device = self._parse_moe_expert_parallel(
  self.moe_num_experts, self.expert_parallel_degree
  )
+ self.is_dummy_moe = False if self.expert_parallel_degree > 1 else True
  else:
  # when moe_group is dummy, we don't need to use all_to_all
  self.moe_group = None
  self.moe_rank = 0
  self.expert_parallel_degree = 1
  self.moe_num_experts_per_device = self.moe_num_experts
+ self.is_dummy_moe = True
 
  self.all_to_all_dropout = all_to_all_dropout
  self.enable_recompute = False
@@ -181,6 +183,7 @@ def __init__(
 
  self.gate = gate
  self.gate.group = self.moe_group
+ self._post_init()
 
  def _parse_moe_expert_parallel(self, moe_num_experts, expert_parallel_degree):
  assert (