fix dpo pp criterion (#9786)

Author: wtmlon

llm/alignment/dpo/run_dpo.py

@@ -127,6 +127,8 @@ def main():
 
  if training_args.pipeline_parallel_degree > 1:
  model_class = AutoModelForCausalLMPipe
+ if not dpo_config.reference_free and not dpo_config.lora:
+ ref_model_config.dpo_config = dpo_config
  model_config.dpo_config = dpo_config
  else:
  model_class = AutoModelForCausalLM