Merge remote-tracking branch 'dev/develop' into uc/speed_check

Author: ZHUI

Makefile

@@ -45,6 +45,7 @@ unit-test:
 
 .PHONY: install
 install:
+pip install paddlepaddle==0.0.0 -f https://www.paddlepaddle.org.cn/whl/linux/cpu-mkl/develop.html
 pip install -r requirements-dev.txt
 pip install -r requirements.txt
 pip install -r paddlenlp/experimental/autonlp/requirements.txt

llm/argument.py

@@ -126,6 +126,14 @@ class ModelArgument:
  lora: bool = field(default=False, metadata={"help": "Whether to use LoRA technique"})
  lora_path: str = field(default=None, metadata={"help": "Initialize lora state dict."})
  lora_rank: int = field(default=8, metadata={"help": "Lora attention dimension"})
+ use_quick_lora: bool = field(
+ default=False,
+ metadata={
+ "help": "Whether to use quick lora, The use of Quick LoRa will only take effect when lora_dropout is set to 0."
+ },
+ )
+ rslora: bool = field(default=False, metadata={"help": "Whether to use RsLoRA"})
+ lora_plus_scale: float = field(default=1.0, metadata={"help": "Lora B scale in LoRA+ technique"})
 
  # prefix tuning related parameters
  prefix_tuning: bool = field(default=False, metadata={"help": "Whether to use Prefix technique"})

llm/finetune_generation.py

@@ -112,6 +112,7 @@ def main():
  weight_double_quant=model_args.weight_double_quant,
  weight_double_quant_block_size=model_args.weight_double_quant_block_size,
  )
+
  if training_args.pipeline_parallel_degree > 1:
  if data_args.eval_with_do_generation and training_args.do_eval:
  raise ValueError("Plese set eval_with_do_generation to false in pipeline parallel mode.")
@@ -418,16 +419,20 @@ def neft_post_hook(module, input, output):
  lora_config = LoRAConfig(
  target_modules=target_modules,
  r=model_args.lora_rank,
- lora_alpha=2 * model_args.lora_rank,
+ lora_alpha=2 * model_args.lora_rank if not model_args.rslora else 4,
+ rslora=model_args.rslora,
+ lora_plus_scale=model_args.lora_plus_scale,
  merge_weights=False,
  tensor_parallel_degree=training_args.tensor_parallel_degree,
  dtype=dtype,
  do_qat=quant_args.do_qat,
  base_model_name_or_path=model_args.model_name_or_path,
+ use_quick_lora=model_args.use_quick_lora,
  )
  model = LoRAModel(model, lora_config)
  else:
  model = LoRAModel.from_pretrained(model=model, lora_path=model_args.lora_path)
+
  model.print_trainable_parameters()
 
  def compute_metrics_do_generation(eval_preds):

llm/llama/auto_parallel/run_pretrain_auto_static.py

@@ -552,7 +552,8 @@ def main():
  # if training_args.bf16:
  # dtype = "bfloat16"
 
- model = model_class._from_config(config)
+ # The `amp` of static graph model can't accept a model initialized with `dtype float16 or bfloat16`
+ model = model_class._from_config(config, dtype="float32")
 
  if training_args.recompute:
 

llm/llama/fused_layers.py

@@ -58,16 +58,18 @@ def backward(ctx, y_grad):
 
  if hasattr(weight, "main_grad") and hasattr(bias, "main_grad"):
  weight.main_grad, bias.main_grad = _C_ops.fused_linear_param_grad_add(
- x, y_grad, weight.main_grad, bias.main_grad, True
+ x, y_grad, weight.main_grad, bias.main_grad, True, True
  )
  return x_grad, None, None
  else:
  if weight.grad is not None:
  assert bias.grad is not None
- weight.grad, bias.grad = _C_ops.fused_linear_param_grad_add(x, y_grad, weight.grad, bias.grad, False)
+ weight.grad, bias.grad = _C_ops.fused_linear_param_grad_add(
+ x, y_grad, weight.grad, bias.grad, False, True
+ )
  return x_grad, None, None
  else:
- weight_grad, bias_grad = _C_ops.fused_linear_param_grad_add(x, y_grad, None, None, False)
+ weight_grad, bias_grad = _C_ops.fused_linear_param_grad_add(x, y_grad, None, None, False, True)
  return x_grad, weight_grad, bias_grad
 
 

model_zoo/gpt-3/benchmarks/test_tipc/gpt/static/new_exec_pp_pir/N1C8/gpt_auto_pir_bs16_fp16O1_DP2-MP1-PP4-SD2-stage1.sh

@@ -0,0 +1,21 @@
+model_item=gpt_auto_pir
+dp_degree=2
+mp_degree=1
+pp_degree=4
+bs_item=16 # micro * dp * pp
+fp_item=fp16O1
+run_mode=DP2-MP1-PP4-SD2-stage1
+device_num=N1C8
+sharding_degree=2 # sharding_degree = dp_degree
+sharding_stage=1
+level=o1
+local_batch_size=8
+
+model=gpt
+micro_bs=2 # local_batch_size / pp_degree
+
+cd ./benchmarks
+bash ./test_tipc/gpt/static/new_exec_pp_pir/benchmark_common/prepare.sh
+# run
+bash ./test_tipc/gpt/static/new_exec_pp_pir/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \
+${sharding_degree} ${sharding_stage} ${level} 2>&1;

model_zoo/gpt-3/benchmarks/test_tipc/gpt/static/new_exec_pp_pir/N1C8/gpt_auto_pir_bs16_fp16O1_DP2-MP1-PP4-SD2-stage2.sh

@@ -0,0 +1,21 @@
+model_item=gpt_auto_pir
+dp_degree=2
+mp_degree=1
+pp_degree=4
+bs_item=16 # micro * dp * pp
+fp_item=fp16O1
+run_mode=DP2-MP1-PP4-SD2-stage2
+device_num=N1C8
+sharding_degree=2 # sharding_degree = dp_degree
+sharding_stage=2
+level=o1
+local_batch_size=8
+
+model=gpt
+micro_bs=2 # local_batch_size / pp_degree
+
+cd ./benchmarks
+bash ./test_tipc/gpt/static/new_exec_pp_pir/benchmark_common/prepare.sh
+# run
+bash ./test_tipc/gpt/static/new_exec_pp_pir/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \
+${sharding_degree} ${sharding_stage} ${level} 2>&1;

model_zoo/gpt-3/benchmarks/test_tipc/gpt/static/new_exec_pp_pir/N1C8/gpt_auto_pir_bs16_fp16O1_DP2-MP2-PP2-SD2-stage1.sh

@@ -0,0 +1,21 @@
+model_item=gpt_auto_pir
+dp_degree=2
+mp_degree=2
+pp_degree=2
+bs_item=16 # micro * dp * pp
+fp_item=fp16O1
+run_mode=DP2-MP2-PP2-SD2-stage1
+device_num=N1C8
+sharding_degree=2 # sharding_degree = dp_degree
+sharding_stage=1
+level=o1
+local_batch_size=8
+
+model=gpt
+micro_bs=4 # local_batch_size / pp_degree
+
+cd ./benchmarks
+bash ./test_tipc/gpt/static/new_exec_pp_pir/benchmark_common/prepare.sh
+# run
+bash ./test_tipc/gpt/static/new_exec_pp_pir/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \
+${sharding_degree} ${sharding_stage} ${level} 2>&1;

model_zoo/gpt-3/benchmarks/test_tipc/gpt/static/new_exec_pp_pir/N1C8/gpt_auto_pir_bs16_fp16O1_DP2-MP2-PP2-SD2-stage2.sh

@@ -0,0 +1,21 @@
+model_item=gpt_auto_pir
+dp_degree=2
+mp_degree=2
+pp_degree=2
+bs_item=16 # micro * dp * pp
+fp_item=fp16O1
+run_mode=DP2-MP2-PP2-SD2-stage2
+device_num=N1C8
+sharding_degree=2 # sharding_degree = dp_degree
+sharding_stage=2
+level=o1
+local_batch_size=8
+
+model=gpt
+micro_bs=4 # local_batch_size / pp_degree
+
+cd ./benchmarks
+bash ./test_tipc/gpt/static/new_exec_pp_pir/benchmark_common/prepare.sh
+# run
+bash ./test_tipc/gpt/static/new_exec_pp_pir/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \
+${sharding_degree} ${sharding_stage} ${level} 2>&1;

model_zoo/gpt-3/benchmarks/test_tipc/gpt/static/new_exec_pp_pir/N1C8/gpt_auto_pir_bs16_fp16O2_DP2-MP1-PP4-SD2-stage1.sh

@@ -0,0 +1,21 @@
+model_item=gpt_auto_pir
+dp_degree=2
+mp_degree=1
+pp_degree=4
+bs_item=16 # micro * dp * pp
+fp_item=fp16O2
+run_mode=DP2-MP1-PP4-SD2-stage1
+device_num=N1C8
+sharding_degree=2 # sharding_degree = dp_degree
+sharding_stage=1
+level=o2
+local_batch_size=8
+
+model=gpt
+micro_bs=2 # local_batch_size / pp_degree
+
+cd ./benchmarks
+bash ./test_tipc/gpt/static/new_exec_pp_pir/benchmark_common/prepare.sh
+# run
+bash ./test_tipc/gpt/static/new_exec_pp_pir/benchmark_common/run_benchmark.sh ${model_item} ${fp_item} ${dp_degree} ${mp_degree} ${pp_degree} ${micro_bs} ${bs_item} ${run_mode} ${device_num} \
+${sharding_degree} ${sharding_stage} ${level} 2>&1;