Skip to content

[AutoParallel] Open dynamic sharding CI test #10793

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jul 10, 2025
Merged

[AutoParallel] Open dynamic sharding CI test #10793

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
afb5ee5
open sharding CI test and remove flag
waliwali777 Jul 2, 2025
796c320
update loss_base
waliwali777 Jul 10, 2025
File filter

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 5 additions & 10 deletions scripts/distribute/ci_case_auto.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ function llm_gpt_case_list_auto() {
llm_gpt_dygraph_auto_bs8_fp32_DP2-MP2
llm_gpt_dygraph_auto_bs8_fp32_DP2-MP2-PP2
llm_gpt_dygraph_auto_bs8_fp16_DP2-MP2-PP2
# llm_gpt_dygraph_auto_bs8_fp16_DP2-MP2-PP2_intermediate
llm_gpt_dygraph_auto_bs8_fp16_DP2-MP2-PP2_intermediate
llm_gpt_pir_auto_bs4_TP2
llm_gpt_pir_auto_bs4_TP2_PP2
llm_gpt_pir_auto_bs8_DP2_TP2_PP2
Expand Down Expand Up @@ -223,7 +223,7 @@ function llama_dygraph_auto_bs4_bf16_SD2() {

export CUDA_DEVICE_MAX_CONNECTIONS=1

flags=("" "FLAGS_fuse_allreduce_in_opt" "FLAGS_fuse_reducescatter_in_opt" "FLAGS_enable_tensor_fusion FLAGS_enable_sharding_overlap")
flags=("" "FLAGS_enable_tensor_fusion FLAGS_enable_sharding_overlap")
for i in "${!flags[@]}"; do
flag="${flags[$i]}"

Expand Down Expand Up @@ -290,7 +290,7 @@ function llama_dygraph_auto_bs4_bf16_SD2() {
--tensor_parallel_degree 1 \
--sharding "stage1" \
--data_parallel_config "enable_allreduce_avg_in_gradinent_scale gradient_sync_after_accumulate" \
--sharding_parallel_config "" \
--sharding_parallel_config "enable_tensor_fusion enable_overlap" \
--to_static 0 \
--amp_custom_black_list "reduce_sum" "c_softmax_with_cross_entropy" \
--amp_custom_white_list "lookup_table" "lookup_table_v2" \
Expand All @@ -302,10 +302,6 @@ function llama_dygraph_auto_bs4_bf16_SD2() {
echo "result: loss=$loss ips=$ips mem=$mem"
echo "flag=$flag acc_step=$acc_step"
if [ -z "$flag" ]; then
loss_base=9.23504791
elif [ "$flag" = "FLAGS_fuse_allreduce_in_opt" ]; then
loss_base=9.23502579
elif [ "$flag" = "FLAGS_fuse_reducescatter_in_opt" ]; then
loss_base=9.23504105
elif [ "$flag" = "FLAGS_enable_tensor_fusion FLAGS_enable_sharding_overlap" ]; then
if [ $acc_step -eq 1 ]; then
Expand Down Expand Up @@ -2653,11 +2649,11 @@ function llm_gpt_dygraph_auto_bs8_fp16_DP2-MP2-PP2_intermediate() {
mem=-1
echo "result: loss=$loss ips=$ips mem=$mem loss_md5=$loss_md5"
# loss_base=10.58456802 # note: need to debug
loss_base=10.56716251
loss_base=10.56668091
ips_base=-1
mem_base=-1
if [ $IS_A100 -ne 0 ];then
loss_base=10.56166935 # after add dropout spmd
loss_base=10.56199837 # after add dropout spmd
fi
check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}
echo "=========== $FUNCNAME run end ==========="
Expand Down Expand Up @@ -3844,7 +3840,6 @@ function llama_baichuan_dygraph_auto_sp_async_reduce_scatter_bs8_bf16_DP4-MP2-SP
export NVIDIA_TF32_OVERRIDE=0

export CUDA_DEVICE_MAX_CONNECTIONS=1
export FLAGS_fuse_reducescatter_in_opt=1
export FLAGS_enable_inplace_master_grad=1
export FLAGS_auto_parallel_align_mode=1
export FLAGS_max_inplace_grad_add=65536
Expand Down
1 change: 0 additions & 1 deletion ...ichuan-inc-baichuan-2-13b_pretrain_dynamic_auto_bs32_bf16_DP1_MP4_PP1_Sharding8_Stage1.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ param+="nnodes=4 "
param+="model_type=baichuan2_13b "
param+='dynamic_auto=_dynamic_auto '

export FLAGS_fuse_reducescatter_in_opt=1
export FLAGS_enable_sharding_overlap=1
export FLAGS_enable_tensor_fusion=1

Expand Down
3 changes: 0 additions & 3 deletions ...ta-llama-Llama-2-13b_pretrain_dynamic_auto_bs32_bf16_DP1_MP1_PP4_VPP5_Sharding8_Stage1.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,6 @@ param+="nnodes=4 "
param+="model_type=llama2_13b "
param+='dynamic_auto=_dynamic_auto '

# This optimization currently only runs in the dynamic automatic parallelism of Llama7B.
export FLAGS_fuse_reducescatter_in_opt=1

# Enable tensor fusion and sharding overlap optimization
export FLAGS_enable_tensor_fusion=1
export FLAGS_enable_sharding_overlap=1
Expand Down
3 changes: 0 additions & 3 deletions ...l/llama2/N4C32/meta-llama-Llama-2-7b_pretrain_dynamic_auto_bs32_bf16_Sharding32_Stage2.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,6 @@ param+="nnodes=4 "
param+="model_type=llama2_7b "
param+='dynamic_auto=_dynamic_auto '

# This optimization currently only runs in the dynamic automatic parallelism of Llama7B.
export FLAGS_fuse_reducescatter_in_opt=1

# Enable tensor fusion and sharding overlap optimization
export FLAGS_enable_tensor_fusion=1
export FLAGS_enable_sharding_overlap=1
Expand Down
1 change: 0 additions & 1 deletion ...parallel/qwen/N4C32/qwen-14b_pretrain_dynamic_auto_bs32_bf16_DP1_MP2_Sharding16_Stage1.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ param+="nnodes=4 "
param+="model_type=qwen_14b "
param+='dynamic_auto=_dynamic_auto '

export FLAGS_fuse_reducescatter_in_opt=1
export FLAGS_enable_tensor_fusion=1
export FLAGS_enable_sharding_overlap=1

Expand Down
Loading