Merge branch 'main' of code.byted.org:ming.li1/ControlNet_Plus_Plus

Author: liming-ai

README.md

@@ -16,6 +16,7 @@ pip3 install -U openmim
 mim install mmengine
 mim install "mmcv==2.1.0"
 pip3 install "mmsegmentation>=1.0.0"
+pip3 install mmdet
 ```
 
 ## 🕹️ Data Preperation

coco_stuff_palette.npy

@@ -16,7 +16,7 @@ accelerate launch --config_file "train/config.yml" \
  --conditioning_image_column="canny" \
  --resolution=512 \
  --train_batch_size=8 \
- --gradient_accumulation_steps=8 \
+ --gradient_accumulation_steps=4 \
  --learning_rate=1e-5 \
  --mixed_precision="fp16" \
  --gradient_checkpointing \
@@ -25,7 +25,7 @@ accelerate launch --config_file "train/config.yml" \
  --lr_scheduler="constant_with_warmup" \
  --lr_warmup_steps=0 \
  --checkpointing_steps=500 \
- --grad_scale=0.5 \
+ --grad_scale=1.0 \
  --use_ema \
  --validation_steps=100 \
  --timestep_sampling_start=0 \

train/reward_canny.sh

@@ -22,7 +22,7 @@ accelerate launch --config_file "train/config.yml" \
  --cache_dir="data/huggingface_datasets" \
  --resolution=512 \
  --train_batch_size=8 \
- --gradient_accumulation_steps=16 \
+ --gradient_accumulation_steps=4 \
  --learning_rate=1e-5 \
  --mixed_precision="fp16" \
  --gradient_checkpointing \

train/reward_cocostuff.sh

@@ -16,7 +16,7 @@ accelerate launch --config_file "train/config.yml" \
  --cache_dir=None \
  --resolution=512 \
  --train_batch_size=8 \
- --gradient_accumulation_steps=16 \
+ --gradient_accumulation_steps=4 \
  --learning_rate=1e-5 \
  --mixed_precision="fp16" \
  --gradient_checkpointing \
@@ -30,4 +30,5 @@ accelerate launch --config_file "train/config.yml" \
  --validation_steps=500 \
  --timestep_sampling_start=0 \
  --timestep_sampling_end=1000 \
+ --min_timestep_rewarding=0 \
  --max_timestep_rewarding=200

train/reward_depth.sh

@@ -1,7 +1,7 @@
 export MODEL_DIR="runwayml/stable-diffusion-v1-5"
 export CONTROLNET_DIR="lllyasviel/control_v11p_sd15_softedge"
 export REWARDMODEL_DIR="https://huggingface.co/lllyasviel/Annotators/resolve/main/ControlNetHED.pth"
-export OUTPUT_DIR="work_dirs/reward_model/MultiGen20M_Hed/reward_controlnet_sd15_lineart_res256_bs256_lr1e-5_warmup100_scale-1.0_iter10k_fp16_train0-1k_reward0-200"
+export OUTPUT_DIR="work_dirs/reward_model/MultiGen20M_Hed/reward_controlnet_sd15_lineart_res512_bs256_lr1e-5_warmup100_scale-1.0_iter10k_fp16_train0-1k_reward0-200"
 
 accelerate launch --config_file "train/config.yml" \
  --main_process_port=23456 controlnet/reward_control.py \
@@ -14,8 +14,8 @@ accelerate launch --config_file "train/config.yml" \
  --caption_column="text" \
  --conditioning_image_column="hed" \
  --resolution=512 \
- --train_batch_size=4 \
- --gradient_accumulation_steps=16 \
+ --train_batch_size=8 \
+ --gradient_accumulation_steps=4 \
  --learning_rate=1e-5 \
  --mixed_precision="fp16" \
  --gradient_checkpointing \

train/reward_hed.sh

@@ -1,7 +1,7 @@
 export MODEL_DIR="runwayml/stable-diffusion-v1-5"
 export CONTROLNET_DIR="lllyasviel/control_v11p_sd15_lineart"
 export REWARDMODEL_DIR="https://huggingface.co/spaces/awacke1/Image-to-Line-Drawings/resolve/main/model.pth"
-export OUTPUT_DIR="work_dirs/reward_model/MultiGen20M_LineDrawing/reward_controlnet_sd15_lineart_res256_bs256_lr1e-5_warmup100_scale-0.5_iter10k_fp16_train0-1k_reward0-200"
+export OUTPUT_DIR="work_dirs/reward_model/MultiGen20M_LineDrawing/reward_controlnet_sd15_lineart_res512_bs256_lr1e-5_warmup100_scale-0.5_iter10k_fp16_train0-1k_reward0-200"
 
 accelerate launch --config_file "train/config.yml" \
  --main_process_port=23156 controlnet/reward_control.py \
@@ -15,7 +15,7 @@ accelerate launch --config_file "train/config.yml" \
  --conditioning_image_column="lineart" \
  --resolution=512 \
  --train_batch_size=8 \
- --gradient_accumulation_steps=8 \
+ --gradient_accumulation_steps=4 \
  --learning_rate=1e-5 \
  --mixed_precision="fp16" \
  --gradient_checkpointing \