Pifpaf accuracy debug and model complexity deployment (#349)

I.Pifpaf accuracy debugging 1.Use same loss weight configuration with original repo 2.Use same optimizer with original repo(SGD with nestrov) 3.Use same pretrain backbone(converted from original repo) 4.Use weight-decay in training II.Model deployment complexity measurement added 1.measure_flops.py allows to measure GFlops of converted model in pb format

Author: Gyx-One

export_pb.py

@@ -25,23 +25,25 @@
  type=str,
  default="MSCOCO",
  help="dataset name,to determine which dataset to use, available options: coco ")
- parser.add_argument("--dataset_path",
- type=str,
- default="data",
- help="dataset path,to determine the path to load the dataset")
- parser.add_argument('--train_type',
- type=str,
- default="Single_train",
- help='train type, available options: Single_train, Parallel_train')
- parser.add_argument('--kf_optimizer',
- type=str,
- default='Sma',
- help='kung fu parallel optimizor,available options: Sync_sgd, Async_sgd, Sma')
  parser.add_argument("--output_dir",
  type=str,
  default="save_dir",
  help="which dir to output the exported pb model")
- 
+ parser.add_argument("--export_batch_size",
+ type=int,
+ default=None,
+ help="the expected input image batch_size of the converted model, set to None to support dynamic shape"
+ )
+ parser.add_argument("--export_h",
+ type=int,
+ default=None,
+ help="the expected input image height of the converted model, set to None to support dynamic shape"
+ )
+ parser.add_argument("--export_w",
+ type=int,
+ default=None,
+ help="the expected input image width of the converted model, set to None to support dynamic shape")
+ 
 
  args=parser.parse_args() 
  Config.set_model_name(args.model_name)
@@ -50,6 +52,9 @@
  config=Config.get_config()
  export_model=Model.get_model(config)
 
+ export_batch_size=args.export_batch_size
+ export_h,export_w=args.export_h,args.export_w
+ print(f"export_batch_size:{export_batch_size} export_h:{export_h} export_w:{export_w}")
  input_path=f"{config.model.model_dir}/newest_model.npz"
  output_dir=f"{args.output_dir}/{config.model.model_name}"
  output_path=f"{output_dir}/frozen_{config.model.model_name}.pb"
@@ -64,9 +69,9 @@
  export_model.load_weights(input_path)
  export_model.eval()
  if(export_model.data_format=="channels_last"):
- input_signature=tf.TensorSpec(shape=(None,None,None,3))
+ input_signature=tf.TensorSpec(shape=(export_batch_size,export_h,export_w,3))
  else:
- input_signature=tf.TensorSpec(shape=(None,3,None,None))
+ input_signature=tf.TensorSpec(shape=(export_batch_size,3,export_h,export_w))
  concrete_function=export_model.infer.get_concrete_function(x=input_signature)
  frozen_graph=convert_variables_to_constants_v2(concrete_function)
  frozen_graph_def=frozen_graph.graph.as_graph_def()

hyperpose/Config/config_pifpaf.py

@@ -28,18 +28,17 @@
 train.n_step = 1000000
 # initial learning rate 
 train.lr_init = 1e-4
-# evey number of step to decay lr
-train.lr_decay_every_step = 136120 
 # decay lr factor
 train.lr_decay_factor = 0.2
-train.lr_decay_steps=[420000,630000]
-train.weight_decay_factor = 0.0
+train.lr_decay_steps=[777920,848640]
+train.lr_decay_duration=35360
+train.weight_decay_factor = 1e-5
 train.train_type=TRAIN.Single_train
 train.vis_dir=f"./save_dir/{model.model_name}/train_vis_dir"
 
 #eval configuration
 eval =edict()
-eval.batch_size=8
+eval.batch_size=4
 eval.vis_dir= f"./save_dir/{model.model_name}/eval_vis_dir"
 
 #test configuration

hyperpose/Dataset/mscoco_dataset/dataset.py

@@ -152,7 +152,7 @@ def official_eval(self,pd_anns,eval_dir=f"./eval_dir"):
  json.dump(gt_json,gt_json_file)
  gt_json_file.close()
 
- pd_json_path=f"{eval_dir}/pd_ann.json"
+ pd_json_path=f"{eval_dir}/person_keypoints_val2017_hyperpose_results.json"
  pd_json_file=open(pd_json_path,"w")
  json.dump(pd_anns,pd_json_file)
  pd_json_file.close()
@@ -186,7 +186,7 @@ def official_eval(self,pd_anns,eval_dir=f"./eval_dir"):
 
  def official_test(self,pd_anns,test_dir="./test_dir"):
  server_url="https://competitions.codalab.org/competitions/12061"
- pd_json_path=f"{test_dir}/pd_ann.json"
+ pd_json_path=f"{test_dir}/person_keypoints_test-dev2017_hyperpose_results.json"
  pd_json_file=open(pd_json_path,mode="w")
  json.dump(pd_anns,pd_json_file)
  pd_json_file.close()

hyperpose/Model/__init__.py

@@ -256,14 +256,14 @@ def get_test(config):
  print(f"testing {model_type.name} model...")
  return test
 
-def get_preprocess(model_type):
- '''get preprocess function based model_type
+def get_preprocessor(model_type):
+ '''get a preprocessor class based on the specified model_type
 
- get the preprocess function of the specified kind of model to help user construct thier own train
- and evaluate pipeline rather than using the integrated train or evaluate pipeline directly when in need.
+ get the preprocessor class of the specified kind of model to help user directly construct their own 
+ train pipeline(rather than using the integrated train pipeline) when in need.
 
- the preprocess function is able to convert the image and annotation to the model output format for training
- or evaluation.
+ the preprocessor class is able to construct a preprocessor object that could convert the image and annotation to 
+ the model output format for training.
 
  Parameters
  ----------
@@ -272,23 +272,26 @@ def get_preprocess(model_type):
  
  Returns
  -------
- function
- a preprocess function of the specified kind of model
+ class
+ a preprocessor class of the specified kind of model
  '''
 
  if model_type == MODEL.Openpose or model_type == MODEL.LightweightOpenpose or model_type==MODEL.MobilenetThinOpenpose:
- from .openpose.utils import preprocess
+ from .openpose import PreProcessor
  elif model_type == MODEL.PoseProposal:
- from .pose_proposal.utils import preprocess
- return preprocess
+ from .pose_proposal import PreProcessor
+ elif model_type == MODEL.Pifpaf:
+ from .pifpaf import PreProcessor
+ return PreProcessor
 
-def get_postprocess(model_type):
- '''get postprocess function based model_type
+def get_postprocessor(model_type):
+ '''get a postprocessor class based on the specified model_type
 
- get the postprocess function of the specified kind of model to help user construct thier own 
- evaluate pipeline rather than using the integrated train or evaluate pipeline directly when in need
+ get the postprocessor class of the specified kind of model to help user directly construct their own 
+ evaluate pipeline(rather than using the integrated evaluate pipeline) or infer pipeline(to check the model utility) 
+ when in need.
 
- the postprocess function is able to parse the model output feature map and output parsed human objects of Human class,
+ the postprocessor is able to parse the model output feature map and output parsed human objects of Human class,
  which contains all dectected keypoints.
 
  Parameters
@@ -299,13 +302,15 @@ def get_postprocess(model_type):
  Returns
  -------
  function
- a postprocess function of the specified kind of model
+ a postprocessor class of the specified kind of model
  '''
  if model_type == MODEL.Openpose or model_type == MODEL.LightweightOpenpose or model_type==MODEL.MobilenetThinOpenpose:
- from .openpose.utils import postprocess
+ from .openpose import PostProcessor
  elif model_type == MODEL.PoseProposal:
- from .pose_proposal.utils import postprocess
- return postprocess
+ from .pose_proposal import PostProcessor
+ elif model_type == MODEL.Pifpaf:
+ from .pifpaf import PostProcessor
+ return PostProcessor
 
 def get_visualize(model_type):
  '''get visualize function based model_type

hyperpose/Model/openpose/__init__.py

@@ -1,3 +1,4 @@
 from .model import *
 from .train import *
-from .eval import *
+from .eval import *
+from .processor import PreProcessor,PostProcessor

hyperpose/Model/pifpaf/__init__.py

@@ -1,3 +1,4 @@
 from .model import *
 from .train import *
-from .eval import *
+from .eval import *
+from .processor import PreProcessor,PostProcessor

hyperpose/Model/pifpaf/model.py

@@ -10,8 +10,8 @@
 
 class Pifpaf(Model):
  def __init__(self,parts,limbs,colors=CocoColor,n_pos=17,n_limbs=19,hin=368,win=368,scale_size=32,backbone=None,pretraining=False,quad_size=2,quad_num=1,
- lambda_pif_conf=30.0,lambda_pif_vec=2.0,lambda_pif_scale=2.0,lambda_paf_conf=50.0,lambda_paf_src_vec=3.0,lambda_paf_dst_vec=3.0,
- lambda_paf_src_scale=2.0,lambda_paf_dst_scale=2.0,data_format="channels_first"):
+ lambda_pif_conf=1.0,lambda_pif_vec=1.0,lambda_pif_scale=1.0,lambda_paf_conf=1.0,lambda_paf_src_vec=1.0,lambda_paf_dst_vec=1.0,
+ lambda_paf_src_scale=1.0,lambda_paf_dst_scale=1.0,data_format="channels_first"):
  super().__init__()
  self.parts=parts
  self.limbs=limbs
@@ -65,6 +65,13 @@ def infer(self,x):
  paf_conf,paf_src_vec,paf_dst_vec,_,_,paf_src_scale,paf_dst_scale=paf_maps
  return pif_conf,pif_vec,pif_scale,paf_conf,paf_src_vec,paf_dst_vec,paf_src_scale,paf_dst_scale
 
+ def soft_clamp(self,x,max_value=5.0):
+ above_mask=tf.where(x>=max_value,1.0,0.0)
+ x_below=x*(1-above_mask)
+ x_soft_above=tf.where(x>=max_value,x,max_value)
+ x_above=(max_value+tf.math.log(1+x_soft_above-max_value))*above_mask
+ return x_below+x_above
+ 
  def Bce_loss(self,pd_conf,gt_conf,focal_gamma=1.0):
  #shape conf:[batch,field,h,w]
  batch_size=pd_conf.shape[0]
@@ -75,15 +82,18 @@ def Bce_loss(self,pd_conf,gt_conf,focal_gamma=1.0):
  gt_conf=gt_conf[valid_mask]
  #calculate loss
  bce_loss=tf.nn.sigmoid_cross_entropy_with_logits(logits=pd_conf,labels=gt_conf)
- bce_loss=tf.clip_by_value(bce_loss,0.02,5.0)
+ bce_loss=self.soft_clamp(bce_loss)
  if(focal_gamma!=0.0):
- focal=(1-tf.exp(-bce_loss))**focal_gamma
- focal=tf.stop_gradient(focal)
- bce_loss=focal*bce_loss
+ p=tf.nn.sigmoid(pd_conf)
+ pt=p*gt_conf+(1-p)*(1-gt_conf)
+ focal=1.0-pt
+ if(focal_gamma!=1.0):
+ focal=(focal+1e-4)**focal_gamma
+ bce_loss=focal*bce_loss*0.5
  bce_loss=tf.reduce_sum(bce_loss)/batch_size
  return bce_loss
 
- def Laplace_loss(self,pd_vec,pd_logb,gt_vec):
+ def Laplace_loss(self,pd_vec,pd_logb,gt_vec,gt_bmin):
  #shape vec: [batch,field,2,h,w]
  #shape logb: [batch,field,h,w]
  batch_size=pd_vec.shape[0]
@@ -98,40 +108,46 @@ def Laplace_loss(self,pd_vec,pd_logb,gt_vec):
  gt_vec_x=gt_vec[:,:,0:1,:,:][valid_mask]
  gt_vec_y=gt_vec[:,:,1:2,:,:][valid_mask]
  gt_vec=tf.stack([gt_vec_x,gt_vec_y])
+ #select gt_bmin
+ gt_bmin=gt_bmin[:,:,np.newaxis,:,:][valid_mask]
  #calculate loss
- norm=tf.norm(pd_vec-gt_vec,axis=0)
- norm=tf.clip_by_value(norm,0.0,5.0)
- pd_logb=tf.clip_by_value(pd_logb,-3.0,np.inf)
- laplace_loss=pd_logb+(norm+0.1)*tf.exp(-pd_logb)
+ norm=tf.norm(tf.stack([pd_vec_x-gt_vec_x,pd_vec_y-gt_vec_y,gt_bmin]),axis=0)
+ pd_logb=3.0*tf.tanh(pd_logb/3.0)
+ scaled_norm=norm*tf.exp(-pd_logb)
+ scaled_norm=self.soft_clamp(scaled_norm)
+ laplace_loss=pd_logb+scaled_norm
  laplace_loss=tf.reduce_sum(laplace_loss)/batch_size
  return laplace_loss
 
  def Scale_loss(self,pd_scale,gt_scale,b=1.0):
  batch_size=pd_scale.shape[0]
  valid_mask=tf.logical_not(tf.math.is_nan(gt_scale))
  pd_scale=pd_scale[valid_mask]
+ pd_scale=tf.nn.softplus(pd_scale)
  gt_scale=gt_scale[valid_mask]
  scale_loss=tf.abs(pd_scale-gt_scale)
- scale_loss=tf.clip_by_value(scale_loss,0.0,5.0)/b
+ denominator=10.0*(0.1+gt_scale)
+ scale_loss=scale_loss/denominator
+ scale_loss=self.soft_clamp(scale_loss)
  scale_loss=tf.reduce_sum(scale_loss)/batch_size
  return scale_loss
 
  def cal_loss(self,pd_pif_maps,pd_paf_maps,gt_pif_maps,gt_paf_maps):
  #calculate pif losses
  pd_pif_conf,pd_pif_vec,pd_pif_logb,pd_pif_scale=pd_pif_maps
- gt_pif_conf,gt_pif_vec,gt_pif_scale=gt_pif_maps
+ gt_pif_conf,gt_pif_vec,gt_pif_bmin,gt_pif_scale=gt_pif_maps
  loss_pif_conf=self.Bce_loss(pd_pif_conf,gt_pif_conf)
- loss_pif_vec=self.Laplace_loss(pd_pif_vec,pd_pif_logb,gt_pif_vec)
+ loss_pif_vec=self.Laplace_loss(pd_pif_vec,pd_pif_logb,gt_pif_vec,gt_pif_bmin)
  loss_pif_scale=self.Scale_loss(pd_pif_scale,gt_pif_scale)
  loss_pif_maps=[loss_pif_conf,loss_pif_vec,loss_pif_scale]
  #calculate paf losses
  pd_paf_conf,pd_paf_src_vec,pd_paf_dst_vec,pd_paf_src_logb,pd_paf_dst_logb,pd_paf_src_scale,pd_paf_dst_scale=pd_paf_maps
- gt_paf_conf,gt_paf_src_vec,gt_paf_dst_vec,gt_paf_src_scale,gt_paf_dst_scale=gt_paf_maps
+ gt_paf_conf,gt_paf_src_vec,gt_paf_dst_vec,gt_paf_src_bmin,gt_paf_dst_bmin,gt_paf_src_scale,gt_paf_dst_scale=gt_paf_maps
  loss_paf_conf=self.Bce_loss(pd_paf_conf,gt_paf_conf)
  loss_paf_src_scale=self.Scale_loss(pd_paf_src_scale,gt_paf_src_scale)
  loss_paf_dst_scale=self.Scale_loss(pd_paf_dst_scale,gt_paf_dst_scale)
- loss_paf_src_vec=self.Laplace_loss(pd_paf_src_vec,pd_paf_src_logb,gt_paf_src_vec)
- loss_paf_dst_vec=self.Laplace_loss(pd_paf_dst_vec,pd_paf_dst_logb,gt_paf_dst_vec)
+ loss_paf_src_vec=self.Laplace_loss(pd_paf_src_vec,pd_paf_src_logb,gt_paf_src_vec,gt_paf_src_bmin)
+ loss_paf_dst_vec=self.Laplace_loss(pd_paf_dst_vec,pd_paf_dst_logb,gt_paf_dst_vec,gt_paf_dst_bmin)
  loss_paf_maps=[loss_paf_conf,loss_paf_src_vec,loss_paf_dst_vec,loss_paf_src_scale,loss_paf_dst_scale]
  #calculate total loss
  total_loss=(loss_pif_conf*self.lambda_pif_conf+loss_pif_vec*self.lambda_pif_vec+loss_pif_scale*self.lambda_pif_scale+
@@ -168,7 +184,7 @@ def forward(self,x,is_train=False):
  if(is_train==False):
  infer_pif_conf=tf.nn.sigmoid(pif_conf)
  infer_pif_vec=(pif_vec[:,:]+self.mesh_grid)*self.stride
- infer_pif_scale=pif_scale*self.stride
+ infer_pif_scale=tf.math.softplus(pif_scale)*self.stride
  return infer_pif_conf,infer_pif_vec,pif_logb,infer_pif_scale
  return pif_conf,pif_vec,pif_logb,pif_scale
 
@@ -204,7 +220,7 @@ def forward(self,x,is_train=False):
  infer_paf_conf=tf.nn.sigmoid(paf_conf)
  infer_paf_src_vec=(paf_src_vec[:,:]+self.mesh_grid)*self.stride
  infer_paf_dst_vec=(paf_dst_vec[:,:]+self.mesh_grid)*self.stride
- infer_paf_src_scale=paf_src_scale*self.stride
- infer_paf_dst_scale=paf_dst_scale*self.stride
+ infer_paf_src_scale=tf.math.softplus(paf_src_scale)*self.stride
+ infer_paf_dst_scale=tf.math.softplus(paf_dst_scale)*self.stride
  return infer_paf_conf,infer_paf_src_vec,infer_paf_dst_vec,paf_src_logb,paf_dst_logb,infer_paf_src_scale,infer_paf_dst_scale
  return paf_conf,paf_src_vec,paf_dst_vec,paf_src_logb,paf_dst_logb,paf_src_scale,paf_dst_scale

hyperpose/Model/pifpaf/processor.py

@@ -24,10 +24,10 @@ def __init__(self,parts,limbs,hin,win,hout,wout,colors=None,data_format="channel
 
  def process(self,annos,mask_valid):
  mask_out=cv2.resize(mask_valid,(self.wout,self.hout))
- pif_conf,pif_vec,pif_scale = get_pifmap(annos, mask_out, self.hin, self.win, self.hout, self.wout, self.parts, self.limbs, data_format=self.data_format)
- paf_conf,paf_src_vec,paf_dst_vec,paf_src_scale,paf_dst_scale = get_pafmap(annos, mask_out, self.hin, self.win, self.hout, self.wout, self.parts, self.limbs, data_format=self.data_format)
- pif_maps=[pif_conf,pif_vec,pif_scale]
- paf_maps=[paf_conf,paf_src_vec,paf_dst_vec,paf_src_scale,paf_dst_scale]
+ pif_conf,pif_vec,pif_bmin,pif_scale = get_pifmap(annos, mask_out, self.hin, self.win, self.hout, self.wout, self.parts, self.limbs, data_format=self.data_format)
+ paf_conf,paf_src_vec,paf_dst_vec,paf_src_bmin,paf_dst_bmin,paf_src_scale,paf_dst_scale = get_pafmap(annos, mask_out, self.hin, self.win, self.hout, self.wout, self.parts, self.limbs, data_format=self.data_format)
+ pif_maps=[pif_conf,pif_vec,pif_bmin,pif_scale]
+ paf_maps=[paf_conf,paf_src_vec,paf_dst_vec,paf_src_bmin,paf_dst_bmin,paf_src_scale,paf_dst_scale]
  return pif_maps,paf_maps
 
 class PostProcessor: