Pifpaf Accuracy+Model export+Pypi setup (#354)

(I)Pifpaf Accuracy debug: (1)pixel shuffle reimplemented (2)input data normalized by mean and std (3)resnet backbone corrected (4)output scale activation corrected (II)Model exportion support (1)Tflite exportion supported (III)Pypi setup support (1)pip install hyperpose python interface supported

Author: Gyx-One

.gitignore

@@ -38,7 +38,14 @@ libstdtensor-prefix
 libstdtracer-prefix
 Makefile
 test.py
+temp.py
 save_dir
+test_dir
+README.rst
+dist
+build
+*.egg*
+*whl*
 Testing
 third_party
 venv

export_tflite.py

@@ -0,0 +1,46 @@
+import pathlib
+import tensorflow as tf
+from functools import partial
+from hyperpose import Config,Model,Dataset
+
+#load model weights from hyperpose
+Config.set_model_name("new_pifpaf")
+Config.set_model_type(Config.MODEL.Pifpaf)
+Config.set_dataset_type(Config.DATA.MSCOCO)
+config=Config.get_config()
+model=Model.get_model(config)
+model.load_weights(f"{config.model.model_dir}/newest_model.npz")
+model.eval()
+#construct representative dataset used for quantization(here using the first 100 validate images)
+scale_image_func=partial(Model.common.scale_image,hin=model.hin,win=model.win,scale_rate=0.95)
+def decode_image(image_file,image_id):
+ image = tf.io.read_file(image_file)
+ image = tf.image.decode_jpeg(image, channels=3) # get RGB with 0~1
+ image = tf.image.convert_image_dtype(image, dtype=tf.float32)
+ scaled_image,pad = tf.py_function(scale_image_func,[image],[tf.float32,tf.float32])
+ return scaled_image
+dataset=Dataset.get_dataset(config)
+val_dataset=dataset.get_eval_dataset()
+rep_dataset=val_dataset.enumerate()
+rep_dataset=rep_dataset.filter(lambda i,image_data : i<=100)
+rep_dataset=rep_dataset.map(lambda i,image_data: image_data)
+rep_dataset=rep_dataset.map(decode_image).batch(1)
+print(f"test rep_dataset:{rep_dataset}")
+#covert to tf-lite using int8-only quantization
+input_signature=tf.TensorSpec(shape=(None,3,None,None))
+converter=tf.lite.TFLiteConverter.from_concrete_functions([model.infer.get_concrete_function(x=input_signature)])
+converter.optimizations=[tf.lite.Optimize.DEFAULT]
+converter.representative_dataset=rep_dataset
+converter.target_spec.supported_ops=[tf.lite.OpsSet.TFLITE_BUILTINS_INT8]
+converter.inference_input_type = tf.uint8
+converter.inference_output_type = tf.uint8
+tflite_model_quant = converter.convert()
+print("model quantized using uint8 quantization!")
+#save the converted quantization model
+save_path=f"./save_dir/{config.model.model_name}.tflite"
+tf.io.write_file(save_path,tflite_model_quant)
+#print(f"export tflite file finished! output file: {save_path}")
+
+
+
+

hyperpose/Model/backbones.py

@@ -508,15 +508,15 @@ def __init__(self,in_channels=64,n_filter=64,strides=(1,1),data_format="channels
  self.downsample=LayerList([
  Conv2d(n_filter=4*self.n_filter,in_channels=self.in_channels,filter_size=(1,1),strides=self.strides,b_init=None,\
  data_format=self.data_format,name=f"{name}_ds_conv1"),
- BatchNorm2d(decay=self.decay,epsilon=self.eps,is_train=True,num_features=4*self.n_filter,data_format=self.data_format,name=f"{name}_ds_bn1")
+ BatchNorm2d(decay=self.decay,epsilon=self.eps,is_train=True,num_features=4*self.n_filter,act=None,data_format=self.data_format,name=f"{name}_ds_bn1")
  ])
  self.main_block=LayerList([
  Conv2d(n_filter=self.n_filter,in_channels=self.in_channels,filter_size=(1,1),strides=(1,1),b_init=None,data_format=self.data_format,name=f"{name}_conv1"),
- BatchNorm2d(decay=self.decay,epsilon=self.eps,is_train=True,num_features=self.n_filter,act=None,data_format=self.data_format,name=f"{name}_bn1"),
+ BatchNorm2d(decay=self.decay,epsilon=self.eps,is_train=True,num_features=self.n_filter,act=tf.nn.relu,data_format=self.data_format,name=f"{name}_bn1"),
  Conv2d(n_filter=self.n_filter,in_channels=self.n_filter,filter_size=(3,3),strides=self.strides,b_init=None,data_format=self.data_format,name=f"{name}_conv2"),
- BatchNorm2d(decay=self.decay,epsilon=self.eps,is_train=True,num_features=self.n_filter,act=None,data_format=self.data_format,name=f"{name}_bn2"),
+ BatchNorm2d(decay=self.decay,epsilon=self.eps,is_train=True,num_features=self.n_filter,act=tf.nn.relu,data_format=self.data_format,name=f"{name}_bn2"),
  Conv2d(n_filter=4*self.n_filter,in_channels=self.n_filter,filter_size=(1,1),strides=(1,1),b_init=None,data_format=self.data_format,name=f"{name}_conv3"),
- BatchNorm2d(decay=self.decay,epsilon=self.eps,is_train=True,num_features=4*self.n_filter,act=tf.nn.relu,data_format=self.data_format,name=f"{name}_bn3")
+ BatchNorm2d(decay=self.decay,epsilon=self.eps,is_train=True,num_features=4*self.n_filter,act=None,data_format=self.data_format,name=f"{name}_bn3")
  ])
 
  def forward(self,x):

hyperpose/Model/common.py

@@ -275,6 +275,15 @@ def pad_image_shape(img,shape,pad_value=0.0):
  padded_image[pad[0]:img_h+pad[0],pad[2]:img_w+pad[2],:]=img
  return padded_image,pad
 
+def scale_image(image,hin,win,scale_rate=0.95):
+ #scale a image into the size of scale_rate*hin and scale_rate*win
+ #used for model inferecne
+ image_h,image_w,_=image.shape
+ scale_h,scale_w=int(scale_rate*image_h),int(scale_rate*image_w)
+ scale_image=cv2.resize(image,(scale_w,scale_h),interpolation=cv2.INTER_CUBIC)
+ padded_image,pad=pad_image_shape(scale_image,shape=(hin,win),pad_value=0.0)
+ return padded_image,pad
+
 def get_optim(optim_type):
  if(optim_type==OPTIM.Adam):
  print("using optimizer Adam!")

hyperpose/Model/pifpaf/eval.py

@@ -95,11 +95,20 @@ def visualize(img,img_id,processed_img,pd_pif_maps,pd_paf_maps,humans,stride=8,s
  plt.savefig(os.path.join(save_dir,f"{img_id}_visualize.png"))
  plt.close()
 
-def _map_fn(image_file,image_id,hin,win):
+def normalize(image):
+ #normalize
+ mean=np.array([0.485, 0.456, 0.406])[np.newaxis,np.newaxis,:]
+ std=np.array([0.229, 0.224, 0.225])[np.newaxis,np.newaxis,:]
+ image=(image-mean)/std
+ return image
+
+def _map_fn(image_file,image_id):
  #load data
  image = tf.io.read_file(image_file)
  image = tf.image.decode_jpeg(image, channels=3) # get RGB with 0~1
  image = tf.image.convert_image_dtype(image, dtype=tf.float32)
+ #normalize
+ image=tf.py_function(normalize,[image],tf.float32)
  return image,image_id
 
 def evaluate(model,dataset,config,vis_num=30,total_eval_num=10000,enable_multiscale_search=False):
@@ -131,7 +140,7 @@ def evaluate(model,dataset,config,vis_num=30,total_eval_num=10000,enable_multisc
  None
  '''
  print(f"enable multiscale_search:{enable_multiscale_search}")
- model.load_weights(os.path.join(config.model.model_dir,"newest_model.npz"))
+ model.load_weights(os.path.join(config.model.model_dir,"newest_model.npz"),format="npz_dict")
  model.eval()
  pd_anns=[]
  vis_dir=config.eval.vis_dir
@@ -141,7 +150,7 @@ def evaluate(model,dataset,config,vis_num=30,total_eval_num=10000,enable_multisc
 
  eval_dataset=dataset.get_eval_dataset()
  dataset_size=dataset.get_eval_datasize()
- paramed_map_fn=partial(_map_fn,hin=model.hin,win=model.win)
+ paramed_map_fn=_map_fn
  eval_dataset=eval_dataset.map(paramed_map_fn,num_parallel_calls=max(multiprocessing.cpu_count()//2,1))
  for eval_num,(img,img_id) in enumerate(eval_dataset):
  img_id=img_id.numpy()
@@ -199,7 +208,7 @@ def test(model,dataset,config,vis_num=30,total_test_num=10000,enable_multiscale_
  None
  '''
  print(f"enable multiscale_search:{enable_multiscale_search}")
- model.load_weights(os.path.join(config.model.model_dir,"newest_model.npz"))
+ model.load_weights(os.path.join(config.model.model_dir,"newest_model.npz"),format="npz_dict")
  model.eval()
  pd_anns=[]
  vis_dir=config.test.vis_dir
@@ -209,7 +218,7 @@ def test(model,dataset,config,vis_num=30,total_test_num=10000,enable_multiscale_
 
  test_dataset=dataset.get_test_dataset()
  dataset_size=dataset.get_test_datasize()
- paramed_map_fn=partial(_map_fn,hin=model.hin,win=model.win)
+ paramed_map_fn=_map_fn
  test_dataset=test_dataset.map(paramed_map_fn,num_parallel_calls=max(multiprocessing.cpu_count()//2,1))
  for test_num,(img,img_id) in enumerate(test_dataset):
  img_id=img_id.numpy()

hyperpose/Model/pifpaf/model.py

@@ -5,6 +5,7 @@
 from tensorlayer.models import Model
 from tensorlayer.layers import BatchNorm2d, Conv2d, DepthwiseConv2d, LayerList, MaxPool2d
 from .define import CocoColor
+from .utils import pixel_shuffle,get_meshgrid
 from ..backbones import Resnet50_backbone
 
 
@@ -40,30 +41,27 @@ def __init__(self,parts,limbs,colors=CocoColor,n_pos=17,n_limbs=19,hin=368,win=3
  self.backbone=backbone(data_format=data_format,scale_size=self.scale_size)
  self.hout=int(hin/self.stride)
  self.wout=int(win/self.stride)
- #generate mesh grid
- x_range=np.linspace(start=0,stop=self.wout-1,num=self.wout)
- y_range=np.linspace(start=0,stop=self.hout-1,num=self.hout)
- mesh_x,mesh_y=np.meshgrid(x_range,y_range)
- self.mesh_grid=np.stack([mesh_x,mesh_y])
  #construct head
  self.pif_head=self.PifHead(input_features=self.backbone.out_channels,n_pos=self.n_pos,n_limbs=self.n_limbs,\
- quad_size=self.quad_size,hout=self.hout,wout=self.wout,stride=self.stride,mesh_grid=self.mesh_grid,data_format=self.data_format)
+ quad_size=self.quad_size,hout=self.hout,wout=self.wout,stride=self.stride,data_format=self.data_format)
  self.paf_head=self.PafHead(input_features=self.backbone.out_channels,n_pos=self.n_pos,n_limbs=self.n_limbs,\
- quad_size=self.quad_size,hout=self.hout,wout=self.wout,stride=self.stride,mesh_grid=self.mesh_grid,data_format=self.data_format)
+ quad_size=self.quad_size,hout=self.hout,wout=self.wout,stride=self.stride,data_format=self.data_format)
 
- @tf.function(experimental_relax_shapes=True)
- def forward(self,x,is_train=False):
- x=self.backbone.forward(x)
- pif_maps=self.pif_head.forward(x,is_train=is_train)
- paf_maps=self.paf_head.forward(x,is_train=is_train)
+# @tf.function(experimental_relax_shapes=True)
+ def forward(self,x,is_train=False,ret_backbone=False):
+ backbone_x=self.backbone.forward(x)
+ pif_maps=self.pif_head.forward(backbone_x,is_train=is_train)
+ paf_maps=self.paf_head.forward(backbone_x,is_train=is_train)
+ if(ret_backbone):
+ return pif_maps,paf_maps,backbone_x
  return pif_maps,paf_maps
 
- @tf.function(experimental_relax_shapes=True)
+# @tf.function(experimental_relax_shapes=True)
  def infer(self,x):
- pif_maps,paf_maps=self.forward(x,is_train=False)
+ pif_maps,paf_maps,backbone_x=self.forward(x,is_train=False,ret_backbone=True)
  pif_conf,pif_vec,_,pif_scale=pif_maps
  paf_conf,paf_src_vec,paf_dst_vec,_,_,paf_src_scale,paf_dst_scale=paf_maps
- return pif_conf,pif_vec,pif_scale,paf_conf,paf_src_vec,paf_dst_vec,paf_src_scale,paf_dst_scale
+ return pif_conf,pif_vec,pif_scale,paf_conf,paf_src_vec,paf_dst_vec,paf_src_scale,paf_dst_scale,backbone_x
 
  def soft_clamp(self,x,max_value=5.0):
  above_mask=tf.where(x>=max_value,1.0,0.0)
@@ -157,7 +155,7 @@ def cal_loss(self,pd_pif_maps,pd_paf_maps,gt_pif_maps,gt_paf_maps):
  return loss_pif_maps,loss_paf_maps,total_loss
 
  class PifHead(Model):
- def __init__(self,input_features=2048,n_pos=19,n_limbs=19,quad_size=2,hout=8,wout=8,stride=8,mesh_grid=None,data_format="channels_first"):
+ def __init__(self,input_features=2048,n_pos=19,n_limbs=19,quad_size=2,hout=8,wout=8,stride=8,data_format="channels_first"):
  super().__init__()
  self.input_features=input_features
  self.n_pos=n_pos
@@ -167,29 +165,32 @@ def __init__(self,input_features=2048,n_pos=19,n_limbs=19,quad_size=2,hout=8,wou
  self.stride=stride
  self.quad_size=quad_size
  self.out_features=self.n_pos*5*(self.quad_size**2)
- self.mesh_grid=mesh_grid
  self.data_format=data_format
  self.tf_data_format="NCHW" if self.data_format=="channels_first" else "NHWC"
  self.main_block=Conv2d(n_filter=self.out_features,in_channels=self.input_features,filter_size=(1,1),data_format=self.data_format)
 
  def forward(self,x,is_train=False):
  x=self.main_block.forward(x)
- x=tf.nn.depth_to_space(x,block_size=self.quad_size,data_format=self.tf_data_format)
- x=tf.reshape(x,[-1,self.n_pos,5,self.hout,self.wout])
+ x=pixel_shuffle(x,scale=2)
+ low_cut=int((self.quad_size-1)//2)
+ high_cut=int(tf.math.ceil((self.quad_size-1)/2.0))
+ hout,wout=x.shape[2],x.shape[3]
+ x=tf.reshape(x,[-1,self.n_pos,5,hout,wout])
  pif_conf=x[:,:,0,:,:]
  pif_vec=x[:,:,1:3,:,:]
  pif_logb=x[:,:,3,:,:]
- pif_scale=tf.exp(x[:,:,4,:,:])
+ pif_scale=x[:,:,4,:,:]
  #restore vec_maps in inference
  if(is_train==False):
+ mesh_grid=get_meshgrid(mesh_h=hout,mesh_w=wout)+np.array([1.5,1.5])[:,np.newaxis,np.newaxis]
  infer_pif_conf=tf.nn.sigmoid(pif_conf)
- infer_pif_vec=(pif_vec[:,:]+self.mesh_grid)*self.stride
+ infer_pif_vec=(pif_vec[:,:]+mesh_grid)*self.stride
  infer_pif_scale=tf.math.softplus(pif_scale)*self.stride
  return infer_pif_conf,infer_pif_vec,pif_logb,infer_pif_scale
  return pif_conf,pif_vec,pif_logb,pif_scale
 
  class PafHead(Model):
- def __init__(self,input_features=2048,n_pos=19,n_limbs=19,quad_size=2,hout=46,wout=46,stride=8,mesh_grid=None,data_format="channels_first"):
+ def __init__(self,input_features=2048,n_pos=19,n_limbs=19,quad_size=2,hout=46,wout=46,stride=8,data_format="channels_first"):
  super().__init__()
  self.input_features=input_features
  self.n_pos=n_pos
@@ -199,27 +200,30 @@ def __init__(self,input_features=2048,n_pos=19,n_limbs=19,quad_size=2,hout=46,wo
  self.wout=wout
  self.stride=stride
  self.out_features=self.n_limbs*9*(self.quad_size**2)
- self.mesh_grid=mesh_grid
  self.data_format=data_format
  self.tf_data_format="NCHW" if self.data_format=="channels_first" else "NHWC"
  self.main_block=Conv2d(n_filter=self.out_features,in_channels=self.input_features,filter_size=(1,1),data_format=self.data_format)
 
  def forward(self,x,is_train=False):
  x=self.main_block.forward(x)
- x=tf.nn.depth_to_space(x,block_size=self.quad_size,data_format=self.tf_data_format)
- x=tf.reshape(x,[-1,self.n_limbs,9,self.hout,self.wout])
+ x=pixel_shuffle(x,scale=2)
+ low_cut=int((self.quad_size-1)//2)
+ high_cut=int(tf.math.ceil((self.quad_size-1)/2.0))
+ hout,wout=x.shape[2],x.shape[3]
+ x=tf.reshape(x,[-1,self.n_limbs,9,hout,wout])
  paf_conf=x[:,:,0,:,:]
  paf_src_vec=x[:,:,1:3,:,:]
  paf_dst_vec=x[:,:,3:5,:,:]
  paf_src_logb=x[:,:,5,:,:]
  paf_dst_logb=x[:,:,6,:,:]
- paf_src_scale=tf.exp(x[:,:,7,:,:])
- paf_dst_scale=tf.exp(x[:,:,8,:,:])
+ paf_src_scale=x[:,:,7,:,:]
+ paf_dst_scale=x[:,:,8,:,:]
  #restore vec_maps in inference
  if(is_train==False):
+ mesh_grid=get_meshgrid(mesh_h=hout,mesh_w=wout)+np.array([1.5,1.5])[:,np.newaxis,np.newaxis]
  infer_paf_conf=tf.nn.sigmoid(paf_conf)
- infer_paf_src_vec=(paf_src_vec[:,:]+self.mesh_grid)*self.stride
- infer_paf_dst_vec=(paf_dst_vec[:,:]+self.mesh_grid)*self.stride
+ infer_paf_src_vec=(paf_src_vec[:,:]+mesh_grid)*self.stride
+ infer_paf_dst_vec=(paf_dst_vec[:,:]+mesh_grid)*self.stride
  infer_paf_src_scale=tf.math.softplus(paf_src_scale)*self.stride
  infer_paf_dst_scale=tf.math.softplus(paf_dst_scale)*self.stride
  return infer_paf_conf,infer_paf_src_vec,infer_paf_dst_vec,paf_src_logb,paf_dst_logb,infer_paf_src_scale,infer_paf_dst_scale

hyperpose/Model/pifpaf/processor.py

@@ -1,4 +1,5 @@
 
+from operator import pos
 import os
 import cv2
 import json
@@ -31,8 +32,8 @@ def process(self,annos,mask_valid):
  return pif_maps,paf_maps
 
 class PostProcessor:
- def __init__(self,parts,limbs,hin,win,hout,wout,colors=None,thresh_pif=0.1,thresh_paf=0.1,thresh_ref_pif=0.1,thresh_ref_paf=0.1,\
- part_num_thresh=4,score_thresh=0.1,reduction=2,min_scale=4,greedy_match=True,reverse_match=True,data_format="channels_first",debug=False):
+ def __init__(self,parts,limbs,hin,win,hout,wout,colors=None,thresh_pif=0.3,thresh_paf=0.1,thresh_ref_pif=0.3,thresh_ref_paf=0.1,\
+ thresh_gen_ref_pif=0.1,part_num_thresh=4,score_thresh=0.1,reduction=2,min_scale=4,greedy_match=True,reverse_match=True,data_format="channels_first",debug=False):
  self.parts=parts
  self.limbs=limbs
  self.colors=colors if (colors!=None) else (len(self.parts)*[[0,255,0]])
@@ -47,6 +48,7 @@ def __init__(self,parts,limbs,hin,win,hout,wout,colors=None,thresh_pif=0.1,thres
  self.thresh_paf=thresh_paf
  self.thresh_ref_pif=thresh_ref_pif
  self.thresh_ref_paf=thresh_ref_paf
+ self.thresh_gen_ref_pif=thresh_gen_ref_pif
  self.part_num_thresh=part_num_thresh
  self.score_thresh=score_thresh
  self.reduction=reduction
@@ -71,7 +73,7 @@ def process(self,pif_maps,paf_maps):
  pif_conf,pif_vec,_,pif_scale=pif_maps
  paf_conf,paf_src_vec,paf_dst_vec,_,_,paf_src_scale,paf_dst_scale=paf_maps
  #get pif_hr_conf
- pif_hr_conf=get_hr_conf(pif_conf,pif_vec,pif_scale,stride=self.stride,thresh=self.thresh_pif,debug=False)
+ pif_hr_conf=get_hr_conf(pif_conf,pif_vec,pif_scale,stride=self.stride,thresh=self.thresh_gen_ref_pif,debug=False)
  self.debug_print(f"test hr_conf")
  for pos_idx in range(0,self.n_pos):
  self.debug_print(f"test hr_conf idx:{pos_idx} max_conf:{np.max(pif_conf[pos_idx])} max_hr_conf:{np.max(pif_hr_conf[pos_idx])}")
@@ -88,7 +90,7 @@ def process(self,pif_maps,paf_maps):
  mask_ref_conf=ref_cs>self.thresh_ref_pif
  for ref_c,x,y,scale in zip(ref_cs[mask_ref_conf],xs[mask_ref_conf],ys[mask_ref_conf],scales[mask_ref_conf]):
  seeds.append((ref_c,pos_idx,x,y,scale))
- self.debug_print(f"seed gen pos_idx:{pos_idx} ref_c:{ref_c} x:{x} y:{y} scale:{scale}")
+ #print(f"seed gen pos_idx:{pos_idx} ref_c:{ref_c} x:{x} y:{y} scale:{scale}")
  self.debug_print(f"test before sort len_seeds:{len(seeds)}")
  seeds=sorted(seeds,reverse=True)
  self.debug_print(f"test after sort len_seeds:{len(seeds)}")
@@ -124,6 +126,7 @@ def process(self,pif_maps,paf_maps):
  self.debug_print(f"test fw_list_gen: limb_idx:{limb_idx} max_score:{np.max(score)} max_cifhr_f:{np.max(cifhr_f)} max_score_f:{np.max(score_f)} mask_num_f:{np.sum(mask_f)}")
  self.debug_print("")
  #greedy assemble
+ #TODO: further check!
  occupied=np.zeros(shape=(self.n_pos,int(pif_hr_conf.shape[1]/self.reduction),int(pif_hr_conf.shape[2]/self.reduction)))
  annotations=[]
  self.debug_print(f"test seeds_num:{len(seeds)}")
@@ -302,7 +305,7 @@ def get_connection(self,ann,src_idx,dst_idx,forward_list,backward_list,connectio
  def grow(self,ann,forward_list,backward_list,reverse_match=True):
  frontier = []
  in_frontier = set()
- #add the point to assemble frontierby_source
+ #add the point to assemble frontier by_source
  def add_frontier(ann,src_idx):
  #traverse all the part that the current part connect to
  for dst_idx,(_,_) in self.by_source[src_idx].items():