added scoring

Author: paulaharder

evaluate.py

@@ -50,6 +50,7 @@ def add_arguments():
  parser.add_argument("--mass_violation", type=bool, default=True)
  parser.add_argument("--factor", type=int, default=2)
  parser.add_argument("--time_sr", default=False)
+ parser.add_argument("--constraints_window_size", default=4, type=int)
 
  return parser.parse_args()
 
@@ -81,7 +82,7 @@ def main(args):
 
  #add_string = args.model_id + '_evaluate_training' 
 
- create_report(scores, args, add_string)
+ #create_report(scores, args, add_string)
  main_scoring(args)
 
 def load_weights(model, model_id):

main.py

@@ -6,7 +6,7 @@
 def add_arguments():
  parser = argparse.ArgumentParser()
  parser.add_argument("--dataset", default='dataset34', help="choose a data set to use")
- parser.add_argument("--scale", default='minmax_fixed', help="standard, minmax, none")
+ parser.add_argument("--scale", default='minmax', help="standard, minmax, none")
  parser.add_argument("--model", default='motifnet_learnable')
  parser.add_argument("--model_id", default='motifbased_exp_ood_softmaxfirst_test')
  parser.add_argument("--number_channels", default=64, type=int)
@@ -34,6 +34,7 @@ def add_arguments():
  parser.add_argument("--test", default=False, type=bool)
  parser.add_argument("--l2_reg", default=False, type=bool)
  parser.add_argument("--dim_channels", default=1, type=int)
+ parser.add_argument("--constraints_window_size", default=4, type=int)
 
 
  return parser.parse_args()

models.py

@@ -43,8 +43,8 @@ def forward(self):
  # (shape[0], shape[1], shape[2])
 
  if self.positivity:
- basis = torch.exp(basis)
- #basis = torch.nn.functional.softplus(basis)
+ #basis = torch.exp(basis)
+ basis = torch.nn.functional.softplus(basis)
 
  # Normalization
  sums = torch.sum(basis, dim=(1, 2))
@@ -364,6 +364,7 @@ def forward(self, x):
  out = out/sum_c
  out = self.mult_in(out, x[:,0,...])
 
+ 
  #print('input', x[0,0,0,0,0])
  #print('after mult', out[0,:,0,0].mean())
  # (n_batch, 16, 32, 32)
@@ -398,7 +399,7 @@ class ResidualBlock(nn.Module):
  def __init__(self, in_channels, out_channels, stride=1, downsample=None):
  super(ResidualBlock, self).__init__()
  self.conv1 = conv3x3(in_channels, out_channels, stride)
- self.relu = nn.ReLU(inplace=False)
+ self.relu = nn.ReLU(inplace=True)
  self.conv2 = conv3x3(out_channels, out_channels)
 
  def forward(self, x):
@@ -444,6 +445,7 @@ def __init__(self, upsampling_factor):
  self.pool = torch.nn.AvgPool2d(kernel_size=upsampling_factor)
  self.upsampling_factor = upsampling_factor
  def forward(self, y, lr):
+ y = y.clone()
  sum_y = self.pool(y)
  out =y+ torch.kron(lr-sum_y, torch.ones((self.upsampling_factor,self.upsampling_factor)).to('cuda'))
  return out
@@ -461,15 +463,18 @@ def forward(self, y, lr):
  return out 
 
 class SoftmaxConstraints(nn.Module):
- def __init__(self, upsampling_factor, exp_factor=1):
+ def __init__(self, upsampling_factor, cwindow_size, exp_factor=1):
  super(SoftmaxConstraints, self).__init__()
- self.pool = torch.nn.AvgPool2d(kernel_size=upsampling_factor)
+ self.pool = torch.nn.AvgPool2d(kernel_size=cwindow_size)
+ self.lr_pool = torch.nn.AvgPool2d(kernel_size=int(cwindow_size/upsampling_factor))
  self.upsampling_factor = upsampling_factor
+ self.cwindow_size = cwindow_size
  self.exp_factor = exp_factor
  def forward(self, y, lr):
  y = torch.exp(y*self.exp_factor)
  sum_y = self.pool(y)
- out = y*torch.kron(lr*1/sum_y, torch.ones((self.upsampling_factor,self.upsampling_factor)).to('cuda'))
+ lr_sum = self.lr_pool(lr)
+ out = y*torch.kron(lr_sum*1/sum_y, torch.ones((self.cwindow_size,self.cwindow_size)).to('cuda'))
  return out
 
 
@@ -624,33 +629,33 @@ def forward(self, x, z=None):
  return out 
 
 class ResNet2(nn.Module):
- def __init__(self, number_channels=64, number_residual_blocks=4, upsampling_factor=2, noise=False, constraints='none', dim=1):
+ def __init__(self, number_channels=64, number_residual_blocks=4, upsampling_factor=2, noise=False, constraints='none', dim=1, cwindow_size=2):
  super(ResNet2, self).__init__()
  # First layer
  if noise:
  self.conv_trans0 = nn.ConvTranspose2d(100, 1, kernel_size=(32,32), padding=0, stride=1)
- self.conv1 = nn.Sequential(nn.Conv2d(dim, number_channels, kernel_size=3, stride=1, padding=1), nn.ReLU(inplace=False))
+ self.conv1 = nn.Sequential(nn.Conv2d(dim, number_channels, kernel_size=3, stride=1, padding=1), nn.ReLU(inplace=True))
  else:
- self.conv1 = nn.Sequential(nn.Conv2d(dim, number_channels, kernel_size=3, stride=1, padding=1), nn.ReLU(inplace=False))
+ self.conv1 = nn.Sequential(nn.Conv2d(dim, number_channels, kernel_size=3, stride=1, padding=1), nn.ReLU(inplace=True))
  #Residual Blocks
  self.res_blocks = nn.ModuleList()
  for k in range(number_residual_blocks):
  self.res_blocks.append(ResidualBlock(number_channels, number_channels))
  # Second conv layer post residual blocks
  self.conv2 = nn.Sequential(
- nn.Conv2d(number_channels, number_channels, kernel_size=3, stride=1, padding=1), nn.ReLU(inplace=False))
+ nn.Conv2d(number_channels, number_channels, kernel_size=3, stride=1, padding=1), nn.ReLU(inplace=True))
  # Upsampling layers
  self.upsampling = nn.ModuleList()
  for k in range(int(np.rint(np.log2(upsampling_factor)))):
  self.upsampling.append(nn.ConvTranspose2d(number_channels, number_channels, kernel_size=2, padding=0, stride=2) )
  # Next layer after upper sampling
- self.conv3 = nn.Sequential(nn.Conv2d(number_channels, number_channels, kernel_size=3, stride=1, padding=1), nn.ReLU(inplace=False))
+ self.conv3 = nn.Sequential(nn.Conv2d(number_channels, number_channels, kernel_size=3, stride=1, padding=1), nn.ReLU(inplace=True))
  # Final output layer
  self.conv4 = nn.Conv2d(number_channels, dim, kernel_size=1, stride=1, padding=0) 
  #optional renomralization layer
  self.is_constraints = False
  if constraints == 'softmax':
- self.constraints = SoftmaxConstraints(upsampling_factor=upsampling_factor)
+ self.constraints = SoftmaxConstraints(upsampling_factor=upsampling_factor, cwindow_size=cwindow_size)
  self.is_constraints = True
  elif constraints == 'enforce_op':
  self.constraints = EnforcementOperator(upsampling_factor=upsampling_factor)

scoring.py

@@ -0,0 +1,147 @@
+from skimage import transform
+import numpy as np
+import torch
+from PIL import Image
+import torch.nn as nn
+import torchgeometry as tgm
+import argparse
+from torch.utils.data import DataLoader, TensorDataset
+import csv
+
+def add_arguments():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--dataset", default='dataset34', help="choose a data set to use")
+ parser.add_argument("--model", default='resnet2_ood_softmax_test')
+ parser.add_argument("--model_id", default='dataset34_resnet2_ood_softmax_test_test')
+ parser.add_argument("--time", default=True)
+ parser.add_argument("--nn", default=True)
+ parser.add_argument("--test", default=True)
+ parser.add_argument("--time_steps", type=int, default=1)
+ parser.add_argument("--mass_violation", type=bool, default=True)
+ parser.add_argument("--factor", type=int, default=4)
+ parser.add_argument("--time_sr", default=False)
+ #args for model loading
+ return parser.parse_args()
+
+def main_scoring(args):
+ #n = 24
+ input_val = torch.load('./data/test/'+args.dataset+'/input_test.pt')
+ target_val = torch.load('./data/test/'+args.dataset+'/target_test.pt')
+ #target_val = torch.load('./data/test/dataset28/target_test.pt')
+ val_data = TensorDataset(input_val, target_val)
+ pred = np.zeros(target_val.shape)
+ print(pred.shape)
+ factor = args.factor
+ mse = 0
+ mae = 0
+ ssim = 0
+ mass_violation = 0
+ l2_crit = nn.MSELoss()
+ l1_crit = nn.L1Loss()
+ ssim_criterion = tgm.losses.SSIM(window_size=11, max_val=130.83, reduction='mean')
+ if args.nn:
+ 
+ pred = torch.load('./data/prediction/'+args.dataset+'_'+args.model_id+'_test.pt')
+ pred = pred.detach().cpu().numpy()
+ print(pred.shape)
+ for i,(lr, hr) in enumerate(val_data):
+ im = lr.numpy()
+ if args.time:
+ #print(hr.shape)
+ for j in range(args.time_steps):
+ if args.model == 'bilinear':
+ pred[i,j,0,:,:] = np.array(Image.fromarray(im[j,0,...]).resize((4*lr.shape[2],4*lr.shape[2]), Image.BILINEAR))
+ elif args.model == 'bicubic':
+ pred[i,j,0,:,:] = np.array(Image.fromarray(im[j,0,...]).resize((factor*lr.shape[2],factor*lr.shape[2]), Image.BICUBIC))
+ elif args.model == 'bicubic_frame':
+ if j == 0:
+ pred[i,j,0,:,:] = np.array(Image.fromarray(im[0,0,...]).resize((4*lr.shape[2],4*lr.shape[2]), Image.BICUBIC))
+ elif j == 2:
+ pred[i,j,0,:,:] = np.array(Image.fromarray(im[1,0,...]).resize((4*lr.shape[2],4*lr.shape[2]), Image.BICUBIC))
+ else:
+ pred[i,j,0,:,:] =np.array(Image.fromarray(0.5*(im[1,0,...]+im[0,0,...])).resize((4*lr.shape[2],4*lr.shape[2]), Image.BICUBIC))
+ elif args.model == 'kronecker':
+ pred[i,j,0,:,:] = np.kron(im[j,0,...], np.ones((4,4)))
+ elif args.model=='frame_inter':
+ print(pred.shape, im.shape)
+ pred[i,j,0,... ] = 0.5*(im[0,0,...]+im[1,0,...])
+ '''
+ mse_loss = l2_crit(torch.Tensor(pred[i,j,...]), hr[j,...]).item()
+ 
+ print(i, mse_loss)'''
+ mse += l2_crit(torch.Tensor(pred[i,j,...]), hr[j,...]).item()
+ mae += l1_crit(torch.Tensor(pred[i,j,...]), hr[j,...]).item()
+ ssim += ssim_criterion(torch.Tensor(pred[i,j:j+1,...]), hr[j:j+1,...]).item()
+ if args.mass_violation:
+ if args.time_sr:
+ if j==0:
+ mass_violation += np.mean( np.abs(transform.downscale_local_mean(pred[i,j,...], (1,args.factor,args.factor)) -im[0,...]))
+ elif j==2:
+ mass_violation += np.mean( np.abs(transform.downscale_local_mean(pred[i,j,...], (1,args.factor,args.factor)) -im[1,...]))
+ else:
+ mass_violation += np.mean( np.abs(transform.downscale_local_mean(pred[i,j,...], (1,args.factor,args.factor)) -im[j,...]))
+ #print(l2_crit(torch.Tensor(pred[i,j,...]), hr[j,...]).item())
+ elif args.model=='frame_inter':
+ pred[i,... ] = 0.5*(im[0:1,...]+im[1:2,...])
+ mse += l2_crit(torch.Tensor(pred[i,...]), hr).item()
+ mae += l1_crit(torch.Tensor(pred[i,...]), hr).item()
+ ssim += ssim_criterion(torch.Tensor(pred[i,...]), hr).item()
+ #print(l1_crit(torch.Tensor(pred[i,...]), hr).item())
+ else:
+ if args.model == 'bilinear':
+ pred[i,:,:] = np.array(Image.fromarray(im).resize((4*lr.shape[1],4*lr.shape[1]), Image.BILINEAR))
+ elif args.model == 'bicubic':
+ pred[i,:,:] = np.array(Image.fromarray(im).resize((4*lr.shape[1],4*lr.shape[1]), Image.BICUBIC))
+ #elif args.model == 'kronecker':
+ 
+ mse += l2_crit(torch.Tensor(pred[i,:,:]), hr).item()
+ mae += l1_crit(torch.Tensor(pred[i,:,:]), hr).item()
+ ssim += ssim_criterion(torch.Tensor(pred[i,:,:]).unsqueeze(0), hr.unsqueeze(0)).item()
+ 
+ 
+ #torch.save(torch.Tensor(pred[:128,:,:]).unsqueeze(1), './data/prediction/'+args.dataset+'_'+args.model_id+'_prediction.pt')
+ if args.time:
+ print(input_val.shape[0])
+ mse *= 1/(input_val.shape[0]*args.time_steps)
+ mae *= 1/(input_val.shape[0]*args.time_steps)
+ ssim *= 1/(input_val.shape[0]*args.time_steps)
+ if args.mass_violation:
+ if args.time_sr:
+ mass_violation *= 1/(input_val.shape[0]*args.time_steps)
+ else:
+ mass_violation *= 1/(input_val.shape[0]*2)
+ else:
+ mse *= 1/input_val.shape[0] 
+ mae *= 1/input_val.shape[0] 
+ ssim *= 1/input_val.shape[0] 
+ psnr = calculate_pnsr(mse, target_val.max() ) 
+ scores = {'MSE':mse, 'RMSE':torch.sqrt(torch.Tensor([mse])), 'PSNR': psnr, 'MAE':mae, 'SSIM':1-ssim, 'Mass_violation': mass_violation}
+ print(scores)
+ create_report(scores, args)
+ #np.save('./data/prediction/bic.npy', pred)
+ 
+ 
+def calculate_pnsr(mse, max_val):
+ return 20 * torch.log10(max_val / torch.sqrt(torch.Tensor([mse])))
+ 
+def create_report(scores, args):
+ args_dict = args_to_dict(args)
+ #combine scorees and args dict
+ args_scores_dict = args_dict | scores
+ #save dict
+ save_dict(args_scores_dict, args)
+ 
+def args_to_dict(args):
+ return vars(args)
+ 
+ 
+def save_dict(dictionary, args):
+ w = csv.writer(open('./data/score_log/'+args.model_id+'.csv', 'w'))
+ # loop over dictionary keys and values
+ for key, val in dictionary.items():
+ # write every key and value to file
+ w.writerow([key, val])
+ 
+if __name__ == '__main__':
+ args = add_arguments()
+ main_scoring(args)

training.py

@@ -121,7 +121,7 @@ def optimizer_step(model, optimizer, criterion, inputs, targets, tepoch, args, c
  else:
  outputs = model(inputs)
  #print(outputs.shape, targets.shape)
- loss = get_loss(outputs, targets, args)#criterion(outputs, targets)
+ loss = get_loss(outputs, targets, inputs,args)#criterion(outputs, targets)
  loss.backward()
  optimizer.step() 
  #print(torch.mean((outputs-targets)**2))
@@ -254,7 +254,7 @@ def validate_model(model, criterion, data, best, patience, epoch, args, discrimi
  outputs, coeff = model(inputs)
  else:
  outputs = model(inputs)
- loss = get_loss(outputs, targets, args) 
+ loss = get_loss(outputs, targets, inputs, args) 
  #print(torch.mean((outputs-targets)**2))
  running_loss += loss.item()
  #print('val:', loss.item())

utils.py

@@ -15,10 +15,11 @@ def load_data(args):
  if args.test:
  input_val = torch.load('./data/test/'+args.dataset+'/input_test.pt')
  target_val = torch.load('./data/test/'+args.dataset+'/target_test.pt')
+ #target_val = torch.load('./data/val/dataset28/target_val.pt')
  else:
  input_val = torch.load('./data/val/'+args.dataset+'/input_val.pt')
+ #target_val = torch.load('./data/val/dataset28/target_val.pt')
  target_val = torch.load('./data/val/'+args.dataset+'/target_val.pt')
- #target_val = torch.load('./data/val/'+args.dataset+'/target_val.pt')
  #define dimesions
  global train_shape_in , train_shape_out, val_shape_in, val_shape_in
  train_shape_in = input_train.shape
@@ -125,8 +126,8 @@ def load_model(args, discriminator=False):
  model = models.MixtureModel(number_channels=args.number_channels, number_residual_blocks=args.number_residual_blocks, upsampling_factor=args.upsampling_factor, noise=args.noise, constraints=args.constraints, dim=1)
  elif args.model == 'gan':
  model = models.ResNet2(number_channels=args.number_channels, number_residual_blocks=args.number_residual_blocks, upsampling_factor=args.upsampling_factor, noise=args.noise, constraints=args.constraints, dim=args.dim_channels)
- else:
- model = models.ResNet2(number_channels=args.number_channels, number_residual_blocks=args.number_residual_blocks, upsampling_factor=args.upsampling_factor, noise=args.noise, constraints=args.constraints, dim=args.dim_channels)
+ elif args.model == 'resnet2':
+ model = models.ResNet2(number_channels=args.number_channels, number_residual_blocks=args.number_residual_blocks, upsampling_factor=args.upsampling_factor, noise=args.noise, constraints=args.constraints, dim=args.dim_channels, cwindow_size= args.constraints_window_size)
  model.to(device)
  return model
 
@@ -141,14 +142,14 @@ def get_criterion(args, discriminator=False):
  criterion = nn.MSELoss()
  return criterion
 
-def mass_loss(output, true_value, args):
+def mass_loss(output, in_val, args):
  ds_out = torch.nn.functional.avg_pool2d(output[:,0,0,:,:], args.upsampling_factor)
- ds_true = torch.nn.functional.avg_pool2d(true_value[:,0,0,:,:], args.upsampling_factor)
- return torch.nn.functional.mse_loss(ds_out, ds_true)
+ #ds_true = torch.nn.functional.avg_pool2d(true_value[:,0,0,:,:], args.upsampling_factor)
+ return torch.nn.functional.mse_loss(ds_out, in_val)
 
-def get_loss(output, true_value, args):
+def get_loss(output, true_value, in_val, args):
  if args.loss == 'mass_constraints':
- return args.alpha*torch.nn.functional.mse_loss(output, true_value) + (1-args.alpha)*mass_loss(output, true_value, args)
+ return args.alpha*torch.nn.functional.mse_loss(output, true_value) + (1-args.alpha)*mass_loss(output, in_val[:,0,0,...], args)
  else:
  return torch.nn.functional.mse_loss(output, true_value)