Updates and bugfix:

- (Major update) Macro-Precision/Recall/F1 score calculations are added into "ner_scorer.py". - (Major update) Trainer function call parameter is shrinked. Instead of giving train/vali/test iterators, I changed it to single "dataloader". - (Major update) Since "macro-x" metrics must be calculated for whole dataset-batch (not batch-based), I changed "single_model_ner_evaluator.py". - (Minor update) A detailed logger is added into "ner_scorer.py". - (Major bugfix) LSTM hidden initialization is changed for "bidirectional=True" case.

Author: Aunavin1

.idea/codeStyles/codeStyleConfig.xml

@@ -98,9 +98,9 @@
  "lstm": {
  "dropout_type": "bernoulli",
  "keep_prob": 0.5,
- "hidden_dim": 300,
- "num_layers": 1,
- "bidirectional": false,
+ "hidden_dim": 100,
+ "num_layers": 2,
+ "bidirectional": true,
  "bias": true
  },
  "transformer_google": {
@@ -127,8 +127,8 @@
  "partial_adam": 0.25,
  "weight_decay": 0,
  "momentum": 0.9,
- "norm_ratio": 0.25,
- "epoch": 20,
+ "norm_ratio": 5,
+ "epoch": 10,
  "print_every_batch_step": 250,
  "save_every_epoch": 1,
  "topk": [

config/config.json

@@ -21,10 +21,10 @@ def __init__(self, args):
  # Matrix of transition parameters. Entry i,j is the score of transitioning *to* i *from* j
  self.transition = nn.Parameter(torch.Tensor(self.tag_size, self.tag_size)).to(self.device)
 
- self.transition.data[self.start_id, :] = -10000. # no transition to SOS
- self.transition.data[:, self.end_id] = -10000. # no transition from EOS except to PAD
- self.transition.data[:, self.pad_id] = -10000. # no transition from PAD except to PAD
- self.transition.data[self.pad_id, :] = -10000. # no transition to PAD except from EOS
+ self.transition.data[self.start_id, :] = -10000. # no transition to start
+ self.transition.data[:, self.end_id] = -10000. # no transition from end except to pad
+ self.transition.data[:, self.pad_id] = -10000. # no transition from pad except to pad
+ self.transition.data[self.pad_id, :] = -10000. # no transition to pad except from end
  self.transition.data[self.pad_id, self.end_id] = 0.
  self.transition.data[self.pad_id, self.pad_id] = 0.
 

crf/CRF.py

@@ -17,6 +17,7 @@ def evaluator_factory(type, device):
  test_evaluator = SingleModelEvaluator(device, is_vali=False)
  return dev_evaluator, test_evaluator
  elif type == "single_model_ner_evaluator":
+ logger.info("Evaluator type is %s", type)
  dev_evaluator = SingleModelNerEvaluator(device, is_vali=True)
  test_evaluator = SingleModelNerEvaluator(device, is_vali=False)
  return dev_evaluator, test_evaluator

evaluation/evaluator.py

@@ -15,9 +15,11 @@ def __init__(self, device, is_vali):
  self.device = device
  self.is_vali = is_vali
 
- def evaluate_iter(self, model, input, save_path, scorer):
+ def evaluate_iter(self, model, input, save_path, scorer, detailed_ner_log=True):
  total_loss = 0
- total_f1 = 0
+ macro_f1 = 0
+ macro_precision = 0
+ macro_recall = 0
  total_token_acc = 0
 
  if not self.is_vali:
@@ -27,6 +29,9 @@ def evaluate_iter(self, model, input, save_path, scorer):
  logger.info("Validation mode!")
  model.eval()
 
+ full_ground_truth_list = list()
+ full_prediction_list = list()
+
  with torch.no_grad():
  for batch in input:
  batch_x = batch.sentence.to(self.device)
@@ -37,13 +42,26 @@ def evaluate_iter(self, model, input, save_path, scorer):
 
  pred_scores, predictions = model.decode(batch_x)
 
- token_level_accuracy = scorer.token_level_accuracy(predictions, batch_y)
+ batch_y = batch_y.permute(1, 0)
+
+ scorer.token_level_accuracy(predictions, batch_y)
+
+ full_ground_truth_list.extend(batch_y.tolist())
+ full_prediction_list.extend(predictions)
+
+ token_level_accuracy = scorer.token_accuracy
 
  total_token_acc += token_level_accuracy
 
  torch.cuda.empty_cache()
 
- current_f1 = total_f1 / len(input)
+ scorer.f1_score(full_prediction_list, full_ground_truth_list)
+ macro_f1 = scorer.avg_macro_f1
+ macro_precision = scorer.avg_macro_precision
+ macro_recall = scorer.avg_macro_recall
  current_token_acc = total_token_acc / len(input)
 
- return current_f1, current_token_acc
+ if detailed_ner_log:
+ scorer.print_detailed_score_log()
+
+ return macro_f1, macro_precision, macro_recall, current_token_acc

evaluation/single_model_ner_evaluator.py

@@ -29,37 +29,29 @@ def initialize_model_and_trainer(model_properties, training_properties, datasetl
  logger.info("Model type is %s", training_properties["learner"])
  if training_properties["learner"] == "text_cnn":
  model = TextCnn(model_properties).to(device)
- trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader.train_iter,
- datasetloader.val_iter, datasetloader.test_iter, device)
+ trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader, device)
  elif training_properties["learner"] == "gru":
  model = GRU(model_properties).to(device)
- trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader.train_iter,
- datasetloader.val_iter, datasetloader.test_iter, device)
+ trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader, device)
  elif training_properties["learner"] == "lstm":
  model = LSTM(model_properties).to(device)
- trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader.train_iter,
- datasetloader.val_iter, datasetloader.test_iter, device)
+ trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader, device)
  elif training_properties["learner"] == "char_cnn":
  model = CharCNN(model_properties).to(device)
- trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader.train_iter,
- datasetloader.val_iter, datasetloader.test_iter, device)
+ trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader, device)
  elif training_properties["learner"] == "vdcnn":
  model = VDCNN(model_properties).to(device)
- trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader.train_iter,
- datasetloader.val_iter, datasetloader.test_iter, device)
+ trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader, device)
  elif training_properties["learner"] == "conv_deconv_cnn":
  model = ConvDeconvCNN(model_properties)
- trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader.train_iter,
- datasetloader.val_iter, datasetloader.test_iter, device)
+ trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader, device)
  elif training_properties["learner"] == "transformer_google":
  model = TransformerGoogle(model_properties).model.to(device)
- trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader.train_iter,
- datasetloader.val_iter, datasetloader.test_iter, device)
+ trainer = Trainer.trainer_factory("single_model_trainer", training_properties, datasetloader, device)
  elif training_properties["learner"] == "lstmcrf":
  assert training_properties["task"] == "ner"
  model = LSTMCRF(model_properties).to(device)
- trainer = Trainer.trainer_factory("single_model_ner_trainer", training_properties, datasetloader.train_iter,
- datasetloader.val_iter, datasetloader.test_iter, device)
+ trainer = Trainer.trainer_factory("single_model_ner_trainer", training_properties, datasetloader, device)
  else:
  raise ValueError("Model is not defined! Available learner values are : 'text_cnn', 'char_cnn', 'vdcnn', 'gru', "
  "'lstm', 'conv_deconv_cnn' and 'transformer_google'")
@@ -176,6 +168,7 @@ def initialize_model_and_trainer(model_properties, training_properties, datasetl
  if category_vocab is not None:
  model_properties["common_model_properties"]["num_class"] = len(category_vocab)
  if ner_vocab is not None:
+ model_properties["common_model_properties"]["ner_vocab"] = ner_vocab
  model_properties["common_model_properties"]["num_tags"] = len(ner_vocab)
  model_properties["common_model_properties"]["start_id"] = ner_vocab.stoi["<start>"]
  model_properties["common_model_properties"]["end_id"] = ner_vocab.stoi["<end>"]
@@ -217,4 +210,4 @@ def initialize_model_and_trainer(model_properties, training_properties, datasetl
  category_vocab_path=category_vocab_path,
  preprocessor=preprocessor.preprocess,
  topk=training_properties["topk"])
- logger.info("")
+ logger.info("Done!")

main.py

@@ -66,11 +66,11 @@ def __init__(self, args):
 
  def init_hidden(self, batch_size):
  if self.bidirectional is True:
- return (Variable(torch.zeros(1, batch_size, self.hidden_dim * 2).to(self.device)),
- Variable(torch.zeros(1, batch_size, self.hidden_dim * 2).to(self.device)))
+ return (Variable(torch.zeros(self.num_layers * 2, batch_size, self.hidden_dim).to(self.device)),
+ Variable(torch.zeros(self.num_layers * 2, batch_size, self.hidden_dim).to(self.device)))
  else:
- return (Variable(torch.zeros(1, batch_size, self.hidden_dim).to(self.device)),
- Variable(torch.zeros(1, batch_size, self.hidden_dim).to(self.device)))
+ return (Variable(torch.zeros(self.num_layers, batch_size, self.hidden_dim).to(self.device)),
+ Variable(torch.zeros(self.num_layers, batch_size, self.hidden_dim).to(self.device)))
 
  def initialize_embeddings(self):
  logger.info("> Embeddings")

models/LSTM.py

@@ -1,11 +1,21 @@
+import logging.config
 
+logging.config.fileConfig(fname='./config/config.logger', disable_existing_loggers=False)
+logger = logging.getLogger("NerScorer")
 
 class NerScorer(object):
- def __init__(self):
+ def __init__(self, ner_vocab):
  super(NerScorer, self).__init__()
+ self.ner_vocab = ner_vocab
+ self.token_accuracy = 0
+ self.avg_macro_precision = 0
+ self.avg_macro_recall = 0
+ self.avg_macro_f1 = 0
+ self.macro_precision = {}
+ self.macro_recall = {}
+ self.macro_f1 = {}
 
- @staticmethod
- def token_level_accuracy(prediction, ground_truth):
+ def token_level_accuracy(self, prediction, ground_truth):
  token_count = 0
  matched = 0
 
@@ -15,4 +25,59 @@ def token_level_accuracy(prediction, ground_truth):
  if p == gt:
  matched += 1
 
- return matched * 100.0 / token_count
+ self.token_accuracy = matched * 100.0 / token_count
+
+ def __initialize_dict(self):
+ d = {}
+ for v in self.ner_vocab.stoi:
+ d[self.ner_vocab.stoi[v]] = 0
+ return d
+
+ def __add_to_dict(self, d, tag):
+ if tag in d:
+ d[tag] += 1
+ else:
+ d[tag] = 1
+ return d
+
+ def __calculate_tag_f1(self, f1, tp, fp, fn):
+ precision = {}
+ recall = {}
+ for tag in tp:
+ precision[tag] = tp[tag] / (tp[tag] + fp[tag] + 1e-16)
+ recall[tag] = tp[tag] / (tp[tag] + fn[tag] + 1e-16)
+ f1[tag] = (2 * precision[tag] * recall[tag] / (precision[tag] + recall[tag] + 1e-16)) * 100
+ self.macro_f1 = f1
+ self.macro_precision = precision
+ self.macro_recall = recall
+
+ def __calculate_mean_f1(self):
+ self.avg_macro_f1 = sum(self.macro_f1.values()) / float(len(self.macro_f1))
+ self.avg_macro_precision = sum(self.macro_precision.values()) / float(len(self.macro_precision))
+ self.avg_macro_recall = sum(self.macro_recall.values()) / float(len(self.macro_recall))
+
+ def f1_score(self, prediction, ground_truth):
+ true_positives = self.__initialize_dict()
+ false_positives = self.__initialize_dict()
+ false_negatives = self.__initialize_dict()
+ f1 = self.__initialize_dict()
+
+ for p_seq, gt_seq in zip(prediction, ground_truth):
+ for p, gt in zip(p_seq, gt_seq):
+ if p == gt:
+ true_positives = self.__add_to_dict(true_positives, gt)
+ else:
+ false_negatives = self.__add_to_dict(false_negatives, gt)
+ false_positives = self.__add_to_dict(false_positives, p)
+ self.__calculate_tag_f1(f1, true_positives, false_positives, false_negatives)
+ self.__calculate_mean_f1()
+
+ def print_detailed_score_log(self):
+ logger.info("--------------------")
+ logger.info("Detailed Tag-Based Score")
+ for tag in self.macro_f1:
+ logger.info("Tag: {} - Precision: {:.4f} - Recall: {:.4f} - F1: {:.4f}".format(self.ner_vocab.itos[tag],
+ self.macro_precision[tag],
+ self.macro_recall[tag],
+ self.macro_f1[tag]))
+ logger.info("--------------------")

scorer/ner_scorer.py

@@ -4,25 +4,23 @@
 
 import torch
 import torch.nn as nn
-import torch.optim as optim
 
-from custom_optimizer import OpenAIAdam, NoamOptimizer, Padam
 from evaluation.evaluator import Evaluator
 from models.GRU import GRU
 from models.LSTM import LSTMBase
+from scorer.ner_scorer import NerScorer
 from training.single_model_trainer import SingleModelTrainer
 from utils.utils import time_since, save_best_model
-from scorer.ner_scorer import NerScorer
 
 logging.config.fileConfig(fname='./config/config.logger', disable_existing_loggers=False)
 logger = logging.getLogger("Trainer")
 
 
 class SingleModelNerTrainer(SingleModelTrainer):
- def __init__(self, training_properties, train_iter, dev_iter, test_iter, device):
- super(SingleModelNerTrainer, self).__init__(training_properties, train_iter, dev_iter, test_iter, device)
+ def __init__(self, training_properties, datasetloader, device):
+ super(SingleModelNerTrainer, self).__init__(training_properties, datasetloader, device)
 
- self.scorer = NerScorer()
+ self.scorer = NerScorer(datasetloader.ner_vocab)
  self.dev_evaluator, self.test_evaluator = Evaluator().evaluator_factory("single_model_ner_evaluator",
  self.device)
 
@@ -57,10 +55,10 @@ def train_iters(self, model, checkpoint=None):
  self.print_epoch(start, e, total_loss, train_f1)
 
  if e % self.eval_every == 0:
- vali_f1, vali_token_acc = self.dev_evaluator.evaluate_iter(model=model,
- input=self.dev_iter,
- save_path=self.save_path,
- scorer=self.scorer)
+ vali_f1, vali_precision, vali_recall, vali_token_acc = self.dev_evaluator.evaluate_iter(model=model,
+  input=self.dev_iter,
+  save_path=self.save_path,
+  scorer=self.scorer)
  if best_vali_f1 < vali_f1:
  best_vali_token_acc = vali_token_acc
  best_vali_f1 = vali_f1
@@ -91,10 +89,10 @@ def train_iters(self, model, checkpoint=None):
  }, out_path)
  old_path = out_path
 
- test_f1, test_token_acc = self.test_evaluator.evaluate_iter(model=model,
- input=self.test_iter,
- save_path=self.save_path,
- scorer=self.scorer)
+ test_f1, test_precision, test_recall, test_token_acc = self.test_evaluator.evaluate_iter(model=model,
+  input=self.test_iter,
+  save_path=self.save_path,
+  scorer=self.scorer)
 
  self.print_test(test_token_acc, test_f1)
 
@@ -182,4 +180,3 @@ def print_test(self, test_token_acc, test_f1):
  logger.info("Test F1: {:.4f} - "
  "Test Token Level Accuracy: {:.4f} - ".format(test_f1,
  test_token_acc))
-

training/single_model_ner_trainer.py

@@ -18,7 +18,7 @@
 
 
 class SingleModelTrainer(object):
- def __init__(self, training_properties, train_iter, dev_iter, test_iter, device):
+ def __init__(self, training_properties, datasetloader, device):
  self.task = training_properties["task"]
  self.optimizer_type = training_properties["optimizer"]
  self.learning_rate = training_properties["learning_rate"]
@@ -36,9 +36,9 @@ def __init__(self, training_properties, train_iter, dev_iter, test_iter, device)
  self.amsgrad = training_properties["amsgrad"]
  self.partial_adam = training_properties["partial_adam"]
 
- self.train_iter = train_iter
- self.dev_iter = dev_iter
- self.test_iter = test_iter
+ self.train_iter = datasetloader.train_iter
+ self.dev_iter = datasetloader.val_iter
+ self.test_iter = datasetloader.test_iter
 
  self.device = device