Update for bugging myself to focus =)

- I separated Google's and OpenAI's versions of Transformer due to better code reading (since they both have many non-intersecting helper classes) - Transformer_OpenAI.py is created. Obviously, model is not ready. - I am using both the paper and the huggingface version of implemen- tation as my resources. But probably, I won't hardly c/p the code of huggingface since I want this part of the project has same architectural flow with other implemented models. - Added gelu activation function to utils.

Author: Aunavin1

config/config.json

@@ -118,8 +118,8 @@
  }
  },
  "training_properties": {
- "learner": "lstmcrf",
- "task": "ner",
+ "learner": "transformer_google",
+ "task": "classification",
  "optimizer": "SGD",
  "learning_rate": 0.05,
  "final_learning_rate": 0.1,

models/Transformer.py

@@ -24,18 +24,6 @@ def forward(self, x):
  return self.a_2 * (x - mean) / (std + self.epsilon) + self.b_2
 
 
-class LayerNormOpenAI(nn.Module):
- def __init__(self, features, epsilon=1e-5):
- self.a_2 = nn.Parameter(torch.ones(features))
- self.b_2 = nn.Parameter(torch.zeros(features))
- self.epsilon = epsilon
-
- def forward(self, x):
- mean = x.mean(-1, keepdim=True)
- std = x.std(-1, keepdim=True)
- return self.a_2 * (x - mean) / torch.sqrt(std + self.epsilon) + self.b_2
-
-
 class EncoderBlockGoogle(nn.Module):
  def __init__(self, layer, num_layers):
  super(EncoderBlockGoogle, self).__init__()
@@ -206,7 +194,7 @@ def forward(self, input):
  return self.dropout(input + Variable(self.pe[:, :input.size(1)], requires_grad=False))
 
 
-class TransformerGoogle():
+class TransformerGoogle:
  def __init__(self, args):
  super(TransformerGoogle, self).__init__()
 
@@ -262,6 +250,8 @@ def create_classifier_transformer(self):
  c = copy.deepcopy
 
  # Initialize individual parts of the full model
+ # attention = torch.nn.MultiheadAttention(num_heads=self.heads, embed_dim=self.embed_dim,
+ # dropout=self.keep_prob_attn)
  attention = MultiHeadedAttentionGoogle(heads=self.heads, d_model=self.embed_dim, keep_prob=self.keep_prob_attn)
 
  ff = PositionalFeedForwardGoogle(d_model=self.embed_dim, d_ff=self.num_hidden_pos_ff,

models/Transformer_OpenAI.py

@@ -0,0 +1,90 @@
+import copy
+import math
+
+import matplotlib.pyplot as plt
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.autograd import Variable
+
+from utils.utils import clones
+
+
+class LayerNorm(nn.Module):
+ def __init__(self, features, epsilon=1e-5):
+ self.a_2 = nn.Parameter(torch.ones(features))
+ self.b_2 = nn.Parameter(torch.zeros(features))
+ self.epsilon = epsilon
+
+ def forward(self, x):
+ mean = x.mean(-1, keepdim=True)
+ std = x.std(-1, keepdim=True)
+ return self.a_2 * (x - mean) / torch.sqrt(std + self.epsilon) + self.b_2
+
+
+class Embeddings(nn.Module):
+ def __init__(self, embed_dim, vocab_size, keep_prob, padding_id, use_pretrained_embed, pretrained_weights):
+ super(Embeddings, self).__init__()
+ # Initialize embeddings
+ self.embedding = nn.Embedding(vocab_size, embed_dim, padding_idx=padding_id).cpu()
+ if use_pretrained_embed:
+ self.load_pretrained_weights()
+ self.embed_drop = nn.Dropout(keep_prob)
+
+ def forward(self, input):
+ x = self.embed_drop(self.embedding(input))
+ out = x.sum(dim=2)
+ return out
+
+
+class LanguageModelHead(nn.Module):
+ def __init__(self):
+ super(LanguageModelHead, self).__init__()
+
+
+class TransformerOpenAI:
+ def __init__(self, args):
+ super(TransformerOpenAI, self).__init__()
+
+ self.args_common = args["common_model_properties"]
+ self.args_specific = args["transformer_openai"]
+
+ # Device
+ self.device = self.args_common["device"]
+
+ # Input/Output dimensions
+ self.vocab_size = self.args_common["vocab_size"]
+ self.embed_dim = self.args_common["embed_dim"]
+ self.num_class = self.args_common["num_class"]
+
+ # Embedding parameters
+ self.padding_id = self.args_common["padding_id"]
+
+ # Condition parameters
+ self.use_pretrained_embed = self.args_common["use_pretrained_embed"]
+
+ # Model/Context size
+ self.d_model = self.args_specific["d_model"]
+
+ # Dropout probabilities for each individual part of the full model.
+ self.keep_prob_embed = self.args_specific["keep_prob_embed"]
+
+ # Number of parallel attention layers for MultiHeadedAttention
+ self.heads = self.args_specific["heads"]
+
+ # Number of layers in terms of Blocks
+ self.num_layers = self.args_specific["num_layers"]
+
+ if self.transformer_type == "classifier":
+ self.model = self.create_classifier_transformer()
+ else:
+ raise ValueError("Transformer can be created as classifier for now!")
+
+ def create_classifier_transformer(self):
+ c = copy.deepcopy
+
+ embedding = Embeddings(self.embed_dim, self.vocab_size, self.keep_prob_embed, self.padding_id,
+ self.use_pretrained_embed, self.pretrained_weights)
+
+

utils/utils.py

@@ -7,6 +7,7 @@
 import numpy as np
 import torch
 import torch.nn as nn
+import torch.nn.functional as F
 
 
 def save_vocabulary(vocab, path):
@@ -75,3 +76,10 @@ def subsequent_mask(size):
  # The Annotated Transformer = https://nlp.seas.hardvard.edu/2018/04/03.attention.html
  sm = np.triu(np.ones((1, size, size)), k=1).astype("uint8")
  return torch.from_numpy(sm) == 0
+
+
+def gelu(x):
+ # Gaussian Error Linear Unit
+ # Ref: https://github.com/pytorch/pytorch/issues/20464
+ return 0.5 * x * (1 + torch.tanh(math.sqrt(math.pi / 2) * (x + 0.044715 * x ** 3)))
+