Remove language features leftovers

Author: adamlerer

OpenNMT/onmt/Dataset.py

@@ -3,18 +3,16 @@
 
 
 class Dataset(object):
- # FIXME: randomize
+
  def __init__(self, srcData, tgtData, batchSize, cuda):
- self.src = srcData['words']
+ self.src = srcData
  if tgtData:
- self.tgt = tgtData['words']
+ self.tgt = tgtData
  assert(len(self.src) == len(self.tgt))
  else:
  self.tgt = None
  self.cuda = cuda
- # FIXME
- # self.srcFeatures = srcData.features
- # self.tgtFeatures = tgtData.features
+
  self.batchSize = batchSize
  self.numBatches = len(self.src) // batchSize
 

OpenNMT/onmt/Models.py

@@ -3,14 +3,6 @@
 from torch.autograd import Variable
 import onmt.modules
 
-
-def _makeFeatEmbedder(opt, dicts):
- return onmt.FeaturesEmbedding(dicts['features'],
- opt.feat_vec_exponent,
- opt.feat_vec_size,
- opt.feat_merge)
-
-
 class Encoder(nn.Module):
 
  def __init__(self, opt, dicts):
@@ -19,40 +11,25 @@ def __init__(self, opt, dicts):
  assert opt.rnn_size % self.num_directions == 0
  self.hidden_size = opt.rnn_size // self.num_directions
  inputSize = opt.word_vec_size
- feat_lut = None
- # Sequences with features.
- if len(dicts['features']) > 0:
- feat_lut = _makeFeatEmbedder(opt, dicts)
- inputSize = inputSize + feat_lut.outputSize
 
  super(Encoder, self).__init__()
- self.word_lut = nn.Embedding(dicts['words'].size(),
+ self.word_lut = nn.Embedding(dicts.size(),
  opt.word_vec_size,
  padding_idx=onmt.Constants.PAD)
  self.rnn = nn.LSTM(inputSize, self.hidden_size,
  num_layers=opt.layers,
  dropout=opt.dropout,
  bidirectional=opt.brnn)
 
-
  # self.rnn.bias_ih_l0.data.div_(2)
  # self.rnn.bias_hh_l0.data.copy_(self.rnn.bias_ih_l0.data)
 
  if opt.pre_word_vecs_enc is not None:
  pretrained = torch.load(opt.pre_word_vecs_enc)
  self.word_lut.weight.copy_(pretrained)
 
- self.has_features = feat_lut is not None
- if self.has_features:
- self.add_module('feat_lut', feat_lut)
-
  def forward(self, input, hidden=None):
- if self.has_features:
- word_emb = self.word_lut(input[0])
- feat_emb = self.feat_lut(input[1])
- emb = torch.cat([word_emb, feat_emb], 1)
- else:
- emb = self.word_lut(input)
+ emb = self.word_lut(input)
 
  if hidden is None:
  batch_size = emb.size(1)
@@ -70,7 +47,6 @@ def __init__(self, num_layers, input_size, rnn_size, dropout):
  super(StackedLSTM, self).__init__()
  self.dropout = nn.Dropout(dropout)
 
-
  self.layers = []
  for i in range(num_layers):
  layer = nn.LSTMCell(input_size, rnn_size)
@@ -104,14 +80,8 @@ def __init__(self, opt, dicts):
  if self.input_feed:
  input_size += opt.rnn_size
 
- feat_lut = None
- # Sequences with features.
- if len(dicts['features']) > 0:
- feat_lut = _makeFeatEmbedder(opt, dicts)
- input_size = input_size + feat_lut.outputSize
-
  super(Decoder, self).__init__()
- self.word_lut = nn.Embedding(dicts['words'].size(),
+ self.word_lut = nn.Embedding(dicts.size(),
  opt.word_vec_size,
  padding_idx=onmt.Constants.PAD)
  self.rnn = StackedLSTM(opt.layers, input_size, opt.rnn_size, opt.dropout)
@@ -127,17 +97,9 @@ def __init__(self, opt, dicts):
  pretrained = torch.load(opt.pre_word_vecs_dec)
  self.word_lut.weight.copy_(pretrained)
 
- self.has_features = feat_lut is not None
- if self.has_features:
- self.add_module('feat_lut', feat_lut)
 
  def forward(self, input, hidden, context, init_output):
- if self.has_features:
- word_emb = self.word_lut(input[0])
- feat_emb = self.feat_lut(input[1])
- emb = torch.cat([word_emb, feat_emb], 1)
- else:
- emb = self.word_lut(input)
+ emb = self.word_lut(input)
 
  batch_size = input.size(1)
 

OpenNMT/onmt/Translator.py

@@ -18,11 +18,8 @@ def __init__(self, opt):
  else:
  self.model.cpu()
 
- self.src_dict = checkpoint['dicts']['src']['words']
- self.tgt_dict = checkpoint['dicts']['tgt']['words']
-
- # if opt.phrase_table.len() > 0:
- # phraseTable = onmt.translate.PhraseTable.new(opt.phrase_table)
+ self.src_dict = checkpoint['dicts']['src']
+ self.tgt_dict = checkpoint['dicts']['tgt']
 
  def buildData(self, srcBatch, goldBatch):
  srcData = [self.src_dict.convertToIdx(b,
@@ -34,9 +31,7 @@ def buildData(self, srcBatch, goldBatch):
  onmt.Constants.BOS_WORD,
  onmt.Constants.EOS_WORD) for b in goldBatch]
 
- return onmt.Dataset(
- {'words': srcData},
- {'words': tgtData} if tgtData else None,
+ return onmt.Dataset(srcData, tgtData,
  self.opt.batch_size, self.opt.cuda)
 
  def buildTargetTokens(self, pred, src, attn):
@@ -53,7 +48,6 @@ def buildTargetTokens(self, pred, src, attn):
 
  def translateBatch(self, batch):
  srcBatch, tgtBatch = batch
- sourceLength = srcBatch.size(0)
  batchSize = srcBatch.size(1)
  beamSize = self.opt.beam_size
 
@@ -179,7 +173,6 @@ def updateActive(t):
 
  def translate(self, srcBatch, goldBatch):
  dataset = self.buildData(srcBatch, goldBatch)
- assert(len(dataset) == 1) # FIXME
  batch = dataset[0]
 
  pred, predScore, attn, goldScore = self.translateBatch(batch)

OpenNMT/onmt/modules/GlobalAttention.py

@@ -24,6 +24,7 @@
 import torch.nn as nn
 import math
 
+
 class GlobalAttention(nn.Module):
  def __init__(self, dim):
  super(GlobalAttention, self).__init__()