Merge branch 'master' of https://github.com/TarrySingh/Artificial-Intelligence-Deep-Learning-Machine-Learning-Tutorials

Author: Tarry Singh

.gitignore

@@ -57,4 +57,8 @@ target/
 .ipynb_checkpoints
 
 # Repo scratch directory
-scratch/
+scratch/
+.DS_Store
+deep-learning/.DS_Store
+.DS_Store
+deep-learning/.DS_Store

deep-learning/NLP/.DS_Store

@@ -0,0 +1,13 @@
+ DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ Version 2, December 2004
+
+ Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
+
+ Everyone is permitted to copy and distribute verbatim or modified
+ copies of this license document, and changing it is allowed as long
+ as the name is changed.
+
+ DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+ TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+ 0. You just DO WHAT THE FUCK YOU WANT TO.

deep-learning/NLP/RNN/Interesting Reading/1803.07770.pdf

@@ -0,0 +1,58 @@
+# Sequence to Sequence models with PyTorch
+
+This repository contains implementations of Sequence to Sequence (Seq2Seq) models in PyTorch
+
+At present it has implementations for : 
+
+ * Vanilla Sequence to Sequence models
+
+ * Attention based Sequence to Sequence models from https://arxiv.org/abs/1409.0473 and https://arxiv.org/abs/1508.04025
+
+ * Faster attention mechanisms using dot products between the **final** encoder and decoder hidden states
+
+ * Sequence to Sequence autoencoders (experimental)
+
+## Sequence to Sequence models
+
+A vanilla sequence to sequence model presented in https://arxiv.org/abs/1409.3215, https://arxiv.org/abs/1406.1078 consits of using a recurrent neural network such as an LSTM (http://dl.acm.org/citation.cfm?id=1246450) or GRU (https://arxiv.org/abs/1412.3555) to encode a sequence of words or characters in a *source* language into a fixed length vector representation and then deocoding from that representation using another RNN in the *target* language.
+
+![Sequence to Sequence](/images/Seq2Seq.png)
+
+An extension of sequence to sequence models that incorporate an attention mechanism was presented in https://arxiv.org/abs/1409.0473 that uses information from the RNN hidden states in the source language at each time step in the deocder RNN. This attention mechanism significantly improves performance on tasks like machine translation. A few variants of the attention model for the task of machine translation have been presented in https://arxiv.org/abs/1508.04025.
+
+![Sequence to Sequence with attention](/images/Seq2SeqAttention.png)
+
+The repository also contains a simpler and faster variant of the attention mechanism that doesn't attend over the hidden states of the encoder at each time step in the deocder. Instead, it computes the a single batched dot product between all the hidden states of the decoder and encoder once after the decoder has processed all inputs in the target. This however comes at a minor cost in model performance. One advantage of this model is that it is possible to use the cuDNN LSTM in the attention based decoder as well since the attention is computed after running through all the inputs in the decoder.
+
+## Results on English - French WMT14
+
+The following presents the model architecture and results obtained when training on the WMT14 English - French dataset. The training data is the english-french bitext from Europral-v7. The validation dataset is newstest2011
+
+The model was trained with following configuration
+
+ * Source and target word embedding dimensions - 512
+
+ * Source and target LSTM hidden dimensions - 1024
+
+ * Encoder - 2 Layer Bidirectional LSTM
+
+ * Decoder - 1 Layer LSTM
+
+ * Optimization - ADAM with a learning rate of 0.0001 and batch size of 80
+
+ * Decoding - Greedy decoding (argmax)
+
+
+| Model | BLEU | Train Time Per Epoch |
+| ------------- | ------------- | ------------- |
+| Seq2Seq | 11.82 | 2h 50min |
+| Seq2Seq FastAttention | 18.89 | 3h 45min |
+| Seq2Seq Attention | 22.60 | 4h 47min |
+
+Times reported are using a Pre 2016 Nvidia GeForce Titan X
+
+## Running
+
+To run, edit the config file and execute python nmt.py --config <your_config_file>
+
+NOTE: This only runs on a GPU for now.

deep-learning/NLP/Seq2Seq-PyTorch/LICENSE

@@ -0,0 +1,119 @@
+"""Beam search implementation in PyTorch."""
+#
+#
+# hyp1#-hyp1---hyp1 -hyp1
+# \ /
+# hyp2 \-hyp2 /-hyp2#hyp2
+# / \
+# hyp3#-hyp3---hyp3 -hyp3
+# ========================
+#
+# Takes care of beams, back pointers, and scores.
+
+# Code borrowed from PyTorch OpenNMT example
+# https://github.com/pytorch/examples/blob/master/OpenNMT/onmt/Beam.py
+
+import torch
+
+
+class Beam(object):
+ """Ordered beam of candidate outputs."""
+
+ def __init__(self, size, vocab, cuda=False):
+ """Initialize params."""
+ self.size = size
+ self.done = False
+ self.pad = vocab['<pad>']
+ self.bos = vocab['<s>']
+ self.eos = vocab['</s>']
+ self.tt = torch.cuda if cuda else torch
+
+ # The score for each translation on the beam.
+ self.scores = self.tt.FloatTensor(size).zero_()
+
+ # The backpointers at each time-step.
+ self.prevKs = []
+
+ # The outputs at each time-step.
+ self.nextYs = [self.tt.LongTensor(size).fill_(self.pad)]
+ self.nextYs[0][0] = self.bos
+
+ # The attentions (matrix) for each time.
+ self.attn = []
+
+ # Get the outputs for the current timestep.
+ def get_current_state(self):
+ """Get state of beam."""
+ return self.nextYs[-1]
+
+ # Get the backpointers for the current timestep.
+ def get_current_origin(self):
+ """Get the backpointer to the beam at this step."""
+ return self.prevKs[-1]
+
+ # Given prob over words for every last beam `wordLk` and attention
+ # `attnOut`: Compute and update the beam search.
+ #
+ # Parameters:
+ #
+ # * `wordLk`- probs of advancing from the last step (K x words)
+ # * `attnOut`- attention at the last step
+ #
+ # Returns: True if beam search is complete.
+
+ def advance(self, workd_lk):
+ """Advance the beam."""
+ num_words = workd_lk.size(1)
+
+ # Sum the previous scores.
+ if len(self.prevKs) > 0:
+ beam_lk = workd_lk + self.scores.unsqueeze(1).expand_as(workd_lk)
+ else:
+ beam_lk = workd_lk[0]
+
+ flat_beam_lk = beam_lk.view(-1)
+
+ bestScores, bestScoresId = flat_beam_lk.topk(self.size, 0, True, True)
+ self.scores = bestScores
+
+ # bestScoresId is flattened beam x word array, so calculate which
+ # word and beam each score came from
+ prev_k = bestScoresId / num_words
+ self.prevKs.append(prev_k)
+ self.nextYs.append(bestScoresId - prev_k * num_words)
+
+ # End condition is when top-of-beam is EOS.
+ if self.nextYs[-1][0] == self.eos:
+ self.done = True
+
+ return self.done
+
+ def sort_best(self):
+ """Sort the beam."""
+ return torch.sort(self.scores, 0, True)
+
+ # Get the score of the best in the beam.
+ def get_best(self):
+ """Get the most likely candidate."""
+ scores, ids = self.sort_best()
+ return scores[1], ids[1]
+
+ # Walk back to construct the full hypothesis.
+ #
+ # Parameters.
+ #
+ # * `k` - the position in the beam to construct.
+ #
+ # Returns.
+ #
+ # 1. The hypothesis
+ # 2. The attention at each time step.
+ def get_hyp(self, k):
+ """Get hypotheses."""
+ hyp = []
+ # print(len(self.prevKs), len(self.nextYs), len(self.attn))
+ for j in range(len(self.prevKs) - 1, -1, -1):
+ hyp.append(self.nextYs[j + 1][k])
+ k = self.prevKs[j][k]
+
+ return hyp[::-1]

deep-learning/NLP/Seq2Seq-PyTorch/README.md

@@ -0,0 +1,39 @@
+{
+ "training": {
+ "optimizer": "adam",
+ "clip_c": 1,
+ "lrate": 0.0002,
+ },
+ "management": {
+ "monitor_loss": 1000,
+ "print_samples": 20000
+ },
+ "data": {
+ "src": "/data/lisatmp4/subramas/datasets/1-billion-words/all-1-billion-words-train.en.train",
+ "test_src": "/data/lisatmp4/subramas/datasets/1-billion-words/all-1-billion-words-train.en.dev",
+ "batch_size": 80,
+ "valid_batch_size": 80,
+ "n_words_src": 30000,
+ "max_src_length": 50,
+ "task": "autoencode",
+ "save_dir": "/data/lisatmp4/subramas/models/torch_seq2seq",
+ "load_dir": false
+ },
+ "model": {
+ "dim": 1024,
+ "dim_trg": 1024,
+ "use_dropout": false,
+ "dim_word_src": 512,
+ "n_words_src": 30000,
+ "n_words": 30000,
+ "dim_word_trg": 512,
+ "n_layers_src": 2,
+ "n_layers_trg": 1,
+ "bidirectional": true,
+ "src_lang": "en",
+ "trg_lang": "en",
+ "decode": "greedy",
+ "seq2seq": "vanilla",
+ "optimizer": "adam"
+ }
+}

deep-learning/NLP/Seq2Seq-PyTorch/beam_search.py

@@ -0,0 +1,43 @@
+{
+ "training": {
+ "optimizer": "adam",
+ "clip_c": 1,
+ "lrate": 0.0001
+ },
+ "management": {
+ "monitor_loss": 1000,
+ "print_samples": 20000
+ },
+ "data": {
+ "src": "/data/lisatmp4/subramas/datasets/nmt/en-fr/data/europarl-v7.fr-en.tok.true.clean.fr",
+ "trg": "/data/lisatmp4/subramas/datasets/nmt/en-fr/data/europarl-v7.fr-en.tok.true.clean.en",
+ "test_src": "/data/lisatmp4/subramas/datasets/nmt/en-fr/data/newstest2011.fr.true.tok",
+ "test_trg": "/data/lisatmp4/subramas/datasets/nmt/en-fr/data/newstest2011.en.true.tok",
+ "batch_size": 80,
+ "n_words_trg": 30000,
+ "valid_batch_size": 80,
+ "n_words_src": 30000,
+ "max_src_length": 50,
+ "max_trg_length": 50,
+ "task": "translation",
+ "save_dir": "/data/lisatmp4/subramas/models/torch_seq2seq",
+ "load_dir": false
+ },
+ "model": {
+ "dim": 1000,
+ "dim_trg": 1000,
+ "use_dropout": false,
+ "dim_word_src": 500,
+ "n_words_src": 30000,
+ "n_words": 30000,
+ "dim_word_trg": 500,
+ "n_layers_src": 2,
+ "n_layers_trg": 1,
+ "bidirectional": true,
+ "src_lang": "fr",
+ "trg_lang": "en",
+ "decode": "greedy",
+ "seq2seq": "attention",
+ "optimizer": "adam"
+ }
+}