exercise 11

Author: jhauschild

4_machine_learning/data_loader.py

@@ -0,0 +1,94 @@
+# Copyright (c) 2012-2018 M. Nielsen (under MIT license)
+# modified version of mnist_loader.py from http://neuralnetworksanddeeplearning.com
+"""
+data_loader
+~~~~~~~~~~~
+
+A library to load the MNIST image data. For details of the data
+structures that are returned, see the doc strings for ``load_data``
+and ``load_data_wrapper``. In practice, ``load_data_wrapper`` is the
+function usually called by our neural network code.
+"""
+
+#### Libraries
+# Standard library
+import pickle
+import gzip
+
+# Third-party libraries
+import numpy as np
+
+def load_data(filename="mnist.pkl.gz"):
+ """Return the MNIST data as a tuple containing the training data,
+ the validation data, and the test data.
+
+ The ``training_data`` is returned as a tuple with two entries.
+ The first entry contains the actual training images. This is a
+ numpy ndarray with 50,000 entries. Each entry is, in turn, a
+ numpy ndarray with 784 values, representing the 28 * 28 = 784
+ pixels in a single MNIST image.
+
+ The second entry in the ``training_data`` tuple is a numpy ndarray
+ containing 50,000 entries. Those entries are just the digit
+ values (0...9) for the corresponding images contained in the first
+ entry of the tuple.
+
+ The ``validation_data`` and ``test_data`` are similar, except
+ each contains only 10,000 images.
+
+ This is a nice data format, but for use in neural networks it's
+ helpful to modify the format of the ``training_data`` a little.
+ That's done in the wrapper function ``load_data_wrapper()``, see
+ below.
+ """
+ with gzip.open(filename, 'rb') as f:
+ training_data, validation_data, test_data = pickle.load(f, encoding="latin1")
+ return (training_data, validation_data, test_data)
+
+def load_data_wrapper(data="mnist.pkl.gz"):
+ """Return a tuple containing ``(training_data, validation_data,
+ test_data)``. Based on ``load_data``, but the format is more
+ convenient for use in our implementation of neural networks.
+
+ In particular, ``training_data`` is a list containing 50,000
+ 2-tuples ``(x, y)``. ``x`` is a 784-dimensional numpy.ndarray
+ containing the input image. ``y`` is a 10-dimensional
+ numpy.ndarray representing the unit vector corresponding to the
+ correct digit for ``x``.
+
+ ``validation_data`` and ``test_data`` are lists containing 10,000
+ 2-tuples ``(x, y)``. In each case, ``x`` is a 784-dimensional
+ numpy.ndarry containing the input image, and ``y`` is the
+ corresponding classification, i.e., the digit values (integers)
+ corresponding to ``x``.
+
+ Obviously, this means we're using slightly different formats for
+ the training data and the validation / test data. These formats
+ turn out to be the most convenient for use in our neural network
+ code.
+
+ The function takes an argument "data", which can be a string
+ giving the filename of a data file, or the data directly.
+ """
+ if isinstance(data, str):
+ data = load_data(data)
+ tr_d, va_d, te_d = data
+ N_labels = 1 + np.max(tr_d[1])
+ print("number of different labels for the output:", N_labels)
+ training_inputs = [np.reshape(x, (784, 1)) for x in tr_d[0]]
+ training_results = [vectorized_result(y, N_labels) for y in tr_d[1]]
+ training_data = list(zip(training_inputs, training_results))
+ validation_inputs = [np.reshape(x, (784, 1)) for x in va_d[0]]
+ validation_data = list(zip(validation_inputs, va_d[1]))
+ test_inputs = [np.reshape(x, (784, 1)) for x in te_d[0]]
+ test_data = list(zip(test_inputs, te_d[1]))
+ return (training_data, validation_data, test_data)
+
+def vectorized_result(j, N=10):
+ """Return a N-dimensional unit vector with a 1.0 in the jth
+ position and zeroes elsewhere. This is used to convert a digit
+ (0...9) into a corresponding desired output from the neural
+ network."""
+ e = np.zeros((N, 1))
+ e[j] = 1.0
+ return e

4_machine_learning/hw11.pdf

@@ -0,0 +1,136 @@
+# Copyright (c) 2012-2018 M. Nielsen (under MIT license)
+# modified version of network.py from http://neuralnetworksanddeeplearning.com
+"""
+network.py
+~~~~~~~~~~
+
+A module to implement the stochastic gradient descent learning
+algorithm for a feedforward neural network. Gradients are calculated
+using backpropagation. Note that I have focused on making the code
+simple, easily readable, and easily modifiable. It is not optimized,
+and omits many desirable features.
+"""
+
+#### Libraries
+# Standard library
+import random
+
+# Third-party libraries
+import numpy as np
+
+class Network:
+
+ def __init__(self, sizes):
+ """The list ``sizes`` contains the number of neurons in the
+ respective layers of the network. For example, if the list
+ was [2, 3, 1] then it would be a three-layer network, with the
+ first layer containing 2 neurons, the second layer 3 neurons,
+ and the third layer 1 neuron. The biases and weights for the
+ network are initialized randomly, using a Gaussian
+ distribution with mean 0, and variance 1. Note that the first
+ layer is assumed to be an input layer, and by convention we
+ won't set any biases for those neurons, since biases are only
+ ever used in computing the outputs from later layers."""
+ self.num_layers = len(sizes)
+ self.sizes = sizes
+ self.biases = [np.random.randn(y, 1) for y in sizes[1:]]
+ self.weights = [np.random.randn(y, x)
+ for x, y in zip(sizes[:-1], sizes[1:])]
+
+ def feedforward(self, a):
+ """Return the output of the network if ``a`` is input."""
+ for b, w in zip(self.biases, self.weights):
+ a = sigmoid(np.dot(w, a)+b)
+ return a
+
+ def SGD(self, training_data, epochs, mini_batch_size, eta,
+ test_data=None):
+ """Train the neural network using mini-batch stochastic
+ gradient descent. The ``training_data`` is a list of tuples
+ ``(x, y)`` representing the training inputs and the desired
+ outputs. The other non-optional parameters are
+ self-explanatory. If ``test_data`` is provided then the
+ network will be evaluated against the test data after each
+ epoch, and partial progress printed out. This is useful for
+ tracking progress, but slows things down substantially."""
+ n = len(training_data)
+ for j in range(epochs):
+ random.shuffle(training_data)
+ mini_batches = [
+ training_data[k:k+mini_batch_size]
+ for k in range(0, n, mini_batch_size)]
+ for mini_batch in mini_batches:
+ self.update_mini_batch(mini_batch, eta)
+ if test_data:
+ eval_correct = self.evaluate(test_data)
+ n_test = len(test_data)
+ print("Epoch {0: 2d}: {1:8d} / {2} = {3:.1f}%".format(
+ j, eval_correct, n_test, 100*eval_correct/n_test))
+ else:
+ print("Epoch {0: 2d} complete".format(j))
+
+ def update_mini_batch(self, mini_batch, eta):
+ """Update the network's weights and biases by applying
+ gradient descent using backpropagation to a single mini batch.
+ The ``mini_batch`` is a list of tuples ``(x, y)``, and ``eta``
+ is the learning rate."""
+ nabla_b = [np.zeros(b.shape) for b in self.biases]
+ nabla_w = [np.zeros(w.shape) for w in self.weights]
+ for x, y in mini_batch:
+ delta_nabla_b, delta_nabla_w = self.backprop(x, y)
+ nabla_b = [nb+dnb for nb, dnb in zip(nabla_b, delta_nabla_b)]
+ nabla_w = [nw+dnw for nw, dnw in zip(nabla_w, delta_nabla_w)]
+ self.weights = [w-(eta/len(mini_batch))*nw
+ for w, nw in zip(self.weights, nabla_w)]
+ self.biases = [b-(eta/len(mini_batch))*nb
+ for b, nb in zip(self.biases, nabla_b)]
+
+ def backprop(self, x, y):
+ """Return a tuple ``(nabla_b, nabla_w)`` representing the
+ gradient for the cost function C_x. ``nabla_b`` and
+ ``nabla_w`` are layer-by-layer lists of numpy arrays, similar
+ to ``self.biases`` and ``self.weights``."""
+ nabla_b = [np.zeros(b.shape) for b in self.biases]
+ nabla_w = [np.zeros(w.shape) for w in self.weights]
+
+ #################
+ raise NotImplementedError("TODO: implementing this is an exercise")
+ #################
+
+ return (nabla_b, nabla_w)
+
+ def evaluate(self, test_data):
+ """Return the number of test inputs for which the neural
+ network outputs the correct result. Note that the neural
+ network's output is assumed to be the index of whichever
+ neuron in the final layer has the highest activation."""
+ test_results = [(np.argmax(self.feedforward(x)), y)
+ for (x, y) in test_data]
+ return sum(int(x == y) for (x, y) in test_results)
+
+ def cost_derivative(self, output_activations, y):
+ """Return the vector of partial derivatives \partial C_x /
+ \partial a for the output activations."""
+ return (output_activations-y)
+
+
+#### Miscellaneous functions
+def sigmoid(z):
+ """The sigmoid function."""
+ return 1.0/(1.0+np.exp(-z))
+
+
+def sigmoid_prime(z):
+ """Derivative of the sigmoid function."""
+ return sigmoid(z)*(1-sigmoid(z))
+
+
+if __name__ == "__main__":
+ # example usage: train a network to recognise digits using the mnist data
+ # first load the data
+ import data_loader
+ training_data, _, test_data = data_loader.load_data_wrapper("mnist.pkl.gz")
+ # then generate the neuronal network
+ net = networks.Network([784, 30, 10])
+ # and train it for 15 epochs
+ net.SGD(training_data, 15, 10, 0.5, test_data=test_data)