import numpy as np # N is batch size; D_in is input dimension; # H is hidden dimension; D_out is output dimension. N, D_in, H, D_out = 64, 1000, 100, 10 # Create random input and output data x = np.random.randn(N, D_in) y = np.random.randn(N, D_out) # Randomly initialize weights w1 = np.random.randn(D_in, H) w2 = np.random.randn(H, D_out) learning_rate = 1e-6 for t in range(500): # Forward pass: compute predicted y h = x.dot(w1) h_relu = np.maximum(h, 0) y_pred = h_relu.dot(w2) # Compute and print loss loss = np.square(y_pred - y).sum() print(t, loss) # Backprop to compute gradients of w1 and w2 with respect to loss grad_y_pred = 2.0 * (y_pred - y) grad_w2 = h_relu.T.dot(grad_y_pred) grad_h_relu = grad_y_pred.dot(w2.T) grad_h = grad_h_relu.copy() grad_h[h < 0] = 0 grad_w1 = x.T.dot(grad_h) # Update weights w1 -= learning_rate * grad_w1 w2 -= learning_rate * grad_w2

import torch dtype = torch.FloatTensor # dtype = torch.cuda.FloatTensor # Uncomment this to run on GPU # N is batch size; D_in is input dimension; # H is hidden dimension; D_out is output dimension. N, D_in, H, D_out = 64, 1000, 100, 10 # Create random input and output data x = torch.randn(N, D_in).type(dtype) y = torch.randn(N, D_out).type(dtype) # Randomly initialize weights w1 = torch.randn(D_in, H).type(dtype) w2 = torch.randn(H, D_out).type(dtype) learning_rate = 1e-6 for t in range(500): # Forward pass: compute predicted y h = x.mm(w1) h_relu = h.clamp(min=0) y_pred = h_relu.mm(w2) # Compute and print loss loss = (y_pred - y).pow(2).sum() print(t, loss) # Backprop to compute gradients of w1 and w2 with respect to loss grad_y_pred = 2.0 * (y_pred - y) grad_w2 = h_relu.t().mm(grad_y_pred) grad_h_relu = grad_y_pred.mm(w2.t()) grad_h = grad_h_relu.clone() grad_h[h < 0] = 0 grad_w1 = x.t().mm(grad_h) # Update weights using gradient descent w1 -= learning_rate * grad_w1 w2 -= learning_rate * grad_w2

import tensorflow as tf import numpy as np # First we set up the computational graph: # N is batch size; D_in is input dimension; # H is hidden dimension; D_out is output dimension. N, D_in, H, D_out = 64, 1000, 100, 10 # Create placeholders for the input and target data; these will be filled # with real data when we execute the graph. x = tf.placeholder(tf.float32, shape=(None, D_in)) y = tf.placeholder(tf.float32, shape=(None, D_out)) # Create Variables for the weights and initialize them with random data. # A TensorFlow Variable persists its value across executions of the graph. w1 = tf.Variable(tf.random_normal((D_in, H))) w2 = tf.Variable(tf.random_normal((H, D_out))) # Forward pass: Compute the predicted y using operations on TensorFlow Tensors. # Note that this code does not actually perform any numeric operations; it # merely sets up the computational graph that we will later execute. h = tf.matmul(x, w1) h_relu = tf.maximum(h, tf.zeros(1)) y_pred = tf.matmul(h_relu, w2) # Compute loss using operations on TensorFlow Tensors loss = tf.reduce_sum((y - y_pred) ** 2.0) # Compute gradient of the loss with respect to w1 and w2. grad_w1, grad_w2 = tf.gradients(loss, [w1, w2]) # Update the weights using gradient descent. To actually update the weights # we need to evaluate new_w1 and new_w2 when executing the graph. Note that # in TensorFlow the the act of updating the value of the weights is part of # the computational graph; in PyTorch this happens outside the computational # graph. learning_rate = 1e-6 new_w1 = w1.assign(w1 - learning_rate * grad_w1) new_w2 = w2.assign(w2 - learning_rate * grad_w2) # Now we have built our computational graph, so we enter a TensorFlow session to # actually execute the graph. with tf.Session() as sess: # Run the graph once to initialize the Variables w1 and w2. sess.run(tf.global_variables_initializer()) # Create numpy arrays holding the actual data for the inputs x and targets y x_value = np.random.randn(N, D_in) y_value = np.random.randn(N, D_out) for _ in range(500): # Execute the graph many times. Each time it executes we want to bind # x_value to x and y_value to y, specified with the feed_dict argument. # Each time we execute the graph we want to compute the values for loss, # new_w1, and new_w2; the values of these Tensors are returned as numpy # arrays. loss_value, _, _ = sess.run([loss, new_w1, new_w2], feed_dict={x: x_value, y: y_value}) print(loss_value)

import torch from torch.autograd import Variable class MyReLU(torch.autograd.Function): """  We can implement our own custom autograd Functions by subclassing  torch.autograd.Function and implementing the forward and backward passes  which operate on Tensors.  """ def forward(self, input): """  In the forward pass we receive a Tensor containing the input and return a  Tensor containing the output. You can cache arbitrary Tensors for use in the  backward pass using the save_for_backward method.  """ self.save_for_backward(input) return input.clamp(min=0) def backward(self, grad_output): """  In the backward pass we receive a Tensor containing the gradient of the loss  with respect to the output, and we need to compute the gradient of the loss  with respect to the input.  """ input, = self.saved_tensors grad_input = grad_output.clone() grad_input[input < 0] = 0 return grad_input

def expr_for_tree(self, tree, decorate=False): if tree.isleaf(): return self.embed(makevar(self.w2i.get(tree.label, 0))) if len(tree.children) == 1: assert(tree.children[0].isleaf()) expr = self.expr_for_tree(tree.children[0]) if decorate: tree._e = expr return expr assert(len(tree.children) == 2), tree.children[0] e1 = self.expr_for_tree(tree.children[0], decorate) e2 = self.expr_for_tree(tree.children[1], decorate) expr = self.TANH(self.WR(torch.cat((e1, e2),1))) if decorate: tree._e = expr return expr

 def _recurrence(node_h,node_c,idx_var): node_info=tf.gather(treestr,idx_var) child_h=tf.gather(node_h,node_info) child_c=tf.gather(node_c,node_info) flat_ = tf.reshape(child_h,[-1]) tmp=tf.matmul(tf.expand_dims(flat_,0),cW) u,o,i,fl,fr=tf.split(1,5,tmp) i=tf.nn.sigmoid(i+bi) o=tf.nn.sigmoid(o+bo) u=tf.nn.tanh(u+bu) fl=tf.nn.sigmoid(fl+bf) fr=tf.nn.sigmoid(fr+bf) f=tf.concat(0,[fl,fr]) c = i * u + tf.reduce_sum(f*child_c,[0]) h = o * tf.nn.tanh(c) node_h = tf.concat(0,[node_h,h]) node_c = tf.concat(0,[node_c,c]) idx_var=tf.add(idx_var,1) return node_h,node_c,idx_var loop_cond = lambda a1,b1,idx_var: tf.less(idx_var,n_inodes) loop_vars=[node_h,node_c,idx_var] node_h,node_c,idx_var=tf.while_loop(loop_cond, _recurrence, loop_vars,parallel_iterations=10)