import numpy as np import matplotlib.pyplot as plt # Create simple dataset - XOR problem X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]]) y = np.array([[0], [1], [1], [0]]) print(f"Input shape: {X.shape}") # (4, 2) print(f"Output shape: {y.shape}") # (4, 1) print(f"Dataset:") for i in range(len(X)): print(f" {X[i]} -> {y[i][0]}") # Print each input pair and its expected output 

# Network architecture: 2 -> 4 -> 1 (input -> hidden -> output) input_size = 2 # Number of input features (x and y coordinates) hidden_size = 4 # Number of neurons in hidden layer output_size = 1 # Number of output values (0 or 1)  # Initialize weights and biases np.random.seed(42) W1 = np.random.randn(input_size, hidden_size) * 0.5 # Weights: connection strengths between layers b1 = np.zeros((1, hidden_size)) # Biases: adjustable offsets for each neuron W2 = np.random.randn(hidden_size, output_size) * 0.5 # Weights for output layer b2 = np.zeros((1, output_size)) # Bias for output layer  print(f"W1 shape: {W1.shape}") # (2, 4) print(f"b1 shape: {b1.shape}") # (1, 4) print(f"W2 shape: {W2.shape}") # (4, 1) print(f"b2 shape: {b2.shape}") # (1, 1) 

# Let's trace through the shapes step by step print("Shape flow through network:") print(f"Input X: {X.shape}") # (4, 2) print(f"Weights W1: {W1.shape}") # (2, 4) print(f"X @ W1: {(X @ W1).shape}") # (4, 4) print(f"Bias b1: {b1.shape}") # (1, 4)  # Broadcasting explanation sample_mult = X @ W1 print(f"\nBroadcasting bias:") print(f"(X @ W1) shape: {sample_mult.shape}") # (4, 4) print(f"b1 shape: {b1.shape}") # (1, 4) print(f"Result shape: {(sample_mult + b1).shape}") # (4, 4) 

def forward_pass(X, W1, b1, W2, b2): """Complete forward pass through the network""" # Hidden layer  z1 = X @ W1 + b1 # Linear transformation  a1 = sigmoid(z1) # Activation  # Output layer  z2 = a1 @ W2 + b2 # Linear transformation  a2 = sigmoid(z2) # Activation  return z1, a1, z2, a2 # Test forward pass z1, a1, z2, predictions = forward_pass(X, W1, b1, W2, b2) print(f"Predictions shape: {predictions.shape}") print(f"Predictions:\n{predictions.flatten()}") print(f"Actual labels:\n{y.flatten()}") 

def sigmoid(x): """Sigmoid activation function""" return 1 / (1 + np.exp(-np.clip(x, -500, 500))) # Clip to prevent overflow  # Test sigmoid on different inputs test_values = np.array([-10, -1, 0, 1, 10]) sigmoid_values = sigmoid(test_values) print("Sigmoid function behavior:") for i, val in enumerate(test_values): print(f" sigmoid({val:3.0f}) = {sigmoid_values[i]:.3f}") # Visualize sigmoid x_range = np.linspace(-10, 10, 100) y_sigmoid = sigmoid(x_range) plt.figure(figsize=(8, 4)) plt.plot(x_range, y_sigmoid, 'b-', linewidth=2) plt.title('Sigmoid Activation Function') plt.xlabel('x') plt.ylabel('sigmoid(x)') plt.grid(True, alpha=0.3) plt.show() 

# Let's understand sigmoid step by step: 1 / (1 + e^(-x)) x = 2.0 print(f"Input: x = {x}") print(f"Step 1: -x = {-x}") print(f"Step 2: e^(-x) = np.exp(-x) = {np.exp(-x):.3f}") print(f"Step 3: 1 + e^(-x) = {1 + np.exp(-x):.3f}") print(f"Step 4: 1 / (1 + e^(-x)) = {1 / (1 + np.exp(-x)):.3f}") print(f"Sigmoid result: {sigmoid(x):.3f}") # Why clipping? print(f"\nWhy we clip large values:") large_x = 1000 print(f"Without clipping: np.exp(-{large_x}) would cause overflow") print(f"With clipping: np.exp(-500) = {np.exp(-500):.2e} (very small, safe)") 

def compute_loss(predictions, targets): """Mean squared error loss""" return np.mean((predictions - targets) ** 2) # Square the difference to penalize large errors 

# Let's see why we use (predictions - targets) ** 2 pred = np.array([0.8, 0.2, 0.9, 0.1]) target = np.array([1.0, 0.0, 1.0, 0.0]) differences = pred - target squared_differences = differences ** 2 print("Understanding squared error:") print(f"Predictions: {pred}") print(f"Targets: {target}") print(f"Differences: {differences}") print(f"Squared: {squared_differences}") print(f"Mean squared error: {np.mean(squared_differences):.4f}") # Why not just absolute difference? abs_differences = np.abs(differences) print(f"\nComparison:") print(f"Absolute differences: {abs_differences}") print(f"Squared differences: {squared_differences}") print("Squared errors penalize large mistakes more heavily!") 

initial_loss = compute_loss(predictions, y) print(f"Initial loss: {initial_loss:.4f}") 

def sigmoid_derivative(x): """Derivative of sigmoid function""" s = sigmoid(x) return s * (1 - s) # Derivative formula: sigmoid(x) * (1 - sigmoid(x))  # Test derivative test_vals = np.array([-2, -1, 0, 1, 2]) sigmoid_vals = sigmoid(test_vals) derivative_vals = sigmoid_derivative(test_vals) print("Sigmoid and its derivative:") for i, val in enumerate(test_vals): print(f" x={val:2.0f}: sigmoid={sigmoid_vals[i]:.3f}, derivative={derivative_vals[i]:.3f}") # Visualize both functions x_range = np.linspace(-6, 6, 100) y_sigmoid = sigmoid(x_range) y_derivative = sigmoid_derivative(x_range) plt.figure(figsize=(10, 4)) plt.plot(x_range, y_sigmoid, 'b-', label='sigmoid(x)', linewidth=2) plt.plot(x_range, y_derivative, 'r--', label="sigmoid'(x)", linewidth=2) plt.title('Sigmoid Function and Its Derivative') plt.xlabel('x') plt.ylabel('y') plt.legend() plt.grid(True, alpha=0.3) plt.show() 

# The chain rule: if y = f(g(x)), then dy/dx = f'(g(x)) * g'(x) # # For our network: Loss = MSE(sigmoid(W2 * sigmoid(W1 * X + b1) + b2), y) # We need: dLoss/dW2, dLoss/db2, dLoss/dW1, dLoss/db1  print("Chain rule breakdown:") print("dLoss/dW2 = dLoss/da2 * da2/dz2 * dz2/dW2") print(" where:") print(" dLoss/da2 = 2 * (predictions - targets) # MSE derivative") print(" da2/dz2 = sigmoid'(z2) # sigmoid derivative") print(" dz2/dW2 = a1 # linear layer derivative") 

def backward_pass(X, y, z1, a1, z2, a2, W1, b1, W2, b2): """Compute gradients using backpropagation""" m = X.shape[0] # Number of samples  # Output layer gradients  dz2 = 2 * (a2 - y) * sigmoid_derivative(z2) # (4, 1)  dW2 = a1.T @ dz2 / m # (4, 1)  db2 = np.mean(dz2, axis=0, keepdims=True) # (1, 1)  # Hidden layer gradients  dz1 = (dz2 @ W2.T) * sigmoid_derivative(z1) # (4, 4)  dW1 = X.T @ dz1 / m # (2, 4)  db1 = np.mean(dz1, axis=0, keepdims=True) # (1, 4)  return dW1, db1, dW2, db2 # Test backpropagation dW1, db1, dW2, db2 = backward_pass(X, y, z1, a1, z2, predictions, W1, b1, W2, b2) print(f"Gradient shapes:") print(f" dW1: {dW1.shape}, dW2: {dW2.shape}") print(f" db1: {db1.shape}, db2: {db2.shape}") 

# Let's break down the output layer gradient calculation print("Output layer gradient breakdown:") print("dz2 = 2 * (a2 - y) * sigmoid_derivative(z2)") # Step by step error = a2 - y # How far off our predictions are print(f"Error (a2 - y) shape: {error.shape}") print(f"Error values:\n{error.flatten()}") mse_gradient = 2 * error # Derivative of MSE print(f"\nMSE gradient (2 * error) shape: {mse_gradient.shape}") sigmoid_grad = sigmoid_derivative(z2) # Derivative of sigmoid print(f"Sigmoid gradient shape: {sigmoid_grad.shape}") dz2_step = mse_gradient * sigmoid_grad # Chain rule print(f"Combined gradient (dz2) shape: {dz2_step.shape}") 

# Hidden layer gradients are more complex due to chain rule print("Hidden layer gradient breakdown:") print("dz1 = (dz2 @ W2.T) * sigmoid_derivative(z1)") # Step by step error_propagated = dz2 @ W2.T # Propagate error backwards print(f"Error propagated shape: {error_propagated.shape}") print(f"This spreads output error to each hidden neuron") hidden_sigmoid_grad = sigmoid_derivative(z1) # Local gradient print(f"Hidden sigmoid gradient shape: {hidden_sigmoid_grad.shape}") dz1_step = error_propagated * hidden_sigmoid_grad # Final gradient print(f"Combined hidden gradient (dz1) shape: {dz1_step.shape}") 

# Weight gradients show how to adjust connections print("Weight gradient calculation:") print("dW2 = a1.T @ dz2 / m") print(f"a1.T shape: {a1.T.shape}") # Transposed hidden activations print(f"dz2 shape: {dz2.shape}") # Output gradients print(f"dW2 shape: {(a1.T @ dz2).shape}") # Weight gradients  # This gives us the gradient for each weight connection print(f"\nWeight gradients tell us:") print(f"- Positive gradient: decrease this weight") print(f"- Negative gradient: increase this weight") print(f"- Large gradient: this weight has big impact on error") 

def train_network(X, y, epochs=1000, learning_rate=1.0): """Train the neural network""" # Initialize weights  np.random.seed(42) W1 = np.random.randn(2, 4) * 0.5 b1 = np.zeros((1, 4)) W2 = np.random.randn(4, 1) * 0.5 b2 = np.zeros((1, 1)) losses = [] for epoch in range(epochs): # Forward pass  z1, a1, z2, predictions = forward_pass(X, W1, b1, W2, b2) # Compute loss  loss = compute_loss(predictions, y) losses.append(loss) # Backward pass  dW1, db1, dW2, db2 = backward_pass(X, y, z1, a1, z2, predictions, W1, b1, W2, b2) # Update weights  W1 -= learning_rate * dW1 b1 -= learning_rate * db1 W2 -= learning_rate * dW2 b2 -= learning_rate * db2 # Print progress  if epoch % 100 == 0: print(f"Epoch {epoch:4d}: Loss = {loss:.6f}") return W1, b1, W2, b2, losses # Train the network W1_trained, b1_trained, W2_trained, b2_trained, loss_history = train_network(X, y) 

# Learning rate controls how big steps we take during optimization # Too small: slow convergence, too large: might overshoot minimum  learning_rates = [0.1, 1.0, 10.0] plt.figure(figsize=(12, 4)) for i, lr in enumerate(learning_rates): plt.subplot(1, 3, i+1) # Train with this learning rate  _, _, _, _, losses = train_network(X, y, epochs=500, learning_rate=lr) plt.plot(losses) plt.title(f'Learning Rate = {lr}') plt.xlabel('Epoch') plt.ylabel('Loss') plt.yscale('log') plt.grid(True, alpha=0.3) plt.tight_layout() plt.show() 

# Plot loss curve plt.figure(figsize=(10, 4)) plt.subplot(1, 2, 1) plt.plot(loss_history) plt.title('Training Loss') plt.xlabel('Epoch') plt.ylabel('Loss') plt.yscale('log') plt.grid(True, alpha=0.3) plt.subplot(1, 2, 2) plt.plot(loss_history[-100:]) # Last 100 epochs plt.title('Training Loss (Last 100 Epochs)') plt.xlabel('Epoch') plt.ylabel('Loss') plt.grid(True, alpha=0.3) plt.tight_layout() plt.show() print(f"Final loss: {loss_history[-1]:.6f}") 

# Test the trained network z1_final, a1_final, z2_final, final_predictions = forward_pass(X, W1_trained, b1_trained, W2_trained, b2_trained) print("Final Results:") print("Input -> Target | Prediction | Rounded") print("-" * 40) for i in range(len(X)): pred = final_predictions[i, 0] rounded = round(pred) target = y[i, 0] print(f"{X[i]} -> {target} | {pred:.4f} | {rounded}") # Calculate accuracy rounded_predictions = np.round(final_predictions) accuracy = np.mean(rounded_predictions == y) print(f"\nAccuracy: {accuracy:.1%}") 

# Create a grid of points to visualize the decision boundary def plot_decision_boundary(W1, b1, W2, b2): """Plot the decision boundary learned by the network""" # Create a grid  xx, yy = np.meshgrid(np.linspace(-0.5, 1.5, 100), np.linspace(-0.5, 1.5, 100)) # Flatten the grid for prediction  grid_points = np.c_[xx.ravel(), yy.ravel()] # Make predictions on the grid  _, _, _, grid_predictions = forward_pass(grid_points, W1, b1, W2, b2) grid_predictions = grid_predictions.reshape(xx.shape) # Plot  plt.figure(figsize=(8, 6)) plt.contourf(xx, yy, grid_predictions, levels=50, alpha=0.8, cmap='RdYlBu') plt.colorbar(label='Network Output') # Plot data points  colors = ['red' if label == 0 else 'blue' for label in y.flatten()] plt.scatter(X[:, 0], X[:, 1], c=colors, s=100, edgecolors='black', linewidth=2) # Add labels  for i, (x, y_val) in enumerate(zip(X, y.flatten())): plt.annotate(f'({x[0]},{x[1]})→{y_val}', (x[0], x[1]), xytext=(5, 5), textcoords='offset points') plt.title('Neural Network Decision Boundary') plt.xlabel('Input 1') plt.ylabel('Input 2') plt.grid(True, alpha=0.3) plt.show() # Visualize the decision boundary plot_decision_boundary(W1_trained, b1_trained, W2_trained, b2_trained) 

class SimpleNeuralNetwork: """A simple 2-layer neural network implementation""" def __init__(self, input_size=2, hidden_size=4, output_size=1): # Initialize weights  self.W1 = np.random.randn(input_size, hidden_size) * 0.5 self.b1 = np.zeros((1, hidden_size)) self.W2 = np.random.randn(hidden_size, output_size) * 0.5 self.b2 = np.zeros((1, output_size)) def sigmoid(self, x): return 1 / (1 + np.exp(-np.clip(x, -500, 500))) def forward(self, X): self.z1 = X @ self.W1 + self.b1 self.a1 = self.sigmoid(self.z1) self.z2 = self.a1 @ self.W2 + self.b2 self.a2 = self.sigmoid(self.z2) return self.a2 def backward(self, X, y): m = X.shape[0] # Output layer gradients  dz2 = 2 * (self.a2 - y) * self.sigmoid(self.z2) * (1 - self.sigmoid(self.z2)) dW2 = self.a1.T @ dz2 / m db2 = np.mean(dz2, axis=0, keepdims=True) # Hidden layer gradients  dz1 = (dz2 @ self.W2.T) * self.sigmoid(self.z1) * (1 - self.sigmoid(self.z1)) dW1 = X.T @ dz1 / m db1 = np.mean(dz1, axis=0, keepdims=True) return dW1, db1, dW2, db2 def train(self, X, y, epochs=1000, learning_rate=1.0): losses = [] for epoch in range(epochs): # Forward pass  predictions = self.forward(X) # Compute loss  loss = np.mean((predictions - y) ** 2) losses.append(loss) # Backward pass  dW1, db1, dW2, db2 = self.backward(X, y) # Update weights  self.W1 -= learning_rate * dW1 self.b1 -= learning_rate * db1 self.W2 -= learning_rate * dW2 self.b2 -= learning_rate * db2 if epoch % 100 == 0: print(f"Epoch {epoch:4d}: Loss = {loss:.6f}") return losses def predict(self, X): return self.forward(X) # Test the class nn = SimpleNeuralNetwork() losses = nn.train(X, y, epochs=1000, learning_rate=1.0) predictions = nn.predict(X) print("\nClass-based Neural Network Results:") for i in range(len(X)): pred = predictions[i, 0] target = y[i, 0] print(f"{X[i]} -> {target} | Prediction: {pred:.4f} | Rounded: {round(pred)}") 

# Test different hidden layer sizes hidden_sizes = [2, 4, 8, 16] results = {} for hidden_size in hidden_sizes: print(f"\nTesting hidden size: {hidden_size}") nn = SimpleNeuralNetwork(input_size=2, hidden_size=hidden_size, output_size=1) losses = nn.train(X, y, epochs=1000, learning_rate=1.0) predictions = nn.predict(X) # Calculate accuracy  accuracy = np.mean(np.round(predictions) == y) results[hidden_size] = { 'final_loss': losses[-1], 'accuracy': accuracy, 'predictions': predictions } print(f" Final loss: {losses[-1]:.6f}") print(f" Accuracy: {accuracy:.1%}") # Plot comparison plt.figure(figsize=(12, 4)) plt.subplot(1, 2, 1) hidden_sizes_list = list(results.keys()) final_losses = [results[hs]['final_loss'] for hs in hidden_sizes_list] plt.bar(hidden_sizes_list, final_losses) plt.title('Final Loss vs Hidden Size') plt.xlabel('Hidden Layer Size') plt.ylabel('Final Loss') plt.yscale('log') plt.subplot(1, 2, 2) accuracies = [results[hs]['accuracy'] for hs in hidden_sizes_list] plt.bar(hidden_sizes_list, accuracies) plt.title('Accuracy vs Hidden Size') plt.xlabel('Hidden Layer Size') plt.ylabel('Accuracy') plt.ylim(0, 1) plt.tight_layout() plt.show()