Objective: Implement the forward and backward pass for a 2-layer network entirely by hand in NumPy, then verify your analytic gradients against a numerical approximation. All in runnable, in-browser Python.
Setup: A 2-Layer Network
We'll use a network with one hidden layer (ReLU) and a sigmoid output, trained with squared-error loss on a single example at a time, to keep every gradient expression simple enough to write out explicitly.
import numpy as np
np.random.seed(42)
W1, b1 = np.random.randn(4, 3) * 0.5, np.zeros(4)
W2, b2 = np.random.randn(1, 4) * 0.5, np.zeros(1)
x = np.array([1.0, -0.5, 2.0])
y = 1.0 # true label
print("W1 shape:", W1.shape, " W2 shape:", W2.shape)
Task 1: Forward Pass, Caching Everything Backprop Will Need
import numpy as np
def relu(z): return np.maximum(0, z)
def sigmoid(z): return 1 / (1 + np.exp(-z))
np.random.seed(42)
W1, b1 = np.random.randn(4, 3) * 0.5, np.zeros(4)
W2, b2 = np.random.randn(1, 4) * 0.5, np.zeros(1)
x = np.array([1.0, -0.5, 2.0])
y = 1.0
def forward(x, W1, b1, W2, b2):
z1 = W1 @ x + b1
a1 = relu(z1)
z2 = W2 @ a1 + b2
y_hat = sigmoid(z2)
cache = {"x": x, "z1": z1, "a1": a1, "z2": z2, "y_hat": y_hat}
return y_hat, cache
y_hat, cache = forward(x, W1, b1, W2, b2)
loss = float((y_hat[0] - y) ** 2)
print("Prediction:", round(float(y_hat[0]), 4))
print("Loss:", round(loss, 4))
Task 2: Backward Pass, One Layer at a Time
Work backward from the loss. First the output layer's gradients, then propagate the gradient back into the hidden layer, exactly mirroring the chain-rule walk-through from the lesson, just with matrices and vectors instead of scalars.
import numpy as np
def relu(z): return np.maximum(0, z)
def relu_grad(z): return (z > 0).astype(float)
def sigmoid(z): return 1 / (1 + np.exp(-z))
def sigmoid_grad(z):
s = sigmoid(z)
return s * (1 - s)
np.random.seed(42)
W1, b1 = np.random.randn(4, 3) * 0.5, np.zeros(4)
W2, b2 = np.random.randn(1, 4) * 0.5, np.zeros(1)
x = np.array([1.0, -0.5, 2.0])
y = 1.0
def forward(x):
z1 = W1 @ x + b1
a1 = relu(z1)
z2 = W2 @ a1 + b2
y_hat = sigmoid(z2)
return y_hat, {"x": x, "z1": z1, "a1": a1, "z2": z2}
def backward(y_hat, y, cache):
x, z1, a1, z2 = cache["x"], cache["z1"], cache["a1"], cache["z2"]
# Output layer
dL_dyhat = 2 * (y_hat - y) # dL/dy_hat, shape (1,)
dyhat_dz2 = sigmoid_grad(z2) # da/dz for the output neuron
dL_dz2 = dL_dyhat * dyhat_dz2 # combine: dL/dz2
dL_dW2 = np.outer(dL_dz2, a1) # gradient for W2, shape matches W2
dL_db2 = dL_dz2
# Propagate into the hidden layer
dL_da1 = W2.T @ dL_dz2 # how the loss depends on hidden activations
dL_dz1 = dL_da1 * relu_grad(z1) # apply ReLU's local gradient elementwise
dL_dW1 = np.outer(dL_dz1, x)
dL_db1 = dL_dz1
return dL_dW1, dL_db1, dL_dW2, dL_db2
y_hat, cache = forward(x)
dW1, db1, dW2, db2 = backward(y_hat, y, cache)
print("dL/dW2 shape:", dW2.shape, " dL/dW1 shape:", dW1.shape)
print("dL/dW1:\n", np.round(dW1, 4))
Two patterns to notice: np.outer(dL_dz, a_prev) appears for every layer's weight gradient, the
gradient with respect to a layer's weights is always the outer product of "how much each output neuron's
pre-activation should change" with "what was fed into this layer." And W2.T @ dL_dz2 is exactly
how the gradient signal travels backward through a layer's weights into the layer before it: the transpose of
the forward weight matrix, multiplying the incoming gradient.
Task 3: Verify With Numerical Gradient Checking
The standard way to confirm hand-derived gradients are correct is to compare them against a numerical
approximation: nudge one weight by a tiny ε, rerun the forward pass, and see how much the
loss changed. If your analytic and numerical gradients agree closely, your backward pass is correct.
import numpy as np
def relu(z): return np.maximum(0, z)
def relu_grad(z): return (z > 0).astype(float)
def sigmoid(z): return 1 / (1 + np.exp(-z))
def sigmoid_grad(z):
s = sigmoid(z)
return s * (1 - s)
np.random.seed(42)
W1, b1 = np.random.randn(4, 3) * 0.5, np.zeros(4)
W2, b2 = np.random.randn(1, 4) * 0.5, np.zeros(1)
x = np.array([1.0, -0.5, 2.0])
y = 1.0
def forward(x, W1, b1, W2, b2):
z1 = W1 @ x + b1
a1 = relu(z1)
z2 = W2 @ a1 + b2
y_hat = sigmoid(z2)
return y_hat, {"x": x, "z1": z1, "a1": a1, "z2": z2}
def loss_fn(W1, b1, W2, b2):
y_hat, _ = forward(x, W1, b1, W2, b2)
return float((y_hat[0] - y) ** 2)
def backward(y_hat, y, cache, W2):
x, z1, a1, z2 = cache["x"], cache["z1"], cache["a1"], cache["z2"]
dL_dz2 = 2 * (y_hat - y) * sigmoid_grad(z2)
dL_dW2 = np.outer(dL_dz2, a1)
dL_da1 = W2.T @ dL_dz2
dL_dz1 = dL_da1 * relu_grad(z1)
dL_dW1 = np.outer(dL_dz1, x)
return dL_dW1, dL_dW2
y_hat, cache = forward(x, W1, b1, W2, b2)
analytic_dW1, analytic_dW2 = backward(y_hat, y, cache, W2)
# Numerical gradient check on a single entry of W1
eps = 1e-5
i, j = 2, 1 # pick one weight to check
W1_plus = W1.copy(); W1_plus[i, j] += eps
W1_minus = W1.copy(); W1_minus[i, j] -= eps
numerical_grad = (loss_fn(W1_plus, b1, W2, b2) - loss_fn(W1_minus, b1, W2, b2)) / (2 * eps)
analytic_grad = analytic_dW1[i, j]
print(f"Analytic gradient: {analytic_grad:.6f}")
print(f"Numerical gradient: {numerical_grad:.6f}")
print(f"Difference: {abs(analytic_grad - numerical_grad):.8f}")
The difference should be tiny, on the order of 1e-6 or smaller. If it's much larger than that,
the most likely culprits are a transpose in the wrong place, a missing elementwise multiply by an activation's
local gradient, or mixing up which cached value (z vs. a) a gradient formula should
use. This numerical check is exactly the technique real deep learning frameworks used historically to validate
new layer implementations before automatic differentiation made hand-written backward passes mostly
unnecessary in practice.