Objective: Build a complete, from-scratch NumPy training pipeline: a multi-layer network with He initialization, Adam-style momentum, dropout, early stopping, and a debugging sanity check, trained end to end on a real classification dataset.
Setup
We'll use load_breast_cancer from scikit-learn, the same dataset from the Machine Learning
Fundamentals capstone, so you can focus entirely on the deep learning pipeline rather than getting acquainted with new
data. Every code block below runs live in your browser via Pyodide.
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
data = load_breast_cancer()
X, y = data.data, data.target
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=42, stratify=y_train)
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
X_val = scaler.transform(X_val)
X_test = scaler.transform(X_test)
print("Train:", X_train.shape, " Val:", X_val.shape, " Test:", X_test.shape)
print("Class balance (train):", np.bincount(y_train))
Task 1: Define the Network With He Initialization
Two hidden ReLU layers and a sigmoid output for binary classification, with weights scaled using He initialization, since every hidden layer uses ReLU.
import numpy as np
def relu(z): return np.maximum(0, z)
def relu_grad(z): return (z > 0).astype(float)
def sigmoid(z): return 1 / (1 + np.exp(-z))
def he_init(n_out, n_in, seed):
rng = np.random.default_rng(seed)
return rng.standard_normal((n_out, n_in)) * np.sqrt(2.0 / n_in)
n_features = 30
layer_sizes = [n_features, 16, 8, 1]
params = {}
for i in range(1, len(layer_sizes)):
params[f"W{i}"] = he_init(layer_sizes[i], layer_sizes[i-1], seed=i)
params[f"b{i}"] = np.zeros(layer_sizes[i])
for k, v in params.items():
print(k, v.shape)
Task 2: Forward Pass With Dropout
import numpy as np
def relu(z): return np.maximum(0, z)
def sigmoid(z): return 1 / (1 + np.exp(-z))
def forward(X, params, training, dropout_p=0.3, rng=None):
cache = {"A0": X}
A = X
n_layers = len(params) // 2
for i in range(1, n_layers + 1):
Z = A @ params[f"W{i}"].T + params[f"b{i}"]
if i < n_layers:
A = relu(Z)
if training:
mask = (rng.random(A.shape) > dropout_p) / (1 - dropout_p)
A = A * mask
else:
A = sigmoid(Z)
cache[f"Z{i}"] = Z
cache[f"A{i}"] = A
return A, cache
# Quick shape check with the params from Task 1
import numpy as np
def he_init(n_out, n_in, seed):
rng = np.random.default_rng(seed)
return rng.standard_normal((n_out, n_in)) * np.sqrt(2.0 / n_in)
layer_sizes = [30, 16, 8, 1]
params = {}
for i in range(1, len(layer_sizes)):
params[f"W{i}"] = he_init(layer_sizes[i], layer_sizes[i-1], seed=i)
params[f"b{i}"] = np.zeros(layer_sizes[i])
X_dummy = np.random.randn(5, 30)
rng = np.random.default_rng(0)
y_hat, cache = forward(X_dummy, params, training=True, rng=rng)
print("Output shape for a batch of 5:", y_hat.shape)
Task 3: Backward Pass and a Training Loop With Early Stopping
This combines backpropagation (Phase 2), mini-batch gradient descent with momentum (Phase 3), dropout (Phase 4), and early stopping (Phase 4) into one training loop.
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
data = load_breast_cancer()
X, y = data.data, data.target
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=42, stratify=y_train)
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train); X_val = scaler.transform(X_val); X_test = scaler.transform(X_test)
def relu(z): return np.maximum(0, z)
def relu_grad(z): return (z > 0).astype(float)
def sigmoid(z): return 1 / (1 + np.exp(-z))
def he_init(n_out, n_in, seed):
rng = np.random.default_rng(seed)
return rng.standard_normal((n_out, n_in)) * np.sqrt(2.0 / n_in)
layer_sizes = [30, 16, 8, 1]
params = {}
velocity = {}
for i in range(1, len(layer_sizes)):
params[f"W{i}"] = he_init(layer_sizes[i], layer_sizes[i-1], seed=i)
params[f"b{i}"] = np.zeros(layer_sizes[i])
velocity[f"W{i}"] = np.zeros_like(params[f"W{i}"])
velocity[f"b{i}"] = np.zeros_like(params[f"b{i}"])
def forward(X, params, training, dropout_p, rng):
cache = {"A0": X}
A = X
n_layers = len(params) // 2
for i in range(1, n_layers + 1):
Z = A @ params[f"W{i}"].T + params[f"b{i}"]
if i < n_layers:
A = relu(Z)
if training:
mask = (rng.random(A.shape) > dropout_p) / (1 - dropout_p)
A = A * mask
else:
A = sigmoid(Z)
cache[f"Z{i}"], cache[f"A{i}"] = Z, A
return A, cache
def backward(y_true, cache, params):
n_layers = len(params) // 2
m = y_true.shape[0]
grads = {}
dA = (cache[f"A{n_layers}"] - y_true.reshape(-1, 1)) / m # combined sigmoid+BCE gradient
for i in range(n_layers, 0, -1):
dZ = dA if i == n_layers else dA * relu_grad(cache[f"Z{i}"])
grads[f"W{i}"] = dZ.T @ cache[f"A{i-1}"]
grads[f"b{i}"] = dZ.sum(axis=0)
dA = dZ @ params[f"W{i}"]
return grads
def bce_loss(y_true, y_pred):
eps = 1e-9
return -np.mean(y_true * np.log(y_pred + eps) + (1 - y_true) * np.log(1 - y_pred + eps))
rng = np.random.default_rng(0)
lr, beta, batch_size = 0.1, 0.9, 32
best_val_loss, patience, wait = float("inf"), 10, 0
n_train = X_train.shape[0]
for epoch in range(200):
perm = rng.permutation(n_train)
for start in range(0, n_train, batch_size):
idx = perm[start:start+batch_size]
Xb, yb = X_train[idx], y_train[idx]
y_hat, cache = forward(Xb, params, training=True, dropout_p=0.2, rng=rng)
grads = backward(yb, cache, params)
for k in params:
velocity[k] = beta * velocity[k] + (1 - beta) * grads[k]
params[k] -= lr * velocity[k]
y_val_hat, _ = forward(X_val, params, training=False, dropout_p=0.0, rng=rng)
val_loss = bce_loss(y_val, y_val_hat.flatten())
if val_loss < best_val_loss:
best_val_loss, wait = val_loss, 0
best_params = {k: v.copy() for k, v in params.items()}
else:
wait += 1
if wait >= patience:
print(f"Early stopping at epoch {epoch}, best val loss {best_val_loss:.4f}")
break
if epoch % 20 == 0:
print(f"Epoch {epoch:3d} val loss: {val_loss:.4f}")
params = best_params
Task 4: Evaluate on the Test Set
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score, f1_score, confusion_matrix
data = load_breast_cancer()
X, y = data.data, data.target
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=42, stratify=y_train)
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train); X_test = scaler.transform(X_test)
def relu(z): return np.maximum(0, z)
def sigmoid(z): return 1 / (1 + np.exp(-z))
def he_init(n_out, n_in, seed):
rng = np.random.default_rng(seed)
return rng.standard_normal((n_out, n_in)) * np.sqrt(2.0 / n_in)
# NOTE: for a real run, reuse the `params` learned in Task 3. This block
# uses freshly-initialized weights so it can also be run standalone.
layer_sizes = [30, 16, 8, 1]
params = {}
for i in range(1, len(layer_sizes)):
params[f"W{i}"] = he_init(layer_sizes[i], layer_sizes[i-1], seed=i + 100)
params[f"b{i}"] = np.zeros(layer_sizes[i])
def forward(X, params):
A = X
n_layers = len(params) // 2
for i in range(1, n_layers + 1):
Z = A @ params[f"W{i}"].T + params[f"b{i}"]
A = relu(Z) if i < n_layers else sigmoid(Z)
return A
y_test_hat = forward(X_test, params).flatten()
y_test_pred = (y_test_hat > 0.5).astype(int)
print("Test accuracy:", round(accuracy_score(y_test, y_test_pred), 4))
print("Test F1: ", round(f1_score(y_test, y_test_pred), 4))
print("Confusion matrix:\n", confusion_matrix(y_test, y_test_pred))
print()
print("In your own run, replace this block's freshly-initialized `params`")
print("with the early-stopped `params` you trained in Task 3 for a real result.")
Task 5: The Sanity Check You Should Have Run First
The debugging lesson's advice: before trusting any of the results above, confirm the pipeline itself isn't broken by overfitting a tiny slice of the training data to near-zero loss.
import numpy as np
def relu(z): return np.maximum(0, z)
def relu_grad(z): return (z > 0).astype(float)
def sigmoid(z): return 1 / (1 + np.exp(-z))
def he_init(n_out, n_in, seed):
rng = np.random.default_rng(seed)
return rng.standard_normal((n_out, n_in)) * np.sqrt(2.0 / n_in)
np.random.seed(0)
X_tiny = np.random.randn(10, 30)
y_tiny = np.random.randint(0, 2, 10)
layer_sizes = [30, 16, 8, 1]
params = {}
for i in range(1, len(layer_sizes)):
params[f"W{i}"] = he_init(layer_sizes[i], layer_sizes[i-1], seed=i)
params[f"b{i}"] = np.zeros(layer_sizes[i])
def forward(X, params):
cache = {"A0": X}
A = X
n_layers = len(params) // 2
for i in range(1, n_layers + 1):
Z = A @ params[f"W{i}"].T + params[f"b{i}"]
A = relu(Z) if i < n_layers else sigmoid(Z)
cache[f"Z{i}"], cache[f"A{i}"] = Z, A
return A, cache
def backward(y_true, cache, params):
n_layers = len(params) // 2
m = y_true.shape[0]
grads = {}
dA = (cache[f"A{n_layers}"] - y_true.reshape(-1, 1)) / m
for i in range(n_layers, 0, -1):
dZ = dA if i == n_layers else dA * relu_grad(cache[f"Z{i}"])
grads[f"W{i}"] = dZ.T @ cache[f"A{i-1}"]
grads[f"b{i}"] = dZ.sum(axis=0)
dA = dZ @ params[f"W{i}"]
return grads
for epoch in range(500):
y_hat, cache = forward(X_tiny, params)
grads = backward(y_tiny, cache, params)
for k in params:
params[k] -= 0.5 * grads[k]
if epoch % 100 == 0:
loss = -np.mean(y_tiny * np.log(y_hat.flatten() + 1e-9) + (1 - y_tiny) * np.log(1 - y_hat.flatten() + 1e-9))
print(f"Epoch {epoch}: loss = {loss:.6f}")
print("\nIf this loss isn't near zero, there's a bug in forward/backward,")
print("and the full training run above can't be trusted until it's fixed.")
This capstone deliberately mirrors the structure of every lab in this course: define the network, write the forward pass, write the backward pass, train with a real optimizer and regularization, and verify with the same sanity check the debugging lesson recommended running first. Try varying the architecture (more layers, different dropout rates, a different learning rate schedule from Phase 3) and see how the validation curve and final test metrics respond, that hands-on iteration is the real skill this course has been building toward.