Objective: Practice the debugging lesson's diagnostic process on five synthetic, deliberately-constructed training runs, each exhibiting a different common failure pattern. All in runnable, in-browser Python.
Task 1: Generate Five Synthetic Training Runs
Each of these is constructed to mimic a specific, recognizable failure pattern from the lesson.
import numpy as np
import matplotlib.pyplot as plt
np.random.seed(0)
epochs = np.arange(40)
runs = {
"A: Learning rate too high": 1.0 + 0.3 * np.sin(epochs) + 0.02 * epochs,
"B: Healthy training": 1.2 * np.exp(-0.12 * epochs) + 0.08,
"C: Underfitting": 1.3 - 0.004 * epochs,
"D: Overfitting": 1.0 * np.exp(-0.13 * epochs) + 0.05,
"E: Stuck from the start": np.full(40, 1.38) + np.random.normal(0, 0.005, 40),
}
val_runs = {
"A: Learning rate too high": runs["A: Learning rate too high"] + 0.05,
"B: Healthy training": 1.25 * np.exp(-0.11 * epochs) + 0.11,
"C: Underfitting": 1.32 - 0.0035 * epochs,
"D: Overfitting": 1.0 * np.exp(-0.1 * epochs) + 0.18 + 0.006 * np.maximum(0, epochs - 12),
"E: Stuck from the start": np.full(40, 1.39) + np.random.normal(0, 0.005, 40),
}
fig, axes = plt.subplots(1, 5, figsize=(17, 3.2))
for ax, name in zip(axes, runs):
ax.plot(epochs, runs[name], label="train")
ax.plot(epochs, val_runs[name], label="val")
ax.set_title(name, fontsize=9); ax.legend(fontsize=7)
plt.tight_layout()
show_plot()
Task 2: Write a Diagnostic Function
Translate the debugging lesson's checklist into a small heuristic function that flags likely failure modes from a pair of loss curves.
import numpy as np
def diagnose(train_loss, val_loss):
train_loss, val_loss = np.array(train_loss), np.array(val_loss)
flags = []
# Stuck from the start: loss barely moves across the whole run
if abs(train_loss[0] - train_loss[-1]) < 0.05 * train_loss[0]:
flags.append("Loss barely moved: check learning rate, loss/label setup, and gradient flow.")
# Oscillating / diverging: loss increases noticeably at any point and stays noisy
diffs = np.diff(train_loss)
if np.mean(diffs > 0) > 0.3 and np.std(diffs) > 0.05:
flags.append("Loss oscillating or trending up: learning rate is likely too high.")
# Underfitting: both curves high and close together, barely decreasing
gap = np.mean(np.abs(train_loss - val_loss))
if gap < 0.05 and (train_loss[0] - train_loss[-1]) < 0.3 * train_loss[0]:
flags.append("Train and val both high and close together: likely underfitting.")
# Overfitting: train keeps falling, val rises in the second half
half = len(val_loss) // 2
if val_loss[-1] > val_loss[half] and train_loss[-1] < train_loss[half]:
flags.append("Val loss rising while train loss falls: likely overfitting.")
if not flags:
flags.append("No major red flags detected: looks like healthy training.")
return flags
# Try it on a "healthy" run from Task 1
epochs = np.arange(40)
train_healthy = 1.2 * np.exp(-0.12 * epochs) + 0.08
val_healthy = 1.25 * np.exp(-0.11 * epochs) + 0.11
print("Diagnosis for healthy run:", diagnose(train_healthy, val_healthy))
Task 3: Run the Diagnostic on All Five Runs
import numpy as np
def diagnose(train_loss, val_loss):
train_loss, val_loss = np.array(train_loss), np.array(val_loss)
flags = []
if abs(train_loss[0] - train_loss[-1]) < 0.05 * train_loss[0]:
flags.append("Loss barely moved: check learning rate, loss/label setup, and gradient flow.")
diffs = np.diff(train_loss)
if np.mean(diffs > 0) > 0.3 and np.std(diffs) > 0.05:
flags.append("Loss oscillating or trending up: learning rate is likely too high.")
gap = np.mean(np.abs(train_loss - val_loss))
if gap < 0.05 and (train_loss[0] - train_loss[-1]) < 0.3 * train_loss[0]:
flags.append("Train and val both high and close together: likely underfitting.")
half = len(val_loss) // 2
if val_loss[-1] > val_loss[half] and train_loss[-1] < train_loss[half]:
flags.append("Val loss rising while train loss falls: likely overfitting.")
if not flags:
flags.append("No major red flags detected: looks like healthy training.")
return flags
np.random.seed(0)
epochs = np.arange(40)
runs = {
"A: Learning rate too high": (1.0 + 0.3 * np.sin(epochs) + 0.02 * epochs,
1.0 + 0.3 * np.sin(epochs) + 0.02 * epochs + 0.05),
"B: Healthy training": (1.2 * np.exp(-0.12 * epochs) + 0.08,
1.25 * np.exp(-0.11 * epochs) + 0.11),
"C: Underfitting": (1.3 - 0.004 * epochs, 1.32 - 0.0035 * epochs),
"D: Overfitting": (1.0 * np.exp(-0.13 * epochs) + 0.05,
1.0 * np.exp(-0.1 * epochs) + 0.18 + 0.006 * np.maximum(0, epochs - 12)),
"E: Stuck from the start": (np.full(40, 1.38), np.full(40, 1.39)),
}
for name, (train_loss, val_loss) in runs.items():
print(f"{name}")
for flag in diagnose(train_loss, val_loss):
print(" ->", flag)
print()
Check the diagnostic's output against what you'd expect from each run's name and the plot from Task 1. The heuristics here are deliberately simple (real debugging always involves judgment, not just thresholds), but they capture the same checklist from the lesson: is the loss moving at all, is it oscillating, is there a persistent train/val gap, and which direction is that gap trending.
Task 4: Inspect Gradient Norms for the "Stuck" Run
For run E ("stuck from the start"), the lesson's advice is to check gradient norms directly. Simulate this with a tiny network whose weights happen to put every ReLU into its dead zone, exactly the kind of root cause a flat loss curve alone can't distinguish from "the learning rate is too small."
import numpy as np
def relu_grad(z): return (z > 0).astype(float)
np.random.seed(0)
# Deliberately bad initialization: large negative bias drives every ReLU dead
x = np.random.randn(5)
W = np.random.randn(6, 5) * 0.3
b_bad = -np.full(6, 10.0) # huge negative bias
b_good = np.zeros(6)
z_bad = W @ x + b_bad
z_good = W @ x + b_good
print("Pre-activations with bad bias:", np.round(z_bad, 2))
print("ReLU gradients (bad bias): ", relu_grad(z_bad), "<- all zero, dead layer")
print()
print("Pre-activations with normal bias:", np.round(z_good, 2))
print("ReLU gradients (normal bias): ", relu_grad(z_good))
A gradient-norm check at the very start of training would immediately reveal the all-zero gradient in the "bad bias" case, distinguishing a genuinely broken layer from a training run that's merely slow, exactly the kind of root-cause check that a loss curve by itself cannot provide.