Neural Networks from Scratch

%matplotlib inline

from pathlib import Path

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import torch
import torch.nn as nn

plt.rcParams.update({"figure.figsize": (9, 4), "axes.grid": True, "grid.alpha": 0.3})
torch.manual_seed(0)
np.random.seed(0)

print("torch", torch.__version__)
print("everything here runs on the CPU in seconds — no GPU required")

torch 2.13.0
everything here runs on the CPU in seconds — no GPU required

def sigmoid(z):
    return 1 / (1 + np.exp(-z))


weights = np.array([0.8, -1.2])      # what the model learned
bias = 0.1
inputs = np.array([1.5, 0.4])        # one application, two features

z = inputs @ weights + bias
print(f"weighted sum + bias: {z:.3f}")
print(f"after sigmoid:       {sigmoid(z):.3f}   <- a probability")

weighted sum + bias: 0.820
after sigmoid:       0.694   <- a probability

z = np.linspace(-5, 5, 200)

plt.plot(z, sigmoid(z), label="sigmoid — squashes to (0, 1)")
plt.plot(z, np.tanh(z), label="tanh — squashes to (-1, 1)")
plt.plot(z, np.maximum(0, z), label="ReLU — max(0, z)")
plt.axhline(0, color="grey", linewidth=0.8)
plt.legend()
plt.title("Three activation functions")
plt.tight_layout()
plt.show()

from sklearn.linear_model import LogisticRegression

X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]], dtype=float)
y = np.array([[0], [1], [1], [0]], dtype=float)

single_neuron = LogisticRegression().fit(X, y.ravel())
print("logistic regression accuracy on XOR:", single_neuron.score(X, y.ravel()))
print("predictions:", single_neuron.predict(X))

logistic regression accuracy on XOR: 0.5
predictions: [0. 0. 0. 0.]

rng = np.random.default_rng(0)
HIDDEN = 8
LEARNING_RATE = 1.0

W1 = rng.normal(0, 1, (2, HIDDEN))      # input -> hidden
b1 = np.zeros((1, HIDDEN))
W2 = rng.normal(0, 1, (HIDDEN, 1))      # hidden -> output
b2 = np.zeros((1, 1))

history = []
for epoch in range(5001):
    # 1. forward
    hidden = np.tanh(X @ W1 + b1)
    output = sigmoid(hidden @ W2 + b2)

    # 2. loss (binary cross-entropy)
    loss = -np.mean(y * np.log(output + 1e-9) + (1 - y) * np.log(1 - output + 1e-9))

    # 3. backward — the chain rule, one layer at a time
    d_output = (output - y) / len(X)              # dLoss/dz for sigmoid + cross-entropy
    dW2 = hidden.T @ d_output
    db2 = d_output.sum(axis=0, keepdims=True)

    d_hidden = (d_output @ W2.T) * (1 - hidden ** 2)   # tanh'(z) = 1 - tanh(z)^2
    dW1 = X.T @ d_hidden
    db1 = d_hidden.sum(axis=0, keepdims=True)

    # 4. update
    W1 -= LEARNING_RATE * dW1
    b1 -= LEARNING_RATE * db1
    W2 -= LEARNING_RATE * dW2
    b2 -= LEARNING_RATE * db2

    history.append(loss)
    if epoch % 1000 == 0:
        print(f"epoch {epoch:>5}   loss {loss:.4f}")

print("\nfinal predictions:", output.ravel().round(3))
print("targets:          ", y.ravel())

epoch     0   loss 0.7414
epoch  1000   loss 0.0013
epoch  2000   loss 0.0006
epoch  3000   loss 0.0004
epoch  4000   loss 0.0003
epoch  5000   loss 0.0002

final predictions: [0. 1. 1. 0.]
targets:           [0. 1. 1. 0.]

plt.plot(history)
plt.xlabel("epoch")
plt.ylabel("loss")
plt.title("Learning XOR: loss against epoch")
plt.tight_layout()
plt.show()

x = torch.tensor(2.0, requires_grad=True)
z = 3 * x ** 2 + 1

z.backward()                 # walk the graph backwards, filling in gradients

print(f"z = 3x² + 1 at x = 2 gives z = {z.item():.1f}")
print(f"dz/dx = 6x = {x.grad.item():.1f}   (computed, not derived by us)")

z = 3x² + 1 at x = 2 gives z = 13.0
dz/dx = 6x = 12.0   (computed, not derived by us)

X_tensor = torch.tensor(X, dtype=torch.float32)
y_tensor = torch.tensor(y, dtype=torch.float32)

model = nn.Sequential(
    nn.Linear(2, 8),      # input -> hidden
    nn.Tanh(),
    nn.Linear(8, 1),      # hidden -> output
)

loss_function = nn.BCEWithLogitsLoss()               # sigmoid + cross-entropy, fused
optimiser = torch.optim.SGD(model.parameters(), lr=1.0)

for epoch in range(5001):
    optimiser.zero_grad()                 # clear last step's gradients
    logits = model(X_tensor)              # forward
    loss = loss_function(logits, y_tensor)
    loss.backward()                       # backward — autograd does section 3
    optimiser.step()                      # update

    if epoch % 1000 == 0:
        print(f"epoch {epoch:>5}   loss {loss.item():.4f}")

print("\npredictions:", torch.sigmoid(model(X_tensor)).detach().numpy().ravel().round(3))

epoch     0   loss 0.7012
epoch  1000   loss 0.0013
epoch  2000   loss 0.0006


epoch  3000   loss 0.0004
epoch  4000   loss 0.0003
epoch  5000   loss 0.0002

predictions: [0. 1. 1. 0.]

class SmallNet(nn.Module):
    def __init__(self, n_features, hidden=32):
        super().__init__()
        self.hidden = nn.Linear(n_features, hidden)
        self.output = nn.Linear(hidden, 1)
        self.activation = nn.ReLU()

    def forward(self, x):
        return self.output(self.activation(self.hidden(x)))


net = SmallNet(n_features=2)
print(net)
print(f"\nparameters: {sum(p.numel() for p in net.parameters()):,}")

SmallNet(
  (hidden): Linear(in_features=2, out_features=32, bias=True)
  (output): Linear(in_features=32, out_features=1, bias=True)
  (activation): ReLU()
)

parameters: 129

from sklearn.compose import ColumnTransformer
from sklearn.impute import SimpleImputer
from sklearn.metrics import average_precision_score, roc_auc_score
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import OneHotEncoder, StandardScaler


def find_repository_root():
    for folder in [Path.cwd(), *Path.cwd().parents]:
        if (folder / "data").is_dir():
            return folder
    raise FileNotFoundError("could not find the repository root")


ROOT = find_repository_root()
loans = pd.read_csv(ROOT / "data" / "loan_default.csv")

NUMERIC = ["loan_amount", "tenure_months", "interest_rate", "previous_loans",
           "days_past_due_history", "annual_income", "credit_score"]
CATEGORICAL = ["sector", "employment_type", "has_collateral"]

preparation = ColumnTransformer([
    ("numeric", Pipeline([("impute", SimpleImputer(strategy="median")),
                          ("scale", StandardScaler())]), NUMERIC),
    ("categorical", Pipeline([("impute", SimpleImputer(strategy="most_frequent")),
                              ("encode", OneHotEncoder(handle_unknown="ignore", drop="first"))]),
     CATEGORICAL),
])

X_train_raw, X_test_raw, y_train, y_test = train_test_split(
    loans[NUMERIC + CATEGORICAL], loans["defaulted"],
    test_size=0.25, stratify=loans["defaulted"], random_state=42,
)

X_train = np.asarray(preparation.fit_transform(X_train_raw), dtype=np.float32)
X_test = np.asarray(preparation.transform(X_test_raw), dtype=np.float32)

print(f"{X_train.shape[0]:,} training rows, {X_train.shape[1]} features after preprocessing")
print("scaling matters more for neural networks than for anything else —")
print("unscaled inputs make the gradients wildly different sizes per feature")

9,000 training rows, 16 features after preprocessing
scaling matters more for neural networks than for anything else —
unscaled inputs make the gradients wildly different sizes per feature

from torch.utils.data import DataLoader, TensorDataset

train_data = TensorDataset(torch.tensor(X_train),
                           torch.tensor(y_train.to_numpy(), dtype=torch.float32).unsqueeze(1))
loader = DataLoader(train_data, batch_size=256, shuffle=True)

net = SmallNet(n_features=X_train.shape[1], hidden=32)
optimiser = torch.optim.Adam(net.parameters(), lr=1e-3)
loss_function = nn.BCEWithLogitsLoss()

for epoch in range(60):
    net.train()
    for batch_X, batch_y in loader:
        optimiser.zero_grad()
        loss = loss_function(net(batch_X), batch_y)
        loss.backward()
        optimiser.step()

net.eval()
with torch.no_grad():
    probabilities = torch.sigmoid(net(torch.tensor(X_test))).numpy().ravel()

logistic = LogisticRegression(max_iter=1000).fit(X_train, y_train)
logistic_probabilities = logistic.predict_proba(X_test)[:, 1]

print(f"neural network      AUC {roc_auc_score(y_test, probabilities):.4f}   "
      f"AP {average_precision_score(y_test, probabilities):.4f}")
print(f"logistic regression AUC {roc_auc_score(y_test, logistic_probabilities):.4f}   "
      f"AP {average_precision_score(y_test, logistic_probabilities):.4f}")

neural network      AUC 0.7795   AP 0.3848
logistic regression AUC 0.7822   AP 0.3817

deeper = nn.Sequential(
    nn.Linear(X_train.shape[1], 64), nn.ReLU(),
    nn.Linear(64, 32), nn.ReLU(),
    nn.Linear(32, 1),
)
optimiser = torch.optim.Adam(deeper.parameters(), lr=1e-3)

train_curve, test_curve = [], []
X_test_tensor = torch.tensor(X_test)
y_test_tensor = torch.tensor(y_test.to_numpy(), dtype=torch.float32).unsqueeze(1)

for epoch in range(60):
    deeper.train()
    for batch_X, batch_y in loader:
        optimiser.zero_grad()
        loss = loss_function(deeper(batch_X), batch_y)
        loss.backward()
        optimiser.step()

    deeper.eval()
    with torch.no_grad():
        train_curve.append(loss_function(deeper(torch.tensor(X_train)),
                                         torch.tensor(y_train.to_numpy(), dtype=torch.float32).unsqueeze(1)).item())
        test_curve.append(loss_function(deeper(X_test_tensor), y_test_tensor).item())

with torch.no_grad():
    deep_probabilities = torch.sigmoid(deeper(X_test_tensor)).numpy().ravel()
print(f"deeper network AUC {roc_auc_score(y_test, deep_probabilities):.4f}")

plt.plot(train_curve, label="training loss")
plt.plot(test_curve, label="validation loss")
plt.xlabel("epoch")
plt.ylabel("binary cross-entropy")
plt.title("The gap that opens is overfitting")
plt.legend()
plt.tight_layout()
plt.show()

deeper network AUC 0.7757

for epoch in range(epochs):
    model.train()                          # enable dropout / batch-norm training behaviour
    for batch_X, batch_y in loader:        # mini-batches, reshuffled each epoch
        optimiser.zero_grad()              # clear old gradients
        loss = loss_function(model(batch_X), batch_y)
        loss.backward()                    # autograd fills every .grad
        optimiser.step()                   # weights move against the gradient

    model.eval()                           # disable dropout for evaluation
    with torch.no_grad():                  # no gradient bookkeeping — faster, less memory
        validate(model)