Galaxy Classification with a Neural Net from Scratch#
Here we implement a neural network from scratch with a single hidden layer, using only NumPy and use it to classify galaxies.
import numpy as np
import matplotlib.pyplot as plt
import h5py
We need the Galaxy class we previously defined
galaxy_types = {0: "disturbed galaxies",
1: "merging galaxies",
2: "round smooth galaxies",
3: "in-between round smooth galaxies",
4: "cigar shaped smooth galaxies",
5: "barred spiral galaxies",
6: "unbarred tight spiral galaxies",
7: "unbarred loose spiral galaxies",
8: "edge-on galaxies without bulge",
9: "edge-on galaxies with bulge"}
class Galaxy:
def __init__(self, data, answer, *, index=-1):
self.data = np.array(data, dtype=np.float32) / 255.0 * 0.99 + 0.01
self.answer = answer
self.out = np.zeros(10, dtype=np.float32) + 0.01
self.out[self.answer] = 0.99
self.index = index
def plot(self, ax=None):
if ax is None:
fig, ax = plt.subplots()
ax.imshow(self.data, interpolation="nearest")
ax.text(0.025, 0.95, f"answer: {self.answer}",
color="white", transform=ax.transAxes)
def validate(self, prediction):
"""check if a categorical prediction matches the answer"""
return np.argmax(prediction) == self.answer
A manager class#
We’ll create a class to manage access to the data. This will do the following:
open the file and store the handles to access the data
partition the data into test and training sets
provide a means to shuffle the data
provide methods to get the next dataset (either training or test)
allow us to coarsen the images to a reduced resolution to reduce the memory requirements and make the training easier.
Tip
Since every input node is connected to every hidden layer node (a dense network), there are a lot of weights to train. By using smaller images, it is easier to train.
class DataManager:
def __init__(self, partition=0.8,
datafile="Galaxy10_DECals.h5",
include_transformation=False,
coarsen=1):
"""manage access to the data
partition: fraction that should be training
datafile: name of the hdf5 file with the data
coarsen: reduce the number of pixels by this factor
"""
self.ds = h5py.File(datafile)
self.ans = np.array(self.ds["ans"])
self.images = np.array(self.ds["images"])
self.coarsen = coarsen
self.include_transformation = include_transformation
N = len(self.ans)
# create a set of indices for the galaxies and randomize
self.indices = np.arange(N, dtype=np.uint32)
self.rng = np.random.default_rng()
self.rng.shuffle(self.indices)
# partition into training and test sets
# these indices will always refer to the index in the original
# unsplit dataset
n_cut = int(partition * N)
self.training_indices = self.indices[0:n_cut]
self.test_indices = self.indices[n_cut:N]
self.n_training = len(self.training_indices)
if self.include_transformation:
self.n_training *= 2
self.n_test = len(self.test_indices)
self.input_size = np.ravel(self._get_galaxy(0).data).size
def _get_galaxy(self, index):
"""return a numpy array containing a single galaxy image, coarsened
if necessary by averaging"""
_tmp = self.images[index, :, :, :]
if self.coarsen > 1:
_tmp = np.mean(_tmp.reshape(_tmp.shape[0]//self.coarsen, self.coarsen,
_tmp.shape[1]//self.coarsen, self.coarsen,
_tmp.shape[2]), axis=(1, 3))
return _tmp
def training_images(self):
self.reset_training()
for idx in self.training_indices:
yield Galaxy(self._get_galaxy(idx), self.ans[idx], index=idx)
if self.include_transformation:
yield Galaxy(np.rot90(self._get_galaxy(idx), axes=(0, 1)),
self.ans[idx], index=idx)
def reset_training(self):
"""prepare for the next epoch: shuffle the training data and
reset the index to point to the start"""
self.rng.shuffle(self.training_indices)
def test_images(self):
for idx in self.test_indices:
yield Galaxy(self._get_galaxy(idx), self.ans[idx], index=idx)
Tip
The training_images() and testing_images() are generators (like range()), so we can iterate like:
d = DataManager()
for g in d.training_images():
# do stuff with g
and g is only converted to 32-bit float as needed.
We can now work with the data as follows. Here we create a DataManager that will coarsen the images by a factor of 4 (so they will be 64x64 pixels with 3 colors).
d = DataManager(coarsen=4, include_transformation=True)
---------------------------------------------------------------------------
FileNotFoundError Traceback (most recent call last)
Cell In[5], line 1
----> 1 d = DataManager(coarsen=4, include_transformation=True)
Cell In[4], line 13, in DataManager.__init__(self, partition, datafile, include_transformation, coarsen)
9 datafile: name of the hdf5 file with the data
10 coarsen: reduce the number of pixels by this factor
11 """
12
---> 13 self.ds = h5py.File(datafile)
14 self.ans = np.array(self.ds["ans"])
15 self.images = np.array(self.ds["images"])
16
File /opt/hostedtoolcache/Python/3.14.10/x64/lib/python3.14/site-packages/h5py/_hl/files.py:555, in File.__init__(self, name, mode, driver, libver, userblock_size, swmr, rdcc_nslots, rdcc_nbytes, rdcc_w0, track_order, fs_strategy, fs_persist, fs_threshold, fs_page_size, page_buf_size, min_meta_keep, min_raw_keep, locking, alignment_threshold, alignment_interval, meta_block_size, track_times, **kwds)
546 fapl = make_fapl(driver, libver, rdcc_nslots, rdcc_nbytes, rdcc_w0,
547 locking, page_buf_size, min_meta_keep, min_raw_keep,
548 alignment_threshold=alignment_threshold,
549 alignment_interval=alignment_interval,
550 meta_block_size=meta_block_size,
551 **kwds)
552 fcpl = make_fcpl(track_order=track_order, track_times=track_times,
553 fs_strategy=fs_strategy, fs_persist=fs_persist,
554 fs_threshold=fs_threshold, fs_page_size=fs_page_size)
--> 555 fid = make_fid(name, mode, userblock_size, fapl, fcpl, swmr=swmr)
557 if isinstance(libver, tuple):
558 self._libver = libver
File /opt/hostedtoolcache/Python/3.14.10/x64/lib/python3.14/site-packages/h5py/_hl/files.py:232, in make_fid(name, mode, userblock_size, fapl, fcpl, swmr)
230 if swmr:
231 flags |= h5f.ACC_SWMR_READ
--> 232 fid = h5f.open(name, flags, fapl=fapl)
233 elif mode == 'r+':
234 fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl)
File h5py/_objects.pyx:54, in h5py._objects.with_phil.wrapper()
---> 54 'Could not get source, probably due dynamically evaluated source code.'
File h5py/_objects.pyx:55, in h5py._objects.with_phil.wrapper()
---> 55 'Could not get source, probably due dynamically evaluated source code.'
File h5py/h5f.pyx:106, in h5py.h5f.open()
--> 106 'Could not get source, probably due dynamically evaluated source code.'
FileNotFoundError: [Errno 2] Unable to synchronously open file (unable to open file: name = 'Galaxy10_DECals.h5', errno = 2, error message = 'No such file or directory', flags = 0, o_flags = 0)
we can see how many images there are in the training and test sets
d.n_training, d.n_test
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[6], line 1
----> 1 d.n_training, d.n_test
NameError: name 'd' is not defined
We can then get the first few training galaxies and look at them. Note that with transformations enabled, the should be pairs with one rotated 90 degrees compared to the previous.
for n, g in enumerate(d.training_images()):
g.plot()
if n == 3:
break
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[7], line 1
----> 1 for n, g in enumerate(d.training_images()):
2 g.plot()
3 if n == 3:
4 break
NameError: name 'd' is not defined
We’ll need a 1-d representation of the data, which the DataManager will provide
d.input_size
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[8], line 1
----> 1 d.input_size
NameError: name 'd' is not defined
Batching#
Training with this data will be very slow. We can speed it up more by using batching and aggregating the linear algebra. Here’s how this works.
Single input recap#
Our basic network does:
where the sizes of the matrices and vectors are:
\({\bf x}^k\) : \(N_\mathrm{in} \times 1\)
\({\bf B}\) : \(N_\mathrm{hidden} \times N_\mathrm{in}\)
\(\tilde{\bf z}^k\) : \(N_\mathrm{hidden}\times 1\)
\({\bf A}\) : \(N_\mathrm{out} \times N_\mathrm{hidden}\)
\({\bf z}^k\) : \(N_\mathrm{out} \times 1\)
we also have the known output, \({\bf y}^k\) corresponding to input \({\bf x}^k\)
\({\bf y}^k\) : \(N_\mathrm{out} \times 1\)
we then compute the errors:
\({\bf e}^k = {\bf z}^k - {\bf y}^k\) (the error on the output layer) : \(N_\mathrm{out} \times 1\)
\(\tilde{\bf e}^k = {\bf A}^\intercal \cdot [{\bf e}^k \circ {\bf z} \circ (1 - {\bf z})]\) (the error backpropagated to the hidden layer) : \(N_\mathrm{hidden} \times 1\)
and finally the corrections due to this single piece of training data, \(({\bf x}^k, {\bf y}^k)\):
\(\Delta {\bf A} = -2\eta \,{\bf e}^k \circ {\bf z}^k \circ (1 - {\bf z}^k) \cdot (\tilde{\bf z}^k)^\intercal\) : \(N_\mathrm{out} \times N_\mathrm{hidden}\)
\(\Delta {\bf B} = -2\eta \,\tilde{\bf e}^k \circ \tilde{\bf z}^k \circ (1 - \tilde{\bf z}^k) \cdot ({\bf x}^k)^\intercal\) : \(N_\mathrm{hidden} \times N_\mathrm{in}\)
A batching approach#
We now want to batch the inputs, by extending \({\bf x}\) to be of size \(N_\mathrm{in} \times S\), where \(S\) is the batch size. This means that each column is a unique input vector \({\bf x}^k\), and \(S\) of them are sandwiched together:
Similarly, we create a batched \({\bf y}_b\) that contains the \({\bf y}^k\) corresponding to the \({\bf x}^k\) in \({\bf x}_b\).
We can propagate this through the network, getting
where \({\bf z}_b\) is now of size \(N_\mathrm{out} \times S\).
Now, we compute the errors from the batched inputs
\({\bf e}_b = {\bf z}_b - {\bf y}_b\) : \(N_\mathrm{out} \times S\)
\(\tilde{\bf e}_b =\underbrace{{\bf A}^\intercal}_{N_\mathrm{hidden} \times N_\mathrm{out}} \cdot \underbrace{[{\bf e}_b \circ {\bf z}_b \circ (1 - {\bf z})]}_{N_\mathrm{out} \times S}\) : \(N_\mathrm{hidden} \times S\)
and the accumulated corrections:
\(\Delta {\bf A} = -\frac{2\eta}{S} \,\underbrace{{\bf e}_b \circ {\bf z}_b \circ (1 - {\bf z}_b)}_{N_\mathrm{out}\times S} \cdot \underbrace{(\tilde{\bf z}_b)^\intercal}_{S\times N_\mathrm{hidden}}\)
\(\Delta {\bf B} = -\frac{2\eta}{S} \,\underbrace{\tilde{\bf e}_b \circ \tilde{\bf z}_b \circ (1 - \tilde{\bf z}_b)}_{N_\mathrm{hidden} \times S} \cdot \underbrace{({\bf x}_b)^\intercal}_{S \times N_\mathrm{in}}\)
In these accumulated corrections, the \(S\) dimensions contract. In essence, this means that each element in \(\Delta {\bf A}\) and \(\Delta {\bf B}\) is the sum of the corrections for each of the \(S\) training data pairs in the batch. For this reason, we normalize by \(S\) to create the average of the gradient.
Tip
Batching also stabilizes the gradient descent, making it easier to find the minimum and allowing us to use a larger learning rate.
Momentum#
The other feature we need for this application is momentum in the gradient descent weight updates.
A popular form of momentum (see, e.g., Momentum: A simple, yet efficient optimizing technique) builds off of the idea of the exponential moving average.
For our gradient descent update, we usually do:
where \(\mathcal{L}\) is our loss function and \(\eta\) is the learning rate.
The basic idea of momentum begins with defining a “velocity”, \({\bf v}^{(0)} = 0\) (no momentum has been built up yet). Then each iteration of training we do the following:
construct the gradient from the current set of training, \(\partial \mathcal{L}/\partial {\bf A}\)
blend this with the previous momentum using an exponential moving average:
\[{\bf v}^{(i)} = \beta {\bf v}^{(i-1)} + (1 - \beta) \frac{\partial \mathcal{L}}{\partial {\bf A}}\]where \(\beta \in [0, 1]\) is the smoothing parameter. It seems like \(\beta = 0.9\) is used often.
Since every gradient is always multiplied by \((1-\beta)\), and each previous gradient picks up a factor of \(\beta\) each iteration, this construction weights the most recent gradients most.
update the weights:
\[{\bf A} = {\bf A} - \eta {\bf v}^{(i)}\]
We would do the same with the other weights, \({\bf B}\).
Tip
Momentum greatly reduces the swings in the “fraction correct” metric from one epoch to the next.
Implementing our neural network#
We’ll write our network to take a DataManager—it can get everything that it needs from there.
Tip
We also have our network do the validation against the test set each epoch so we can see how well we are doing.
import time
class NeuralNetwork:
"""A neural network class with a single hidden layer."""
def __init__(self, data_manager, *, hidden_layer_size=20):
self.data_manager = data_manager
# the number of nodes/neurons on the output layer
self.N_out = 10
# the number of nodes/neurons on the input layer
self.N_in = d.input_size
# the number of nodes/neurons on the hidden layer
self.N_hidden = hidden_layer_size
# we will initialize the weights with Gaussian normal random
# numbers centered on 0 with a width of 1/sqrt(n), where n is
# the length of the input state
rng = np.random.default_rng()
# A is the set of weights between the hidden layer and output layer
self.A = np.zeros((self.N_out, self.N_hidden), dtype=np.float32)
self.A[:, :] = rng.normal(0.0, 1.0/np.sqrt(self.N_hidden), self.A.shape)
# B is the set of weights between the input layer and hidden layer
self.B = np.zeros((self.N_hidden, self.N_in), dtype=np.float32)
self.B[:, :] = rng.normal(0.0, 1.0/np.sqrt(self.N_in), self.B.shape)
# reset the training
self.data_manager.reset_training()
self.n_trained = 0
self.training_time = 0
def sigmoid(self, xi):
"""our sigmoid function that operates on the hidden layer"""
return 1.0/(1.0 + np.exp(-xi))
def _batch_update(self, x_batch, y_batch):
# batch size
S = len(x_batch)
x = np.array(x_batch).T
y = np.array(y_batch).T
# propagate the input through the network
z_tilde = self.sigmoid(self.B @ x)
z = self.sigmoid(self.A @ z_tilde)
# compute the errors (backpropagate to the hidden layer)
e = z - y
e_tilde = self.A.T @ (e * z * (1 - z))
# corrections
grad_A = (2/S) * e * z * (1 - z) @ z_tilde.T
grad_B = (2/S) * e_tilde * z_tilde * (1 - z_tilde) @ x.T
self.n_trained += S
return grad_A, grad_B
def assess(self):
"""Run through the test data and return the fraction correct
with the currently trained network"""
n_correct = 0
for g in self.data_manager.test_images():
ans = self.predict(g)
if g.validate(ans):
n_correct += 1
return n_correct / self.data_manager.n_test
def train(self, *, n_epochs=1,
learning_rate=0.2, beta_momentum=0.9,
batch_size=64):
"""Train the neural network by doing gradient descent with back
propagation to set the matrix elements in B (the weights
between the input and hidden layer) and A (the weights between
the hidden layer and output layer)
"""
v_A = np.zeros_like(self.A)
v_B = np.zeros_like(self.B)
for i in range(n_epochs):
start = time.perf_counter()
self.data_manager.reset_training()
# storage for our batches
x_batch = []
y_batch = []
for g in self.data_manager.training_images():
# make a 1-d representation of the input, called x, and call
# the output y
x_batch.append(np.ravel(g.data))
y_batch.append(g.out)
if len(x_batch) == batch_size:
# batch is full -- do the training
grad_A, grad_B = self._batch_update(x_batch, y_batch)
v_A[...] = beta_momentum * v_A + (1.0 - beta_momentum) * grad_A
v_B[...] = beta_momentum * v_B + (1.0 - beta_momentum) * grad_B
self.A[...] += -learning_rate * v_A
self.B[...] += -learning_rate * v_B
x_batch = []
y_batch = []
# we may have run out of data without filling up the
# last batch, so take care of that now
if x_batch:
grad_A, grad_B = self._batch_update(x_batch, y_batch)
v_A[...] = beta_momentum * v_A + (1.0 - beta_momentum) * grad_A
v_B[...] = beta_momentum * v_B + (1.0 - beta_momentum) * grad_B
self.A[...] += -learning_rate * v_A
self.B[...] += -learning_rate * v_B
epoch_time = time.perf_counter() - start
self.training_time += epoch_time
frac_correct = self.assess()
print(f"epoch {i+1:3} | " +
f"test set correct: {frac_correct:5.3f}; " +
f"training time: {epoch_time:7.3f} s")
def predict(self, model):
""" predict the outcome using our trained matrix A """
x_in = np.ravel(model.data)[:, np.newaxis]
y = self.sigmoid(self.A @ (self.sigmoid(self.B @ x_in)))
return y
nn = NeuralNetwork(d, hidden_layer_size=500)
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[11], line 1
----> 1 nn = NeuralNetwork(d, hidden_layer_size=500)
NameError: name 'd' is not defined
nn.train(n_epochs=100)
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[12], line 1
----> 1 nn.train(n_epochs=100)
NameError: name 'nn' is not defined
Note
We are able to get about 65% correct here, when using the coarsened data. It does appear that more training would help—there are a lot of weights and (relatively) few training images.
Ultimately, a major issue is that the backpropagated errors get really small when using the sigmoid (this is called the vanishing gradient problem). Our network is just too simple for this problem.