Application: Galaxy Classification with Keras#
Now we’ll try to do the galaxy classification with Keras. This will allow us to explore more features in the network.
In particular we’ll use a convolutional neural network.
import numpy as np
import matplotlib.pyplot as plt
import h5py
We’ll need the Galaxy class we defined earlier.
galaxy_types = {0: "disturbed galaxies",
1: "merging galaxies",
2: "round smooth galaxies",
3: "in-between round smooth galaxies",
4: "cigar shaped smooth galaxies",
5: "barred spiral galaxies",
6: "unbarred tight spiral galaxies",
7: "unbarred loose spiral galaxies",
8: "edge-on galaxies without bulge",
9: "edge-on galaxies with bulge"}
class Galaxy:
def __init__(self, data, answer, *, index=-1):
self.data = np.array(data, dtype=np.float32) / 255.0
self.answer = answer
self.index = index
def plot(self, ax=None):
if ax is None:
fig, ax = plt.subplots()
ax.imshow(self.data, interpolation="nearest")
ax.text(0.025, 0.95, f"answer: {self.answer}",
color="white", transform=ax.transAxes)
def validate(self, prediction):
"""check if a categorical prediction matches the answer"""
return np.argmax(prediction) == self.answer
A data manager class#
We’ll create a class to manage access to the data. This is similar to the version in our “from scratch” implementation, except we do the batching in the generator. This will do the following:
open the file and store the handles to access the data
partition the data into test and training sets
allow for transformations (rotation, flipping)
provide a means to shuffle the data
provide a generator to get the next batch of training data
allow us to coarsen the images to a reduced resolution to make the training easier.
Tip
We will read all of the data once, which will take about 3.5 GB of memory. It is kept
as a uint8 until needed.
from keras.utils import to_categorical
class DataManager:
def __init__(self, partition=0.8,
batch_size=32,
n_transforms=1,
datafile="Galaxy10_DECals.h5",
coarsen=1):
"""manage access to the data
partition: fraction that should be training
datafile: name of the hdf5 file with the data
coarsen: reduce the number of pixels by this factor
"""
self.ds = h5py.File(datafile)
self.ans = np.array(self.ds["ans"])
self.images = np.array(self.ds["images"])
self.coarsen = coarsen
self.n_transforms = n_transforms
self.batch_size = batch_size
N = len(self.ans)
# create a set of indices for the galaxies and randomize
self.indices = np.arange(N, dtype=np.uint32)
self.rng = np.random.default_rng()
self.rng.shuffle(self.indices)
# partition into training and test sets
# these indices will always refer to the index in the original
# unsplit dataset
n_cut = int(partition * N)
# we want this to be a multiple of the batch size
n_cut -= n_cut % self.batch_size
self.training_indices = self.indices[0:n_cut]
self.test_indices = self.indices[n_cut:N]
self.n_training = len(self.training_indices)
self.n_test = len(self.test_indices)
# number of batches for the training
self.n_batches = n_cut // self.batch_size
self.n_batches *= (n_transforms + 1)
# shape information
self.input_shape = self._get_galaxy(0).data.shape
def _get_galaxy(self, index):
"""return a numpy array containing a single galaxy image, coarsened
if necessary by averaging"""
_tmp = self.images[index, :, :, :]
if self.coarsen > 1:
_tmp = np.mean(_tmp.reshape(_tmp.shape[0]//self.coarsen, self.coarsen,
_tmp.shape[1]//self.coarsen, self.coarsen,
_tmp.shape[2]), axis=(1, 3))
return _tmp
def training_images(self):
self.reset_training()
for idx in self.training_indices:
yield Galaxy(self._get_galaxy(idx), self.ans[idx], index=idx)
def batched_training_generator(self):
while True:
self.reset_training()
batch_x = []
batch_y = []
for idx in self.training_indices:
g = Galaxy(self._get_galaxy(idx), self.ans[idx], index=idx)
batch_x.append(g.data)
batch_y.append(to_categorical(g.answer, 10))
if self.n_transforms > 0:
# rotation
batch_x.append(np.rot90(g.data, axes=(0, 1)))
batch_y.append(to_categorical(g.answer, 10))
if self.n_transforms > 1:
# flipping first axis
batch_x.append(g.data[::-1, :, :])
batch_y.append(to_categorical(g.answer, 10))
if self.n_transforms > 2:
# flipping second axis
batch_x.append(g.data[:, ::-1, :])
batch_y.append(to_categorical(g.answer, 10))
if len(batch_x) == self.batch_size:
yield (np.array(batch_x), np.array(batch_y))
batch_x = []
batch_y = []
def reset_training(self):
"""prepare for the next epoch: shuffle the training data and
reset the index to point to the start"""
self.rng.shuffle(self.training_indices)
def testing_images(self):
for idx in self.test_indices:
yield Galaxy(self._get_galaxy(idx), self.ans[idx], index=idx)
Tip
The training_images() and testing_images() are generators (like range()), so we can iterate like:
d = DataManager()
for g in d.training_images():
# do stuff with g
and g is only converted to 32-bit float as needed.
Here we create a DataManager that will coarsen the images by a factor of 2 (so they will be 128x128 pixels with 3 colors).
We also need to specify the batch size.
d = DataManager(coarsen=2, batch_size=128)
---------------------------------------------------------------------------
FileNotFoundError Traceback (most recent call last)
Cell In[6], line 1
----> 1 d = DataManager(coarsen=2, batch_size=128)
Cell In[5], line 14, in DataManager.__init__(self, partition, batch_size, n_transforms, datafile, coarsen)
10 datafile: name of the hdf5 file with the data
11 coarsen: reduce the number of pixels by this factor
12 """
13
---> 14 self.ds = h5py.File(datafile)
15 self.ans = np.array(self.ds["ans"])
16 self.images = np.array(self.ds["images"])
17
File /opt/hostedtoolcache/Python/3.14.10/x64/lib/python3.14/site-packages/h5py/_hl/files.py:555, in File.__init__(self, name, mode, driver, libver, userblock_size, swmr, rdcc_nslots, rdcc_nbytes, rdcc_w0, track_order, fs_strategy, fs_persist, fs_threshold, fs_page_size, page_buf_size, min_meta_keep, min_raw_keep, locking, alignment_threshold, alignment_interval, meta_block_size, track_times, **kwds)
546 fapl = make_fapl(driver, libver, rdcc_nslots, rdcc_nbytes, rdcc_w0,
547 locking, page_buf_size, min_meta_keep, min_raw_keep,
548 alignment_threshold=alignment_threshold,
549 alignment_interval=alignment_interval,
550 meta_block_size=meta_block_size,
551 **kwds)
552 fcpl = make_fcpl(track_order=track_order, track_times=track_times,
553 fs_strategy=fs_strategy, fs_persist=fs_persist,
554 fs_threshold=fs_threshold, fs_page_size=fs_page_size)
--> 555 fid = make_fid(name, mode, userblock_size, fapl, fcpl, swmr=swmr)
557 if isinstance(libver, tuple):
558 self._libver = libver
File /opt/hostedtoolcache/Python/3.14.10/x64/lib/python3.14/site-packages/h5py/_hl/files.py:232, in make_fid(name, mode, userblock_size, fapl, fcpl, swmr)
230 if swmr:
231 flags |= h5f.ACC_SWMR_READ
--> 232 fid = h5f.open(name, flags, fapl=fapl)
233 elif mode == 'r+':
234 fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl)
File h5py/_objects.pyx:54, in h5py._objects.with_phil.wrapper()
---> 54 'Could not get source, probably due dynamically evaluated source code.'
File h5py/_objects.pyx:55, in h5py._objects.with_phil.wrapper()
---> 55 'Could not get source, probably due dynamically evaluated source code.'
File h5py/h5f.pyx:106, in h5py.h5f.open()
--> 106 'Could not get source, probably due dynamically evaluated source code.'
FileNotFoundError: [Errno 2] Unable to synchronously open file (unable to open file: name = 'Galaxy10_DECals.h5', errno = 2, error message = 'No such file or directory', flags = 0, o_flags = 0)
for n, (x, y) in enumerate(d.batched_training_generator()):
print(x.shape)
break
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[7], line 1
----> 1 for n, (x, y) in enumerate(d.batched_training_generator()):
2 print(x.shape)
3 break
NameError: name 'd' is not defined
d.input_shape
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[8], line 1
----> 1 d.input_shape
NameError: name 'd' is not defined
we can see how many images there are in the training and test sets
d.n_training, d.n_test
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[9], line 1
----> 1 d.n_training, d.n_test
NameError: name 'd' is not defined
d.n_batches
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[10], line 1
----> 1 d.n_batches
NameError: name 'd' is not defined
We can then get loop over training galaxies and look at them (we’ll break after 5):
for n, g in enumerate(d.training_images()):
g.plot()
if n == 4:
break
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[11], line 1
----> 1 for n, g in enumerate(d.training_images()):
2 g.plot()
3 if n == 4:
4 break
NameError: name 'd' is not defined
Note
Each time we access the generator it randomizes the galaxies in the training set.
Implementing our neural network#
We’ll use just a single hidden layer. If we use more hidden layers, then there are so many parameters that we will need to do a lot of training (epochs).
Tip
Our DataManager will tell us the size of the input layer, the number of batches, the batch size, etc.
from keras.models import Sequential
from keras.layers import Input, Dense, Dropout, Conv2D, MaxPooling2D, Flatten
model = Sequential()
model.add(Input(shape=d.input_shape))
model.add(Conv2D(32, kernel_size=(3, 3), activation="relu"))
model.add(Conv2D(32, kernel_size=(3, 3), activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, kernel_size=(3, 3), activation="relu"))
model.add(Conv2D(64, kernel_size=(3, 3), activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(128, kernel_size=(3, 3), activation="relu"))
model.add(Conv2D(128, kernel_size=(3, 3), activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(10, activation="softmax"))
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[13], line 2
1 model = Sequential()
----> 2 model.add(Input(shape=d.input_shape))
3
4 model.add(Conv2D(32, kernel_size=(3, 3), activation="relu"))
5 model.add(Conv2D(32, kernel_size=(3, 3), activation="relu"))
NameError: name 'd' is not defined
model.summary()
Model: "sequential"
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓ ┃ Layer (type) ┃ Output Shape ┃ Param # ┃ ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩ └─────────────────────────────────┴────────────────────────┴───────────────┘
Total params: 0 (0.00 B)
Trainable params: 0 (0.00 B)
Non-trainable params: 0 (0.00 B)
We’ll use a different optimizer now, the Adam optimizer. This is supposed to be one of the best that Keras provides.
from keras.optimizers import Adam
rms = Adam()
model.compile(loss='categorical_crossentropy',
optimizer=rms, metrics=['accuracy'])
The keras fit() method can work with a generator directly, so we just pass in the generator.
model.fit(d.batched_training_generator(),
batch_size=d.batch_size,
steps_per_epoch=d.n_batches,
epochs=25)
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[16], line 1
----> 1 model.fit(d.batched_training_generator(),
2 batch_size=d.batch_size,
3 steps_per_epoch=d.n_batches,
4 epochs=25)
NameError: name 'd' is not defined
Assessing#
Now try the test set that the training did not see
n_correct = 0
for g in d.testing_images():
res = model.predict(np.expand_dims(g.data, axis=0), verbose=0)
if np.argmax(res) == g.answer:
n_correct += 1
print(f"fraction correct = {n_correct / d.n_test}")
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
Cell In[17], line 2
1 n_correct = 0
----> 2 for g in d.testing_images():
3 res = model.predict(np.expand_dims(g.data, axis=0), verbose=0)
4 if np.argmax(res) == g.answer:
5 n_correct += 1
NameError: name 'd' is not defined
We see that we are getting about 70% of the galaxies we didn’t train on correct.