import gzip, os
import numpy as np################ Step No.1 to well manage the dataset. ################
class MNIST:# Images size is told in the official website 28*28 px.image_size = 28image_size_flat = image_size * image_size# Let the validation set flexible when making an instance.def __init__(self, val_ratio=0.1, data_dir='MNIST_data'):self.val_ratio = val_ratioself.data_dir = data_dir# Load 4 files to individual lists with one string pixels.img_train = self.load_flat_images('train-images-idx3-ubyte.gz')lab_train = self.load_labels('train-labels-idx1-ubyte.gz')img_test = self.load_flat_images('t10k-images-idx3-ubyte.gz')lab_test = self.load_labels('t10k-labels-idx1-ubyte.gz')# Determine the actual number of training / validation sets.self.val_train_num = round(len(img_train) * self.val_ratio)self.main_train_num = len(img_train) - self.val_train_num# The normalized image pixels value can be more convenient when training.# dtype=np.int64 would be more general when applying to Tensorflow.self.img_train = img_train[0:self.main_train_num] / 255.0self.lab_train = lab_train[0:self.main_train_num].astype(np.int)self.img_train_val = img_train[self.main_train_num:] / 255.0self.lab_train_val = lab_train[self.main_train_num:].astype(np.int)# Also convert the format of testing set.self.img_test = img_test / 255.0self.lab_test = lab_test.astype(np.int)# Extract the same codes from "load_flat_images" and "load_labels".# This method won't be called during training procedure.def load_binary_to_num(self, dataset_name, offset):path = os.path.join(self.data_dir, dataset_name)with gzip.open(path, 'rb') as binary_file:# The datasets files are stored in 8 bites, mind the format.data = np.frombuffer(binary_file.read(), np.uint8, offset=offset)return data# This method won't be called during training procedure.def load_flat_images(self, dataset_name):# Images offset position is 16 by default formatdata = self.load_binary_to_num(dataset_name, offset=16)images_flat_all = data.reshape(-1, self.image_size_flat)return images_flat_all# This method won't be called during training procedure.def load_labels(self, dataset_name):# Labels offset position is 8 by default format.labels_all = self.load_binary_to_num(dataset_name, offset=8)return labels_all# This method would be called for training usage.def one_hot(self, labels):# Properly use numpy module to mimic the one hot effect.class_num = np.max(self.lab_test) + 1convert = np.eye(class_num, dtype=float)[labels]return convert
#---------------------------------------------------------------------#path = '/home/abc/MNIST_data'
data = MNIST(val_ratio=0.1, data_dir=path)
import tensorflow as tfflat_size = data.image_size_flat
label_num = np.max(data.lab_test) + 1################ Step No.2 to construct tensor graph. ################
x_train= tf.placeholder(dtype=tf.float32, shape=[None, flat_size])
t_label_oh = tf.placeholder(dtype=tf.float32, shape=[None, label_num])
t_label = tf.placeholder(dtype=tf.int64, shape=[None])################ These are the values ################
# Initialize the beginning weights and biases by random_normal method.
weights = tf.Variable(tf.random_normal([flat_size, label_num], mean=0.0, stddev=1.0, dtype=tf.float32))
biases = tf.Variable(tf.random_normal([label_num], mean=0.0, stddev=1.0, dtype=tf.float32))
########### that we wish to get by training ##########logits = tf.matmul(x_train, weights) + biases # < Annotation No.1 >
# Shrink the distances between values into 0 to 1 by softmax formula.
p_label_soh = tf.nn.softmax(logits)
# Pick the position of largest value along y axis.
p_label = tf.argmax(p_label_soh, axis=1)
#---------------------------------------------------------------------######## Step No.3 to get a loss value by certain loss function. #######
# This softmax function can not accept input being "softmaxed" before.
CE = tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=t_label_oh)
# Shrink all loss values in a matrix to only one averaged loss.
loss = tf.reduce_mean(CE)
#---------------------------------------------------------------------##### Step No.4 get a minimized loss value using gradient descent. ####
# Decrease this only averaged loss to a minimum value by using gradient descent.
optimizer = tf.train.AdamOptimizer(learning_rate=0.5).minimize(loss)
#---------------------------------------------------------------------## First return a boolean list values by tf.equal function
correct_predict = tf.equal(p_label, t_label)
# And cast them into 0 and 1 values so that its average value would be accuracy.
accuracy = tf.reduce_mean(tf.cast(correct_predict, dtype=tf.float32))sess = tf.Session()
sess.run(tf.global_variables_initializer())###### Step No.5 iterate the training set and check the accuracy. #####
# The trigger to train the linear model with a defined cycles.
def optimize(iteration, batch_size=32):for i in range(iteration):total = len(data.lab_train)random = np.random.randint(0, total, size=batch_size)# Randomly pick training images / labels with a defined batch size.x_train_batch = data.img_train[random]t_label_batch_oh = data.one_hot(data.lab_train[random])batch_dict = {x_train: x_train_batch, t_label_oh: t_label_batch_oh}sess.run(optimizer, feed_dict=batch_dict)# The trigger to check the current accuracy value
def Accuracy():# Use the totally separate dataset to test the trained modeltest_dict = {x_train: data.img_test,t_label_oh: data.one_hot(data.lab_test),t_label: data.lab_test}Acc = sess.run(accuracy, feed_dict=test_dict)print('Accuracy on Test Set: {0:.2%}'.format(Acc))
#---------------------------------------------------------------------#### Step No.6 plot wrong predicted pictures with its predicted label.##
import matplotlib.pyplot as plt# We can decide how many wrong predicted images are going to be shown up.
# We can focus on the specific wrong predicted labels
def wrong_predicted_images(pic_num=[3, 4], label_number=None):test_dict = {x_train: data.img_test,t_label_oh: data.one_hot(data.lab_test),t_label: data.lab_test}correct_pred, p_lab = sess.run([correct_predict, p_label], feed_dict=test_dict)# To reverse the boolean value in order to pick up wrong labelswrong_pred = (correct_pred == False)# Pick up the wrong doing elements from the corresponding placeswrong_img_test = data.img_test[wrong_pred]wrong_t_label = data.lab_test[wrong_pred]wrong_p_label = p_lab[wrong_pred]fig, axes = plt.subplots(pic_num[0], pic_num[1])fig.subplots_adjust(hspace=0.3, wspace=0.3)edge = data.image_sizefor ax in axes.flat:# If we were not interested in certain label number,# pick up the wrong predicted images randomly.if label_number is None:i = np.random.randint(0, len(wrong_t_label), size=None, dtype=np.int)pic = wrong_img_test[i].reshape(edge, edge)ax.imshow(pic, cmap='binary')xlabel = "True: {0}, Pred: {1}".format(wrong_t_label[i], wrong_p_label[i])# If we are interested in certain label number,# pick up the specific wrong images number randomly.else:# Mind that np.where return a "tuple" that should be indexing.specific_idx = np.where(wrong_t_label==label_number)[0]i = np.random.randint(0, len(specific_idx), size=None, dtype=np.int)pic = wrong_img_test[specific_idx[i]].reshape(edge, edge)ax.imshow(pic, cmap='binary')xlabel = "True: {0}, Pred: {1}".format(wrong_t_label[specific_idx[i]], wrong_p_label[specific_idx[i]])ax.set_xlabel(xlabel)# Pictures don't need any ticks, so we remove them in both dimensionsax.set_xticks([])ax.set_yticks([])plt.show()
#---------------------------------------------------------------------#
Accuracy() # Accuracy before doing anything
optimize(10); Accuracy() # Iterate 10 times
optimize(1000); Accuracy() # Iterate 10 + 1000 times
optimize(10000); Accuracy() # Iterate 10 + 1000 + 10000 times### ----- Results are shown below ----- ###
Accuracy on Test Set: 11.51%
Accuracy on Test Set: 68.37%
Accuracy on Test Set: 86.38%
Accuracy on Test Set: 89.34%
