1) LeNET scratch (59%)
2) Use MaxPool instead of Average pool (Slight improvement : 61% )
3) Use ReLu instead of TanH (Fast training but not much improvement)
4) L2 regularizer with Relu (Slight improvement : 63%)
5) Increase convolution filters (Improves to 70%, which is the benchmark for model of this size and its training settings)
import tensorflow as tf
import numpy as np
from PIL import Image
import matplotlib.pyplot as plt
import glob
%matplotlib inline
def get_accuracy(predict:'eg: [2,4,1,...]',true: 'eg: [2,4,1,...]') -> int:
correct_pred = tf.equal(predict,true)
#We have to cast [True,False,True,...] --> [1,0,1...]
acc = tf.reduce_mean(tf.cast(correct_pred,tf.float32))
return acc
file_path = "datasets/cifar-10-batches-bin/data_batch_*"
train_bins = glob.glob(file_path)
def cifar_dataset(files_list:list) -> tf.data.Dataset:
data = tf.data.FixedLengthRecordDataset(files_list,1+3*32*32)
data = data.map(lambda x: tf.decode_raw(x,tf.uint8),num_parallel_calls=4)
data = data.map(lambda x: (x[1:],tf.expand_dims(x[0],0)),num_parallel_calls=4)
data = data.map(lambda x,y: (tf.reshape(x,(3,32,32)),y),num_parallel_calls=4)
data = data.map(lambda x,y: (tf.transpose(x,(1,2,0)),y),num_parallel_calls=4)
data = data.map(lambda x,y: (tf.image.convert_image_dtype(x,tf.float32),y),num_parallel_calls=4)
return data
with tf.device('/cpu:0'):
train_dataset = cifar_dataset(train_bins)
train_dataset = train_dataset.shuffle(20000)
train_dataset = train_dataset.repeat(20)
train_dataset = train_dataset.batch(10)
train_dataset = train_dataset.prefetch(2)
handle = tf.placeholder(tf.string, shape=[])
iterator = tf.data.Iterator.from_string_handle(handle,train_dataset.output_types,train_dataset.output_shapes)
model_input,label = iterator.get_next()
train_iterator = train_dataset.make_one_shot_iterator()
with tf.device('/cpu:0'):
test_dataset = cifar_dataset(['datasets/cifar-10-batches-bin/test_batch.bin'])
test_dataset = test_dataset.batch(10)
test_dataset = test_dataset.prefetch(2)
test_iterator = test_dataset.make_one_shot_iterator()
train_dataset.output_shapes,train_dataset.output_types
DROPOUT = tf.placeholder(tf.bool)
def trainval():
import time
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
#Training
hdl = sess.run(train_iterator.string_handle())
start = time.time()
try:
i = 1
tmp = []
while True:
i = i+1
l,_ = sess.run([loss,train],{handle:hdl,DROPOUT:True})
tmp.append(l)
if i%5000 == 0:
avg_loss = np.array(tmp).mean()
print("Batch: ",i,avg_loss)
tmp = []
except tf.errors.OutOfRangeError:
pass
end = time.time()
elapsed = end-start
print("Elapsed time : ", elapsed, " s")
#Testing
print("Testing : ")
hdl = sess.run(test_iterator.string_handle())
acc = get_accuracy(tf.argmax(classifier,axis=1),tf.transpose(tf.argmax(tf.one_hot(label,10),axis=2)))
try:
i = 0
acc_list = []
while True:
i = i+1
a = sess.run(acc,{handle:hdl,DROPOUT:False})
acc_list.append(a)
if i%100 == 0:
print(i, "Mean Acc : ", np.array(acc_list).mean())
acc_list = []
except tf.errors.OutOfRangeError:
pass
Base Accuracy : 59%
def cnn_features(model_input):
conv1 = tf.layers.conv2d(model_input,6,(5,5),(1,1),activation=tf.nn.tanh,name="Conv1")
avgpool1 = tf.layers.average_pooling2d(conv1,(2,2),(2,2),name="AvgPool1")
conv2 = tf.layers.conv2d(avgpool1,16,(5,5),(1,1),activation=tf.nn.tanh,name="Conv2")
avgpool2 = tf.layers.average_pooling2d(conv2,(2,2),(2,2),name="AvgPool2")
conv3 = tf.layers.conv2d(avgpool2,120,(5,5),(1,1),activation=tf.nn.tanh,name="Conv3")
flatten = tf.layers.Flatten()(conv3)
features = tf.layers.dense(flatten,84,activation=tf.nn.tanh,name="Dense_84")
return features
features = cnn_features(model_input)
classifier = tf.layers.dense(features,10,name="Dense_10")
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=tf.one_hot(label,10),logits=classifier))
optimizer = tf.train.GradientDescentOptimizer(0.01)
train = optimizer.minimize(loss)
trainval()
Slight improvement (61%)
def cnn_features(model_input):
conv1 = tf.layers.conv2d(model_input,6,(5,5),(1,1),activation=tf.nn.tanh,name="Conv1")
maxpool1 = tf.layers.max_pooling2d(conv1,(2,2),(2,2),name="MaxPool1")
conv2 = tf.layers.conv2d(maxpool1,16,(5,5),(1,1),activation=tf.nn.tanh,name="Conv2")
maxpool2 = tf.layers.max_pooling2d(conv2,(2,2),(2,2),name="MaxPool2")
conv3 = tf.layers.conv2d(maxpool2,120,(5,5),(1,1),activation=tf.nn.tanh,name="Conv3")
flatten = tf.layers.Flatten()(conv3)
features = tf.layers.dense(flatten,84,activation=tf.nn.tanh,name="Dense_84")
return features
features = cnn_features(model_input)
classifier = tf.layers.dense(features,10,name="Dense_10")
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=tf.one_hot(label,10),logits=classifier))
optimizer = tf.train.GradientDescentOptimizer(0.01)
train = optimizer.minimize(loss)
trainval()
ReLU if fast, but does not show improvement without regularization
def cnn_features(model_input):
conv1 = tf.layers.conv2d(model_input,6,(5,5),(1,1),activation=tf.nn.relu,name="Conv1")
maxpool1 = tf.layers.max_pooling2d(conv1,(2,2),(2,2),name="MaxPool1")
conv2 = tf.layers.conv2d(maxpool1,16,(5,5),(1,1),activation=tf.nn.relu,name="Conv2")
maxpool2 = tf.layers.max_pooling2d(conv2,(2,2),(2,2),name="MaxPool2")
conv3 = tf.layers.conv2d(maxpool2,120,(5,5),(1,1),activation=tf.nn.relu,name="Conv3")
flatten = tf.layers.Flatten()(conv3)
features = tf.layers.dense(flatten,84,activation=tf.nn.relu,name="Dense_84")
return features
features = cnn_features(model_input)
classifier = tf.layers.dense(features,10,name="Dense_10")
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=tf.one_hot(label,10),logits=classifier))
optimizer = tf.train.GradientDescentOptimizer(0.01)
train = optimizer.minimize(loss)
trainval()
Slight improvement (63%)
def cnn_features(model_input):
conv1 = tf.layers.conv2d(model_input,6,(5,5),(1,1),
kernel_regularizer= tf.contrib.layers.l2_regularizer(scale=0.001),
activation=tf.nn.relu,
name="Conv1")
maxpool1 = tf.layers.max_pooling2d(conv1,(2,2),(2,2),name="MaxPool1")
conv2 = tf.layers.conv2d(maxpool1,16,(5,5),(1,1),
kernel_regularizer= tf.contrib.layers.l2_regularizer(scale=0.01),
activation=tf.nn.relu,
name="Conv2")
maxpool2 = tf.layers.max_pooling2d(conv2,(2,2),(2,2),name="MaxPool2")
conv3 = tf.layers.conv2d(maxpool2,120,(5,5),(1,1),
kernel_regularizer= tf.contrib.layers.l2_regularizer(scale=0.01),
activation=tf.nn.relu,
name="Conv3")
flatten = tf.layers.Flatten()(conv3)
features = tf.layers.dense(flatten,84,
kernel_regularizer= tf.contrib.layers.l2_regularizer(scale=0.01),
activation=tf.nn.relu,
name="Dense_84")
return features
features = cnn_features(model_input)
classifier = tf.layers.dense(features,10,name="Dense_10")
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=tf.one_hot(label,10),logits=classifier))
loss += tf.losses.get_regularization_loss()
optimizer = tf.train.GradientDescentOptimizer(0.01)
train = optimizer.minimize(loss)
trainval()
Achieved 70% accuracy
GPU Util : 60%, GPU Temp: 70 degree
def cnn_features(model_input):
conv1 = tf.layers.conv2d(model_input,32,(5,5),(1,1),
kernel_regularizer= tf.contrib.layers.l2_regularizer(scale=0.001),
activation=tf.nn.relu,
name="Conv1")
maxpool1 = tf.layers.max_pooling2d(conv1,(2,2),(2,2),name="MaxPool1")
conv2 = tf.layers.conv2d(maxpool1,64,(5,5),(1,1),
kernel_regularizer= tf.contrib.layers.l2_regularizer(scale=0.01),
activation=tf.nn.relu,
name="Conv2")
maxpool2 = tf.layers.max_pooling2d(conv2,(2,2),(2,2),name="MaxPool2")
conv3 = tf.layers.conv2d(maxpool2,120,(5,5),(1,1),
kernel_regularizer= tf.contrib.layers.l2_regularizer(scale=0.01),
activation=tf.nn.relu,
name="Conv3")
flatten = tf.layers.Flatten()(conv3)
features = tf.layers.dense(flatten,592,
kernel_regularizer= tf.contrib.layers.l2_regularizer(scale=0.01),
activation=tf.nn.relu,
name="Dense_84")
return features
features = cnn_features(model_input)
classifier = tf.layers.dense(features,10,name="Dense_10")
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=tf.one_hot(label,10),logits=classifier))
loss += tf.losses.get_regularization_loss()
optimizer = tf.train.GradientDescentOptimizer(0.01)
train = optimizer.minimize(loss)
trainval()