kacey@ieee.org

Getting familiar with TensorFlow

TensorFlow is one of the most popular deep learning framework developed by Google. If you are new to TensorFlow, please read and play with the sample in Getting started with TensorFlow to get started.

Import libraries

In [1]:
# Import required libraries
# Add whatever you want
import os
import time
os.environ["KERAS_BACKEND"] = "plaidml.keras.backend"
import keras 
# os.environ["KMP_BLOCKTIME"] = str(FLAGS.kmp_blocktime)
# os.environ["KMP_SETTINGS"] = str(FLAGS.kmp_settings)
# os.environ["KMP_AFFINITY"]= FLAGS.kmp_affinity
# if FLAGS.num_intra_threads > 0:
#   os.environ["OMP_NUM_THREADS"]= str(FLAGS.num_intra_threads)

import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

from lib.datasets import CIFAR10_tf
from lib.datasets import CIFAR10_data
# for auto-reloading external modules
# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython
%load_ext autoreload
%autoreload 2

# We recommend to use tensorflow==1.14.0
print("TensorFlow Version {}".format(tf.__version__))
TensorFlow Version 1.14.0

Load datasets

Download CIFAR-10 and load the dataset. In this exercise, we will use the standard 50,000 images for training and 10,000 images for test.

In [2]:
# Configuration
num_training = 49000
num_validation = 50000 - num_training
num_test = 10000

data = CIFAR10_tf(num_training=num_training,
                  num_validation=num_validation,
                  num_test=num_test)

# Load cifar-10 data
X_train, Y_train = data['data_train'], data['labels_train']
X_val, Y_val = data['data_val'], data['labels_val']
X_test, Y_test = data['data_test'], data['labels_test']

# Check the shape of the dataset
assert X_train.shape == (num_training, 32, 32, 3)
assert Y_train.shape == (num_training, )
assert X_val.shape == (num_validation, 32, 32, 3)
assert Y_val.shape == (num_validation, )
assert X_test.shape == (num_test, 32, 32, 3)
assert Y_test.shape == (10000, )

Implementing a neural network architecture with an optimization routine according to the specification provided below:

Model:

  • Input image with the size 32x32x3
  • 7x7 convolutional layer with 32 filters, stride of 1, and padding 'SAME'
  • ReLU activation layer
  • 3x3 max pooling layer with a stride of 2
  • 5x5 convolutional layer with 64 filters, stride of 1, and padding 'SAME'
  • ReLU activation layer
  • 3x3 max pooling layer with a stride of 2
  • Flatten layer (8x8x64 -> 4096)
  • Fully-connected layer with 384 output units (4096 -> 384)
  • ReLU activation layer
  • Fully-connected layer with 10 output units (384 -> 10)
  • Output logits (10)

Optimizer:

  • Adam optimizer

Learning rate:

  • Set start learning rate as 5e-4 and apply exponential decay every 500 steps with a base of 0.96
  • Use 'tf.train.exponential_decay' and 'tf.train.AdamOptimizer'

Loss:

  • Softmax cross entropy loss
  • Use 'tf.nn.softmax_cross_entropy_with_logits_v2'

This model should achieve about 55% accuracy on test set in 5 epochs using provided evaluation code.

Define your layers

In [3]:
# Define max pooling and conv layers

def conv2d(input, kernel_size, stride, num_filter):
    stride_shape = [1, stride, stride, 1]
    filter_shape = [kernel_size, kernel_size, input.get_shape()[3], num_filter]

    W = tf.get_variable('w', filter_shape, tf.float32, tf.random_normal_initializer(0.0, 0.02))
    b = tf.get_variable('b', [1, 1, 1, num_filter], initializer=tf.constant_initializer(0.0))
    return tf.nn.conv2d(input, W, stride_shape, padding='SAME') + b

def max_pool(input, kernel_size, stride):
    ksize = [1, kernel_size, kernel_size, 1]
    strides = [1, stride, stride, 1]
    return tf.nn.max_pool(input, ksize=ksize, strides=strides, padding='SAME')

def flatten(input):
    """
        - input: input tensors
        
    """
    #return tf.keras.layers.Flatten()(input)
    return tf.layers.flatten(input)

def fc(input, num_output):
    """
        - input: input tensors
        - num_output: int, the output dimension
    """
    #return tf.keras.layers.Dense(num_output)(input)
    return tf.layers.dense(input, num_output)

Sample convolutional neural network

In [4]:
class BaseModel(object):
    def __init__(self):
        self.num_epoch = 5
        self.batch_size = 64
        self.log_step = 50
        self._build_model()

    def _model(self):
        print('-' * 5 + '  Sample model  ' + '-' * 5)

        print('intput layer: ' + str(self.X.get_shape()))

        with tf.variable_scope('conv1'):
            self.conv1 = conv2d(self.X, 7, 1, 32)
            self.relu1 = tf.nn.relu(self.conv1)
            self.pool1 = max_pool(self.relu1, 3, 2)            
            print('conv1 layer: ' + str(self.pool1.get_shape()))

        with tf.variable_scope('conv2'):

            self.conv2 = conv2d(self.pool1, 5, 1, 64)
            self.relu2 = tf.nn.relu(self.conv2)
            self.pool2 = max_pool(self.relu2, 3, 2)            
            print('conv2 layer: ' + str(self.pool2.get_shape()))

     
        with tf.variable_scope('flat'):
            self.flat= flatten(self.pool2)
            
            print('flat layer: ' + str(self.flat.get_shape()))

        with tf.variable_scope('fc3'):
            # Fully-connected layer with 384 output units (4096 -> 384)
            self.fc3=fc(self.flat, 384)
            self.relu3=tf.nn.relu(self.fc3)

            print('fc3 layer: ' + str(self.relu3.get_shape()))
            
        with tf.variable_scope('fc4'):
          
            self.fc4=fc(self.relu3, 10)

            print('fc4 layer: ' + str(self.fc4.get_shape()))
            
        # Return the last layer
        return self.fc4

    def _input_ops(self):
        # Placeholders
        self.X = tf.placeholder(tf.float32, [None, 32, 32, 3])
        self.Y = tf.placeholder(tf.int64, [None])

    def _build_optimizer(self):
        #Adam optimizer 'self.train_op' that minimizes 'self.loss_op'  
        # **Learning rate:**
        #- Set start learning rate as 5e-4 and apply exponential decay every 500 steps with a base of 0.96
        # - Use 'tf.train.exponential_decay' and 'tf.train.AdamOptimizer'
        initial_lr=.0005
        decay_every=500
        base=0.96
        global_step = tf.Variable(0, trainable=False)
        #learning rate:
        lr=tf.train.exponential_decay(initial_lr, global_step, decay_every, base, staircase=True)
        #learning step:
        self.train_op = tf.train.AdamOptimizer(lr).minimize(self.loss_op,global_step=global_step)
        
    def _loss(self, labels, logits):

        loss_tensor = tf.nn.softmax_cross_entropy_with_logits_v2(
    labels,
    logits)
        self.loss_op =tf.reduce_mean(loss_tensor)
        
    def _build_model(self):
        # Define input variables
        self._input_ops()

        # Convert Y to one-hot vector
        labels = tf.one_hot(self.Y, 10)

        # Build a model and get logits
        logits = self._model()

        # Compute loss
        self._loss(labels, logits)
        
        # Build optimizer
        self._build_optimizer()

        # Compute accuracy
        predict = tf.argmax(logits, 1)
        correct = tf.equal(predict, self.Y)
        self.accuracy_op = tf.reduce_mean(tf.cast(correct, tf.float32))
        
    def train(self, sess, X_train, Y_train, X_val, Y_val):
        sess.run(tf.global_variables_initializer())
        #self.is_train=True
        start_time = time.time()
        step = 0
        losses = []
        accuracies = []
        print('-' * 5 + '  Start training  ' + '-' * 5)
        cnt=0
        for epoch in range(self.num_epoch):
            print('train for epoch %d' % epoch)
            for i in range(num_training // self.batch_size):
                X_ = X_train[i * self.batch_size:(i + 1) * self.batch_size][:]
                Y_ = Y_train[i * self.batch_size:(i + 1) * self.batch_size]

                feed_dict = {self.X:X_, self.Y:Y_}
                
                fetches = [self.train_op, self.loss_op, self.accuracy_op]

                _, loss, accuracy = sess.run(fetches, feed_dict=feed_dict)
                losses.append(loss)
                accuracies.append(accuracy)

                if step % self.log_step == 0:
                    print('iteration (%d)(%.3f s): loss = %.3f, accuracy = %.3f' %(step,(time.time() - start_time), loss, accuracy))
#                     print('step: ',step)
#                     print('loss:', np.sum(loss)
#                     print('accuracy: ',accuracy)
                    #print("--- %s seconds ---" % )
                step += 1

            # Print validation results
            print('validation for epoch %d' % epoch)
            val_accuracy = self.evaluate(sess, X_val, Y_val)
            print('-  epoch %d: validation accuracy = %.3f' % (epoch, val_accuracy))
            
        #############################################################################
        # Plot training curve                                                 #
        #############################################################################
        # Graph 1. X: iteration (training step), Y: training loss

        # Graph 2. X: iteration (training step), Y: training accuracy
        iterations=np.arange(step)
        
        # Plot the learning curves
        plt.subplot(2, 1, 1)
        plt.tight_layout()
        plt.title('Training loss')
        #loss_hist_ = losses[1::100] # sparse the curve a bit
        #plt.plot(loss_hist_, '-o')
        plt.plot(iterations,losses, '-o')
        plt.xlabel('Iteration')

        plt.subplot(2, 1, 2)
        plt.title('Training Accuracy')
        plt.plot(iterations,accuracies, '-o') #, label='Training')
        #plt.plot([0.5] * len(val_acc_hist), 'k--')
        plt.xlabel('Iteration')
        #plt.gcf().set_size_inches(15, 12)
        plt.show()
        
    def evaluate(self, sess, X_eval, Y_eval):
        
        eval_accuracy = 0.0
        eval_iter = 0
        for i in range(X_eval.shape[0] // self.batch_size):
            X_ = X_eval[i * self.batch_size:(i + 1) * self.batch_size][:]
            Y_ = Y_eval[i * self.batch_size:(i + 1) * self.batch_size]

            
            feed_dict = {self.X:X_, self.Y:Y_}#,self.is_train:False}
            
            accuracy = sess.run(self.accuracy_op, feed_dict=feed_dict)
            eval_accuracy += accuracy
            eval_iter += 1
        return eval_accuracy / eval_iter
In [5]:
# Clear old computation graphs
tf.reset_default_graph()

start_time = time.time()
# Train our sample model
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=True)) as sess:
    with tf.device('/cpu:0'):
        model = BaseModel()
        for var in [X_train, Y_train, X_val, Y_val]:
            print(var.shape)
        model.train(sess, X_train, Y_train, X_val, Y_val)
        accuracy = model.evaluate(sess, X_test, Y_test)
        print('***** test accuracy: %.3f' % accuracy)
        saver = tf.train.Saver()
        model_path = saver.save(sess, "lib/tf_models/problem2/csci-599_sample.ckpt")
        print("Model saved in %s" % model_path)
        print("--- %s seconds ---" % (time.time() - start_time))
***** test accuracy: 0.619
Model saved in lib/tf_models/problem2/csci-599_sample.ckpt
--- 220.3107008934021 seconds ---

Tweaking the model

You can modify the template code as you want and you can use GPU for fast training. For GPU usage, simply change the following line of the training block:
from with tf.device('/cpu:0') to with tf.device('/GPU:0') and you can set your desired device number.

These are the techniques that you can try:

  • Data preprocessing
  • Data augmentation
  • Batch normalization
  • Dropout
  • More convolutional layers
  • More training epochs
  • Learning rate decay
  • Any other models and techniqes
In [6]:
class YourModel(BaseModel):
    def __init__(self):
        super(YourModel, self).__init__()
        self.num_epoch = 15

    def _model(self):
        print('-' * 5 + '  Your model  ' + '-' * 5)
        
        with tf.variable_scope('conv1'):
            self.conv1 = conv2d(self.X, 3, 1, 32)
            self.relu1 = tf.nn.relu(self.conv1)
            self.batch_norm1=tf.compat.v1.keras.layers.BatchNormalization(axis=1)(self.relu1)
            #self.pool1 = max_pool(self.batch_norm1, 2, 1) 
            #self.drop1= tf.nn.dropout(self.pool1,rate=.3)
            print('conv1 layer: ' + str(self.batch_norm1.get_shape()))

        with tf.variable_scope('conv2'):
            #self.conv2 = conv2d(self.drop1, 3, 1,32)
            self.conv2 = conv2d(self.batch_norm1, 3, 1,32)
            self.relu2 = tf.nn.relu(self.conv2)
            self.batch_norm2=tf.keras.layers.BatchNormalization(axis=1)(self.relu2)
            self.pool2 = max_pool(self.batch_norm2, 2, 2)     
            self.drop2= tf.nn.dropout(self.pool2,rate=.2)
            print('conv2 layer: ' + str(self.drop2.get_shape()))

        with tf.variable_scope('conv3'):
            self.conv3 = conv2d(self.drop2, 3, 1,64)
            self.relu3 = tf.nn.relu(self.conv3)
            self.batch_norm3=tf.keras.layers.BatchNormalization(axis=1)(self.relu3)
            self.pool3 = max_pool(self.batch_norm3, 2, 2)     
            self.drop3= tf.nn.dropout(self.pool2,rate=.3)
            print('conv3 layer: ' + str(self.drop3.get_shape()))
            
        with tf.variable_scope('conv4'):
            self.conv4 = conv2d(self.drop3, 3, 1, 128)
            self.relu4 = tf.nn.relu(self.conv4)
            self.batch_norm4=tf.keras.layers.BatchNormalization(axis=1)(self.relu4)
            #self.pool1 = max_pool(self.batch_norm1, 2, 1) 
            #self.drop1= tf.nn.dropout(self.pool1,rate=.3)
            print('conv4 layer: ' + str(self.batch_norm4.get_shape()))

        with tf.variable_scope('conv5'):
            #self.conv2 = conv2d(self.drop1, 3, 1,32)
            self.conv5 = conv2d(self.batch_norm4, 3, 1,128)
            self.relu5 = tf.nn.relu(self.conv5)
            self.batch_norm5=tf.keras.layers.BatchNormalization(axis=1)(self.relu5)
            self.pool5 = max_pool(self.batch_norm5, 2, 2)     
            self.drop5= tf.nn.dropout(self.pool5,rate=.4)
            print('conv5 layer: ' + str(self.drop5.get_shape()))
        with tf.variable_scope('flat'):
            self.flat= flatten(self.drop5)     
            print('flat layer: ' + str(self.flat.get_shape()))

        with tf.variable_scope('fc3'):
            # Fully-connected layer with 384 output units (4096 -> 384)
            self.fc3=fc(self.flat, 2048)
            self.relu3=tf.nn.relu(self.fc3)
            print('fc3 layer: ' + str(self.relu3.get_shape()))
            
        with tf.variable_scope('fc4'):
            self.fc4=fc(self.relu3, 10)
            print('fc4 layer: ' + str(self.fc4.get_shape()))
            
        # Return the last layer
        return self.fc4
    def _input_ops(self):
        # Placeholders
        self.X = tf.placeholder(tf.float32, [None, 32, 32, 3])
        self.Y = tf.placeholder(tf.int64, [None])
In [7]:
# Clear old computation graphs
%env KMP_BLOCKTIME=0
%env KMP_AFFINITY=granularity=fine,verbose,compact,1,0 
%env KMP_SETTINGS=1 

tf.reset_default_graph()
config = tf.ConfigProto(device_count={"CPU": 8},allow_soft_placement=True, log_device_placement=True)
config.intra_op_parallelism_threads = 1
config.inter_op_parallelism_threads = 8
#start_time = time.time()
with tf.Session(config=config) as sess:
    
    with tf.device('/cpu:0'):
        model = YourModel()
        
        model.train(sess, X_train, Y_train, X_val, Y_val)
        accuracy = model.evaluate(sess, X_test, Y_test)
        print('***** test accuracy: %.3f' % accuracy)
       #print("--- %s seconds ---" % (time.time() - start_time))
        # Save your model
        saver = tf.train.Saver()
        model_path = saver.save(sess, "lib/tf_models/problem2/csci-599_mine.ckpt")
        print("Model saved in %s" % model_path)
***** test accuracy: 0.768
Model saved in lib/tf_models/problem2/csci-599_mine.ckpt

class YourModel(tf.keras.Model): def init(self): super(YourModel, self).init()

    self.conv1 = Conv2D(32, 3, activation='relu')
    self.flatten = Flatten()
    self.d1 = Dense(128, activation='relu')
    self.d2 = Dense(10, activation='softmax')
def call(self, x):
    x = self.conv1(x)
    x = self.flatten(x)
    x = self.d1(x)
    return self.d2(x)
model = MyModel()

with tf.GradientTape() as tape:
  logits = model(images)
  loss_value = loss(logits, labels)
grads = tape.gradient(loss_value, model.trainable_variables)
optimizer.apply_gradients(zip(grads, model.trainable_variables))

from keras import layers inputs = tf.keras.Input(shape=(None, 32,32,3)) # Returns a placeholder tensor

A layer instance is callable on a tensor, and returns a tensor.

x = layers.Dense(64, activation='relu')(inputs) x = layers.Dense(64, activation='relu')(x) predictions = layers.Dense(10, activation='softmax')(x)

model = tf.keras.Model(inputs=inputs, outputs=predictions)

The compile step specifies the training configuration.

model.compile(optimizer=tf.train.AdamOptimizer(0.001), loss='categorical_crossentropy', metrics=['accuracy'])

Trains for 5 epochs

model.fit(x_train, y_train, batch_size=32, epochs=5) model.evaluate(x_val, y_val)

In [8]:
tf.reset_default_graph()

# Load your model
ckpt_filepath="path/to/saved/checkpoint/file.ckpt"
model = YourModel()
sess = tf.Session()
saver = tf.train.Saver()
saver.restore(sess, ckpt_filepath)
-----  Your model  -----
conv1 layer: (?, 32, 32, 32)
conv2 layer: (?, 16, 16, 32)
conv3 layer: (?, 16, 16, 32)
conv4 layer: (?, 16, 16, 128)
conv5 layer: (?, 8, 8, 128)
flat layer: (?, 8192)
fc3 layer: (?, 2048)
fc4 layer: (?, 10)
INFO:tensorflow:Restoring parameters from lib/tf_models/problem2/csci-599_mine.ckpt