Code:
`import tensorflow as tf
from morph_net.network_regularizers import flop_regularizer
from morph_net.network_regularizers import latency_regularizer
from morph_net.tools import structure_exporter
import numpy as np
from datetime import datetime
import time
def conv2d(input, w, b):
return tf.nn.relu(tf.nn.bias_add(tf.nn.conv2d(input, w, strides=[1, 1, 1, 1], padding='SAME'),
b))
def max_pool(input):
return tf.nn.max_pool(input, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1],
padding='SAME')
def fc(input, w, b):
return tf.nn.relu(tf.add(tf.matmul(input, w), b))
def alex_net(_input_r, _weights, _biases, _keep_prob):
with tf.name_scope('conv1'):
_conv1 = conv2d(_input_r, _weights['wc1'], _biases['bc1'])
with tf.name_scope('pool1'):
_pool1 = max_pool(_conv1)
with tf.name_scope('conv2'):
_conv2 = conv2d(_pool1, _weights['wc2'], _biases['bc2'])
with tf.name_scope('pool2'):
_pool2 = max_pool(_conv2)
with tf.name_scope('conv3'):
_conv3 = conv2d(_pool2, _weights['wc3'], _biases['bc3'])
with tf.name_scope('conv4'):
_conv4 = conv2d(_conv3, _weights['wc4'], _biases['bc4'])
with tf.name_scope('conv5'):
_conv5 = conv2d(_conv4, _weights['wc5'], _biases['bc5'])
with tf.name_scope('pool3'):
_pool3 = max_pool(_conv5)
_densel = tf.reshape(_pool3,
[-1, _weights['wd1'].get_shape().as_list()[0]])
with tf.name_scope('fc1'):
_fc1 = fc(_densel, _weights['wd1'], _biases['bd1'])
_fc1_drop = tf.nn.dropout(_fc1, _keep_prob)
with tf.name_scope('fc2'):
_fc2 = fc(_fc1_drop, _weights['wd2'], _biases['bd2'])
_fc2_drop = tf.nn.dropout(_fc2, _keep_prob)
with tf.name_scope('out'):
_out = tf.add(tf.matmul(_fc2_drop, _weights['wd3']), _biases['bd3'])
return _out
n_output = 10
learning_rate = 0.001
dropout = 0.75
train_data, test_data = tf.keras.datasets.mnist.load_data()
X_train, X_test = train_data[0], test_data[0]
Y_train, Y_test = train_data[1], test_data[1]
train_dir = "./Morph_save/"
print("CNN READY")
x = tf.placeholder(tf.float32, [None, 28, 28, 1])
y = tf.placeholder(tf.float32, [None, 10])
keep_prob = tf.placeholder(tf.float32)
weights = {
'wc1': tf.Variable(tf.truncated_normal([3, 3, 1, 64], dtype=tf.float32, stddev=0.1), name='weights1'),
'wc2': tf.Variable(tf.truncated_normal([3, 3, 64, 128], dtype=tf.float32, stddev=0.1), name='weights2'),
'wc3': tf.Variable(tf.truncated_normal([3, 3, 128, 256], dtype=tf.float32, stddev=0.1), name='weights3'),
'wc4': tf.Variable(tf.truncated_normal([3, 3, 256, 256], dtype=tf.float32, stddev=0.1), name='weights4'),
'wc5': tf.Variable(tf.truncated_normal([3, 3, 256, 128], dtype=tf.float32, stddev=0.1), name='weights5'),
'wd1': tf.Variable(tf.truncated_normal([4*4*128, 1024], dtype=tf.float32, stddev=0.1), name='weights_fc1'),
'wd2': tf.Variable(tf.random_normal([1024, 1024], dtype=tf.float32, stddev=0.1), name='weights_fc2'),
'wd3': tf.Variable(tf.random_normal([1024, n_output], dtype=tf.float32, stddev=0.1), name='weights_output')
}
biases = {
'bc1': tf.Variable(tf.constant(0.0, shape=[64], dtype=tf.float32), trainable=True, name='biases1'),
'bc2': tf.Variable(tf.constant(0.0, shape=[128], dtype=tf.float32), trainable=True, name='biases2'),
'bc3': tf.Variable(tf.constant(0.0, shape=[256], dtype=tf.float32), trainable=True, name='biases3'),
'bc4': tf.Variable(tf.constant(0.0, shape=[256], dtype=tf.float32), trainable=True, name='biases4'),
'bc5': tf.Variable(tf.constant(0.0, shape=[128], dtype=tf.float32), trainable=True, name='biases5'),
'bd1': tf.Variable(tf.constant(0.0, shape=[1024], dtype=tf.float32), trainable=True, name='biases_fc1'),
'bd2': tf.Variable(tf.constant(0.0, shape=[1024], dtype=tf.float32), trainable=True, name='biases_fc2'),
'bd3': tf.Variable(tf.constant(0.0, shape=[n_output], dtype=tf.float32), trainable=True, name='biases_output')
}
print("CNN READY")
pred = alex_net(x, weights, biases, keep_prob)
network_regularizer = flop_regularizer.GroupLassoFlopsRegularizer(output_boundary=[pred.op],input_boundary=[x.op, y.op],threshold=1e-3)
regularization_strength = 1e-8
regularizer_loss = (network_regularizer.get_regularization_term() * regularization_strength)
model_loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(y, pred))
cost_op = network_regularizer.get_cost()
optimizer = tf.train.MomentumOptimizer(learning_rate=0.01, momentum=0.9)
train_op = optimizer.minimize(model_loss + regularizer_loss)
exporter = structure_exporter.StructureExporter(network_regularizer.op_regularizer_manager)
init = tf.global_variables_initializer()
print("FUNCTIONS READY")
training_epochs = 1000
batch_size = 500
display_step = 10
sess = tf.Session()
sess.run(init)
for epoch in range(training_epochs):
avg_cost = 0.
total_batch = int(60000/batch_size)
start_time = time.time()
for i in range(total_batch):
batch_xs, batch_ys = X_train[ibatch_size:ibatch_size+batch_size], Y_train[ibatch_size:ibatch_size+batch_size]
_, get_cost, loss_val, regloss_val, structure_exporter_tensors=\
sess.run([train_op,cost_op,model_loss,regularizer_loss,exporter.tensors], feed_dict={x: batch_xs, y: batch_ys, keep_prob:dropout})
if epoch % display_step == 0:
train_accuracy = sess.run(model_loss, feed_dict={x: batch_xs, y: batch_ys, keep_prob: 1.0})
test_accuracy = sess.run(model_loss, feed_dict={x: X_train, y: Y_test, keep_prob:1.0})
exporter.populate_tensor_values(structure_exporter_tensors)
exporter.create_file_and_save_alive_counts(train_dir, epoch)
print("Epoch: %03d/%03d cost: %.9f TRAIN ACCURACY: %.3f TEST ACCURACY: %.3f" % (epoch, training_epochs, avg_cost, train_accuracy, test_accuracy))
duration = time.time() - start_time
print('%s: step %d, duration = %.3f' % (datetime.now(), epoch, duration))
print("DONE")
`