Tensorflow实现进阶的神经网络

今天对照Tensorflow的书，实现了一个进阶的卷积神经网络。基于CIFAR-10数据集。

CIFAR-10数据集官网：http://www.cs.toronto.edu/~kriz/cifar.html

(这里选择二进制版本进行下载)

与之前的简单的神经网络相比，此进阶的神经网络，使用了一些新的技巧：

1. 对weights进行了L2正则化

2. 对图片进行了翻转、剪切等图片增强

3. 使用了LRN层，增强了图片的泛化能力

代码如下：

1.  
   import cifar10, cifar10_input
2.  
   import tensorflow as tf
3.  
   import numpy as np
4.  
   import time
5.  
    
6.  
   # 定义一个batch的大小，定义训练轮数maxstep
7.  
   max_step = 3000
8.  
   batch_size = 128
9.  
   data_dir = '/home/heyang/databaseImg/cifar-10-batches-bin'
10.  
     
11.  
     
12.  
    # 定义初始化weight函数
13.  
    # 依然使用截断的truncated_normal分布来初始化权重
14.  
    # 加一个L2的loss防止过拟合
15.  
    def variable_with_weight_loss(shape, stddev, w1):
16.  
    var = tf.Variable(tf.truncated_normal(shape, stddev=stddev))
17.  
    if w1 is not None:
18.  
    weight_loss = tf.multiply(tf.nn.l2_loss(var), w1, name="weight_loss")
19.  
    tf.add_to_collection('losses', weight_loss)
20.  
    return var
21.  
     
22.  
     
23.  
    # 使用distorted_input产生更多的输入数据，进行数据增强
24.  
    images_train, labels_train = cifar10_input.distorted_inputs(
25.  
    data_dir=data_dir, batch_size=batch_size
26.  
    )
27.  
     
28.  
    # 生成测试数据，inputs方法里只有裁剪
29.  
    images_test, labels_test = cifar10_input.inputs(eval_data=True,
30.  
    data_dir=data_dir,
31.  
    batch_size=batch_size)
32.  
     
33.  
    # 创建输入数据的placeholder[batchsize，尺寸，尺寸，通道数]
34.  
    image_holder = tf.placeholder(tf.float32, [batch_size, 24, 24, 3])
35.  
    label_holder = tf.placeholder(tf.int32, [batch_size])
36.  
     
37.  
    # 第一个卷积层
38.  
    # 其中，池化层为3×3,步长为2×2
39.  
    # 增加了一个LRN层，归一化局部特征
40.  
    weight1 = variable_with_weight_loss(shape=[5, 5, 3, 64], stddev=5e-2,
41.  
    w1=0.0)
42.  
    kernel1 = tf.nn.conv2d(image_holder, weight1, [1, 1, 1, 1], padding='SAME')
43.  
    bias1 = tf.Variable(tf.constant(0.0, shape=[64]))
44.  
    conv1 = tf.nn.relu(tf.nn.bias_add(kernel1, bias1))
45.  
    pool1 = tf.nn.max_pool(conv1, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1],
46.  
    padding='SAME')
47.  
    norm1 = tf.nn.lrn(pool1, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75)
48.  
     
49.  
    # 第二个卷积层
50.  
    # 次层的LRN层在池化层之前
51.  
    weight2 = variable_with_weight_loss(shape=[5, 5, 64, 64], stddev=5e-2,
52.  
    w1=0.0)
53.  
    kernel2 = tf.nn.conv2d(norm1, weight2, [1, 1, 1, 1], padding='SAME')
54.  
    bias2 = tf.Variable(tf.constant(0.1, shape=[64]))
55.  
    conv2 = tf.nn.relu(tf.nn.bias_add(kernel2, bias2))
56.  
    norm2 = tf.nn.lrn(conv2, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75)
57.  
    pool2 = tf.nn.max_pool(norm2, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1],
58.  
    padding='SAME')
59.  
     
60.  
    # 使用全连接层3
61.  
    # reshape第二个卷积层的输出
62.  
    # getshape得到数据扁平化后的长度
63.  
    # 设置了一个非0的weight_loss保证此层都被L2正则约束
64.  
    reshape = tf.reshape(pool2, [batch_size, -1])
65.  
    dim = reshape.get_shape()[1].value
66.  
    weight3 = variable_with_weight_loss(shape=[dim, 384], stddev=0.04, w1=0.004)
67.  
    bias3 = tf.Variable(tf.constant(0.1, shape=[384]))
68.  
    local3 = tf.nn.relu(tf.matmul(reshape, weight3) + bias3)
69.  
     
70.  
    # 全连接层4
71.  
    weight4 = variable_with_weight_loss(shape=[384, 192], stddev=0.04, w1=0.004)
72.  
    bias4 = tf.Variable(tf.constant(0.1, shape=[192]))
73.  
    local4 = tf.nn.relu(tf.matmul(local3, weight4) + bias4)
74.  
     
75.  
    # 最后一层，正态分布标准差设为上一层层数的倒数
76.  
    # 不过在此处不使用softmax输出最终结果
77.  
    weight5 = variable_with_weight_loss(shape=[192, 10], stddev=1 / 192.0, w1=0.0)
78.  
    bias5 = tf.Variable(tf.constant(0.0, shape=[10]))
79.  
    logits = tf.add(tf.matmul(local4, weight5), bias5)
80.  
     
81.  
     
82.  
    # 计算loss
83.  
    # 把softmax与cross_entropy联合在一起使用
84.  
    # 然后计算均值
85.  
    # 然后添加到整体的losses中，得到最终的loss（里面包含两个全连接层的L2）
86.  
    def loss(logits, labels):
87.  
    labels = tf.cast(labels, tf.int64)
88.  
    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
89.  
    logits=logits, labels=labels, name='cross_entropy_per_example'
90.  
    )
91.  
    cross_entropy_mean = tf.reduce_mean(cross_entropy,
92.  
    name='cross_entropy')
93.  
    tf.add_to_collection('losses', cross_entropy_mean)
94.  
     
95.  
    return tf.add_n(tf.get_collection('losses'), name='total_loss')
96.  
     
97.  
     
98.  
    # 设置优化器
99.  
    loss = loss(logits, label_holder)
100.  
     train_op = tf.train.AdamOptimizer(1e-3).minimize(loss)
101.  
     top_k_op = tf.nn.in_top_k(logits, label_holder, 1)
102.  
      
103.  
     # 创建默认的Session
104.  
     sess = tf.InteractiveSession()
105.  
     tf.global_variables_initializer().run()
106.  
      
107.  
     # 启动线程队列（用于图片数据增强）
108.  
     tf.train.start_queue_runners()
109.  
      
110.  
     # 正式训练开始
111.  
     for step in range(max_step):
112.  
     start_time = time.time()
113.  
     image_batch, label_batch = sess.run([images_train, labels_train])
114.  
     _, loss_value = sess.run([train_op, loss],
115.  
     feed_dict={image_holder: image_batch, label_holder: label_batch})
116.  
     duration = time.time() - start_time
117.  
     if step % 10 == 0:
118.  
     examples_per_sec = batch_size / duration
119.  
     sec_per_batch = float(duration)
120.  
      
121.  
     format_str = ('step %d, loss = %.2f (%.1f examples/sec; %.3f sec/batch)')
122.  
     print(format_str % (step, loss_value, examples_per_sec, sec_per_batch))
123.  
      
124.  
     # 评测准确率
125.  
     # 以top数相同为正确，除以总数为正确率
126.  
     num_examples = 10000
127.  
     import math
128.  
      
129.  
     num_iter = int(math.ceil(num_examples / batch_size))
130.  
     true_count = 0
131.  
     total_sample_count = num_iter * batch_size
132.  
     step = 0
133.  
     while step < num_iter:
134.  
     image_batch, label_batch = sess.run([images_test, labels_test])
135.  
     predictions = sess.run([top_k_op], feed_dict={image_holder: image_batch,
136.  
     label_holder: label_batch})
137.  
     true_count += np.sum(predictions)
138.  
     step += 1
139.  
      
140.  
     precision = true_count / total_sample_count
141.  
     print('precision @ 1 = %.3f' % precision)
142.  
      

结果如下：