TensorFlow神经网络优化策略学习

# 比较L1正则化和L2正则化函数的作用效果 
w = tf.constant([[1.0, -2.0], [-3.0, 4.0]]) 
with tf.Session() as sess: 
  # 0.5*(|1|+|-2|+|-3|+|4|=5.0) 
  print(sess.run(tf.contrib.layers.l1_regularizer(0.5)(w))) # 5.0 
  # 0.5*[(1+4+9+16)/2]=7.5 TensorFlow会将L2正则化项除以2使得求导的结果更简洁 
  print(sess.run(tf.contrib.layers.l2_regularizer(0.5)(w))) # 7.5 

import tensorflow as tf 
 
''''' 
# 比较L1正则化和L2正则化函数的作用效果 
w = tf.constant([[1.0, -2.0], [-3.0, 4.0]]) 
with tf.Session() as sess: 
  # 0.5*(|1|+|-2|+|-3|+|4|=5.0) 
  print(sess.run(tf.contrib.layers.l1_regularizer(0.5)(w))) # 5.0 
  # 0.5*[(1+4+9+16)/2]=7.5 TensorFlow会将L2正则化项除以2使得求导的结果更简洁 
  print(sess.run(tf.contrib.layers.l2_regularizer(0.5)(w))) # 7.5 
''' 
 
# 复杂神经网络结构权重L2正则化方法 
# 定义各层的权重，并将该权重的L2正则化项加入至名称为‘losses'的集合 
def get_weight(shape, lambda1): 
  var = tf.Variable(tf.random_normal(shape), dtype=tf.float32) 
  tf.add_to_collection('losses', tf.contrib.layers.l2_regularizer(lambda1)(var)) 
  return var 
 
x = tf.placeholder(tf.float32, (None, 2)) 
y_ = tf.placeholder(tf.float32, (None, 1)) 
 
layer_dimension = [2,10,5,3,1] # 定义了神经网络每层的节点数 
n_layers = len(layer_dimension) 
 
current_layer = x # 将当前层设置为输入层 
in_dimension = layer_dimension[0] 
 
# 通过循环生成一个5层全连接的神经网络结构 
for i in range(1,n_layers): 
  out_dimension = layer_dimension[i] 
  weight = get_weight([in_dimension,out_dimension], 0.003) 
  bias = tf.Variable(tf.constant(0.1, shape=[out_dimension])) 
  current_layer = tf.nn.relu(tf.matmul(current_layer, weight) + bias) 
  in_dimension = layer_dimension[i] 
 
mse_loss = tf.reduce_mean(tf.square(y_ - current_layer)) 
tf.add_to_collection('losses', mse_loss) 
loss = tf.add_n(tf.get_collection('losses')) # 包含所有参数正则化项的损失函数 

import tensorflow as tf 
 
# tf.train.ExponentialMovingAverage使用样例 
v1 = tf.Variable(0, dtype=tf.float32) 
step = tf.Variable(0, trainable=False) # 此处step模拟神经网络迭代的轮数 
# 定义一个滑动平均的类对象，初始化衰减率decay=0.99，用于动态控制衰减率的参数num_updates 
ema = tf.train.ExponentialMovingAverage(0.99, num_updates=step) 
 
# apply方法返回一个对var_list进行更新滑动平均的操作，var_list必须是list的Variable或Tensor 
# 该操作执行会更新var_list的影子变量shadow variable 
maintain_averages_op = ema.apply(var_list=[v1]) 
 
with tf.Session() as sess: 
  init_op = tf.global_variables_initializer() 
  sess.run(init_op) 
  # average方法可获取滑动平均后变量的取值 
  print(sess.run([v1, ema.average(v1)])) # [0.0, 0.0] 
 
  sess.run(tf.assign(v1, 5)) 
  # min{0.99, (1+step)(10+step)=0.1}=0.1 
  # 更新v1的滑动平均值为 0.1*0.0+0.9*5=4.5 
  sess.run(maintain_averages_op) 
  print(sess.run([v1, ema.average(v1)])) # [5.0, 4.5] 
 
  sess.run(tf.assign(step, 10000)) 
  sess.run(tf.assign(v1, 10)) 
  # min{0.99, (1+step)(10+step)=0.999}=0.99 
  # 更新v1的滑动平均值为 0.99*4.5+0.01*10=4.555 
  sess.run(maintain_averages_op) 
  print(sess.run([v1, ema.average(v1)])) # [10.0, 4.5549998] 
 
  # 更新v1的滑动平均值为 0.99*4.555+0.01*10=4.60945 
  sess.run(maintain_averages_op) 
  print(sess.run([v1, ema.average(v1)])) # [10.0, 4.6094499]