深度学习通用流程

import tensorflow as tf
from tensorflow.keras import datasets  # 导入经典数据集

# 加载自带数据集
# 加载MNIST数据集，返回两个元祖

#x y为训练组得元素和标签

#x_test y_test 测试集得元素和标签
(x, y), (x_test, y_test) = datasets.mnist.load_data()
print('x:', x.shape, 'y:', y.shape, 'x_test:', x_test.shape, 'y_test:', y_test.shape)

# 使用`tf.data.Dataset.from_tensor_slices`将NumPy数组转换为TensorFlow Dataset
train = tf.data.Dataset.from_tensor_slices((x, y))
test = tf.data.Dataset.from_tensor_slices((x_test, y_test))

# 加载CSV文件
import pandas as pd

titanic_file = pd.read_csv('../data/titanic_file.csv')
titanic_slices = tf.data.Dataset.from_tensor_slices(dict(titanic_file))
for feature_batch in titanic_slices.take(1):  # 采用take函数在列轴上的位置1处取值
for key, value in feature_batch.items():  # 返回遍历的键和值
print('{!r:20s}: {}'.format(key, value))  # 打印键与值

titanic_batches = tf.data.experimental.make_csv_dataset('../data/titanic_file.csv', batch_size=4,
label_name='survived')
for feature_batch, label_batch in titanic_batches.take(1):
print('survived: {}'.format(label_batch))
print('features:')
for key, value in feature_batch.items():
print('{!r:20s}: {}'.format(key, value))

# 加载TFRecord文件
dataset = tf.data.TFRecordDataset(filenames = ['../data/fsns.tfrec'])
print(dataset)

raw_example = next(iter(dataset))
parsed = tf.train.Example.FromString(raw_example.numpy())  # 解析TFRecord
print(parsed.features.feature['image/text'])  # 输出检查

# 加载文本文件
cowper = tf.data.TextLineDataset('../data/cowper.txt')
for line in cowper.take(5):
print(line.numpy())

# 加载文件集
import random
import pathlib

data_path = pathlib.Path('../data/flower_photos')
all_image_paths = list(data_path.glob('*/*'))  
all_image_paths = [str(path) for path in all_image_paths]  # 所有图片路径的列表
random.shuffle(all_image_paths)  # 打散数据

image_count = len(all_image_paths)
print('数据大小：', image_count)
# 查看5张图片
print('5张图片', all_image_paths[:5])

# 提取分类名
label_names = sorted(item.name for item in data_path.glob('*/') if item.is_dir())
print('分类名', label_names)

# 创建标签
label_to_index = dict((name, index) for index, name in enumerate(label_names))
print('标签', label_to_index)

# 将图片与标签对应
all_image_labels = [label_to_index[pathlib.Path(
path).parent.name] for path in all_image_paths]
for image, label in zip(all_image_paths[:5], all_image_labels[:5]):
print(image, ' --->  ', label)

# 转化为Dataset对象实例
ds = tf.data.Dataset.from_tensor_slices((all_image_paths, all_image_labels))

# 数据预处理
# 图像数据预处理
import os
import matplotlib.pylab as plt

# 在URL上下载数据集
flowers_root = tf.keras.utils.get_file(
'flower_photos',
'https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz',
untar=True)
flowers_root = pathlib.Path(flowers_root)
# 获取每个类下的文件数据
list_ds = tf.data.Dataset.list_files(str(flowers_root/'*/*'))

# 自定义函数用于操作数据集元素
def parse_image(filename):
# 分割数据
parts = tf.strings.split(filename, os.sep)
label = parts[-2]
# 读取并输出输入文件名的全部内容
image = tf.io.read_file(filename)
# 编码解码处理
image = tf.image.decode_jpeg(image)
# 转换为float类型
image = tf.image.convert_image_dtype(image, tf.float32)
# 尺寸调整为128*128
image = tf.image.resize(image, [128, 128])
return image, label

# 测试函数是否有效
file_path = next(iter(list_ds))
image, label = parse_image(file_path)

# 自定义函数绘制图像图形
plt.figure()
plt.imshow(image)
plt.title(label.numpy().decode('utf-8'))
plt.axis('off')
plt.show()

# 运用map()对文件数据进行预处理
images_ds = list_ds.map(parse_image)
for image, label in images_ds.take(1):
plt.figure()
plt.imshow(image)
plt.title(label.numpy().decode('utf-8'))
plt.axis('off')
plt.show()



# 文本数据预处理
import tensorflow_datasets as tfds

# 分割数据
tokenizer = tfds.deprecated.text.Tokenizer()
# 自定义空集合
vocabulary_set = set()
# 循环获取词汇
for text_tensor in cowper:
some_tokens = tokenizer.tokenize(text_tensor.numpy())
vocabulary_set.update(some_tokens)
# 查看词汇表大小
vocab_size = len(vocabulary_set)
print('词汇表大小：', vocab_size)
# 构建编码器
encoder = tfds.deprecated.text.TokenTextEncoder(vocabulary_set)
# 输出查看词汇样式
example_text = next(iter(cowper)).numpy()
print('样式：', example_text)
# 样式编码 
encoded_example = encoder.encode(example_text)
print('样式编码：', encoded_example)

# MNIST数据集预处理
train = train.shuffle(10000)  # 随机打散样本，不会打乱样本与标签的映射关系

# 自定义预处理函数
def train_preprocess(x, y):
# 调用此函数时会自动传入x、y对象
# 标准化到0-1
x = tf.cast(x, dtype=tf.float32) / 255.
x = tf.reshape(x, [-1, 28*28])
y = tf.cast(y, dtype=tf.int32)  # 转成整型张量
# one_hot接受的输入为int32,输出为float32
y = tf.one_hot(y, depth=10)
# 返回的x、y将替换传入的x、y参数，从而实现数据的预处理功能
return x, y

def test_preprocess(x_test, y_test):
x_test = tf.cast(x_test, dtype=tf.float32) / 255.
y_test = tf.cast(y_test, dtype=tf.int32)
return x_test, y_test

# 在preprocess函数中实现预处理函数，传入函数名即可
train = train.map(train_preprocess)
test = test.map(test_preprocess)

# 构建网络
# 构建输入层
x = tf.keras.Input(shape=(32,))
y = tf.keras.layers.Dense(16, activation='softmax')(x)
model = tf.keras.Model(x, y)
print(model)

model = tf.keras.Sequential()
# 展平为一维数组
model.add(tf.keras.layers.Flatten(input_shape=(28, 28)))
model.add(tf.keras.layers.Dense(10, activation='softmax'))

# 构建隐藏层
# 构建单个全连接层
model = tf.keras.models.Sequential()
# 输入矩阵的大小为 (None, 16)
model.add(tf.keras.Input(shape=(16,)))
model.add(tf.keras.layers.Dense(32, activation='relu'))
# 输出的大小为 (None, 32)
print('输出大小：', model.output_shape)

# 构建多个全连接层
model = tf.keras.models.Sequential()
# 输入矩阵的大小为 (None, 1)
model.add(tf.keras.Input(shape=(1,)))
# 定义第一个全连接层
model.add(tf.keras.layers.Dense(5, activation='sigmoid'))
# 定义第二个全连接层
model.add(tf.keras.layers.Dense(1, activation='sigmoid'))

# 构建输出层
# 使用tanh激活函数
x = tf.linspace(-6., 6., 10)
tf.tanh(x)  # tanh激活函数

model = tf.keras.models.Sequential()
model.add(tf.keras.Input(shape=(10,)))
# 定义全连接层
model.add(tf.keras.layers.Dense(12, activation='relu'))
# 输出层
model.add(tf.keras.layers.Dense(1, activation='softmax'))

# 构建Sequential网络
import numpy as np
# 读取文件数据
data = np.load('../data/mnist.npz')
# 查看数据集合
print(data.files)
train_images, train_labels, test_images, test_labels = data['x_train'], data[
'y_train'], data['x_test'], data['y_test']

# 构建网络
model = tf.keras.models.Sequential()
# 添加特征（28*28）展平为一维数组
model.add(tf.keras.layers.Flatten(input_shape=(28, 28)))
# 定义第一层
model.add(tf.keras.layers.Dense(128, activation='relu'))
# 定义第二层
model.add(tf.keras.layers.Dense(10, activation='softmax'))
# 打印模型
print(model.summary())

# 编译网络
# 设置损失函数
# 计算绝对差的平均值
y_true = [[0., 0.], [0., 1.]]
y_pred = [[1., 0.], [1., 1.]]
mae = tf.keras.losses.MeanAbsoluteError()
print(mae(y_true, y_pred).numpy())

# 计算平均绝对误差
y_true = np.random.randint(0, 2, size=(2, 3))
y_pred = np.random.random(size=(2, 3))
loss = tf.keras.losses.MAE(y_true, y_pred)
print(mae(y_true, y_pred).numpy())

# 计算误差平方的均值
y_true = [[0., 1.], [0., 0.]]
y_pred = [[1., 1.], [1., 0.]]
mse = tf.keras.losses.MeanSquaredError()
print(mse(y_true, y_pred).numpy())

# 计算均方误差
y_true = np.random.randint(0, 2, size=(2, 3))
y_pred = np.random.random(size=(2, 3))
loss = tf.keras.losses.MSE(y_true, y_pred)
print(mae(y_true, y_pred).numpy())

# 计算二进制交叉熵损失
y_true = [[0, 1], [0, 0]]
y_pred = [[0.6, 0.4], [0.4, 0.6]]
loss = tf.keras.losses.binary_crossentropy(y_true, y_pred)
assert loss.shape == (2,)
print(loss.numpy())

#计算稀疏分类交叉熵损失
y_true = [1, 2]
y_pred = [[0.05, 0.95, 0], [0.1, 0.8, 0.1]]
loss = tf.keras.losses.sparse_categorical_crossentropy(y_true, y_pred)
assert loss.shape == (2,)
print(loss.numpy())

# 编译Sequential网络
model.compile(optimizer='adam', loss='sparse_categorical_crossentropy',
metrics=['accuracy'])

# 训练网络
model.fit(train_images, train_labels, verbose=1, epochs=20, batch_size=128,
validation_data=(test_images, test_labels))

# 性能评估
# 评估指标
# 计算AUC值
m = tf.keras.metrics.AUC(num_thresholds=3)
m.update_state([0, 0, 1, 1], [0, 0.5, 0.3, 0.9])
print(m.result().numpy())

# 计算准确率
m = tf.keras.metrics.BinaryAccuracy()
m.update_state([[1], [1], [0], [0]], [[0.98], [1], [0], [0.6]])
print(m.result().numpy())

m = tf.keras.metrics.CategoricalAccuracy()
m.update_state([[0, 0, 1], [0, 1, 0]], [[0.1, 0.9, 0.8],
[0.05, 0.95, 0]])
print(m.result().numpy())

model1 = tf.keras.models.Sequential()
model1.add(tf.keras.layers.Flatten(input_shape=(28, 28)))
model1.add(tf.keras.layers.Dense(128, activation='relu'))
model1.add(tf.keras.layers.Dense(10, activation='softmax'))

model1.compile(optimizer='adam', loss='sparse_categorical_crossentropy',
metrics=[tf.keras.metrics.CategoricalAccuracy()])
model1.fit(train_images, train_labels, verbose=1, epochs=20, batch_size=128,
validation_data=(test_images, test_labels))

# 计算精确率
m = tf.keras.metrics.Precision()
m.update_state([0, 1, 1, 1], [1, 0, 1, 1])
print(m.result().numpy())

# 计算平均绝对误差
m = tf.keras.metrics.MeanAbsoluteError()
m.update_state([[0, 1], [0, 0]], [[1, 1], [0, 0]])
print(m.result().numpy())

# 计算均方误差
m = tf.keras.metrics.MeanSquaredError()
m.update_state([[0, 0], [0, 1]], [[1, 0], [1, 1]])
print(m.result().numpy())

# 模型评价
test_loss, test_acc = model.evaluate(test_images, test_labels)
print('损失值为：', test_loss)
print('准确率为：%.2f%%'% (test_acc * 100.0))

# 回调检查
# TensorBoard日志的路径要用os.path.join生成，不然在windows下会报错
log_dir = os.path.join('D:\logs')
if not os.path.exists(log_dir):
os.mkdir(log_dir)

from tensorflow.keras import callbacks
# 把训练过程需要可视化的数据保存在log_dir目录
my_callbacks = [callbacks.TensorBoard(log_dir=log_dir), ]

# 拟合模型
model.fit(train_images, train_labels, validation_data=(test_images, test_labels), 
batch_size=128, epochs=20, callbacks=my_callbacks)

# 模型保存与调用
# 张量方式
# 保存模型参数到文件中
model.save_weights('../tmp/checkpoints/mannul_checkpoint')
# 重新创建相同的网络结构
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Flatten(input_shape=(28, 28)))
model.add(tf.keras.layers.Dense(128, activation='relu'))
model.add(tf.keras.layers.Dense(10, activation='softmax'))
model.compile(optimizer='adam', loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
# 从参数文件中读取数据并写入当前网络
model.load_weights('../tmp/checkpoints/mannul_checkpoint')
print(model.summary())

# HDF5格式文件
model.save('../tmp/model_mnist.h5')
# 调用模型
new_model = tf.keras.models.load_model('../tmp/model_mnist.h5')
new_model.summary()  # 查看模型基本信息

# saved_model格式
import time
saved_model_path = '../tmp/saved_models/{}'.format(int(time.time()))
tf.keras.models.save_model(model, saved_model_path)
# 调用模型
new_model1 = tf.keras.models.load_model(saved_model_path)
new_model1.summary()  # 查看模型基本信息

# 调用保存好的模型对新样本进行预测
for i in range(30):
image = plt.imread('../data/testimages/'+str(i)+'.jpg')  # 读取照片
image_new = image.reshape([1, 28, 28])  # 对照片进行维度转化
result = new_model.predict(image_new)[0].argmax()  # 输出预测结果
print('第', i, ' 张照片中的数字是：', result)