Class13预测房价代码

import hashlib # 校验文件完整性、密码存储
import os # 提供与操作系统交互的功能
import tarfile # 用于处理压缩归档文件
import zipfile # 用于处理.zip压缩文件
import requests # 第三方库，用于发送http请求

# 创建名为DATA_HUB的字典
DATA_HUB = dict()
# 构建完整下载链接
DATA_URL = 'http://d2l-data.s3-accelerate.amazonaws.com/'

# 定义一个函数下载数据文件并校验其完整性
# name:下载的文件名称
# cache_dir:本地缓存目录
def download(name,cache_dir = os.path.join('..','data')):"""下载一个DATA_HUB的文件名，返回本地文件名"""# 检查name是否存在于DATA_HUBassert name in DATA_HUB,f"{name} 不存在于 {DATA_HUB}"# url:文件相对路径# sha1_hash:预期的SHA-1值，用于文件完整性校验url,sha1_hash = DATA_HUB[name]# 如果目录不存在则创建，否则就跳过os.makedirs(cache_dir,exist_ok=True)# 读取文件目标路径fname = os.path.join(cache_dir,url.split('/')[-1])# 如果文件已存在if os.path.exists(fname):# 对整个文件进行哈希处理sha1 = hashlib.sha1()with open(fname,'rb') as f:while True:# 每次读取1MBdata = f.read(1048576)if not data:breaksha1.update(data)# 如果文件的SHA-1值与预期相同if sha1.hexdigest() == sha1_hash:# 文件完整不需要下载return fname# 流模式下载，适用于大文件print(f'正在从{url}下载{fname}...')# 启用HTTPS证书验证r = requests.get(url,stream=True,verify=True)with open(fname,'wb') as f:# 下载后写入文件f.write(r.content)# 返回本地文件名return fname

%matplotlib inline
import numpy as np
import pandas as pd
import torch
from torch import nn
from d2l import torch as d2l

# 从网络下载Kaggle房价预测数据集
# 下载地址：训练集CSV文件
# 哈希值：用来校验文件完整性
DATA_HUB['kaggle_house_train'] = (DATA_URL + 'kaggle_house_pred_train.csv','585e9cc93e70b39160e7921475f9bcd7d31219ce')# 下载地址：测试集CSV文件
# 哈希值：用来校验文件完整性
DATA_HUB['kaggle_house_test'] = (DATA_URL + 'kaggle_house_pred_test.csv','fa19780a7b011d9b009e8bff8e99922a8ee2eb90')# 下载数据并加载为DateFrame
train_data = pd.read_csv(download('kaggle_house_train'))
test_data = pd.read_csv(download('kaggle_house_test'))

# 打印训练数据集形状
print(train_data.shape)
# 打印测试数据集形状
print(test_data.shape)

# 打印数据集前4条记录，显示前3列和后3列数据
print(train_data.iloc[0:4,[0,1,2,3,-3,-2,-1]])

# 把训练集和测试集所有特征在行上拼接为一个特征表
all_features = pd.concat((train_data.iloc[:,1:-1],test_data.iloc[:,1:]))

# 选出数值型特征的列名
# all_features.dtypes:返回每一列的数据类型
# all_features.dtypes!='object':筛选非object类型
# .index:取出列名
numeric_features = all_features.dtypes[all_features.dtypes!='object'].index
# 标准化数值特征
# .apply(lambda x:):对每一列应用标准化函数
all_features[numeric_features] = all_features[numeric_features].apply(lambda x: (x - x.mean()) / (x.std()))
# 填补缺失值为0
all_features[numeric_features] = all_features[numeric_features].fillna(0)

# 将所有类别特征进行one-hot(独热编码)
# pd.get_dummies():用于将分类变量转换为one-hot形式
all_features = pd.get_dummies(all_features,dummy_na=True)
# 查看特征表维度
all_features.shape

# 获取训练样本行数
n_train = train_data.shape[0]
# 训练特征处理
# all_features[:n_train]:选取前n_train行
# .values:转为Numpy数组
# .astype(np.float32):强制转换成float32
# torch.tensor():转换为PyTorch的Tensor
train_features = torch.tensor(all_features[:n_train].values.astype(np.float32))
# 测试特征处理
test_features = torch.tensor(all_features[n_train:].values.astype(np.float32))
# 训练标签处理
# train_data.SalePrice.values:取出目标变量SalePrice
# .reshape(-1,1):将一维数组变成二维向量
# torch.tensor(dtype=torch.float32):转为float32类型
train_labels = torch.tensor(train_data.SalePrice.values.reshape(-1, 1), dtype=torch.float32)

# 定义损失函数
loss = nn.MSELoss()
# 获取输入特征的维度
in_features = train_features.shape[1]# 返回单层线性回归模型的神经网络
def get_net():net = nn.Sequential(nn.Linear(in_features,1))return net

# 计算模型的RMSE均方根误差 
# net:训练好的模型
# features:输入特征标签
# labels:真实标签
def log_rmse(net,features,labels):# 预测值裁剪clipped_preds = torch.clamp(net(features),1,float('inf'))# 计算log空间下的RMSErmse = torch.sqrt(loss(torch.log(clipped_preds),torch.log(labels)))# 返回Python标量return rmse.item()

# 训练函数
# net:线性回归模型或其他神经网络模型
# train_features:训练特征
# train_labels:训练标签
# test_features：测试特征
# test_labels:测试标签
# num_epochs:训练轮数
# learning_rate:学习率
# weight_decay:权重衰减项
# batch_size:批次大小
def train(net,train_features,train_labels,test_features,test_labels,num_epochs,learning_rate,weight_decay,batch_size):train_ls, test_ls = [], []# 构造训练数据迭代器train_iter = d2l.load_array((train_features,train_labels),batch_size)# 优化器定义(Adam+权重衰减)optimizer = torch.optim.Adam(net.parameters(),lr = learning_rate,weight_decay=weight_decay)# 训练循环for epoch in range(num_epochs):# 训练批次过程for X,y in train_iter:# 清空梯度optimizer.zero_grad()# 计算损失l = loss(net(X),y)# 反向传播l.backward()# 更新参数optimizer.step()# 每轮后进行性能记录train_ls.append(log_rmse(net,train_features,train_labels))if test_labels is not None:test_ls.append(log_rmse(net,test_features,test_labels))# 返回记录和训练的测试误差return train_ls,test_ls

# k折交叉验证函数
# 将数据集X,y划分为k折，并取第i折作为验证集，其余作为训练集
def get_k_fold_data(k,i,X,y):# 防止只划分为1折assert k > 1# 计算每折大小fold_size = X.shape[0] // k# 初始化训练集X_train,y_train = None,None# 循环分割数据for j in range(k):# 把数据在第j折上切出来# 用slice提高可读性idx = slice(j * fold_size,(j+1) * fold_size)X_part,y_part = X[idx,:],y[idx]# 构建训练集和验证集if j == i:# 第i折作为验证集X_valid,y_valid = X_part,y_part# 第一次构建，直接赋值elif X_train is None:X_train,y_train = X_part,y_partelse:# 后续折数与已有训练集拼接起来X_train = torch.cat([X_train,X_part],0)y_train = torch.cat([y_train,y_part],0)# 返回值return X_train,y_train,X_valid,y_valid

# k这交叉验证训练函数
# k:总折数
# x_train,y_train:原始训练集
# num_epochs:每折训练轮数
# learning_rate:学习率
# weight_decay:权重衰减
# batch_size:批量大小
def k_fold(k,X_train,y_train,num_epochs,learning_rate,weight_decay,batch_size):# 初始化总损失器train_l_sum,valid_l_sum = 0,0# 每折遍历for i in range(k):# 获取当前折的数据划分data = get_k_fold_data(k,i,X_train,y_train)# 初始化模型net = get_net()# 训练并记录损失曲线train_ls,valid_ls = train(net,*data,num_epochs,learning_rate,weight_decay,batch_size)# 累加最后一个epoch的训练和验证误差train_l_sum += train_ls[-1]valid_l_sum += valid_ls[-1]# 第1折画图if i == 0:d2l.plot(list(range(1,num_epochs + 1)),[train_ls,valid_ls],xlabel='epoch',ylabel='rmse',xlim=[1,num_epochs],legend=['train','valid'],yscale='log')# 打印每折训练结果print(f'折{i + 1},训练log rmse{float(train_ls[-1]):f},'f'验证log rmse{float(valid_ls[-1]):f}')# 返回平均误差return train_l_sum / k,valid_l_sum / k

# k:折数
# num_epochs:训练轮数
# lr:学习率
# weight_decay:无权重衰减
# batch_size:批量大小
k, num_epochs, lr, weight_decay, batch_size = 5, 100, 5, 0, 64
# 调用k_fold函数进行训练和评估
train_l, valid_l = k_fold(k, train_features, train_labels, num_epochs, lr,weight_decay, batch_size)
# 打印平均误差
print(f'{k}-折验证: 平均训练log rmse: {float(train_l):f}, 'f'平均验证log rmse: {float(valid_l):f}')