1
import hashlib
2
import os
3
import tarfile
4
import zipfile
5
import requests # 下载
6

7
DATA_HUB = dict() # DATA_HUB的元素为url和hash组成的键值对
8
DATA_URL = 'http://d2l-data.s3-accelerate.amazonaws.com/'
9

10
# 从指定的url下载数据，并验证哈希函数，存储到../data/文件名 中。
11
def download_data(name, cache_dir=os.path.join('..', 'data')):
12
    assert name in DATA_HUB, f"{name} 不存在于 {DATA_HUB}"
13
    url, sha1_hash = DATA_HUB[name]
14
    os.makedirs(cache_dir, exist_ok=True)
15
    fname = os.path.join(cache_dir, url.split('/')[-1]) # 拼接文件存储的相对路劲
16
    if os.path.exists(fname):
17
        sha1 = hashlib.sha1()
18
        with open(fname, 'rb') as f:
19
            while True:
20
                data = f.read(1048576)
21
                if not data:
22
                    break
23
                sha1.update(data)
24
        if sha1.hexdigest() == sha1_hash:
25
            return fname
26
    print(f'正在从{url}下载{fname}...')
27
    r = requests.get(url, stream=True, verify=True)
28
    with open(fname, 'wb') as f:
29
        f.write(r.content)
30
    return fname
31

32
# 下载文件，并且解压缩
33
def download_extract(name, folder=None):
34
    fname = download(name)
35
    base_dir = os.path.dirname(fname)
36
    data_dir, ext = os.path.splitext(fname)
37
    if ext == '.zip':
38
        fp = zipfile.ZipFile(fname, 'r')
39
    elif ext in ('.tar', '.gz'):
40
        fp = tarfile.open(fname, 'r')
41
    else:
42
        assert False, '只有zip和tar，gz文件可以被解压缩'
43
    fp.extractall(base_dir) # 解压到base_dir
44
    return os.path.join(base_dir, folder) if folder else data_dir
45

46
def download_all():
47
    for name in DATA_HUB:
48
        download_data(name)

1
%matplotlib inline
2
import numpy as np
3
import pandas as pd
4
import torch
5
from torch import nn
6
from d2l import torch as d2l
7

8
DATA_HUB['kaggle_house_train'] = (DATA_URL + 'kaggle_house_pred_train.csv',
9
 '585e9cc93e70b39160e7921475f9bcd7d31219ce')
10
DATA_HUB['kaggle_house_test'] = (DATA_URL + 'kaggle_house_pred_test.csv',
11
 'fa19780a7b011d9b009e8bff8e99922a8ee2eb90')
12

13
train_data = pd.read_csv(download_data('kaggle_house_train'))
14
test_data = pd.read_csv(download_data('kaggle_house_test'))
15

16
print(train_data.shape)
17
print(test_data.shape)
18

19
print(train_data.iloc[0:4,[0,1,2,3,4,-4,-3,-2,-1]])
20
print(test_data.iloc[0:4,[0,1,2,3,4,-4,-3,-2,-1]])

1
# 获取数字类型特征的下标
2
numeric_features = all_features.dtypes[all_features.dtypes != 'object'].index
3
print(numeric_features)
4

5
# 将每个特征的数据，进行归一化操作
6
all_features[numeric_features] = all_features[numeric_features].apply(
7
    lambda x : (x - x.mean()) / (x.std())
8
)
9

10
# 处理离散值
11
all_features = pd.get_dummies(all_features, dummy_na=True)
12
print(all_features.shape)
13
print(all_features.iloc[0:5, :])
14

15
# 将缺失值设置为0
16
all_features[numeric_features] = all_features[numeric_features].fillna(0)
17
print(all_features[numeric_features])
18

19

20
# 将训练数据和测试数据分开
21
n_train = train_data.shape[0]
22
# 前n_train个为训练数据，后面的我测试数据
23
train_features = torch.tensor(all_features[:n_train].values, dtype=torch.float32)
24
test_features = torch.tensor(all_features[n_train:].values, dtype=torch.float32)
25
train_labels = torch.tensor(
26
    train_data.SalePrice.values.reshape(-1, 1), dtype=torch.float32
27
)

1
loss = nn.MSELoss()
2
in_features = train_features.shape[1]
3

4
def get_net():
5
    net = nn.Sequential(nn.Linear(in_features, 1))
6
    return net

1
def log_rmse(net, features, labels):
2
    clipped_preds = torch.clamp(net(features), 1, float('inf'))
3
    rmse = torch.sqrt(loss(torch.log(clipped_preds), torch.log(labels)))
4
    return rmse.item()

1
def get_k_fold_data(k, i, X, y):
2
    assert k > 1
3
    fold_size = X.shape[0] // k # 整除k
4
    X_train, y_train = None, None
5
    for j in range(k):
6
        idx = slice(j * fold_size, (j + 1) * fold_size) # slice切片对象
7
        X_part, y_part = X[idx, :], y[idx] # idx用于切片
8
        if j == i: # _valid用于返回第i折的数据
9
            X_valid, y_valid = X_part, y_part
10
        elif X_train is None:
11
            X_train, y_train = X_part, y_part
12
        else:
13
            X_train = torch.cat([X_train, X_part], 0) # 将两个张量在第0维拼接
14
            y_train = torch.cat([y_train, y_part], 0)
15
    return X_train, y_train, X_valid, y_valid
16

17
def k_fold(k, X_train, y_train, num_epochs, learning_rate, weight_decay, batch_size):
18
    train_l_sum, valid_l_sum = 0, 0
19
    for i in range(k):
20
        # 获取单折的数据
21
        data = get_k_fold_data(k, i, X_train, y_train)
22
        net = get_net()
23
        # ls用于存放，每一折训练的结果
24
        train_ls, valid_ls = train(net, *data, num_epochs, learning_rate,
25
                                  weight_decay, batch_size)
26
        # 计算训练结果的sum
27
        train_l_sum += train_ls[-1]
28
        valid_l_sum += valid_ls[-1]
29
        # 挥着epoch和ls中对应的数据
30
        if i == 0:
31
            d2l.plot(list(range(1, num_epochs + 1)), [train_ls, valid_ls],
32
                     xlabel='epoch', ylabel='rmse', xlim=[1, num_epochs],
33
                     legend=['train', 'valid'], yscale='log')
34
        print(f'折{i + 1}, 训练log rmse{float(train_ls[-1]):f}, '
35
             f'验证 log rmse{float(valid_ls[-1]):f}')
36
    return train_l_sum / k, valid_l_sum / k

1
def train(net, train_features, train_labels, test_features, test_labels,
2
         num_epochs, learning_rate, weight_decay, batch_size):
3
    train_ls, test_ls = [], []
4
    train_iter = d2l.load_array((train_features, train_labels), batch_size)
5
    # 使用Adam优化器
6
    optimizer = torch.optim.Adam(net.parameters(),lr = learning_rate, weight_decay=weight_decay)
7
    for epoch in range(num_epochs):
8
        for X, y in train_iter:
9
            optimizer.zero_grad() # 清空梯度
10
            l = loss(net(X), y) # 确定损失
11
            l.backward() # 求导
12
            optimizer.step() # 继续迭代
13
        train_ls.append(log_rmse(net, train_features, train_labels)) # 记录结果
14
        if test_labels is not None:
15
            test_ls.append(log_rmse(net, test_features, test_labels))
16
    return train_ls, test_ls

1
k = 5
2
num_epochs = 100
3
lr = 5
4
weight_decay = 0
5
batch_size = 64
6

7
train_l, valid_l = k_fold(k, train_features, train_labels, num_epochs,
8
                          lr, weight_decay, batch_size)
9
print(f'{k}-折验证：平均训练log rmse:{float(train_l):f},'
10
     f'平均训练log rmse:{float(valid_l):f}')