我正在尝试为boston
scikit-learn 中的数据集构建多元线性回归模型。
我使用随机梯度下降 (SGD) 来优化模型。似乎我必须使用非常小的学习率(0.000000001)来让模型学习。如果我使用更大的学习率,模型将无法学习并发散到 NaN 或 inf。
所以,这是我的问题:
- 使用这么小的学习率可以吗?或者我下面的代码有什么问题吗?
- 似乎验证数据集的损失减少了,增加了一段时间,然后又减少了。这是我的模型陷入过度拟合问题的情况,但由于 SGD 的不稳定性与批量梯度下降方法相比,幸运地逃脱了?
这是我的代码:
from sklearn import datasets
import numpy as np
import matplotlib.pyplot as plt
def loss(x, y, w):
predict_y = x @ w
return np.sqrt(np.mean(np.square((y - predict_y))))
def status(w):
w_ = np.squeeze(w)
print("w = [", end="")
for i in range(14):
if(i == 13):
print(w_[i], end="]")
else:
print(w_[i], end=", ")
print()
training_loss = loss(training_x, training_y, w)
validation_loss = loss(validation_x, validation_y, w)
print("Training Loss = " + str(training_loss))
print("Validation Loss = " + str(validation_loss))
training_predict_y = training_x @ w
validation_predict_y = validation_x @ w
print("{:^40s}|{:^40s}".format("training", "validation"))
print("{:^20s}{:^20s}|{:^20s}{:^20s}".format("predict_y", "true_y", "predict_y", "true_y"))
for i in range(10):
print("{:^20f}{:^20f}|{:^20f}{:^20f}".format(float(training_predict_y[i]), float(training_y[i]), float(validation_predict_y[i]), float(validation_y[i])))
print()
def plot(title, data):
plt.title(title)
plt.plot(range(len(data)), data)
plt.savefig(title + ".png", dpi = 300)
plt.show()
np.random.seed(2020) # for consistency
# data
dataset = datasets.load_boston()
x = dataset.data
y = dataset.target
# reformat the data
x_ = np.concatenate((np.ones((x.shape[0], 1)), x), axis=1) # x0 = 1인 열 추가
y_ = np.expand_dims(y, axis=1)
# divide data into training set and validation set
training_x = x_[ 0:406, : ]
training_y = y_[ 0:406, : ]
validation_x = x_[ 406:506, : ]
validation_y = y_[ 406:506, : ]
# initialize w
w = np.random.rand(x_.shape[1], 1)
print("Before Training...")
status(w)
# hyperparameter
epochs = 100000
lr = 0.000000001
training_losses = []
validation_losses = []
data_num = training_x.shape[0]
for epoch in range(epochs):
for i in range(data_num):
sample = training_x[ i:i + 1, : ]
true_y = training_y[ i:i + 1, : ]
predict_y = sample @ w
# calculate gradient
gradient = -(2 / sample.shape[0]) * sample.T @ (true_y - predict_y)
# update w
w = w - lr * gradient
training_loss = loss(training_x, training_y, w)
validation_loss = loss(validation_x, validation_y, w)
training_losses.append(training_loss)
validation_losses.append(validation_loss)
print("After Training...")
status(w)
plot("Training Loss - SGD", training_losses)
plot("Validation Loss - SGD", validation_losses)