【PyTorch】正则化：L1范数，L2范数，Dropout |
您所在的位置：网站首页 › pytorch正则化 › 【PyTorch】正则化：L1范数，L2范数，Dropout |
|
正则化 当数据集较小或者网络模型较简单，在进行一些处理的时候，很容易形成训练精度很高了，但是测试的误差仍然较大，为了解决这种过拟合的问题，需要进行正则化处理，有以下几种方式： 目标函数添加正则项：L1范数目标函数添加正则项：L2f范数当增强数据集后可使用dropout，获得更好的效果正则化的详细的内容可参考：https://www.cnblogs.com/pinking/p/9310728.html 代码实现 L2正则化： #高维线性回归 import torch import torch.nn as nn import numpy as np import sys import matplotlib.pyplot as plt %matplotlib inline n_train,n_test,num_inputs = 20,100,200 true_w,true_b = torch.ones(num_inputs,1)*0.01,0.05 features = torch.randn((n_train +n_test,num_inputs)) labels = torch.matmul(features,true_w)+true_b labels += torch.tensor(np.random.normal(0,0.01,size = labels.size()),dtype = torch.float) train_features ,test_features = features[:n_train,:],features[n_train:,:] train_labels,test_labels = labels[:n_train],labels[n_train:] #通过目标函数添加L2惩罚项来实现权重衰减 #初始化 def init_params(): w = torch.randn((num_inputs,1),requires_grad = True) b = torch.zeros(1,requeires_grad = True) return [w,b] #定义L2范数 1/2(w1+w2+..)^2 def l2_penaalty(w): return (w**2).sum()/2 #定义训练和测试模型 plt.semilogy对y轴使用了对数尺度 #x_vals：横坐标,y_vals：纵坐标 训练损失,x_label：横坐标的表头,y_label：纵坐标的表头, #x2_vals=None,y2_vals=None 测试损失, legend=None：提示线型,figsize=(3.5,2.5）图像大小 def semilogy(x_vals,y_vals,x_label,y_label,x2_vals=None,y2_vals=None, legend=None,figsize=(3.5,2.5)): plt.xlabel(x_label) plt.ylabel(y_label) plt.semilogy(x_vals,y_vals) if x2_vals and y2_vals: plt.semilogy(x2_vals,y2_vals,linestyle=':') plt.legend(legend) #定义训练和测试 batch_size, num_epochs, lr,loss = 1, 100, 0.003,torch.nn.MSELoss() dataset = torch.utils.data.TensorDataset(train_features, train_labels) train_iter = torch.utils.data.DataLoader(dataset, batch_size, shuffle=True) def fit_and_plot_pytorch(wd): # 对权重参数衰减。权重名称一般是以weight结尾 net = nn.Linear(num_inputs, 1) nn.init.normal_(net.weight, mean=0, std=1) nn.init.normal_(net.bias, mean=0, std=1) # 对权重参数衰减 optimizer_w = torch.optim.SGD(params=[net.weight], lr=lr, weight_decay=wd) # 不对偏差参数衰减 optimizer_b = torch.optim.SGD(params=[net.bias], lr=lr) train_ls, test_ls = [], [] for _ in range(num_epochs): for X, y in train_iter: #最小均方误差损失 l = loss(net(X), y).mean() optimizer_w.zero_grad() optimizer_b.zero_grad() l.backward() # 对两个optimizer实例分别调用step函数，从而分别更新权重和偏差 optimizer_w.step() optimizer_b.step() #将训练和测试损失添加到列表中 train_ls.append(loss(net(train_features), train_labels).mean().item()) test_ls.append(loss(net(test_features), test_labels).mean().item()) semilogy(range(1, num_epochs + 1), train_ls, 'epochs', 'loss', range(1, num_epochs + 1), test_ls, ['train', 'test']) print('L2 norm of w:', net.weight.data.norm().item()) #通过加大超参数 使得惩罚项的比重增大 fit_and_plot_pytorch(5)Dropout #dropout 不改变输入的期望值 import torch import torch.nn as nn import numpy as np import sys import torchvision import torchvision.transforms as transforms #获取fashion_mnist数据集 mnist_train = torchvision.datasets.FashionMNIST(root='~/Desktop/OpenCV_demo/Datasets/FashionMNIST', train=True, download=False, transform=transforms.ToTensor()) mnist_test = torchvision.datasets.FashionMNIST(root='~/Desktop/OpenCV_demo/Datasets/FashionMNIST', train=False, download=False, transform=transforms.ToTensor()) #获得训练和测试数据 if sys.platform.startswith('win'): num_workers = 0 else: num_workers = 4 train_iter = torch.utils.data.DataLoader( mnist_train,batch_size = batch_size,shuffle = True,num_workers = num_workers) test_iter = torch.utils.data.DataLoader( mnist_test,batch_size = batch_size,shuffle = True,num_workers = num_workers) #定义dropout def dropout(X,drop_prob): X = X.float() assert 0 |
今日新闻 |
点击排行 |
|
推荐新闻 |
图片新闻 |
|
专题文章 |
| CopyRight 2018-2019 实验室设备网 版权所有 win10的实时保护怎么永久关闭 |