Day 07_ridge-regression_gradient_descent

# -*- coding: utf-8 -*-
"""Day 10_ridge-regression_gradient_descent.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1efMZze-KFlXWkRqqwAfI7Sbt6Hp-Xp60
"""
 
from google.colab import drive
drive.mount('/gdrive')
 
PATH = "/gdrive/My Drive/Colab Notebooks/resources/"
 
# %matplotlib inline
 
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
 
from sklearn import linear_model
from sklearn.metrics import mean_squared_error
 
matplotlib.rc('xtick', labelsize=14) 
matplotlib.rc('ytick', labelsize=14)
 
def generate_data(n):
    d = 100
    w = np.zeros(d)
    for i in range(0,10):
        w[i] = 1.0
    
    #
    trainx = np.random.normal(size=(n,d))
    e = np.random.normal(size=(n))
    trainy = np.dot(trainx, w) + e
    #
    
    return trainx, trainy
 
def ridge_regression_GD(x,y,C):
    
    w = np.zeros(x.shape[1])
    loss= []
    b = np.zeros(y.shape)
 
    residual = y -( x.dot(w) + b )
    deriv_w = -2 * ( residual.dot(x)  ) + (2 * C * w)
 
    ada = 0.001
 
    
    while 0.15 < np.sqrt(deriv_w.T.dot(deriv_w)) or np.sqrt(deriv_w.T.dot(deriv_w)) < -0.15:
        residual = y -( x.dot(w) + b )
        deriv_w = -2 * ( residual.dot(x)  ) + (2 * C * w)
 
        deriv_b = -2 * residual
        
        w = w - ada * deriv_w
        b = b - ada * deriv_b
        
        loss.append(residual.dot(residual) + C * w.T.dot(w)) 
 
 
 
    return w,b,loss
 
# Generate 200 data points
n = 200
x,y = generate_data(n)
# Set regularization constant
C = 10.0
# Run gradient descent solver
w, b, losses = ridge_regression_GD(x,y,C)
 
 
# Plot the losses
plt.plot(losses,'r')
plt.xlabel('Iterations', fontsize=14)
plt.ylabel('Loss', fontsize=14)
plt.show()
 
def compute_mse(w,b,x,y):
    residuals = y - (np.dot(x, w) + b)
    return np.dot(residuals, residuals)/n
 
# Generate 200 data points
n = 200
x,y = generate_data(n)
 
# Set regularization constant
C = 10.0
 
# Run gradient descent solver and compute its MSE
w, b, losses = ridge_regression_GD(x,y,C)
 
 
# Use built-in routine for ridge regression and compute MSE
regr = linear_model.Ridge(alpha=C)
regr.fit(x, y)
 
 
# Print MSE values and L2 distance between the regression functions
print ("MSE of gradient descent solver: ", compute_mse(w,b,x,y))
print ("MSE of built-in solver: ", mean_squared_error(regr.predict(x), y)) 
print ("Distance between w-coefficients: ", np.linalg.norm(w-regr.coef_))