- implementing linear regression with tensorflow (using normal equation)
- implementing gradient descent (manually computing gradients)
- using tensorflow autodiff to automatically compute gradients
- using optimizers

import numpy as np from sklearn.datasets import fetch_california_housing # Download the data. California housing is a standard sklearn dataset, so we'll just use it from there. housing = fetch_california_housing() m, n = housing.data.shape # Add a bias column (with all ones) housing_data_with_bias = np.c_[np.ones((m, 1)), housing.data] # Initialize X and y constants in tensorflow X = tf.constant(housing_data_with_bias, dtype = tf.float32, name='X') y = tf.constant(housing.target.reshape(-1, 1), dtype = tf.float32, name='y') # Define the value of theta with normal equation XT = tf.transpose(X) XTdotX = tf.matmul(XT, X) XTdotX_inverse = tf.matrix_inverse(XTdotX) XTdotY = tf.matmul(XT, y) theta = tf.matmul(XTdotX_inverse, XTdotY) # Evaluate theta with tf.Session() as sess: theta_value = theta.eval() print(theta_value)

import numpy as np from sklearn.datasets import fetch_california_housing # Download the data. California housing is a standard sklearn dataset, so we'll just use it from there. housing = fetch_california_housing() m, n = housing.data.shape # Add a bias column (with all ones) housing_data_with_bias = np.c_[np.ones((m, 1)), housing.data] # Normalize input features from sklearn.preprocessing import StandardScaler housing_data_with_bias_scaled = StandardScaler().fit_transform(housing_data_with_bias) n_epochs = 1000 learning_rate = 0.01 # Define X, y, theta X = tf.constant(housing_data_with_bias_scaled, dtype = tf.float32, name = 'X') y = tf.constant(housing.target.reshape(-1, 1), dtype = tf.float32, name = 'y') theta = tf.Variable(tf.random_uniform([n+1, 1], -1.0, 1.0), name = 'theta') y_prediction = tf.matmul(X, theta, name = 'y_prediction') # Compute mean squared error error = y_prediction - y mse = tf.reduce_mean(tf.square(error), name = 'mse') # Compute gradients gradients = 2/m * tf.matmul(tf.transpose(X), error) # Update theta theta_new = theta - learning_rate * gradients theta_update_op = tf.assign(theta, theta_new) init = tf.global_variables_initializer() # Run with tf.Session() as sess: sess.run(init) for epoch in range(n_epochs): if epoch % 100 == 0: print('Epoch', epoch, 'MSE =', mse.eval()) sess.run(theta_update_op) best_theta = theta.eval() print(best_theta)

# Compute gradients w.r.t. theta with Tensorflow' autodiff gradients = tf.gradients(mse, [theta])[0] # Update theta theta_new = theta - learning_rate * gradients theta_update_op = tf.assign(theta, theta_new)

# Use an optimizer to perform gradient descent (compute gradients, compute and update theta values) optimizer = tf.train.GradientDescentOptimizer(learning_rate = learning_rate) training_op = optimizer.minimize(mse)

# Use an optimizer to perform gradient descent (compute gradients, compute and update theta values) optimizer = tf.train.MomentumOptimizer(learning_rate = learning_rate, momentum = 0.9) training_op = optimizer.minimize(mse)