code for lecture 3. solutions uploaded after class

Author: chiphuyen

examples/02_lazy_loading.py

@@ -1,5 +1,5 @@
 """ Example of lazy vs normal loading
-Created by Chip Huyen (huyenn@stanford.edu)
+Created by Chip Huyen (chiphuyen@cs.stanford.edu)
 CS20: "TensorFlow for Deep Learning Research"
 cs20.stanford.edu
 Lecture 02

examples/02_placeholder.py

@@ -1,5 +1,5 @@
 """ Placeholder and feed_dict example
-Created by Chip Huyen (huyenn@stanford.edu)
+Created by Chip Huyen (chiphuyen@cs.stanford.edu)
 CS20: "TensorFlow for Deep Learning Research"
 cs20.stanford.edu
 Lecture 02

examples/02_simple_tf.py

@@ -1,5 +1,5 @@
 """ Simple TensorFlow's ops
-Created by Chip Huyen (huyenn@stanford.edu)
+Created by Chip Huyen (chiphuyen@cs.stanford.edu)
 CS20: "TensorFlow for Deep Learning Research"
 cs20.stanford.edu
 """

examples/02_variables.py

@@ -1,5 +1,5 @@
 """ Variable exmaples
-Created by Chip Huyen (huyenn@stanford.edu)
+Created by Chip Huyen (chiphuyen@cs.stanford.edu)
 CS20: "TensorFlow for Deep Learning Research"
 cs20.stanford.edu
 Lecture 02

examples/03_linreg_dataset.py

@@ -0,0 +1,74 @@
+""" Solution for simple linear regression example using tf.data
+Created by Chip Huyen ([email protected])
+CS20: "TensorFlow for Deep Learning Research"
+cs20.stanford.edu
+Lecture 03
+"""
+import os
+os.environ['TF_CPP_MIN_LOG_LEVEL']='2'
+import time
+
+import numpy as np
+import matplotlib.pyplot as plt
+import tensorflow as tf
+
+import utils
+
+DATA_FILE = 'data/birth_life_2010.txt'
+
+# Step 1: read in the data
+data, n_samples = utils.read_birth_life_data(DATA_FILE)
+
+# Step 2: create Dataset and iterator
+dataset = tf.data.Dataset.from_tensor_slices((data[:,0], data[:,1]))
+
+iterator = dataset.make_initializable_iterator()
+X, Y = iterator.get_next()
+
+# Step 3: create weight and bias, initialized to 0
+w = tf.get_variable('weights', initializer=tf.constant(0.0))
+b = tf.get_variable('bias', initializer=tf.constant(0.0))
+
+# Step 4: build model to predict Y
+Y_predicted = X * w + b
+
+# Step 5: use the square error as the loss function
+loss = tf.square(Y - Y_predicted, name='loss')
+# loss = utils.huber_loss(Y, Y_predicted)
+
+# Step 6: using gradient descent with learning rate of 0.001 to minimize loss
+optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001).minimize(loss)
+
+start = time.time()
+with tf.Session() as sess:
+    # Step 7: initialize the necessary variables, in this case, w and b
+    sess.run(tf.global_variables_initializer()) 
+    writer = tf.summary.FileWriter('./graphs/linear_reg', sess.graph)
+    
+    # Step 8: train the model for 100 epochs
+    for i in range(100):
+        sess.run(iterator.initializer) # initialize the iterator
+        total_loss = 0
+        try:
+            while True:
+                _, l = sess.run([optimizer, loss]) 
+                total_loss += l
+        except tf.errors.OutOfRangeError:
+            pass
+            
+        print('Epoch {0}: {1}'.format(i, total_loss/n_samples))
+
+    # close the writer when you're done using it
+    writer.close() 
+    
+    # Step 9: output the values of w and b
+    w_out, b_out = sess.run([w, b]) 
+    print('w: %f, b: %f' %(w_out, b_out))
+print('Took: %f seconds' %(time.time() - start))
+
+# plot the results
+plt.plot(data[:,0], data[:,1], 'bo', label='Real data')
+plt.plot(data[:,0], data[:,0] * w_out + b_out, 'r', label='Predicted data with squared error')
+# plt.plot(data[:,0], data[:,0] * (-5.883589) + 85.124306, 'g', label='Predicted data with Huber loss')
+plt.legend()
+plt.show()

examples/03_linreg_starter.py

@@ -0,0 +1,94 @@
+""" Starter code for simple linear regression example using placeholders
+Created by Chip Huyen ([email protected])
+CS20: "TensorFlow for Deep Learning Research"
+cs20.stanford.edu
+Lecture 03
+"""
+import os
+os.environ['TF_CPP_MIN_LOG_LEVEL']='2'
+import time
+
+import numpy as np
+import matplotlib.pyplot as plt
+import tensorflow as tf
+
+import utils
+
+DATA_FILE = 'data/birth_life_2010.txt'
+
+# Step 1: read in data from the .txt file
+data, n_samples = utils.read_birth_life_data(DATA_FILE)
+
+# Step 2: create placeholders for X (birth rate) and Y (life expectancy)
+# Remember both X and Y are scalars with type float
+X, Y = None, None
+#############################
+########## TO DO ############
+#############################
+
+# Step 3: create weight and bias, initialized to 0.0
+# Make sure to use tf.get_variable
+w, b = None, None
+#############################
+########## TO DO ############
+#############################
+
+# Step 4: build model to predict Y
+# e.g. how would you derive at Y_predicted given X, w, and b
+Y_predicted = None
+#############################
+########## TO DO ############
+#############################
+
+# Step 5: use the square error as the loss function
+loss = None
+#############################
+########## TO DO ############
+#############################
+
+# Step 6: using gradient descent with learning rate of 0.001 to minimize loss
+optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001).minimize(loss)
+
+start = time.time()
+
+# Create a filewriter to write the model's graph to TensorBoard
+#############################
+########## TO DO ############
+#############################
+
+with tf.Session() as sess:
+    # Step 7: initialize the necessary variables, in this case, w and b
+    #############################
+    ########## TO DO ############
+    #############################
+
+    # Step 8: train the model for 100 epochs
+    for i in range(100):
+        total_loss = 0
+        for x, y in data:
+            # Execute train_op and get the value of loss.
+            # Don't forget to feed in data for placeholders
+            _, loss = ########## TO DO ############
+            total_loss += loss
+
+        print('Epoch {0}: {1}'.format(i, total_loss/n_samples))
+
+    # close the writer when you're done using it
+    #############################
+    ########## TO DO ############
+    #############################
+    writer.close()
+    
+    # Step 9: output the values of w and b
+    w_out, b_out = None, None
+    #############################
+    ########## TO DO ############
+    #############################
+
+print('Took: %f seconds' %(time.time() - start))
+
+# uncomment the following lines to see the plot 
+# plt.plot(data[:,0], data[:,1], 'bo', label='Real data')
+# plt.plot(data[:,0], data[:,0] * w_out + b_out, 'r', label='Predicted data')
+# plt.legend()
+# plt.show()

examples/03_logreg_placeholder.py

@@ -0,0 +1,93 @@
+""" Solution for simple logistic regression model for MNIST
+with placeholder
+MNIST dataset: yann.lecun.com/exdb/mnist/
+Created by Chip Huyen ([email protected])
+CS20: "TensorFlow for Deep Learning Research"
+cs20.stanford.edu
+Lecture 03
+"""
+import os
+os.environ['TF_CPP_MIN_LOG_LEVEL']='2'
+
+import numpy as np
+import tensorflow as tf
+from tensorflow.examples.tutorials.mnist import input_data
+import time
+
+import utils
+
+# Define paramaters for the model
+learning_rate = 0.01
+batch_size = 128
+n_epochs = 30
+
+# Step 1: Read in data
+# using TF Learn's built in function to load MNIST data to the folder data/mnist
+mnist = input_data.read_data_sets('data/mnist', one_hot=True)
+X_batch, Y_batch = mnist.train.next_batch(batch_size)
+
+# Step 2: create placeholders for features and labels
+# each image in the MNIST data is of shape 28*28 = 784
+# therefore, each image is represented with a 1x784 tensor
+# there are 10 classes for each image, corresponding to digits 0 - 9. 
+# each lable is one hot vector.
+X = tf.placeholder(tf.float32, [batch_size, 784], name='image') 
+Y = tf.placeholder(tf.int32, [batch_size, 10], name='label')
+
+# Step 3: create weights and bias
+# w is initialized to random variables with mean of 0, stddev of 0.01
+# b is initialized to 0
+# shape of w depends on the dimension of X and Y so that Y = tf.matmul(X, w)
+# shape of b depends on Y
+w = tf.get_variable(name='weights', shape=(784, 10), initializer=tf.random_normal_initializer())
+b = tf.get_variable(name='bias', shape=(1, 10), initializer=tf.zeros_initializer())
+
+# Step 4: build model
+# the model that returns the logits.
+# this logits will be later passed through softmax layer
+logits = tf.matmul(X, w) + b 
+
+# Step 5: define loss function
+# use cross entropy of softmax of logits as the loss function
+entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=Y, name='loss')
+loss = tf.reduce_mean(entropy) # computes the mean over all the examples in the batch
+# loss = tf.reduce_mean(-tf.reduce_sum(tf.nn.softmax(logits) * tf.log(Y), reduction_indices=[1]))
+
+# Step 6: define training op
+# using gradient descent with learning rate of 0.01 to minimize loss
+optimizer = tf.train.AdamOptimizer(learning_rate).minimize(loss)
+
+# Step 7: calculate accuracy with test set
+preds = tf.nn.softmax(logits)
+correct_preds = tf.equal(tf.argmax(preds, 1), tf.argmax(Y, 1))
+accuracy = tf.reduce_sum(tf.cast(correct_preds, tf.float32))
+
+writer = tf.summary.FileWriter('./graphs/logreg_placeholder', tf.get_default_graph())
+with tf.Session() as sess:
+	start_time = time.time()
+	sess.run(tf.global_variables_initializer())	
+	n_batches = int(mnist.train.num_examples/batch_size)
+	
+	# train the model n_epochs times
+	for i in range(n_epochs): 
+		total_loss = 0
+
+		for j in range(n_batches):
+			X_batch, Y_batch = mnist.train.next_batch(batch_size)
+			_, loss_batch = sess.run([optimizer, loss], {X: X_batch, Y:Y_batch}) 
+			total_loss += loss_batch
+		print('Average loss epoch {0}: {1}'.format(i, total_loss/n_batches))
+	print('Total time: {0} seconds'.format(time.time() - start_time))
+
+	# test the model
+	n_batches = int(mnist.test.num_examples/batch_size)
+	total_correct_preds = 0
+
+	for i in range(n_batches):
+		X_batch, Y_batch = mnist.test.next_batch(batch_size)
+		accuracy_batch = sess.run(accuracy, {X: X_batch, Y:Y_batch})
+		total_correct_preds += accuracy_batch	
+
+	print('Accuracy {0}'.format(total_correct_preds/mnist.test.num_examples))
+
+writer.close()