add loading

Author: Shashank Tyagi

.DS_Store

@@ -1,27 +1,28 @@
 import numpy as np 
 import tensorflow as tf
-from skimage import io
+#from skimage import io
 import sqlite3
-import cv2
+#import cv2
 import matplotlib.pyplot as plt
 import os
 import random
+from tqdm import tqdm
 
-select_string = "faceimages.filepath, faces.face_id, facepose.roll, facepose.pitch, facepose.yaw, facerect.x, facerect.y, facerect.w, facerect.h"
-from_string = "faceimages, faces, facepose, facerect"
-where_string = "faces.face_id = facepose.face_id and faces.file_id = faceimages.file_id and faces.face_id = facerect.face_id"
-query_string = "SELECT " + select_string + " FROM " + from_string + " WHERE " + where_string
+# select_string = "faceimages.filepath, faces.face_id, facepose.roll, facepose.pitch, facepose.yaw, facerect.x, facerect.y, facerect.w, facerect.h"
+# from_string = "faceimages, faces, facepose, facerect"
+# where_string = "faces.face_id = facepose.face_id and faces.file_id = faceimages.file_id and faces.face_id = facerect.face_id"
+# query_string = "SELECT " + select_string + " FROM " + from_string + " WHERE " + where_string
 
-conn = sqlite3.connect('/home/shashank/Documents/CSE-252C/AFLW/aflw/data/aflw.sqlite')
-c = conn.cursor()
+# conn = sqlite3.connect('/home/shashank/Documents/CSE-252C/AFLW/aflw/data/aflw.sqlite')
+# c = conn.cursor()
 
 img_path = '/home/shashank/Documents/CSE-252C/AFLW/'
 
-tfrecords_train_filename = 'aflw_train.tfrecords'
-tfrecords_test_filename = 'aflw_test.tfrecords'
-
-writer_train = tf.python_io.TFRecordWriter(tfrecords_train_filename)
-writer_test = tf.python_io.TFRecordWriter(tfrecords_test_filename)
+# tfrecords_train_filename = 'aflw_train.tfrecords'
+# tfrecords_test_filename = 'aflw_test.tfrecords'
+tfrecords_filename = 'aflw_train.tfrecords'
+# writer_train = tf.python_io.TFRecordWriter(tfrecords_train_filename)
+# writer_test = tf.python_io.TFRecordWriter(tfrecords_test_filename)
 
 def _bytes_feature(value):
 	return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
@@ -103,9 +104,10 @@ def make_tfrecord(test_images):
 	writer_train.close()
 	writer_test.close()
 
-def extract_tfrecord():
+def extract_tfrecord(session):
 	record_iterator = tf.python_io.tf_record_iterator(path=tfrecords_filename)
-
+	save_data = None
+	save_euler = []
 	for string_record in record_iterator:
 		example = tf.train.Example()
 		example.ParseFromString(string_record)
@@ -118,13 +120,29 @@ def extract_tfrecord():
 		loc_y = int(example.features.feature['loc_y'].int64_list.value[0])
 		loc_w = int(example.features.feature['loc_w'].int64_list.value[0])
 		loc_h = int(example.features.feature['loc_h'].int64_list.value[0])
-		cv2.rectangle(img_1d,(loc_x,loc_y),(loc_x+loc_w,loc_y+loc_h),(0,255,0),3)
-		cv2.imshow('result',img_1d)
-		cv2.waitKey(0)
-		
+		roll = float(example.features.feature['roll'].float_list.value[0])
+		yaw = float(example.features.feature['yaw'].float_list.value[0])
+		pitch = float(example.features.feature['pitch'].float_list.value[0])
+
+		boxes = np.asarray([[loc_y/float(img_height),loc_x/float(img_width),(loc_y+loc_h)/float(img_height),(loc_x+loc_w)/float(img_width)]])
+		resized_and_cropped_image = tf.image.crop_and_resize(img_1d[np.newaxis,:,:,:].astype(np.float32), boxes.astype(np.float32), [0]*1, crop_size=[227,227])
+		if save_data is not None:
+			save_data = np.concatenate([save_data,resized_and_cropped_image.eval(session=session)],axis=0)
+		else:
+			save_data = resized_and_cropped_image.eval(session=session)
+		save_euler.append([roll,yaw,pitch])
+
+	np.save('truth_data.npy',save_data)
+	np.save('annotations.npy',np.asarray(save_euler))
+	
+		# cv2.rectangle(img_1d,(loc_x,loc_y),(loc_x+loc_w,loc_y+loc_h),(0,255,0),3)
+		# cv2.imshow('result',img_1d)
+		# cv2.waitKey(0)
+				
 
 if __name__ == '__main__':
-	test_images = test_names()
-	make_tfrecord(test_images)
-	#extract_tfrecord()
+	#test_images = test_names()
+	#make_tfrecord(test_images)
+	session = tf.Session()
+	extract_tfrecord(session)
 

aflw_train.tfrecords

@@ -2,6 +2,10 @@
 import os
 from model import *
 
+
+weights_path = '/Users/shashank/Tensorflow/SPN/weights/'
+imgs_path = '/Users/shashank/Tensorflow/CSE252C-Hyperface/git/truth_data.npy'
+
 if not os.path.exists('./logs'):
 	os.makedirs('./logs')
 
@@ -14,5 +18,6 @@
 		print 'Graph Built!'
 		sess.run(tf.global_variables_initializer())
 		net.print_variables()
-		# net.train()
+		net.load_weights(weights_path)
+		net.predict(imgs_path)
 

annotations.npy

@@ -1,6 +1,8 @@
 import tensorflow as tf
 import tensorflow.contrib.slim as slim
 import numpy as np
+from tqdm import tqdm
+from pdb import set_trace as brk
 
 
 class HyperFace(object):
@@ -62,7 +64,10 @@ def train(self):
 		writer = tf.summary.FileWriter('./logs', self.sess.graph)
 		loss_summ = tf.summary.scalar('loss', self.loss)
 
-	def network(self,inputs):
+	def network(self,inputs,reuse=False):
+
+		if reuse:
+			tf.get_variable_scope().reuse_variables()
 
 		with slim.arg_scope([slim.conv2d, slim.fully_connected],
 							 activation_fn = tf.nn.relu,
@@ -87,21 +92,40 @@ def network(self,inputs):
 			conv_all = slim.conv2d(concat_feat, 192, [1,1], 1, padding= 'VALID', scope='conv_all')
 
 			shape = int(np.prod(conv_all.get_shape()[1:]))
-			fc_full = slim.fully_connected(tf.reshape(conv_all, [-1, shape]), 3072, scope='fc_full')
+			fc_full = slim.fully_connected(tf.reshape(tf.transpose(conv_all, [0,3,1,2]), [-1, shape]), 3072, scope='fc_full')
+
+			fc_detection = slim.fully_connected(fc_full, 512, scope='fc_detection1')
+			fc_landmarks = slim.fully_connected(fc_full, 512, scope='fc_landmarks1')
+			fc_visibility = slim.fully_connected(fc_full, 512, scope='fc_visibility1')
+			fc_pose = slim.fully_connected(fc_full, 512, scope='fc_pose1')
+			fc_gender = slim.fully_connected(fc_full, 512, scope='fc_gender1')
+
+			out_detection = slim.fully_connected(fc_detection, 2, scope='fc_detection2', activation_fn = None)
+			out_landmarks = slim.fully_connected(fc_landmarks, 42, scope='fc_landmarks2', activation_fn = None)
+			out_visibility = slim.fully_connected(fc_visibility, 21, scope='fc_visibility2', activation_fn = None)
+			out_pose = slim.fully_connected(fc_pose, 3, scope='fc_pose2', activation_fn = None)
+			out_gender = slim.fully_connected(fc_gender, 2, scope='fc_gender2', activation_fn = None)
+
+		return [tf.nn.softmax(out_detection), out_landmarks, out_visibility, out_pose, tf.nn.softmax(out_gender)]
+
 
-			fc_detection = slim.fully_connected(fc_full, 512, scope='fc_detection')
-			fc_landmarks = slim.fully_connected(fc_full, 512, scope='fc_landmarks')
-			fc_visibility = slim.fully_connected(fc_full, 512, scope='fc_visibility')
-			fc_pose = slim.fully_connected(fc_full, 512, scope='fc_pose')
-			fc_gender = slim.fully_connected(fc_full, 512, scope='fc_gender')
 
-			out_detection = slim.fully_connected(fc_detection, 2, scope='out_detection')
-			out_landmarks = slim.fully_connected(fc_landmarks, 42, scope='out_landmarks')
-			out_visibility = slim.fully_connected(fc_visibility, 21, scope='out_visibility')
-			out_pose = slim.fully_connected(fc_pose, 3, scope='out_pose')
-			out_gender = slim.fully_connected(fc_gender, 2, scope='out_gender')
+	def predict(self, imgs_path):
+		print 'Running inference...'
+		np.set_printoptions(suppress=True)
+		imgs = (np.load(imgs_path) - 127.5)/128.0
+		shape = imgs.shape
+		self.X = tf.placeholder(tf.float32, [shape[0], self.img_height, self.img_width, self.channel], name='images')
+		pred = self.network(self.X, reuse = True)
+
+		net_preds = self.sess.run(pred, feed_dict={self.X: imgs})
+
+		print 'gender: \n', net_preds[-1]
+		import matplotlib.pyplot as plt
+		plt.imshow(imgs[-1]);plt.show()
+
+		brk()
 
-		return [out_detection, out_landmarks, out_visibility, out_pose, out_gender]
 
 	def load_from_tfRecord(self,filename_queue):
 
@@ -129,6 +153,18 @@ def load_from_tfRecord(self,filename_queue):
 
 		return images
 
+	def load_weights(self, path):
+		variables = slim.get_model_variables()
+		print 'Loading weights...'
+		for var in tqdm(variables):
+			if ('conv' in var.name) and ('weights' in var.name):
+				self.sess.run(var.assign(np.load(path+var.name.split('/')[0]+'/W.npy').transpose((2,3,1,0))))
+			elif ('fc' in var.name) and ('weights' in var.name):
+				self.sess.run(var.assign(np.load(path+var.name.split('/')[0]+'/W.npy').T))
+			elif 'biases' in var.name:
+				self.sess.run(var.assign(np.load(path+var.name.split('/')[0]+'/b.npy')))
+		print 'Weights loaded!!'
+
 	def print_variables(self):
 		variables = slim.get_model_variables()
 		print 'Model Variables:'

data_prep.py

@@ -1,6 +1,7 @@
 import tensorflow as tf
 import numpy as np
 import dlib
+from pdb import set_trace as brk
 
 tf_record_file = 'aflw_train.tfrecords'
 
@@ -45,7 +46,8 @@ def perform_selective_search(img,w,h,ground_truth):
 
 	return np.asarray(filter_positive_rects),np.asarray(filter_negative_rects)
 
-def split_(filename_queue):
+def split_(filename_queue, sess):
+	brk()
 	reader = tf.TFRecordReader()
 	_, serialized_example = reader.read(filename_queue)
 
@@ -72,17 +74,19 @@ def split_(filename_queue):
 
 	image_shape = tf.pack([height, width, 3])
 	image = tf.reshape(image, image_shape)
-	boxes,box_ind = perform_selective_search(,tf.cast(width,tf.float32),tf.cast(height,tf.float32),(loc_x,loc_y,loc_x+loc_w,loc_y+loc_h))
-
-	resized_and_cropped_image = tf.image.crop_and_resize(image, boxes, box_ind, crop_size=[227,227])
+	height,width,loc_x,loc_y,loc_h,loc_w = sess.run([height,width,loc_x,loc_y,loc_h,loc_w])
+	# boxes,box_ind = perform_selective_search(,tf.cast(width,tf.float32),tf.cast(height,tf.float32),(loc_x,loc_y,loc_x+loc_w,loc_y+loc_h))
+	boxes = np.asarray([[loc_y/float(height),loc_x/float(width),(loc_y+loc_h)/float(height),(loc_x+loc_w)/float(width)]])
+	resized_and_cropped_image = tf.image.crop_and_resize(image.astype(np.float32), boxes.astype(np.float32), [0]*1, crop_size=[227,227])
 
+
 	images = tf.train.shuffle_batch([resized_and_cropped_image],batch_size=10,num_threads=1,capacity=50,min_after_dequeue=10)
 
 	return images
 
 filename_queue = tf.train.string_input_producer([tf_record_file], num_epochs=1)
 
-images = split_(filename_queue)
+
 
 init_op = tf.group(tf.global_variables_initializer(),tf.local_variables_initializer())
 
@@ -91,6 +95,8 @@ def split_(filename_queue):
 with tf.Session() as sess:
 
 	sess.run(init_op)
+	images = split_(filename_queue, sess)
+	
 	coord = tf.train.Coordinator()
 	threads = tf.train.start_queue_runners(coord=coord)
 	op_images = sess.run([images])

main.py

@@ -15,6 +15,7 @@
 with tf.Session() as sess:
 		print 'Building Graph...'
 		model = Network(sess,tf_record_file_path)
-		print 'Graph Built!'
+		print 'Done!\nInitializing variables...'
 		sess.run(tf.global_variables_initializer())
+		print 'Done!'
 		model.train()

model.py

@@ -4,6 +4,7 @@
 from spatial_transformer import transformer
 from tqdm import tqdm
 from pdb import set_trace as brk
+import time
 
 class Network(object):
 
@@ -82,8 +83,11 @@ def train(self):
 		loss_summ = tf.summary.scalar('loss', self.loss)
 		img_summ = tf.summary.image('cropped_image', self.cropped)
 
-		print self.sess.run(self.T_mat, feed_dict={self.X: np.random.randn(self.batch_size, self.img_height, self.img_width, self.channel)})
 
+		tic = time.time()
+		print self.sess.run(self.T_mat, feed_dict={self.X: np.random.randn(self.batch_size, self.img_height, self.img_width, self.channel)})
+		toc = time.time()
+		print toc-tic
 		images = self.load_from_tfRecord(self.filename_queue)
 
 		coord = tf.train.Coordinator()
@@ -122,8 +126,7 @@ def hyperface(self,inputs, reuse = False):
 			conv_all = slim.conv2d(concat_feat, 192, [1,1], 1, padding= 'VALID', scope='conv_all')
 
 			shape = int(np.prod(conv_all.get_shape()[1:]))
-			# transposed for weight loading from chainer model
-			fc_full = slim.fully_connected(tf.reshape(tf.transpose(conv_all, [0,3,1,2]), [-1, shape]), 3072, scope='fc_full')
+			fc_full = slim.fully_connected(tf.reshape(conv_all, [-1, shape]), 3072, scope='fc_full')
 
 			fc_detection = slim.fully_connected(fc_full, 512, scope='fc_detection1')
 			fc_landmarks = slim.fully_connected(fc_full, 512, scope='fc_landmarks1')
@@ -191,8 +194,10 @@ def localization_squeezenet(self, inputs):
 				fire9 = self.fire_module(pool8, 64, 256, scope = 'fire9', res_connection=True)
 				conv10 = slim.conv2d(fire9, 128, [1,1], 1, scope='conv10')
 				shape = int(np.prod(conv10.get_shape()[1:]))
-				fc11 = slim.fully_connected(tf.reshape(conv10, [-1, shape]), 6, biases_initializer = tf.constant_initializer(np.array([[1., 0., 0.],
-																										  [0., 1., 0.]])) , scope='fc11')
+				identity = np.array([[1., 0., 0.],
+									[0., 1., 0.]])
+				identity = identity.flatten()
+				fc11 = slim.fully_connected(tf.reshape(conv10, [-1, shape]), 6, biases_initializer = tf.constant_initializer(identity), scope='fc11')
 		return fc11