update

Author: AndyShen105

bin/ps.sh

@@ -61,7 +61,7 @@ do
             then
                 sleep 0.5
             fi
-            ssh $ip python /root/code/disML_Framwork/disML_Framwork.py $ps $worker --job_name=worker --task_index=$index --Learning_rate=$4 --ML_model=$6 --optimizer=$3 --n_intra_threads=$7 --Batch_size=$8 --n_partitions=$9 --num_Features=${10} --targeted_loss=${11} >> /root/code/$index".temp"
+            ssh $ip python /root/code/disML_Framwork/disML_Framwork.py $ps $worker --job_name=worker --task_index=$index --Learning_rate=$4 --ML_model=$6 --optimizer=$3 --n_intra_threads=$7 --Batch_size=$8 --n_partitions=$9 --num_Features=${10} --targeted_loss=0.5 >> /root/code/$index".temp"
 	    echo "worker"$index" complated"
             echo "1"> /root/code/disML_Framwork/bin/temp$index
         fi

start_experiment.py

@@ -10,7 +10,7 @@
 import logging
 import sys
 
-id = ""
+
 logging.basicConfig(level=logging.DEBUG,
                     format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
                     datefmt='%m-%d %H:%M',
@@ -19,12 +19,15 @@
 # n_workers, n_ps, n_intra, n_iter, n_partition, optimizer, batch_size, learning_rate
 Optimizer=["SGD","Adadelta","Adagrad","Ftrl","Adam","Momentum","RMSProp"]
 
-def wait_finish():
+def wait_finish(model, id):
     start_time = time.time()
-    dir = os.path.join("temp0")
-    while os.path.exists("/root/code/disML_Framwork/bin/temp0"):
+    if model != "CNN":
+	path = "/root/code/disML_Framwork/bin/temp0"
+    else:
+	path = "/root/code/disCNN_cifar/bin/temp0"
+    while os.path.exists(path):
         logging.info("The job %s is not finish" % id)
-        time.sleep(10)
+        time.sleep(3)
 	if (time.time()-start_time)>18000:
 	    os.system("./bin/kill_cluster_pid.sh 36 72 22222")
 
@@ -65,7 +68,7 @@ def execute(model, n_workers, n_ps, n_intra, n_partition, optimizer, batch_size,
 									batch_size,
 									n_partition)
     else:
-        cmd = "~/code/disCNN_cifar/bin/ps.sh %d  %s %f 22222 %s %d %d %d " % (n_workers,
+        cmd = "~/code/disCNN_cifar/bin/ps.sh %d %d %s %f 22222 %s %d %d %d " % (n_workers,
                                                                         n_ps,
                                                                         optimizer,
                                                                         learning_rate,
@@ -74,31 +77,33 @@ def execute(model, n_workers, n_ps, n_intra, n_partition, optimizer, batch_size,
                                                                         batch_size,
                                                                         n_partition)
     logging.info("run command: %s" % cmd)
-    id = model+"_"+str(n_workers)+"_"+str(n_intra)+"_"+optimizer+"_"+str(learning_rate)+"_"+str(batch_size)+"_"+str(n_partition)
     #p = pexpect.spawn(cmd)
-    #os.system(cmd+" >> running_log.out")
-    return id
+    os.system(cmd+" >> running_log.out")
+   
 
 def run(n_samples, model):
     for i in range(0, n_samples):
 	n_workers = np.random.randint(1, 35)
 	n_ps = 36-n_workers
 	n_intra = np.random.randint(1, 15)
-	n_partition = int(np.random.randint(0.1, 2)*n_ps)
+	n_partition = int(np.random.randint(1, 50)*n_ps/10)
+	if n_partition == 0:
+	    n_partition=1
 	optimizer = Optimizer[np.random.randint(0, 6)]
 	if model != "CNN":
 	    batch_size = np.random.randint(10, 50)*100
 	else:
 	    batch_size = np.random.randint(1, 10)*100
-	learning_rate = np.random.randint(1, 1000)/10000.0
+	learning_rate = np.random.randint(1, 10)/100000.0
 	threads = []
+	id = model+"_"+str(n_workers)+"_"+str(n_intra)+"_"+optimizer+"_"+str(learning_rate)+"_"+str(batch_size)+"_"+str(n_partition)
 	t1 = threading.Thread(target=execute,args=(model, n_workers, n_ps, n_intra, n_partition, optimizer, batch_size, learning_rate))
 	threads.append(t1)
 	#job_id = execute(model, n_workers, n_ps, n_intra, n_partition, optimizer, batch_size, learning_rate)
         t1.setDaemon(True)
         t1.start()
 	time.sleep(10)
-	wait_finish()
+	wait_finish(model, id)
 def main():
     run(int(sys.argv[1]), sys.argv[2])
 if __name__=="__main__":