From c5208681e35cec2f9c339126a96d3cee709813a4 Mon Sep 17 00:00:00 2001 From: Anthony Walker Date: Thu, 23 Apr 2020 20:29:54 -0700 Subject: [PATCH] Cluster batch file --- src/Makefile | 6 +- src/hsweep-all-batch.sh | 101 ++++++++++++++++++ src/launch/OSU_cluster/hSweep.sh | 10 +- src/launch/slurm-batch-files/hsweep-batch.sh | 26 +++++ .../slurm-batch-files/test-hsweep-batch.sh | 28 +++++ src/launch/slurm-batch-files/test-nrg-nodes | 2 + src/nrg-nodes | 2 + src/oneD/Makefile | 2 +- 8 files changed, 168 insertions(+), 9 deletions(-) create mode 100644 src/hsweep-all-batch.sh create mode 100644 src/launch/slurm-batch-files/hsweep-batch.sh create mode 100644 src/launch/slurm-batch-files/test-hsweep-batch.sh create mode 100644 src/launch/slurm-batch-files/test-nrg-nodes create mode 100644 src/nrg-nodes diff --git a/src/Makefile b/src/Makefile index 904add8..5f245fd 100644 --- a/src/Makefile +++ b/src/Makefile @@ -4,10 +4,10 @@ include config.mk -CODES :=oneD #twoD +CODES :=/oneD #twoD CODE_DIR :=$(addprefix $(SOURCEPATH), $(CODES)) -BIN_DIR :=$(SOURCEPATH)bin -UTIL_DIR :=$(SOURCEPATH)utilities +BIN_DIR :=$(SOURCEPATH)/bin +UTIL_DIR :=$(SOURCEPATH)/utilities UTIL_BIN :=$(BIN_DIR)/util TESTDIR :=$(UTIL_BIN)/tests EQ_BIN :=$(BIN_DIR)/eqobj diff --git a/src/hsweep-all-batch.sh b/src/hsweep-all-batch.sh new file mode 100644 index 0000000..ec68e3a --- /dev/null +++ b/src/hsweep-all-batch.sh @@ -0,0 +1,101 @@ +#!/bin/bash + +#SBATCH -J walker # name of job + +#SBATCH -A niemeyek # name of my sponsored account, e.g. class or research group + +#SBATCH -p mime4 # name of partition or queue + +#SBATCH -F ./nrg-nodes + +#SBATCH -N 2 + +#SBATCH --ntasks-per-node=1 + +#SBATCH --cpus-per-task=20 + +#SBATCH --time=7-00:00:00 + +#SBATCH -o walker.out # name of output file for this submission script + +#SBATCH -e walker.err # name of error file for this submission script + +#SBATCH --mail-type=BEGIN,END,FAIL # send email when job begins, ends or aborts + +#SBATCH --mail-user=walkanth@oregonstate.edu # send email to this address + +# load any software environment module required for app + +# run my jobs + +#/scratch/a1/mpich3i/bin/mpirun -bootstrap slurm --host compute-e-1,compute-e-2 -n 40 ./bin/euler S ./oneD/tests/eulerTest.json ./rslts tpb 64 gpuA 20 nX 200000 lx 21 tf 1 + +echo $SLURM_JOB_ID + +hostname +ls rslts +rm rslts/* || true +ls rslts +tfile="./trslt/otime.dat" +opath="./rslts" +rm $tfile || true + +mkdir -p $opath +mkdir -p $(dirname $tfile) + +tfs=(0.06 1.2) +nxStart=100000 + +for eq in euler heat +do + if [ $eq == 'euler' ] + then + tf=0.06 + else + tf=1.2 + fi + for sc in S C + do + for t in $(seq 6 10) + do + for dvt in $(seq 0 1) + do + tpbz=$((2**$t)) + tpb=$(($tpbz + $dvt*$tpbz/2)) + for g in $(seq 0 5 100) + do + snx0=$SECONDS + for x in $(seq 0 7) + do + if [[ $x -gt 0 && $x -lt 7 ]] + then + midpt=1 + else + midpt=0 + fi + for dvx in $(seq 0 $midpt) + do + echo -------- START ------------ + nxo=$(($nxStart * 2**$x)) + nx=$(($nxo + $dvx*$nxo/2)) + lx=$(($nxo/10000 + 1)) + S0=$SECONDS + /scratch/a1/mpich3i/bin/mpirun -n 40 --hostfile ./nrg-nodes ./bin/$eq $sc ./oneD/tests/"$eq"Test.json $opath tpb $tpb gpuA $g nX $nx lx $lx tf $tf + echo len, eq, sch, tpb, gpuA, nX, tf + s1=$(($SECONDS-$S0)) + echo $lx, $eq, $sc, $tpb, $g, $nx $tf took $s1 + echo -------- END ------------ + sleep 0.05 + done + done + snx1=$(($SECONDS-$snx0)) + snxout=$(($snx1/60)) + echo "----------------------WRITING TIME-------------------------" + echo $snxout + echo $eq "|" $sc "|" $tpb "|" $g :: $snxout >> $tfile + echo "----------------------END WRITING TIME----------------------" + done + done + done + done +done diff --git a/src/launch/OSU_cluster/hSweep.sh b/src/launch/OSU_cluster/hSweep.sh index 17a2581..23370be 100755 --- a/src/launch/OSU_cluster/hSweep.sh +++ b/src/launch/OSU_cluster/hSweep.sh @@ -22,8 +22,8 @@ rm rslts/* || true ls rslts tfile="trslt/otime.dat" -opath="../../rslts" - +opath="$MSCRATCH/rslts" +testdir=$HSWEEPDIR/src/oneD/tests rm $tfile || true eqs=(euler heat) @@ -39,13 +39,13 @@ do tf=$tfs[$ix] for sc in S C do - for t in $(seq 6 9) + for t in $(seq 6 10) do for dvt in $(seq 0 1) do tpbz=$((2**$t)) tpb=$(($tpbz + 0.5*$dvt*$tpbz)) - for g in $(seq 0 2 20) + for g in $(seq 0 10 100) do snx0=$SECONDS for x in $(seq 0 7) @@ -64,7 +64,7 @@ do lx=$(($nxo/10000 + 1)) S0=$SECONDS - $MPIPATH/bin/mpirun -np $NSLOTS -machinefile $TMPDIR/machines $bindir/$eq $sc $testdir/"$eq"Test.json $opath tpb $tpb gpuA $g nX $nx lx $lx tf $tf + $MPIPATH/bin/mpirun -np 40 -machinefile $HSWEEPDIR/src/nrg-nodes $HSWEEPDIR/src/bin/$eq $sc $testdir/"$eq"Test.json $opath tpb $tpb gpuA $g nX $nx lx $lx tf $tf echo len, eq, sch, tpb, gpuA, nX s1=$(($SECONDS-$S0)) diff --git a/src/launch/slurm-batch-files/hsweep-batch.sh b/src/launch/slurm-batch-files/hsweep-batch.sh new file mode 100644 index 0000000..70cc225 --- /dev/null +++ b/src/launch/slurm-batch-files/hsweep-batch.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +#SBATCH -J shsweep # name of job + +#SBATCH -A niemeyek # name of my sponsored account, e.g. class or research group + +#SBATCH -p mime4 # name of partition or queue + +#SBATCH -F ./nrg-nodes + +#SBATCH -o shsweep.out # name of output file for this submission script + +#SBATCH -e shsweep.err # name of error file for this submission script + +#SBATCH --mail-type=BEGIN,END,FAIL # send email when job begins, ends or aborts + +#SBATCH --mail-user=walkanth@oregonstate.edu # send email to this address + +#SBATCH --time=7-00:00:00 +​ +#SBATCH --ntasks=40 + +# load any software environment module required for app +# run my jobs +/scratch/a1/mpich3i/bin/mpirun -bootstrap slurm --host compute-e-1,compute-e-2 -n 40 ./bin/euler S ./oneD/tests/eulerTest.json ./rslts tpb 64 gpuA 20 nX 200000 lx 21 tf 1 + diff --git a/src/launch/slurm-batch-files/test-hsweep-batch.sh b/src/launch/slurm-batch-files/test-hsweep-batch.sh new file mode 100644 index 0000000..c77632e --- /dev/null +++ b/src/launch/slurm-batch-files/test-hsweep-batch.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +#SBATCH -J testjob # name of job + +#SBATCH -A niemeyek # name of my sponsored account, e.g. class or research group + +#SBATCH -p mime4 # name of partition or queue + +#SBATCH -F ./test-nrg-nodes + +#SBATCH -o testjob.out # name of output file for this submission script + +#SBATCH -e testjob.err # name of error file for this submission script + +#SBATCH --mail-type=BEGIN,END,FAIL # send email when job begins, ends or aborts + +#SBATCH --mail-user=walkanth@oregonstate.edu # send email to this address + +#SBATCH --time=7-00:00:00 +​ +#SBATCH --ntasks=40 + +# load any software environment module required for app +# run my jobs +for i in 10000 20000 +do +/scratch/a1/mpich3i/bin/mpirun -bootstrap slurm --host compute-e-3,compute-e-4 -n 40 ./bin/euler S ./oneD/tests/eulerTest.json ./rslts tpb 64 gpuA 0 nX $i lx 21 tf 0.5 +done diff --git a/src/launch/slurm-batch-files/test-nrg-nodes b/src/launch/slurm-batch-files/test-nrg-nodes new file mode 100644 index 0000000..5dfebea --- /dev/null +++ b/src/launch/slurm-batch-files/test-nrg-nodes @@ -0,0 +1,2 @@ +compute-e-3 +compute-e-4 diff --git a/src/nrg-nodes b/src/nrg-nodes new file mode 100644 index 0000000..726e25d --- /dev/null +++ b/src/nrg-nodes @@ -0,0 +1,2 @@ +compute-e-1 +compute-e-2 diff --git a/src/oneD/Makefile b/src/oneD/Makefile index d505f5e..e7eedb6 100644 --- a/src/oneD/Makefile +++ b/src/oneD/Makefile @@ -3,7 +3,7 @@ # run make DEF="-DNOS" -MAKEPATH:=$(SOURCEPATH)oneD +MAKEPATH:=$(SOURCEPATH)/oneD # LPATH+=