pdsfDiskUsageMonitor/run.sh at master · jthaeder/pdsfDiskUsageMonitor · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
#!/bin/bash
#
# Author: Jochen Thaeder (LBNL) <jochen@thaeder.de>
# Run script to produce webpage of data usage
#
###################################################

BASEPATH=${HOME}/pdsfDiskUsageMonitor

reCreateTree=0

# -- Setup scratch space for reduced GPFS dump output
if [ -z "$SCRATCH" ] ; then
    SCRATCH="/scratch/`whoami`"
fi

SCRATCHPATH=${SCRATCH}/pdsfDiskUsageMonitor

if [ ! -d $SCRATCHPATH ] ; then
    mkdir -p $SCRATCHPATH
fi

# -- Get input data
pushd ${BASEPATH} > /dev/null
inputPath=/project/statistics/LIST

# -- get latest PROJECT/PROJECTA folder
for prjFolder in project projecta  ; do

    if [ "${prjFolder}" == "project" ] ; then
	prjPath=tlproject2
    elif [ "${prjFolder}" == "projecta" ] ; then
	prjPath=tlprojecta
    fi

    projectFolders=`ls -t ${inputPath}/${prjPath}/ | head -n 10 | sort -r 2> /dev/null`

    for folder in $projectFolders ; do

	# -- Get file in latest folder
	inFile=`find ${inputPath}/${prjPath}/${folder}/ -name "*list.allfiles" | sort | head -n 1 2> /dev/null`
	if [[ "${inFile}" == "" || ! -f ${inFile} ]] ; then
	    continue
	fi

	# -- Check the latest file is completed
	if [ ! -f ${inFile}.completed ] ; then
	    continue
	fi

	# -- Get old modification dates and check if tree recreation has to be run
	if [ -f modDate_${prjFolder}.txt ] ; then
	    oldmodDate=`cat modDate_${prjFolder}.txt`
	    if [ "$oldmodDate" == "$folder" ] ; then
		break
	    fi
	fi

	reCreateTree=1

	# -- Create input files for parsing
	# ----------------------------------------------

	# -- Get folders for local input files
	prjFolderLocal=${SCRATCHPATH}/${prjFolder}
	if [ -d ${prjFolderLocal} ] ; then
	    rm -rf ${prjFolderLocal}
	fi

	mkdir -p ${prjFolderLocal}
	ln -sf ${prjFolderLocal} ${prjFolder}

	# -- Get input files for parsing

	if [ "${prjFolder}" == "project" ] ; then
	    cat ${inFile} | grep "%2Fprojectdirs%2Fstarprod%2F" > ${prjFolder}/prj-starprod.list
	    sed -i "s/%2F${prjFolder}%2F.snapshots%2F${folder}%2Fprojectdirs%2F//" ${prjFolder}/prj-starprod.list

	    cat ${inFile} | grep "%2Fprojectdirs%2Fstar%2F" > ${prjFolder}/prj-star.list
	    sed -i "s/%2F${prjFolder}%2F.snapshots%2F${folder}%2Fprojectdirs%2F//" ${prjFolder}/prj-star.list

	    cat ${inFile} | grep "%2Fprojectdirs%2Falice%2F" > ${prjFolder}/prj-alice.list
	    sed -i "s/%2F${prjFolder}%2F.snapshots%2F${folder}%2Fprojectdirs%2F//" ${prjFolder}/prj-alice.list
	else
	    cat ${inFile} | grep "%2Fprojectdirs%2Fstarprod%2F" > ${prjFolder}/prj-starprod.list
	    sed -i "s/%2Fglobal%2F${prjFolder}%2F.snapshots%2F${folder}%2Fprojectdirs%2F//" ${prjFolder}/prj-starprod.list
	fi

	echo $folder > modDate_${prjFolder}.txt

	break
    done
done

# -- recreate input Trees
# -------------------------------------------------------
if [ $reCreateTree -eq 1 ] ; then

    if [ -d output ] ; then
	rm -rf output
    fi

    mkdir -p output

    # -- run script
    ${BASEPATH}/parseGPFSDump.tcsh ${BASEPATH} 2
fi

# -- create html
${BASEPATH}/runDiskUsage.sh ${BASEPATH}

popd > /dev/null