Skip to content

Commit a3d6d55

Browse files
michaeltrybymichaeltryby
authored
Merge pull request #7 from OpenWaterAnalytics/dev
Initial release for ci-tools
2 parents 36f8777 + 6a636ba commit a3d6d55

21 files changed

+1395
-2
lines changed

‎AUTHORS

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# Authors ordered by first contribution
2+
3+
Michael Tryby <[email protected]> (public domain)
4+
Caleb Buahin <[email protected]>

‎LICENSE

+19-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,22 @@
1-
Creative Commons Legal Code
1+
2+
3+
4+
Copyright Notice
5+
6+
This project contains material prepared by the United States Government for
7+
which domestic copyright protection is not available under 17 USC § 105.
8+
9+
10+
No Copyright Restrictions 2020 U.S. Federal Government
11+
12+
13+
14+
License Notice
15+
16+
This project is distributed under the Creative Commons Zero (CC0) Universal
17+
version 1.0 (see below).
18+
19+
220

321
CC0 1.0 Universal
422

‎README.md

+51-1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,52 @@
1+
2+
<!---
3+
README.md
4+
5+
Created: May 3, 2020
6+
Updated:
7+
8+
Author: See AUTHORS
9+
--->
10+
111
# ci-tools
2-
Tools for continuous integration and local testing
12+
Tools for continuous integration and local testing for SWMM and EPANET
13+
14+
15+
### Dependencies
16+
17+
Before the project can be built and tested the required dependencies must be installed.
18+
19+
**Summary of Build Dependencies: Windows**
20+
21+
- Build
22+
- Build Tools for Visual Studio 2017
23+
- CMake 3.17
24+
25+
- Regression Test
26+
- Python 3.7 64 bit
27+
- curl
28+
- git
29+
- 7z
30+
31+
Once Python is present, the following command installs the required packages for regression testing.
32+
```
33+
\> cd < PROJECT_ROOT >
34+
\>pip install -r tools\requirements-< PROJECT >.txt
35+
```
36+
37+
38+
### Build
39+
40+
EPANET can be built with one simple command.
41+
```
42+
\>tools\make.cmd
43+
```
44+
45+
46+
### Regression Test
47+
48+
This command runs regression tests for the local build and compares them to the latest benchmark.
49+
```
50+
\>tools\before-nrtest.cmd
51+
\>tools\run-nrtest.cmd
52+
```

‎darwin/app-config.zsh

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#!/usr/bin/env zsh
2+
3+
#
4+
# app-config.sh - Generates nrtest app configuration file for test executable
5+
#
6+
# Date Created: 11/15/2017
7+
# Updated: 08/21/2020
8+
#
9+
# Author: See AUTHORS
10+
#
11+
# Requires:
12+
# git
13+
#
14+
# Environment Variables:
15+
# PROJECT
16+
#
17+
# Arguments:
18+
# 1 - absolute path to test executable
19+
# 2 - Platform
20+
# 3 - (SUT build id)
21+
#
22+
23+
# Check requirements
24+
where git &> /dev/null
25+
[[ ! $? ]] && { echo "ERROR: git not installed"; return 1 }
26+
27+
# check that env variables are set
28+
[[ ! -v PROJECT ]] && { echo "ERROR: PROJECT must be defined"; return 1 }
29+
30+
31+
# check if project is swmm otherwise EPANET
32+
TEST_CMD="run${PROJECT}"
33+
34+
# path to executable in cmake build tree
35+
ABS_BUILD_PATH=$1
36+
37+
# process optional arguments
38+
if [ ! -z "$2" ]; then
39+
PLATFORM=$2
40+
else
41+
PLATFORM="unknown"
42+
fi
43+
44+
if [ ! -z "$3" ]; then
45+
BUILD_ID=$3
46+
else
47+
BUILD_ID="unknown"
48+
fi
49+
50+
51+
# determine version
52+
VERSION=$( git rev-parse --short HEAD )
53+
[[ ! -v VERSION ]] && { echo "ERROR: VERSION must be determined"; return 1 }
54+
55+
cat<<EOF
56+
{
57+
"name" : "${PROJECT}",
58+
"version" : "${VERSION}",
59+
"description" : "${PLATFORM} ${BUILD_ID}",
60+
"setup_script" : "",
61+
"exe" : "${ABS_BUILD_PATH}/${TEST_CMD}"
62+
}
63+
EOF

‎darwin/before-nrtest.zsh

+120
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
#!/usr/bin/env zsh
2+
#
3+
# before-nrtest.sh - Runs before numerical regression test
4+
#
5+
# Date Created: 11/15/2017
6+
# Updated: 08/21/2020
7+
#
8+
# Author: See AUTHORS
9+
#
10+
# Dependencies:
11+
# curl
12+
# tar
13+
#
14+
# Environment Variables:
15+
# PROJECT
16+
# BUILD_HOME - relative path
17+
# PLATFORM
18+
# NRTESTS_URL
19+
#
20+
# Arguments:
21+
# 1 - (RELEASE_TAG) - Release tag
22+
#
23+
# Note:
24+
# Tests and benchmark files are stored in the swmm-example-networks repo.
25+
# This script retreives them using a stable URL associated with a release on
26+
# GitHub and stages the files for nrtest to run. The script assumes that
27+
# before-test.sh and app-config.sh are located together in the same folder.
28+
#
29+
30+
export TEST_HOME="nrtests"
31+
32+
# check that env variables are set
33+
REQUIRED_VARS=(PROJECT BUILD_HOME PLATFORM)
34+
for i in ${REQUIRED_VARS}; do
35+
[[ ! -v ${i} ]] && { echo "ERROR: $i must be defined"; return 1 }
36+
done
37+
38+
# determine project directory
39+
CUR_DIR=${PWD}
40+
SCRIPT_HOME=${0:a:h}
41+
cd ${SCRIPT_HOME}
42+
cd ./../../
43+
PROJECT_DIR=${PWD}
44+
45+
# set URL to github repo with nrtest files
46+
if [[ -z "${NRTESTS_URL}" ]]
47+
then
48+
NRTESTS_URL="https://github.com/OpenWaterAnalytics/${PROJECT}-nrtestsuite"
49+
fi
50+
51+
echo INFO: Staging files for regression testing
52+
53+
# use release tag arg else determine latest hard coded for now.
54+
if [[ ! -z "$1" ]]
55+
then
56+
RELEASE_TAG=$1
57+
else
58+
LATEST_URL="${NRTESTS_URL}/releases/latest"
59+
RELEASE_TAG=$( basename $( curl -Ls -o /dev/null -w %{url_effective} ${LATEST_URL} ) )
60+
echo INFO: Latest nrtestsuite release: ${RELEASE_TAG}
61+
fi
62+
63+
64+
# build URLs for test and benchmark files
65+
if [[ ! -v RELEASE_TAG ]]
66+
then
67+
echo "ERROR: tag RELEASE_TAG is invalid" ; return 1
68+
else
69+
TESTFILES_URL="${NRTESTS_URL}/archive/${RELEASE_TAG}.tar.gz"
70+
BENCHFILES_URL="${NRTESTS_URL}/releases/download/${RELEASE_TAG}/benchmark-${PLATFORM}.tar.gz"
71+
fi
72+
73+
echo INFO: Staging files for regression testing
74+
75+
# create a clean directory for staging regression tests
76+
if [[ -d ${TEST_HOME} ]]; then
77+
rm -rf ${TEST_HOME}
78+
fi
79+
80+
mkdir ${TEST_HOME}
81+
cd ${TEST_HOME}
82+
83+
# retrieve tests and benchmarks for regression testing
84+
curl -fsSL -o nrtestfiles.tar.gz ${TESTFILES_URL}
85+
# retrieve swmm benchmark results
86+
curl -fsSL -o benchmarks.tar.gz ${BENCHFILES_URL}
87+
88+
# extract tests and setup symlink
89+
tar xzf nrtestfiles.tar.gz
90+
ln -s ${PROJECT}-nrtestsuite-${RELEASE_TAG:1}/public tests
91+
92+
93+
# create benchmark dir and extract benchmarks
94+
mkdir benchmark
95+
tar xzf benchmarks.tar.gz -C benchmark
96+
97+
98+
#determine ref_build_id
99+
MANIFEST_FILE=$( find . -name manifest.json )
100+
101+
while read line; do
102+
if [[ $line == *"${PLATFORM} "* ]]; then
103+
REF_BUILD_ID=${line#*"${PLATFORM} "}
104+
REF_BUILD_ID=${REF_BUILD_ID//"\","/""}
105+
fi
106+
done < $MANIFEST_FILE
107+
108+
109+
if [[ -z "${REF_BUILD_ID}" ]]
110+
then
111+
echo "ERROR: REF_BUILD_ID could not be determined" ; exit 1
112+
fi
113+
114+
export REF_BUILD_ID=$REF_BUILD_ID
115+
116+
# GitHub Actions
117+
echo "REF_BUILD_ID=$REF_BUILD_ID" >> $GITHUB_ENV
118+
119+
# return user to current dir
120+
cd ${CUR_DIR}

‎darwin/make.zsh

+104
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
#!/usr/bin/env zsh
2+
#
3+
#
4+
# make.sh - Builds swmm/epanet executable
5+
#
6+
# Date Created: 06/29/2020
7+
# Updated: 08/21/2020
8+
#
9+
# Authors: See AUTHORS
10+
#
11+
# Environment Variables:
12+
# PROJECT
13+
#
14+
# Optional Arguments:
15+
# -g ("GENERATOR") defaults to "Ninja"
16+
# -t builds and runs unit tests (requires Boost)
17+
18+
19+
setopt extendedglob
20+
21+
export BUILD_HOME="build"
22+
23+
# determine project directory
24+
CUR_DIR=${PWD}
25+
SCRIPT_HOME=${0:a:h}
26+
cd ${SCRIPT_HOME}
27+
cd ./../../
28+
PROJECT_DIR=${PWD}
29+
30+
31+
# determine project
32+
if [[ ! -v PROJECT ]]
33+
then
34+
[[ $( basename $PROJECT_DIR ) = ((#i)'STO'*|(#i)'SWM'*) ]] && { export PROJECT="swmm" }
35+
[[ $( basename $PROJECT_DIR ) = ((#i)'WAT'*|(#i)'EPA'*) ]] && { export PROJECT="epanet" }
36+
fi
37+
# check that PROJECT is defined
38+
[[ ! -v PROJECT ]] && { echo "ERROR: PROJECT must be defined"; return 1 }
39+
40+
41+
# prepare for artifact upload
42+
if [ ! -d upload ]; then
43+
mkdir upload
44+
fi
45+
46+
echo INFO: Building ${PROJECT} ...
47+
48+
GENERATOR="Xcode"
49+
TESTING=0
50+
51+
52+
POSITIONAL=()
53+
54+
while [[ $# -gt 0 ]]
55+
do
56+
key="$1"
57+
case $key in
58+
-g|--gen)
59+
GENERATOR="$2"
60+
shift # past argument
61+
shift # past value
62+
;;
63+
-t|--test)
64+
TESTING=1
65+
shift # past argument
66+
;;
67+
*) # unknown option
68+
shift # past argument
69+
;;
70+
esac
71+
done
72+
73+
set -- "${POSITIONAL[@]}" # restore positional parameters
74+
75+
# perform the build
76+
cmake -E make_directory ${BUILD_HOME}
77+
78+
RESULT=$?
79+
80+
if [ ${TESTING} -eq 1 ];
81+
then
82+
echo "Building debug"
83+
cmake -E chdir ./${BUILD_HOME} cmake -G "${GENERATOR}" -DBUILD_TESTS=ON .. \
84+
&& cmake --build ./${BUILD_HOME} --config Debug \
85+
&& cmake -E chdir ./${BUILD_HOME} ctest -C Debug --output-on-failure
86+
RESULT=$?
87+
else
88+
echo "Building release"
89+
cmake -E chdir ./${BUILD_HOME} cmake -G "${GENERATOR}" -DBUILD_TESTS=OFF .. \
90+
&& cmake --build ./${BUILD_HOME} --config Release --target package
91+
RESULT=$?
92+
cp ./${BUILD_HOME}/*.tar.gz ./upload >&1
93+
fi
94+
95+
export PLATFORM="darwin"
96+
97+
#GitHub Actions
98+
echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV
99+
100+
# return user to current dir
101+
cd ${CUR_DIR}
102+
103+
104+
return $RESULT

‎darwin/requirements-epanet.txt

Whitespace-only changes.

‎darwin/requirements-swmm.txt

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#
2+
# requirements-win.txt - Python requirements for running nrtest on Win32/Win64
3+
#
4+
# Date Created: 10/17/2019
5+
# Date Updated: 11/26/2019
6+
#
7+
# Author: See AUTHORS
8+
#
9+
# Useful for configuring a python environment to run nrtests on swmm.
10+
#
11+
# usage:
12+
# pip install -r tools/requirements-win.txt
13+
#
14+
15+
nrtest
16+
17+
-f https://github.com/SWMM-Project/swmm-python/releases/download/v0.6.0-rc.1/swmm_toolkit-0.5.0-cp37-cp37m-macosx_10_10_x86_64.whl
18+
swmm-toolkit
19+
20+
-f https://github.com/SWMM-Project/swmm-python/releases/download/v0.6.0-rc.1/nrtest_swmm-0.6.0-py3-none-any.whl
21+
nrtest-swmm

0 commit comments

Comments
 (0)