modify api functions

Author: hasithadkr7

.gitignore

@@ -1,4 +1,4 @@
-venv/
 .idea/
 template/
 output/
+venv/

config.py

@@ -15,12 +15,8 @@
 
 BACK_DAYS = 2
 
-# MYSQL_USER = 'curw'
-# MYSQL_PASSWORD = 'curw'
-# MYSQL_HOST = '124.43.13.195'
-# MYSQL_PORT = '5036'
-# MYSQL_DB = 'curw'
 RAIN_FALL_FILE_NAME = '/home/hasitha/PycharmProjects/hechms_distributed/DailyRain-{}.csv'
+DISCHARGE_FILE_NAME = '/home/hasitha/PycharmProjects/hechms_distributed/DailyDischarge-{}.csv'
 
 GAGE_MANAGER_TEMPLATE = 'Gage Manager: {MODEL_NAME}\n     Version: 4.2.1\n     Filepath Separator: \ \nEnd:'
 
@@ -55,3 +51,21 @@
                     'End:'
 
 RUN_FILE_NAME = '/home/hasitha/PycharmProjects/hechms_distributed/{MODEL_NAME}.run'
+
+HEC_HMS_HOME = '/home/curw/distributed_hec/hec-hms-421'
+HEC_HMS_SH = 'hec-hms.sh'
+HEC_DSSVUE_HOME = '/home/curw/distributed_hec/hec-dssvue201'
+HEC_DSSVUE_SH = 'hec-dssvue.sh'
+HEC_HMS_MODEL_DIR = '/home/curw/distributed_hec/distributed_model'
+BASIN_STATES_DIR = 'basinStates'
+HEC_EVENT = 'hec_event'
+PRE_PROCESSING_SCRIPT = 'CSVTODSS.py'
+POST_PROCESSING_SCRIPT = 'DSSTOCSV.py'
+HEC_INPUT_DSS = '{MODEL_NAME}_input.dss'
+HEC_OUTPUT_DSS = '{MODEL_NAME}_run.dss'
+CONTROL_FILE_NAME = '{MODEL_NAME}.control'
+GAGE_FILE_NAME = '{MODEL_NAME}.gage'
+RUN_FILE_NAME = '{MODEL_NAME}.run'
+STATE_INDEX_NAME = HEC_EVENT + '.stateIndex'
+HEC_EVENT_SCRIPT = '{MODEL_NAME}.script'
+

controller.py

@@ -2,13 +2,15 @@
 from flask_json import FlaskJSON, JsonError, json_response
 from flask_uploads import UploadSet, configure_uploads
 from os import path
-from config import UPLOADS_DEFAULT_DEST, HECHMS_LIBS_DIR, DISTRIBUTED_MODEL_TEMPLATE_DIR, INIT_DATE_TIME_FORMAT, RAIN_FALL_FILE_NAME
-from input.shape_util.polygon_util import get_sub_ratios, get_timeseris, get_sub_catchment_rain_files, get_rain_files
+from datetime import datetime,timedelta
+from pathlib import Path
+
+from config import UPLOADS_DEFAULT_DEST, INIT_DATE_TIME_FORMAT, RAIN_FALL_FILE_NAME, HEC_HMS_MODEL_DIR
+from input.shape_util.polygon_util import get_rain_files
 from input.gage.model_gage import create_gage_file, create_gage_file_by_rain_file
 from input.control.model_control import create_control_file, create_control_file_by_rain_file
 from input.run.model_run import create_run_file
-from datetime import datetime,timedelta
-from pathlib import Path
+from model.model_execute import execute_pre_dssvue, execute_post_dssvue, execute_hechms
 
 
 app = Flask(__name__)
@@ -74,6 +76,70 @@ def init_run():
     return json_response(status_=200, run_id=run_id, description='Successfully saved files.')
 
 
+@app.route('/HECHMS/distributed/init', methods=['GET', 'POST'])
+@app.route('/HECHMS/distributed/init/<string:run_datetime>',  methods=['GET', 'POST'])
+@app.route('/HECHMS/distributed/init/<string:run_datetime>/<int:back_days>/<int:forward_days>',  methods=['GET', 'POST'])
+def prepare_input_files(run_datetime=datetime.now().strftime('%Y-%m-%d %H:%M:%S'), back_days=2, forward_days=3):
+    print('prepare_input_files.')
+    print('run_datetime : ', run_datetime)
+    print('back_days : ', back_days)
+    print('forward_days : ', forward_days)
+    run_datetime = datetime.strptime(run_datetime, '%Y-%m-%d %H:%M:%S')
+    to_date = run_datetime + timedelta(days=forward_days)
+    from_date = run_datetime - timedelta(days=back_days)
+    file_date = run_datetime.strftime('%Y-%m-%d')
+    from_date = from_date.strftime('%Y-%m-%d %H:%M:%S')
+    to_date = to_date.strftime('%Y-%m-%d %H:%M:%S')
+    file_name = RAIN_FALL_FILE_NAME.format(file_date)
+    print('file_name : ', file_name)
+    print('{from_date, to_date} : ', {from_date, to_date})
+    get_rain_files(file_name, run_datetime.strftime('%Y-%m-%d %H:%M:%S'), forward_days, back_days)
+    rain_fall_file = Path(file_name)
+    if rain_fall_file.is_file():
+        create_gage_file_by_rain_file('distributed_model', file_name)
+        create_control_file_by_rain_file('distributed_model', file_name)
+    else:
+        create_gage_file('distributed_model', from_date, to_date)
+        create_control_file('distributed_model', from_date, to_date)
+    create_run_file('distributed_model', run_datetime.strftime('%Y-%m-%d %H:%M:%S'))
+
+
+@app.route('/HECHMS/distributed/pre-process', methods=['GET', 'POST'])
+@app.route('/HECHMS/distributed/pre-process/<string:run_datetime>',  methods=['GET', 'POST'])
+def pre_processing(run_datetime=datetime.now().strftime('%Y-%m-%d %H:%M:%S')):
+    print('pre_processing.')
+    print('run_datetime : ', run_datetime)
+    run_datetime = datetime.strptime(run_datetime, '%Y-%m-%d %H:%M:%S')
+    ret_code = execute_pre_dssvue('distributed_model', run_datetime.strftime('%Y-%m-%d %H:%M:%S'))
+    if ret_code == 0:
+        return jsonify({'Result': 'Success'})
+    else:
+        return jsonify({'Result': 'Fail'})
+
+
+@app.route('/HECHMS/distributed/run', methods=['GET', 'POST'])
+def run_hec_hms_model():
+    print('run_hec_hms_model.')
+    ret_code = execute_hechms('distributed_model', HEC_HMS_MODEL_DIR)
+    if ret_code == 0:
+        return jsonify({'Result': 'Success'})
+    else:
+        return jsonify({'Result': 'Fail'})
+
+
+@app.route('/HECHMS/distributed/post-process', methods=['GET', 'POST'])
+@app.route('/HECHMS/distributed/post-process/<string:run_datetime>',  methods=['GET', 'POST'])
+def post_processing(run_datetime=datetime.now().strftime('%Y-%m-%d %H:%M:%S')):
+    print('post_processing.')
+    print('run_datetime : ', run_datetime)
+    run_datetime = datetime.strptime(run_datetime, '%Y-%m-%d %H:%M:%S')
+    ret_code = execute_post_dssvue('distributed_model', run_datetime.strftime('%Y-%m-%d %H:%M:%S'))
+    if ret_code == 0:
+        return jsonify({'Result': 'Success'})
+    else:
+        return jsonify({'Result': 'Fail'})
+
+
 @app.route('/HECHMS/distributed/rain-fall', methods=['GET', 'POST'])
 @app.route('/HECHMS/distributed/rain-fall/<string:run_datetime>',  methods=['GET', 'POST'])
 @app.route('/HECHMS/distributed/rain-fall/<string:run_datetime>/<int:back_days>/<int:forward_days>',  methods=['GET', 'POST'])
@@ -111,7 +177,7 @@ def get_gage_file(run_datetime=datetime.now().strftime('%Y-%m-%d %H:%M:%S'), bac
     file_date = run_datetime.strftime('%Y-%m-%d')
     from_date = from_date.strftime('%Y-%m-%d %H:%M:%S')
     to_date = to_date.strftime('%Y-%m-%d %H:%M:%S')
-    file_name = 'output/DailyRain-{}.csv'.format(file_date)
+    file_name = RAIN_FALL_FILE_NAME.format(file_date)
     rain_fall_file = Path(file_name)
     if rain_fall_file.is_file():
         create_gage_file_by_rain_file('distributed_model', file_name)

dags/__init__.py

@@ -0,0 +1,4 @@
+import airflow
+from airflow import DAG
+from airflow.operators.python_operator import PythonOperator
+

dags/hechms_distributed_dag.py

@@ -0,0 +1,39 @@
+import subprocess
+from os import path
+
+from config import HEC_HMS_HOME, HEC_HMS_SH, HEC_DSSVUE_HOME, HEC_DSSVUE_SH, HEC_EVENT_SCRIPT,\
+    PRE_PROCESSING_SCRIPT,POST_PROCESSING_SCRIPT, RAIN_FALL_FILE_NAME,DISCHARGE_FILE_NAME, \
+    HEC_INPUT_DSS, HEC_OUTPUT_DSS
+
+
+def execute_pre_dssvue(model_name, run_date_time):
+    python_script_fp = PRE_PROCESSING_SCRIPT
+    rain_csv_fp = RAIN_FALL_FILE_NAME.replace('{MODEL_NAME}', model_name).format(run_date_time)
+    input_dss_fp = HEC_INPUT_DSS
+    return _execute_hec_dssvue(python_script_fp, rain_csv_fp, input_dss_fp)
+
+
+def execute_post_dssvue(model_name, run_date_time):
+    python_script_fp = POST_PROCESSING_SCRIPT
+    discharge_csv_fp = DISCHARGE_FILE_NAME.replace('{MODEL_NAME}', model_name).format(run_date_time)
+    output_dss_fp = HEC_OUTPUT_DSS
+    return _execute_hec_dssvue(python_script_fp, discharge_csv_fp, output_dss_fp)
+
+
+def _execute_hec_dssvue(python_script_fp, csv_fp, dss_fp):
+    dssvue_sh = path.join(HEC_DSSVUE_HOME, HEC_DSSVUE_SH)
+    bash_command = '{dssvue_sh} {python_script} --csvfp {csv_fp} --dssfp {dss_fp}' \
+        .format(dssvue_sh=dssvue_sh, python_script=python_script_fp, csv_fp=csv_fp, dss_fp=dss_fp)
+    print('execute_hec_dssvue|bash_command : ', bash_command)
+    ret_code = subprocess.call(bash_command, shell=True)
+    return ret_code
+
+
+def execute_hechms(model_name, run_path):
+    hec_hms_sh_fp = path.join(HEC_HMS_HOME, HEC_HMS_SH)
+    model_event_script_fp = path.join(run_path, HEC_EVENT_SCRIPT.replace('{MODEL_NAME}', model_name))
+    bash_command = "{hec_hms_sh} -s {hec_event_script}" \
+        .format(hec_hms_sh=hec_hms_sh_fp, hec_event_script=model_event_script_fp)
+    print('execute_hechms|bash_command : ', bash_command)
+    ret_code = subprocess.call(bash_command, shell=True)
+    return ret_code