Merge pull request #199 from guydavis/develop

Improvements around Alerts.

Author: guydavis

CREDITS.md

@@ -18,6 +18,7 @@ A huge thank-you to the great teams/devs behind these projects, being used by Ma
 * [Other Icons](https://www.shareicon.net): Images and icons for web apps.
 * [Boostrap Sidebars](https://dev.to/codeply/bootstrap-5-sidebar-examples-38pb): Used for sidebar menu layout.
 * [Toplevel](https://github.com/and-semakin/marshmallow-toplevel): Used for list of objects sent to REST API.
+* [DataTables.js](https://datatables.net/): Filter/search/pagination of dynamic tables, as per [this tutorial](https://blog.miguelgrinberg.com/post/beautiful-interactive-tables-for-your-flask-templates).
 
 ## Blockchain Forks
 * [Flax](https://github.com/Flax-Network/flax-blockchain)

api/commands/chiadog_cli.py

@@ -15,7 +15,6 @@
 from flask import Flask, jsonify, abort, request, flash, g
 from subprocess import Popen, TimeoutExpired, PIPE
 
-from api.models import chiadog
 from api import app
 
 def load_config(blockchain):
@@ -46,10 +45,6 @@ def get_chiadog_pid(blockchain):
             return proc.info['pid']
     return None
 
-def get_notifications(since):
-    return chiadog.Notification.query.filter(chiadog.Notification.created_at >= since). \
-        order_by(chiadog.Notification.created_at.desc()).limit(20).all()
-
 def dispatch_action(job):
     service = job['service']
     if service != 'monitoring':

api/commands/log_parser.py

@@ -39,7 +39,7 @@ def recent_challenges(blockchain):
         app.logger.debug(
             "Skipping challenges parsing as no such log file: {0}".format(log_file))
         return []
-    proc = Popen("grep -i eligible {0} | tail -n {1}".format(log_file, CHALLENGES_TO_LOAD),
+    proc = Popen("grep --text -i eligible {0} | tail -n {1}".format(log_file, CHALLENGES_TO_LOAD),
                  stdout=PIPE, stderr=PIPE, shell=True)
     try:
         outs, errs = proc.communicate(timeout=90)

api/commands/plotman_cli.py

@@ -41,10 +41,9 @@ def load_plotting_summary():
     except TimeoutExpired:
         proc.kill()
         proc.communicate()
-        abort(500, description="The timeout is expired!")
+        raise Exception("The timeout expired during plotman status.")
     if errs:
-        app.logger.error(errs.decode('utf-8'))
-        abort(500, description=errs.decode('utf-8'))
+        raise Exception("Errors during plotman status:\n {0}".format(errs.decode('utf-8')))
     cli_stdout = outs.decode('utf-8')
     return plotman.PlottingSummary(cli_stdout.splitlines(), get_plotman_pid())
 
@@ -76,14 +75,16 @@ def action_plots(job):
     #app.logger.info("About to {0} plots: {1}".format(action, plot_ids))
     for plot_id in plot_ids:
         try:
-            suffix = ""
+            param = ""
             if action == "kill":
-                suffix = "--force"
+                param = "--force"
             logfile = "/root/.chia/plotman/logs/plotman.log"
             log_fd = os.open(logfile, os.O_RDWR | os.O_CREAT)
             log_fo = os.fdopen(log_fd, "a+")
-            proc = Popen("{0} {1} {2} {3}".format(PLOTMAN_SCRIPT, action, suffix, plot_id),
+            proc = Popen("{0} {1} {2} {3}".format(PLOTMAN_SCRIPT, action, param, plot_id),
                          shell=True, universal_newlines=True, stdout=log_fo, stderr=log_fo)
+            # Plotman regressed on cleaning temp after kill so do it here:
+            clean_tmp_dirs_after_kill(plot_id)
         except:
             app.logger.info('Failed to {0} selected plot {1}.'.format(action, plot_id))
             app.logger.info(traceback.format_exc())
@@ -98,16 +99,12 @@ def get_plotman_pid():
 def start_plotman():
     app.logger.info("Starting Plotman run...")
     check_config()
-    try:
-        if len(load_plotting_summary().rows) == 0:  # No plots running
-            clean_tmp_dirs_before_run()  
-        logfile = "/root/.chia/plotman/logs/plotman.log"
-        proc = Popen("nohup {0} {1} < /dev/tty >> {2} 2>&1 &".format(PLOTMAN_SCRIPT, 'plot', logfile),
-                     shell=True, stdin=DEVNULL, stdout=None, stderr=None, close_fds=True)
-        app.logger.info("Completed launch of plotman.")
-    except:
-        app.logger.info('Failed to start Plotman plotting run!')
-        app.logger.info(traceback.format_exc())
+    if len(load_plotting_summary().rows) == 0:  # No plots running
+        clean_tmp_dirs_before_run()  
+    logfile = "/root/.chia/plotman/logs/plotman.log"
+    proc = Popen("nohup {0} {1} < /dev/tty >> {2} 2>&1 &".format(PLOTMAN_SCRIPT, 'plot', logfile),
+                    shell=True, stdin=DEVNULL, stdout=None, stderr=None, close_fds=True)
+    app.logger.info("Completed launch of plotman.")
 
 def clean_tmp_dirs_before_run():
     try:
@@ -126,15 +123,28 @@ def clean_tmp_dirs_before_run():
                         p.unlink()
     except Exception as ex:
         app.logger.info("Skipping deletion of temp files due to {0}.".format(traceback.format_exc()))
-
+    
+def clean_tmp_dirs_after_kill(plot_id):
+    try:
+        with open("/root/.chia/plotman/plotman.yaml") as f:
+            config = yaml.safe_load(f)
+            if 'directories' in config:
+                if 'tmp' in config['directories']:
+                    for tmp_dir in config['directories']['tmp']:
+                        for p in pathlib.Path(tmp_dir).glob("*{0}*.tmp".format(plot_id)):
+                            app.logger.info("After kill, deleting stale tmp file: {0}".format(p))
+                            p.unlink()
+                if 'tmp2' in config['directories']:
+                    tmp_dir = config['directories']['tmp2']
+                    for p in pathlib.Path(tmp_dir).glob("*{0}*.tmp".format(plot_id)):
+                        app.logger.info("After kill, deleting stale tmp file: {0}".format(p))
+                        p.unlink()
+    except Exception as ex:
+        app.logger.info("Skipping deletion of temp files due to {0}.".format(traceback.format_exc()))
 
 def stop_plotman():
     app.logger.info("Stopping Plotman run...")
-    try:
-        os.kill(get_plotman_pid(), signal.SIGTERM)
-    except:
-        app.logger.info('Failed to stop Plotman plotting run!')
-        app.logger.info(traceback.format_exc())
+    os.kill(get_plotman_pid(), signal.SIGTERM)
 
 def get_archiver_pid():
     for proc in psutil.process_iter(['pid', 'name', 'cmdline']):
@@ -145,49 +155,39 @@ def get_archiver_pid():
 def start_archiver():
     app.logger.info("Starting archiver run...")
     check_config()
-    try:
-        logfile = "/root/.chia/plotman/logs/archiver.log"
-        app.logger.info("About to start archiver...")
-        proc = Popen("nohup {0} {1} < /dev/tty >> {2} 2>&1 &".format(PLOTMAN_SCRIPT, 'archive', logfile),
-                     shell=True, stdin=DEVNULL, stdout=None, stderr=None, close_fds=True)
-        app.logger.info("Completed launch of archiver.")
-    except:
-        app.logger.info('Failed to start Plotman archiving run!')
-        app.logger.info(traceback.format_exc())
+    logfile = "/root/.chia/plotman/logs/archiver.log"
+    app.logger.info("About to start archiver...")
+    proc = Popen("nohup {0} {1} < /dev/tty >> {2} 2>&1 &".format(PLOTMAN_SCRIPT, 'archive', logfile),
+                    shell=True, stdin=DEVNULL, stdout=None, stderr=None, close_fds=True)
+    app.logger.info("Completed launch of archiver.")
 
 def stop_archiver():
     app.logger.info("Stopping Archiver run...")
-    try:
-        os.kill(get_archiver_pid(), signal.SIGTERM)
-    except:
-        app.logger.info('Failed to stop Plotman archiving run!')
-        app.logger.info(traceback.format_exc())
+    os.kill(get_archiver_pid(), signal.SIGTERM)
 
 def load_config():
     return open('/root/.chia/plotman/plotman.yaml','r').read()
 
 def save_config(config):
-    try:
-        # Validate the YAML first
-        yaml.safe_load(config)
-        # Save a copy of the old config file
-        src = "/root/.chia/plotman/plotman.yaml"
-        dst = "/root/.chia/plotman/plotman." + \
-            time.strftime("%Y%m%d-%H%M%S")+".yaml"
-        shutil.copy(src, dst)
-        # Now save the new contents to main config file
-        with open(src, 'w') as writer:
-            writer.write(config)
-    except Exception as ex:
-        app.logger.info(traceback.format_exc())
-        raise Exception('Updated plotman.yaml failed validation!\n' + str(ex))
-    else: # Restart services if running
-        if get_plotman_pid():
-            stop_plotman()
-            start_plotman()
-        if get_archiver_pid():
-            stop_archiver()
-            start_archiver()
+    # Validate the YAML first
+    yaml.safe_load(config)
+    # Save a copy of the old config file
+    src = "/root/.chia/plotman/plotman.yaml"
+    dst = "/root/.chia/plotman/plotman." + \
+        time.strftime("%Y%m%d-%H%M%S")+".yaml"
+    shutil.copy(src, dst)
+    # Now save the new contents to main config file
+    with open(src, 'w') as writer:
+        writer.write(config)
+    # Now try to validate config by calling plotman status
+    load_plotting_summary()
+    # Finally restart plotman and archiver if they are running
+    if get_plotman_pid():
+        stop_plotman()
+        start_plotman()
+    if get_archiver_pid():
+        stop_archiver()
+        start_archiver()
 
 def find_plotting_job_log(plot_id):
     dir_path = '/root/.chia/plotman/logs'

api/default_settings.py

@@ -13,8 +13,7 @@ class DefaultConfig:
     SQLALCHEMY_TRACK_MODIFICATIONS = False
     SQLALCHEMY_DATABASE_URI = 'sqlite:////root/.chia/machinaris/dbs/machinaris.db'
     SQLALCHEMY_BINDS = {
-        'stats':      'sqlite:////root/.chia/machinaris/dbs/stats.db',
-        'chiadog':    'sqlite:////root/.chia/chiadog/dbs/chiadog.db',
+        'stats':      'sqlite:////root/.chia/machinaris/dbs/stats.db'
     }
     SQLALCHEMY_ECHO = True if 'FLASK_ENV' in os.environ and os.environ['FLASK_ENV'] == "development" else False
     ETAG_DISABLED = True # https://flask-smorest.readthedocs.io/en/latest/etag.html

api/models/chiadog.py

@@ -14,10 +14,10 @@
 
 from flask import g
 
+from common.models import alerts as a
 from common.config import globals
 from api.commands import chiadog_cli
-from api import app
-from api import utils
+from api import app, utils
 
 first_run = True
 
@@ -26,15 +26,19 @@ def update():
     if not globals.farming_enabled() and not globals.harvesting_enabled():
         #app.logger.info("Skipping alerts status collection on plotting-only instance.")
         return
+    if globals.load()['is_controller']:
+        #app.logger.info("Skipping alerts polling on fullnode are already placed in database directly via chiadog_notifier.sh script.")
+        return
     with app.app_context():
         try:
+            from api import db
             hostname = utils.get_hostname()
             if first_run:  # On first launch, load last week of notifications
                 since = (datetime.datetime.now() - datetime.timedelta(weeks=1)).strftime("%Y-%m-%d %H:%M:%S.000")
                 first_run = False
             else: # On subsequent schedules, load only last 5 minutes.
                 since = (datetime.datetime.now() - datetime.timedelta(minutes=5)).strftime("%Y-%m-%d %H:%M:%S.000")
-            alerts = chiadog_cli.get_notifications(since)
+            alerts = db.session.query(a.Alert).filter(a.Alert.created_at >= since).order_by(a.Alert.created_at.desc()).limit(20).all()
             payload = []
             for alert in alerts:
                 payload.append({

api/schedules/status_alerts.py

@@ -51,4 +51,4 @@ def post(self):
             return make_response("Action completed.", 200)
         except Exception as ex:
             app.logger.info(traceback.format_exc())
-            abort("Failed during {0} action.".format(service), 500)
+            return str(ex), 400

api/views/actions/resources.py

@@ -59,8 +59,9 @@ def put(self, type):
             response = make_response("Successfully saved config.", 200)
             return response
         except Exception as ex:
-            app.logger.error(traceback.format_exc())
-            abort(400, str(ex))
+            app.logger.info("Failed to save a validated Plotman config.")
+            app.logger.info(traceback.format_exc())
+            return str(ex), 400
 
     def clean_config(self, req_data):
         # First decode the bytes

api/views/configs/resources.py

@@ -6,8 +6,12 @@
 event_priority_name="$1"
 event_service_name="$2"
 event_message="$3"
+hostname="$(hostname -s)"
+now="$(date +'%Y-%m-%d %H:%M:%S')"
+unique_id="${hostname}_chia_${now}"
+echo "Creating alert ${hostname}_chia_${now} ${event_service_name} ${event_priority_name}: ${event_message}"
 
-cd /root/.chia/chiadog/dbs
-sqlite3 chiadog.db <<EOF
-INSERT INTO notification (blockchain,priority,service,message,created_at) VALUES ('chia','${event_priority_name//\'/\'\'}','${event_service_name//\'/\'\'}','${event_message//\'/\'\'}', strftime('%Y-%m-%d %H:%M:%S','now'));
+cd /root/.chia/machinaris/dbs
+sqlite3 -cmd '.timeout 5000' machinaris.db <<EOF
+INSERT INTO alerts (unique_id,hostname,blockchain,priority,service,message,created_at) VALUES ('${unique_id}', '${hostname}', 'chia','${event_priority_name//\'/\'\'}','${event_service_name//\'/\'\'}','${event_message//\'/\'\'}', '${now}');
 EOF