diff --git a/CREDITS.md b/CREDITS.md index e4848972..80f911e7 100644 --- a/CREDITS.md +++ b/CREDITS.md @@ -18,6 +18,7 @@ A huge thank-you to the great teams/devs behind these projects, being used by Ma * [Other Icons](https://www.shareicon.net): Images and icons for web apps. * [Boostrap Sidebars](https://dev.to/codeply/bootstrap-5-sidebar-examples-38pb): Used for sidebar menu layout. * [Toplevel](https://github.com/and-semakin/marshmallow-toplevel): Used for list of objects sent to REST API. +* [DataTables.js](https://datatables.net/): Filter/search/pagination of dynamic tables, as per [this tutorial](https://blog.miguelgrinberg.com/post/beautiful-interactive-tables-for-your-flask-templates). ## Blockchain Forks * [Flax](https://github.com/Flax-Network/flax-blockchain) diff --git a/api/commands/chiadog_cli.py b/api/commands/chiadog_cli.py index f390776b..02fe895c 100644 --- a/api/commands/chiadog_cli.py +++ b/api/commands/chiadog_cli.py @@ -15,7 +15,6 @@ from flask import Flask, jsonify, abort, request, flash, g from subprocess import Popen, TimeoutExpired, PIPE -from api.models import chiadog from api import app def load_config(blockchain): @@ -46,10 +45,6 @@ def get_chiadog_pid(blockchain): return proc.info['pid'] return None -def get_notifications(since): - return chiadog.Notification.query.filter(chiadog.Notification.created_at >= since). \ - order_by(chiadog.Notification.created_at.desc()).limit(20).all() - def dispatch_action(job): service = job['service'] if service != 'monitoring': diff --git a/api/commands/log_parser.py b/api/commands/log_parser.py index a7398158..22a96365 100644 --- a/api/commands/log_parser.py +++ b/api/commands/log_parser.py @@ -39,7 +39,7 @@ def recent_challenges(blockchain): app.logger.debug( "Skipping challenges parsing as no such log file: {0}".format(log_file)) return [] - proc = Popen("grep -i eligible {0} | tail -n {1}".format(log_file, CHALLENGES_TO_LOAD), + proc = Popen("grep --text -i eligible {0} | tail -n {1}".format(log_file, CHALLENGES_TO_LOAD), stdout=PIPE, stderr=PIPE, shell=True) try: outs, errs = proc.communicate(timeout=90) diff --git a/api/commands/plotman_cli.py b/api/commands/plotman_cli.py index 7eb5da6c..5c1ddb8e 100644 --- a/api/commands/plotman_cli.py +++ b/api/commands/plotman_cli.py @@ -41,10 +41,9 @@ def load_plotting_summary(): except TimeoutExpired: proc.kill() proc.communicate() - abort(500, description="The timeout is expired!") + raise Exception("The timeout expired during plotman status.") if errs: - app.logger.error(errs.decode('utf-8')) - abort(500, description=errs.decode('utf-8')) + raise Exception("Errors during plotman status:\n {0}".format(errs.decode('utf-8'))) cli_stdout = outs.decode('utf-8') return plotman.PlottingSummary(cli_stdout.splitlines(), get_plotman_pid()) @@ -76,14 +75,16 @@ def action_plots(job): #app.logger.info("About to {0} plots: {1}".format(action, plot_ids)) for plot_id in plot_ids: try: - suffix = "" + param = "" if action == "kill": - suffix = "--force" + param = "--force" logfile = "/root/.chia/plotman/logs/plotman.log" log_fd = os.open(logfile, os.O_RDWR | os.O_CREAT) log_fo = os.fdopen(log_fd, "a+") - proc = Popen("{0} {1} {2} {3}".format(PLOTMAN_SCRIPT, action, suffix, plot_id), + proc = Popen("{0} {1} {2} {3}".format(PLOTMAN_SCRIPT, action, param, plot_id), shell=True, universal_newlines=True, stdout=log_fo, stderr=log_fo) + # Plotman regressed on cleaning temp after kill so do it here: + clean_tmp_dirs_after_kill(plot_id) except: app.logger.info('Failed to {0} selected plot {1}.'.format(action, plot_id)) app.logger.info(traceback.format_exc()) @@ -98,16 +99,12 @@ def get_plotman_pid(): def start_plotman(): app.logger.info("Starting Plotman run...") check_config() - try: - if len(load_plotting_summary().rows) == 0: # No plots running - clean_tmp_dirs_before_run() - logfile = "/root/.chia/plotman/logs/plotman.log" - proc = Popen("nohup {0} {1} < /dev/tty >> {2} 2>&1 &".format(PLOTMAN_SCRIPT, 'plot', logfile), - shell=True, stdin=DEVNULL, stdout=None, stderr=None, close_fds=True) - app.logger.info("Completed launch of plotman.") - except: - app.logger.info('Failed to start Plotman plotting run!') - app.logger.info(traceback.format_exc()) + if len(load_plotting_summary().rows) == 0: # No plots running + clean_tmp_dirs_before_run() + logfile = "/root/.chia/plotman/logs/plotman.log" + proc = Popen("nohup {0} {1} < /dev/tty >> {2} 2>&1 &".format(PLOTMAN_SCRIPT, 'plot', logfile), + shell=True, stdin=DEVNULL, stdout=None, stderr=None, close_fds=True) + app.logger.info("Completed launch of plotman.") def clean_tmp_dirs_before_run(): try: @@ -126,15 +123,28 @@ def clean_tmp_dirs_before_run(): p.unlink() except Exception as ex: app.logger.info("Skipping deletion of temp files due to {0}.".format(traceback.format_exc())) - + +def clean_tmp_dirs_after_kill(plot_id): + try: + with open("/root/.chia/plotman/plotman.yaml") as f: + config = yaml.safe_load(f) + if 'directories' in config: + if 'tmp' in config['directories']: + for tmp_dir in config['directories']['tmp']: + for p in pathlib.Path(tmp_dir).glob("*{0}*.tmp".format(plot_id)): + app.logger.info("After kill, deleting stale tmp file: {0}".format(p)) + p.unlink() + if 'tmp2' in config['directories']: + tmp_dir = config['directories']['tmp2'] + for p in pathlib.Path(tmp_dir).glob("*{0}*.tmp".format(plot_id)): + app.logger.info("After kill, deleting stale tmp file: {0}".format(p)) + p.unlink() + except Exception as ex: + app.logger.info("Skipping deletion of temp files due to {0}.".format(traceback.format_exc())) def stop_plotman(): app.logger.info("Stopping Plotman run...") - try: - os.kill(get_plotman_pid(), signal.SIGTERM) - except: - app.logger.info('Failed to stop Plotman plotting run!') - app.logger.info(traceback.format_exc()) + os.kill(get_plotman_pid(), signal.SIGTERM) def get_archiver_pid(): for proc in psutil.process_iter(['pid', 'name', 'cmdline']): @@ -145,49 +155,39 @@ def get_archiver_pid(): def start_archiver(): app.logger.info("Starting archiver run...") check_config() - try: - logfile = "/root/.chia/plotman/logs/archiver.log" - app.logger.info("About to start archiver...") - proc = Popen("nohup {0} {1} < /dev/tty >> {2} 2>&1 &".format(PLOTMAN_SCRIPT, 'archive', logfile), - shell=True, stdin=DEVNULL, stdout=None, stderr=None, close_fds=True) - app.logger.info("Completed launch of archiver.") - except: - app.logger.info('Failed to start Plotman archiving run!') - app.logger.info(traceback.format_exc()) + logfile = "/root/.chia/plotman/logs/archiver.log" + app.logger.info("About to start archiver...") + proc = Popen("nohup {0} {1} < /dev/tty >> {2} 2>&1 &".format(PLOTMAN_SCRIPT, 'archive', logfile), + shell=True, stdin=DEVNULL, stdout=None, stderr=None, close_fds=True) + app.logger.info("Completed launch of archiver.") def stop_archiver(): app.logger.info("Stopping Archiver run...") - try: - os.kill(get_archiver_pid(), signal.SIGTERM) - except: - app.logger.info('Failed to stop Plotman archiving run!') - app.logger.info(traceback.format_exc()) + os.kill(get_archiver_pid(), signal.SIGTERM) def load_config(): return open('/root/.chia/plotman/plotman.yaml','r').read() def save_config(config): - try: - # Validate the YAML first - yaml.safe_load(config) - # Save a copy of the old config file - src = "/root/.chia/plotman/plotman.yaml" - dst = "/root/.chia/plotman/plotman." + \ - time.strftime("%Y%m%d-%H%M%S")+".yaml" - shutil.copy(src, dst) - # Now save the new contents to main config file - with open(src, 'w') as writer: - writer.write(config) - except Exception as ex: - app.logger.info(traceback.format_exc()) - raise Exception('Updated plotman.yaml failed validation!\n' + str(ex)) - else: # Restart services if running - if get_plotman_pid(): - stop_plotman() - start_plotman() - if get_archiver_pid(): - stop_archiver() - start_archiver() + # Validate the YAML first + yaml.safe_load(config) + # Save a copy of the old config file + src = "/root/.chia/plotman/plotman.yaml" + dst = "/root/.chia/plotman/plotman." + \ + time.strftime("%Y%m%d-%H%M%S")+".yaml" + shutil.copy(src, dst) + # Now save the new contents to main config file + with open(src, 'w') as writer: + writer.write(config) + # Now try to validate config by calling plotman status + load_plotting_summary() + # Finally restart plotman and archiver if they are running + if get_plotman_pid(): + stop_plotman() + start_plotman() + if get_archiver_pid(): + stop_archiver() + start_archiver() def find_plotting_job_log(plot_id): dir_path = '/root/.chia/plotman/logs' diff --git a/api/default_settings.py b/api/default_settings.py index c4ab0a9b..f33e28c0 100644 --- a/api/default_settings.py +++ b/api/default_settings.py @@ -13,8 +13,7 @@ class DefaultConfig: SQLALCHEMY_TRACK_MODIFICATIONS = False SQLALCHEMY_DATABASE_URI = 'sqlite:////root/.chia/machinaris/dbs/machinaris.db' SQLALCHEMY_BINDS = { - 'stats': 'sqlite:////root/.chia/machinaris/dbs/stats.db', - 'chiadog': 'sqlite:////root/.chia/chiadog/dbs/chiadog.db', + 'stats': 'sqlite:////root/.chia/machinaris/dbs/stats.db' } SQLALCHEMY_ECHO = True if 'FLASK_ENV' in os.environ and os.environ['FLASK_ENV'] == "development" else False ETAG_DISABLED = True # https://flask-smorest.readthedocs.io/en/latest/etag.html diff --git a/api/models/chiadog.py b/api/models/chiadog.py deleted file mode 100644 index 2391acb8..00000000 --- a/api/models/chiadog.py +++ /dev/null @@ -1,19 +0,0 @@ -import os - -from datetime import datetime -from flask import Flask -from flask_sqlalchemy import SQLAlchemy -from sqlalchemy import Column, Integer, DateTime, ForeignKey, func - -from common.extensions.database import db - -from api import app - -class Notification(db.Model): - __bind_key__ = 'chiadog' - id = db.Column(db.Integer, primary_key=True, nullable=False) - blockchain = db.Column(db.String(64), nullable=True) - priority = db.Column(db.String(40), nullable=False) - service = db.Column(db.String(60), nullable=False) - message = db.Column(db.String(255), nullable=False) - created_at = db.Column(db.DateTime, server_default=func.now(), nullable=False) diff --git a/api/schedules/status_alerts.py b/api/schedules/status_alerts.py index 9d0459b2..bc1c4a0f 100644 --- a/api/schedules/status_alerts.py +++ b/api/schedules/status_alerts.py @@ -14,10 +14,10 @@ from flask import g +from common.models import alerts as a from common.config import globals from api.commands import chiadog_cli -from api import app -from api import utils +from api import app, utils first_run = True @@ -26,15 +26,19 @@ def update(): if not globals.farming_enabled() and not globals.harvesting_enabled(): #app.logger.info("Skipping alerts status collection on plotting-only instance.") return + if globals.load()['is_controller']: + #app.logger.info("Skipping alerts polling on fullnode are already placed in database directly via chiadog_notifier.sh script.") + return with app.app_context(): try: + from api import db hostname = utils.get_hostname() if first_run: # On first launch, load last week of notifications since = (datetime.datetime.now() - datetime.timedelta(weeks=1)).strftime("%Y-%m-%d %H:%M:%S.000") first_run = False else: # On subsequent schedules, load only last 5 minutes. since = (datetime.datetime.now() - datetime.timedelta(minutes=5)).strftime("%Y-%m-%d %H:%M:%S.000") - alerts = chiadog_cli.get_notifications(since) + alerts = db.session.query(a.Alert).filter(a.Alert.created_at >= since).order_by(a.Alert.created_at.desc()).limit(20).all() payload = [] for alert in alerts: payload.append({ diff --git a/api/views/actions/resources.py b/api/views/actions/resources.py index 4556b683..5919b621 100644 --- a/api/views/actions/resources.py +++ b/api/views/actions/resources.py @@ -51,4 +51,4 @@ def post(self): return make_response("Action completed.", 200) except Exception as ex: app.logger.info(traceback.format_exc()) - abort("Failed during {0} action.".format(service), 500) + return str(ex), 400 diff --git a/api/views/configs/resources.py b/api/views/configs/resources.py index c3ab8e01..c3e1185c 100644 --- a/api/views/configs/resources.py +++ b/api/views/configs/resources.py @@ -59,8 +59,9 @@ def put(self, type): response = make_response("Successfully saved config.", 200) return response except Exception as ex: - app.logger.error(traceback.format_exc()) - abort(400, str(ex)) + app.logger.info("Failed to save a validated Plotman config.") + app.logger.info(traceback.format_exc()) + return str(ex), 400 def clean_config(self, req_data): # First decode the bytes diff --git a/scripts/chiadog_notifier.sh b/scripts/chiadog_notifier.sh index 1389dbff..c5666f18 100644 --- a/scripts/chiadog_notifier.sh +++ b/scripts/chiadog_notifier.sh @@ -6,8 +6,12 @@ event_priority_name="$1" event_service_name="$2" event_message="$3" +hostname="$(hostname -s)" +now="$(date +'%Y-%m-%d %H:%M:%S')" +unique_id="${hostname}_chia_${now}" +echo "Creating alert ${hostname}_chia_${now} ${event_service_name} ${event_priority_name}: ${event_message}" -cd /root/.chia/chiadog/dbs -sqlite3 chiadog.db <{0}".format(response.content.decode('utf-8')), 'danger') def action_plots(action, plot_ids): plots_by_worker = group_plots_by_worker(plot_ids) app.logger.info("About to {0} plots: {1}".format(action, plots_by_worker)) error = False + error_message = "" for hostname in plots_by_worker.keys(): try: plotter = w.get_worker_by_hostname(hostname) plot_ids = plots_by_worker[hostname] - utils.send_post(plotter, "/actions/", debug=False, + response = utils.send_post(plotter, "/actions/", debug=False, payload={"service": "plotting","action": action, "plot_ids": plot_ids} ) + if response.status_code != 200: + error_message += response.content.decode('utf-8') + "\n" except: error = True app.logger.info(traceback.format_exc()) if error: flash('Failed to action all plots!', 'danger') - flash('Please see the log file(s) on your plotter(s).', 'warning') + flash('
{0}
'.format(error_message), 'warning') else: flash('Plotman was able to {0} the selected plots successfully.'.format( action), 'success') @@ -90,35 +96,44 @@ def group_plots_by_worker(plot_ids): def stop_plotman(plotter): app.logger.info("Stopping Plotman run...") try: - utils.send_post(plotter, "/actions/", payload={"service": "plotting","action": "stop"}, debug=False) + response = utils.send_post(plotter, "/actions/", payload={"service": "plotting","action": "stop"}, debug=False) except: app.logger.info(traceback.format_exc()) flash('Failed to stop Plotman plotting run!', 'danger') flash('Please see /root/.chia/plotman/logs/plotman.log', 'warning') else: - flash('Plotman stopped successfully. No new plots will be started, but existing ones will continue on.', 'success') + if response.status_code == 200: + flash('Plotman stopped successfully. No new plots will be started, but existing ones will continue on.', 'success') + else: + flash("
{0}
".format(response.content.decode('utf-8')), 'danger') def start_archiving(plotter): app.logger.info("Starting Archiver....") try: - utils.send_post(plotter, "/actions/", {"service": "archiving","action": "start"}, debug=False) + response = utils.send_post(plotter, "/actions/", {"service": "archiving","action": "start"}, debug=False) except: app.logger.info(traceback.format_exc()) flash('Failed to start Plotman archiver!', 'danger') flash('Please see log files.', 'warning') else: - flash('Archiver started successfully.', 'success') + if response.status_code == 200: + flash('Archiver started successfully.', 'success') + else: + flash("
{0}
".format(response.content.decode('utf-8')), 'danger') def stop_archiving(plotter): app.logger.info("Stopping Archiver run....") try: - utils.send_post(plotter, "/actions/", payload={"service": "archiving","action": "stop"}, debug=False) + response = utils.send_post(plotter, "/actions/", payload={"service": "archiving","action": "stop"}, debug=False) except: app.logger.info(traceback.format_exc()) flash('Failed to stop Plotman archiver', 'danger') flash('Please see /root/.chia/plotman/logs/archiver.log', 'warning') else: - flash('Archiver stopped successfully.', 'success') + if response.status_code == 200: + flash('Archiver stopped successfully.', 'success') + else: + flash("
{0}
".format(response.content.decode('utf-8')), 'danger') def load_key_pk(type): keys = c.load_keys_show() @@ -129,24 +144,27 @@ def load_key_pk(type): def load_pool_contract_address(): plotnfts = c.load_plotnfts() - m = re.search('P2 singleton address .*: (\w+)'.format(type), plotnfts.rows[0]['details']) - if m: - return m.group(1) + if len(plotnfts.rows) == 1: + m = re.search('P2 singleton address .*: (\w+)'.format(type), plotnfts.rows[0]['details']) + if m: + return m.group(1) + elif len(plotnfts.rows) > 1: + app.logger.info("Did not find a unique P2 singleton address as multiple plotnfts exist. Not replacing in plotman.yaml.") return None def load_config_replacements(): replacements = [] farmer_pk = load_key_pk('Farmer') if farmer_pk: - app.logger.info("FARMER_PK: {0}".format(farmer_pk)) + #app.logger.info("FARMER_PK: {0}".format(farmer_pk)) replacements.append([ 'farmer_pk:.*$', 'farmer_pk: '+ farmer_pk]) pool_pk = load_key_pk('Pool') if pool_pk: - app.logger.info("POOL_PK: {0}".format(pool_pk)) + #app.logger.info("POOL_PK: {0}".format(pool_pk)) replacements.append([ 'pool_pk:.*$', 'pool_pk: '+ pool_pk]) pool_contract_address = load_pool_contract_address() if pool_contract_address: - app.logger.info("POOL_CONTRACT_ADDRESS: {0}".format(pool_contract_address)) + #app.logger.info("POOL_CONTRACT_ADDRESS: {0}".format(pool_contract_address)) replacements.append([ 'pool_contract_address:.*$', 'pool_contract_address: '+ pool_contract_address]) return replacements @@ -173,12 +191,15 @@ def save_config(plotter, config): flash('Updated plotman.yaml failed validation! Fix and save or refresh page.', 'danger') flash(str(ex), 'warning') try: - utils.send_put(plotter, "/configs/plotting", config, debug=False) + response = utils.send_put(plotter, "/configs/plotting", config, debug=False) except Exception as ex: flash('Failed to save config to plotter. Please check log files.', 'danger') flash(str(ex), 'warning') else: - flash('Nice! Plotman\'s plotman.yaml validated and saved successfully.', 'success') + if response.status_code == 200: + flash('Nice! Plotman\'s plotman.yaml validated and saved successfully.', 'success') + else: + flash("
{0}
".format(response.content.decode('utf-8')), 'danger') def analyze(plot_file, plotters): # Don't know which plotter might have the plot result so try them in-turn diff --git a/web/routes.py b/web/routes.py index ce41a370..63474bd9 100644 --- a/web/routes.py +++ b/web/routes.py @@ -113,11 +113,14 @@ def plots_check(): def alerts(): gc = globals.load() if request.method == 'POST': - w = worker.get_worker_by_hostname(request.form.get('hostname')) if request.form.get('action') == 'start': + w = worker.get_worker_by_hostname(request.form.get('hostname')) chiadog.start_chiadog(w) elif request.form.get('action') == 'stop': + w = worker.get_worker_by_hostname(request.form.get('hostname')) chiadog.stop_chiadog(w) + elif request.form.get('action') == 'remove': + chiadog.remove_alerts(request.form.getlist('unique_id')) else: app.logger.info("Unknown alerts form: {0}".format(request.form)) return redirect(url_for('alerts')) # Force a redirect to allow time to update status diff --git a/web/templates/alerts.html b/web/templates/alerts.html index 1b5e71eb..4a23b25b 100644 --- a/web/templates/alerts.html +++ b/web/templates/alerts.html @@ -33,14 +33,15 @@