diff --git a/.github/workflows/develop.yaml b/.github/workflows/develop.yaml index 985c873e..1a729a5c 100644 --- a/.github/workflows/develop.yaml +++ b/.github/workflows/develop.yaml @@ -39,7 +39,7 @@ jobs: platforms: linux/amd64 push: true build-args: | - "CHIA_BRANCH=1.2.0" + "CHIA_BRANCH=1.2.2" "FLAX_BRANCH=main" tags: | ${{ secrets.DOCKERHUB_USERNAME }}/machinaris:develop diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 7b8b3d17..5bf38741 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -40,7 +40,7 @@ jobs: platforms: linux/amd64 push: true build-args: | - "CHIA_BRANCH=1.2.0" + "CHIA_BRANCH=1.2.2" "FLAX_BRANCH=main" tags: | ${{ secrets.DOCKERHUB_USERNAME }}/machinaris:latest diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 03f0900b..537e8a00 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -40,7 +40,7 @@ jobs: platforms: linux/amd64 push: true build-args: | - "CHIA_BRANCH=1.2.0" + "CHIA_BRANCH=1.2.2" "FLAX_BRANCH=main" tags: | ${{ secrets.DOCKERHUB_USERNAME }}/machinaris:test diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c1f3cc1..76736ef6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.5.1] - 2021-07-22 + +- Wizard on Workers page to create a Docker run/compose based on your settings. [Issue #97](https://github.com/guydavis/machinaris/issues/97) +- Update to patch release of Chia 1.2.2, including a fix for harvester cache updates. See their [changelog for details](https://github.com/Chia-Network/chia-blockchain/releases/tag/1.2.2). +- Latest Madmax plotter with support for n_buckets3 and n_rmulti2 settings in Plotman. + ## [0.5.0] - 2021-07-09 - Support for [official Chia pools](https://github.com/guydavis/machinaris/issues/131). Chia and Madmax plotters can create portable plots. diff --git a/CREDITS.md b/CREDITS.md index e4848972..80f911e7 100644 --- a/CREDITS.md +++ b/CREDITS.md @@ -18,6 +18,7 @@ A huge thank-you to the great teams/devs behind these projects, being used by Ma * [Other Icons](https://www.shareicon.net): Images and icons for web apps. * [Boostrap Sidebars](https://dev.to/codeply/bootstrap-5-sidebar-examples-38pb): Used for sidebar menu layout. * [Toplevel](https://github.com/and-semakin/marshmallow-toplevel): Used for list of objects sent to REST API. +* [DataTables.js](https://datatables.net/): Filter/search/pagination of dynamic tables, as per [this tutorial](https://blog.miguelgrinberg.com/post/beautiful-interactive-tables-for-your-flask-templates). ## Blockchain Forks * [Flax](https://github.com/Flax-Network/flax-blockchain) diff --git a/VERSION b/VERSION index 79a2734b..5d4294b9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.5.0 \ No newline at end of file +0.5.1 \ No newline at end of file diff --git a/api/commands/chiadog_cli.py b/api/commands/chiadog_cli.py index f390776b..0f701419 100644 --- a/api/commands/chiadog_cli.py +++ b/api/commands/chiadog_cli.py @@ -15,7 +15,6 @@ from flask import Flask, jsonify, abort, request, flash, g from subprocess import Popen, TimeoutExpired, PIPE -from api.models import chiadog from api import app def load_config(blockchain): @@ -46,10 +45,6 @@ def get_chiadog_pid(blockchain): return proc.info['pid'] return None -def get_notifications(since): - return chiadog.Notification.query.filter(chiadog.Notification.created_at >= since). \ - order_by(chiadog.Notification.created_at.desc()).limit(20).all() - def dispatch_action(job): service = job['service'] if service != 'monitoring': @@ -72,6 +67,9 @@ def start_chiadog(): for blockchain in blockchains: try: workdir = "/{0}dog".format(blockchain) + offset_file = "{0}/debug.log.offset".format(workdir) + if os.path.exists(offset_file): + os.remove(offset_file) configfile = "/root/.chia/{0}dog/config.yaml".format(blockchain) logfile = "/root/.chia/{0}dog/logs/{0}dog.log".format(blockchain) proc = Popen("nohup /{0}-blockchain/venv/bin/python3 -u main.py --config {1} >> {2} 2>&1 &".format(blockchain, configfile, logfile), \ diff --git a/api/commands/log_parser.py b/api/commands/log_parser.py index a7398158..22a96365 100644 --- a/api/commands/log_parser.py +++ b/api/commands/log_parser.py @@ -39,7 +39,7 @@ def recent_challenges(blockchain): app.logger.debug( "Skipping challenges parsing as no such log file: {0}".format(log_file)) return [] - proc = Popen("grep -i eligible {0} | tail -n {1}".format(log_file, CHALLENGES_TO_LOAD), + proc = Popen("grep --text -i eligible {0} | tail -n {1}".format(log_file, CHALLENGES_TO_LOAD), stdout=PIPE, stderr=PIPE, shell=True) try: outs, errs = proc.communicate(timeout=90) diff --git a/api/commands/plotman_cli.py b/api/commands/plotman_cli.py index 67a1944d..5c1ddb8e 100644 --- a/api/commands/plotman_cli.py +++ b/api/commands/plotman_cli.py @@ -41,10 +41,9 @@ def load_plotting_summary(): except TimeoutExpired: proc.kill() proc.communicate() - abort(500, description="The timeout is expired!") + raise Exception("The timeout expired during plotman status.") if errs: - app.logger.error(errs.decode('utf-8')) - abort(500, description=errs.decode('utf-8')) + raise Exception("Errors during plotman status:\n {0}".format(errs.decode('utf-8'))) cli_stdout = outs.decode('utf-8') return plotman.PlottingSummary(cli_stdout.splitlines(), get_plotman_pid()) @@ -76,14 +75,16 @@ def action_plots(job): #app.logger.info("About to {0} plots: {1}".format(action, plot_ids)) for plot_id in plot_ids: try: - prefix = "" + param = "" if action == "kill": - prefix = "printf 'y\n' |" + param = "--force" logfile = "/root/.chia/plotman/logs/plotman.log" log_fd = os.open(logfile, os.O_RDWR | os.O_CREAT) log_fo = os.fdopen(log_fd, "a+") - proc = Popen("{0} {1} {2} {3}".format(prefix, PLOTMAN_SCRIPT, action, plot_id), + proc = Popen("{0} {1} {2} {3}".format(PLOTMAN_SCRIPT, action, param, plot_id), shell=True, universal_newlines=True, stdout=log_fo, stderr=log_fo) + # Plotman regressed on cleaning temp after kill so do it here: + clean_tmp_dirs_after_kill(plot_id) except: app.logger.info('Failed to {0} selected plot {1}.'.format(action, plot_id)) app.logger.info(traceback.format_exc()) @@ -98,36 +99,52 @@ def get_plotman_pid(): def start_plotman(): app.logger.info("Starting Plotman run...") check_config() - try: - if len(load_plotting_summary().rows) == 0: # No plots running - clean_tmp_dirs_before_run() - logfile = "/root/.chia/plotman/logs/plotman.log" - proc = Popen("nohup {0} {1} < /dev/tty >> {2} 2>&1 &".format(PLOTMAN_SCRIPT, 'plot', logfile), - shell=True, stdin=DEVNULL, stdout=None, stderr=None, close_fds=True) - app.logger.info("Completed launch of plotman.") - except: - app.logger.info('Failed to start Plotman plotting run!') - app.logger.info(traceback.format_exc()) + if len(load_plotting_summary().rows) == 0: # No plots running + clean_tmp_dirs_before_run() + logfile = "/root/.chia/plotman/logs/plotman.log" + proc = Popen("nohup {0} {1} < /dev/tty >> {2} 2>&1 &".format(PLOTMAN_SCRIPT, 'plot', logfile), + shell=True, stdin=DEVNULL, stdout=None, stderr=None, close_fds=True) + app.logger.info("Completed launch of plotman.") def clean_tmp_dirs_before_run(): try: with open("/root/.chia/plotman/plotman.yaml") as f: config = yaml.safe_load(f) - for tmp_dir in config['directories']['tmp']: - app.logger.info("No running plot jobs found so deleting {0}/*.tmp before starting plotman.".format(tmp_dir)) - for p in pathlib.Path(tmp_dir).glob("*.tmp"): - p.unlink() + if 'directories' in config: + if 'tmp' in config['directories']: + for tmp_dir in config['directories']['tmp']: + app.logger.info("No running plot jobs found so deleting {0}/*.tmp before starting plotman.".format(tmp_dir)) + for p in pathlib.Path(tmp_dir).glob("*.tmp"): + p.unlink() + if 'tmp2' in config['directories']: + tmp_dir = config['directories']['tmp2'] + app.logger.info("No running plot jobs found so deleting {0}/*.tmp before starting plotman.".format(tmp_dir)) + for p in pathlib.Path(tmp_dir).glob("*.tmp"): + p.unlink() except Exception as ex: - app.logger.info(traceback.format_exc()) - raise Exception('Updated plotman.yaml failed validation!\n' + str(ex)) + app.logger.info("Skipping deletion of temp files due to {0}.".format(traceback.format_exc())) + +def clean_tmp_dirs_after_kill(plot_id): + try: + with open("/root/.chia/plotman/plotman.yaml") as f: + config = yaml.safe_load(f) + if 'directories' in config: + if 'tmp' in config['directories']: + for tmp_dir in config['directories']['tmp']: + for p in pathlib.Path(tmp_dir).glob("*{0}*.tmp".format(plot_id)): + app.logger.info("After kill, deleting stale tmp file: {0}".format(p)) + p.unlink() + if 'tmp2' in config['directories']: + tmp_dir = config['directories']['tmp2'] + for p in pathlib.Path(tmp_dir).glob("*{0}*.tmp".format(plot_id)): + app.logger.info("After kill, deleting stale tmp file: {0}".format(p)) + p.unlink() + except Exception as ex: + app.logger.info("Skipping deletion of temp files due to {0}.".format(traceback.format_exc())) def stop_plotman(): app.logger.info("Stopping Plotman run...") - try: - os.kill(get_plotman_pid(), signal.SIGTERM) - except: - app.logger.info('Failed to stop Plotman plotting run!') - app.logger.info(traceback.format_exc()) + os.kill(get_plotman_pid(), signal.SIGTERM) def get_archiver_pid(): for proc in psutil.process_iter(['pid', 'name', 'cmdline']): @@ -138,49 +155,39 @@ def get_archiver_pid(): def start_archiver(): app.logger.info("Starting archiver run...") check_config() - try: - logfile = "/root/.chia/plotman/logs/archiver.log" - app.logger.info("About to start archiver...") - proc = Popen("nohup {0} {1} < /dev/tty >> {2} 2>&1 &".format(PLOTMAN_SCRIPT, 'archive', logfile), - shell=True, stdin=DEVNULL, stdout=None, stderr=None, close_fds=True) - app.logger.info("Completed launch of archiver.") - except: - app.logger.info('Failed to start Plotman archiving run!') - app.logger.info(traceback.format_exc()) + logfile = "/root/.chia/plotman/logs/archiver.log" + app.logger.info("About to start archiver...") + proc = Popen("nohup {0} {1} < /dev/tty >> {2} 2>&1 &".format(PLOTMAN_SCRIPT, 'archive', logfile), + shell=True, stdin=DEVNULL, stdout=None, stderr=None, close_fds=True) + app.logger.info("Completed launch of archiver.") def stop_archiver(): app.logger.info("Stopping Archiver run...") - try: - os.kill(get_archiver_pid(), signal.SIGTERM) - except: - app.logger.info('Failed to stop Plotman archiving run!') - app.logger.info(traceback.format_exc()) + os.kill(get_archiver_pid(), signal.SIGTERM) def load_config(): return open('/root/.chia/plotman/plotman.yaml','r').read() def save_config(config): - try: - # Validate the YAML first - yaml.safe_load(config) - # Save a copy of the old config file - src = "/root/.chia/plotman/plotman.yaml" - dst = "/root/.chia/plotman/plotman." + \ - time.strftime("%Y%m%d-%H%M%S")+".yaml" - shutil.copy(src, dst) - # Now save the new contents to main config file - with open(src, 'w') as writer: - writer.write(config) - except Exception as ex: - app.logger.info(traceback.format_exc()) - raise Exception('Updated plotman.yaml failed validation!\n' + str(ex)) - else: # Restart services if running - if get_plotman_pid(): - stop_plotman() - start_plotman() - if get_archiver_pid(): - stop_archiver() - start_archiver() + # Validate the YAML first + yaml.safe_load(config) + # Save a copy of the old config file + src = "/root/.chia/plotman/plotman.yaml" + dst = "/root/.chia/plotman/plotman." + \ + time.strftime("%Y%m%d-%H%M%S")+".yaml" + shutil.copy(src, dst) + # Now save the new contents to main config file + with open(src, 'w') as writer: + writer.write(config) + # Now try to validate config by calling plotman status + load_plotting_summary() + # Finally restart plotman and archiver if they are running + if get_plotman_pid(): + stop_plotman() + start_plotman() + if get_archiver_pid(): + stop_archiver() + start_archiver() def find_plotting_job_log(plot_id): dir_path = '/root/.chia/plotman/logs' diff --git a/api/default_settings.py b/api/default_settings.py index c4ab0a9b..f33e28c0 100644 --- a/api/default_settings.py +++ b/api/default_settings.py @@ -13,8 +13,7 @@ class DefaultConfig: SQLALCHEMY_TRACK_MODIFICATIONS = False SQLALCHEMY_DATABASE_URI = 'sqlite:////root/.chia/machinaris/dbs/machinaris.db' SQLALCHEMY_BINDS = { - 'stats': 'sqlite:////root/.chia/machinaris/dbs/stats.db', - 'chiadog': 'sqlite:////root/.chia/chiadog/dbs/chiadog.db', + 'stats': 'sqlite:////root/.chia/machinaris/dbs/stats.db' } SQLALCHEMY_ECHO = True if 'FLASK_ENV' in os.environ and os.environ['FLASK_ENV'] == "development" else False ETAG_DISABLED = True # https://flask-smorest.readthedocs.io/en/latest/etag.html diff --git a/api/migrations/versions/bdfe8db75307_.py b/api/migrations/versions/bdfe8db75307_.py new file mode 100644 index 00000000..d5d87701 --- /dev/null +++ b/api/migrations/versions/bdfe8db75307_.py @@ -0,0 +1,85 @@ +"""empty message + +Revision ID: bdfe8db75307 +Revises: 51f75e96b994 +Create Date: 2021-07-14 16:11:24.451771 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.sql import func + + +# revision identifiers, used by Alembic. +revision = 'bdfe8db75307' +down_revision = '51f75e96b994' +branch_labels = None +depends_on = None + + +def upgrade(engine_name): + globals()["upgrade_%s" % engine_name]() + + +def downgrade(engine_name): + globals()["downgrade_%s" % engine_name]() + + + + + +def upgrade_(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('plots') + op.create_table('plots', + sa.Column('hostname', sa.String(length=255), primary_key=True), + sa.Column('plot_id', sa.String(length=16), primary_key=True), + sa.Column('dir', sa.String(length=255), nullable=False), + sa.Column('file', sa.String(length=255), nullable=False), + sa.Column('size', sa.Integer, nullable=False), + sa.Column('created_at', sa.String(length=64), nullable=False), + sa.Column('updated_at', sa.DateTime(), onupdate=func.now()), + sa.PrimaryKeyConstraint('hostname', 'plot_id') + ) + # ### end Alembic commands ### + + +def downgrade_(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('plots') + op.create_table('plots', + sa.Column('plot_id', sa.String(length=16), primary_key=True), + sa.Column('hostname', sa.String(length=255), nullable=False), + sa.Column('dir', sa.String(length=255), nullable=False), + sa.Column('file', sa.String(length=255), nullable=False), + sa.Column('size', sa.Integer, nullable=False), + sa.Column('created_at', sa.String(length=64), nullable=False), + sa.Column('updated_at', sa.DateTime(), onupdate=func.now()), + sa.PrimaryKeyConstraint('plot_id') + ) + # ### end Alembic commands ### + + +def upgrade_stats(): + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade_stats(): + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def upgrade_chiadog(): + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade_chiadog(): + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + diff --git a/api/models/chia.py b/api/models/chia.py index 1155a5f6..e41bbd80 100644 --- a/api/models/chia.py +++ b/api/models/chia.py @@ -15,6 +15,7 @@ class FarmSummary: def __init__(self, cli_stdout=None, farm_plots=None): if cli_stdout: next_line_local_harvester = False + self.plot_count = 0 self.plots_size = None for line in cli_stdout: if next_line_local_harvester: diff --git a/api/models/chiadog.py b/api/models/chiadog.py deleted file mode 100644 index 2391acb8..00000000 --- a/api/models/chiadog.py +++ /dev/null @@ -1,19 +0,0 @@ -import os - -from datetime import datetime -from flask import Flask -from flask_sqlalchemy import SQLAlchemy -from sqlalchemy import Column, Integer, DateTime, ForeignKey, func - -from common.extensions.database import db - -from api import app - -class Notification(db.Model): - __bind_key__ = 'chiadog' - id = db.Column(db.Integer, primary_key=True, nullable=False) - blockchain = db.Column(db.String(64), nullable=True) - priority = db.Column(db.String(40), nullable=False) - service = db.Column(db.String(60), nullable=False) - message = db.Column(db.String(255), nullable=False) - created_at = db.Column(db.DateTime, server_default=func.now(), nullable=False) diff --git a/api/schedules/status_alerts.py b/api/schedules/status_alerts.py index 9d0459b2..bc1c4a0f 100644 --- a/api/schedules/status_alerts.py +++ b/api/schedules/status_alerts.py @@ -14,10 +14,10 @@ from flask import g +from common.models import alerts as a from common.config import globals from api.commands import chiadog_cli -from api import app -from api import utils +from api import app, utils first_run = True @@ -26,15 +26,19 @@ def update(): if not globals.farming_enabled() and not globals.harvesting_enabled(): #app.logger.info("Skipping alerts status collection on plotting-only instance.") return + if globals.load()['is_controller']: + #app.logger.info("Skipping alerts polling on fullnode are already placed in database directly via chiadog_notifier.sh script.") + return with app.app_context(): try: + from api import db hostname = utils.get_hostname() if first_run: # On first launch, load last week of notifications since = (datetime.datetime.now() - datetime.timedelta(weeks=1)).strftime("%Y-%m-%d %H:%M:%S.000") first_run = False else: # On subsequent schedules, load only last 5 minutes. since = (datetime.datetime.now() - datetime.timedelta(minutes=5)).strftime("%Y-%m-%d %H:%M:%S.000") - alerts = chiadog_cli.get_notifications(since) + alerts = db.session.query(a.Alert).filter(a.Alert.created_at >= since).order_by(a.Alert.created_at.desc()).limit(20).all() payload = [] for alert in alerts: payload.append({ diff --git a/api/schedules/status_plots.py b/api/schedules/status_plots.py index 23b4d309..5f93ed97 100644 --- a/api/schedules/status_plots.py +++ b/api/schedules/status_plots.py @@ -21,16 +21,24 @@ def update(): try: hostname = utils.get_hostname() plots_farming = chia_cli.load_plots_farming() + plots_by_id = {} payload = [] for plot in plots_farming.rows: - payload.append({ - "plot_id": plot['plot_id'], - "hostname": hostname, - "dir": plot['dir'], - "file": plot['file'], - "created_at": plot['created_at'], - "size": plot['size'], - }) + plot_id = plot['plot_id'] + if plot_id in plots_by_id: + other_plot = plots_by_id[plot_id] + app.logger.info("Skipping addition of plot at {0}/{1} because same plot_id found at {2}/{3}".format( + plot['dir'], plot['file'], other_plot['dir'], other_plot['file'])) + else: # No conflict so add it to plots list + plots_by_id[plot_id] = plot + payload.append({ + "plot_id": plot_id, + "hostname": hostname, + "dir": plot['dir'], + "file": plot['file'], + "created_at": plot['created_at'], + "size": plot['size'], + }) if len(payload) > 0: utils.send_post('/plots/', payload, debug=False) else: diff --git a/api/views/actions/resources.py b/api/views/actions/resources.py index 4556b683..5919b621 100644 --- a/api/views/actions/resources.py +++ b/api/views/actions/resources.py @@ -51,4 +51,4 @@ def post(self): return make_response("Action completed.", 200) except Exception as ex: app.logger.info(traceback.format_exc()) - abort("Failed during {0} action.".format(service), 500) + return str(ex), 400 diff --git a/api/views/configs/resources.py b/api/views/configs/resources.py index c3ab8e01..c3e1185c 100644 --- a/api/views/configs/resources.py +++ b/api/views/configs/resources.py @@ -59,8 +59,9 @@ def put(self, type): response = make_response("Successfully saved config.", 200) return response except Exception as ex: - app.logger.error(traceback.format_exc()) - abort(400, str(ex)) + app.logger.info("Failed to save a validated Plotman config.") + app.logger.info(traceback.format_exc()) + return str(ex), 400 def clean_config(self, req_data): # First decode the bytes diff --git a/common/models/plots.py b/common/models/plots.py index 93ae4309..d1ddb8aa 100644 --- a/common/models/plots.py +++ b/common/models/plots.py @@ -9,12 +9,10 @@ class Plot(db.Model): __tablename__ = "plots" - plot_id = sa.Column(sa.String(length=8), primary_key=True) - hostname = sa.Column(sa.String(length=255), nullable=False) + hostname = sa.Column(sa.String(length=255), primary_key=True) + plot_id = sa.Column(sa.String(length=16), primary_key=True) dir = sa.Column(sa.String(length=255), nullable=False) file = sa.Column(sa.String(length=255), nullable=False) size = sa.Column(sa.Integer, nullable=False) created_at = sa.Column(sa.String(length=64), nullable=False) updated_at = sa.Column(sa.DateTime(), onupdate=func.now()) - - \ No newline at end of file diff --git a/config/chiadog.sample.yaml b/config/chiadog.sample.yaml index 5558c78c..a0211a19 100644 --- a/config/chiadog.sample.yaml +++ b/config/chiadog.sample.yaml @@ -21,7 +21,7 @@ chia_logs: # on your farm performance at the specified time of the day. daily_stats: enable: true - time_of_day: 21 + time_of_day: 20 # We support a lot of notifiers, please check the README for more # information. You can delete the sections which you aren't using. diff --git a/config/plotman.sample.yaml b/config/plotman.sample.yaml index a50dac7f..adc98c3d 100644 --- a/config/plotman.sample.yaml +++ b/config/plotman.sample.yaml @@ -28,19 +28,6 @@ directories: tmp: - /plotting - # Optional: Allows overriding some characteristics of certain tmp - # directories. This contains a map of tmp directory names to - # attributes. If a tmp directory and attribute is not listed here, - # it uses the default attribute setting from the main configuration. - # - # Currently support override parameters: - # - tmpdir_max_jobs - #tmp_overrides: - # In this example, /plotting3 is larger than the other tmp - # dirs and it can hold more plots than the default. - #/plotting3: - # tmpdir_max_jobs: 5 - # Optional: tmp2 directory. If specified, will be passed to # the chia and madmax plotters as the '-2' param. # tmp2: /plotting2 @@ -90,29 +77,43 @@ scheduling: # How often the daemon wakes to consider starting a new plot job, in seconds. polling_time_s: 20 + # Optional: Allows overriding some characteristics of certain tmp + # directories. This contains a map of tmp directory names to + # attributes. If a tmp directory and attribute is not listed here, + # it uses the default attribute setting from the main configuration. + # + # Currently support override parameters: + # - tmpdir_max_jobs + #tmp_overrides: + # In this example, /plotting3 is larger than the other tmp + # dirs and it can hold more plots than the default. + #/plotting3: + # tmpdir_max_jobs: 5 # Configure the plotter. See: https://github.com/guydavis/machinaris/wiki/Plotman#plotting plotting: - # Your farmer public key is always required for plotting - # farmer_pk: YOUR_FARMER_PUBLIC_KEY (always required) - # Your pool public key is only required for solo plotting, else comment it out! - #pool_pk: YOUR_POOL_PUBLIC_KEY (comment this out if portable pool plotting) - # Your pool contract address is only required for portable pool plotting, else comment it out! - #pool_contract_address: YOUR_POOL_CONTRACT_ADDRESS (comment this out if solo plotting) + # See Keys page, for 'Farmer public key' value + farmer_pk: REPLACE_WITH_THE_REAL_VALUE + # ONLY FOR OLD SOLO PLOTS, COMMENT OUT IF PORTABLE PLOTTING!!! + pool_pk: REPLACE_WITH_THE_REAL_VALUE + # See 'Settings | Pools' page, for 'P2 singleton address' value, UNCOMMENT IF PORTABLE PLOTTING!!! + #pool_contract_address: REPLACE_WITH_THE_REAL_VALUE # If you enable 'chia', plot in *parallel* with higher tmpdir_max_jobs and global_max_jobs # If you enable 'madmax', plot in *sequence* with very low tmpdir_max_jobs and global_max_jobs type: chia - # The chia plotter: https://github.com/Chia-Network/chia-blockchain chia: + # The stock plotter: https://github.com/Chia-Network/chia-blockchain k: 32 # k-size of plot, leave at 32 most of the time - e: False # Disable bitfield back sorting (default is True) + e: False # Use -e plotting option n_threads: 2 # Threads per job n_buckets: 128 # Number of buckets to split data into job_buffer: 3389 # Per job memory - # The madmax plotter: https://github.com/madMAx43v3r/chia-plotter madmax: + # madMAx plotter: https://github.com/madMAx43v3r/chia-plotter n_threads: 4 # Default is 4, SLOWLY crank up if you have many cores n_buckets: 256 # Default is 256 + n_buckets3: 256 # Default is 256 + n_rmulti2: 1 # Default is 1 diff --git a/dockerfile b/dockerfile index a319a4e9..b0ea979b 100644 --- a/dockerfile +++ b/dockerfile @@ -110,10 +110,17 @@ ENV AUTO_PLOT=false VOLUME [ "/id_rsa" ] -# ports -EXPOSE 8555 +# Chia protocol port - forward at router EXPOSE 8444 +# Chia RPC port - DO NOT forward at router +EXPOSE 8447 +# Flax protocol port - forward at router +EXPOSE 6888 +# Flax RPC port - DO NOT forward at router +EXPOSE 6885 +# Machinaris WebUI - DO NOT forward at router, proxy if needed EXPOSE 8926 +# Machinaris API - DO NOT forward at router EXPOSE 8927 WORKDIR /chia-blockchain diff --git a/scripts/chiadog_install.sh b/scripts/chiadog_install.sh index dd0f2174..114f8b0d 100644 --- a/scripts/chiadog_install.sh +++ b/scripts/chiadog_install.sh @@ -9,6 +9,12 @@ cd / git clone https://github.com/martomi/chiadog.git +# Temporary patch for spam about partial proofs +# https://github.com/martomi/chiadog/issues/252#issuecomment-877416135 + +cd /chiadog/src/chia_log/handlers/ +sed -i 's/FoundProofs(),//g' harvester_activity_handler.py + cd /chia-blockchain/ # Chia-blockchain needs PyYAML=5.4.1 but Chiadog wants exactly 5.4 diff --git a/scripts/chiadog_launch.sh b/scripts/chiadog_launch.sh index 57d15188..898f2fc3 100644 --- a/scripts/chiadog_launch.sh +++ b/scripts/chiadog_launch.sh @@ -6,7 +6,7 @@ cp -f /machinaris/scripts/chiadog_notifier.sh /root/.chia/chiadog/notifier.sh && echo 'Starting Chiadog...' cd /chiadog chiadog_pids=$(pidof python3) -if [ ! -z $chiadog_pids ]; then +if [[ ! -z $chiadog_pids ]]; then kill $chiadog_pids fi /chia-blockchain/venv/bin/python3 -u main.py --config /root/.chia/chiadog/config.yaml > /root/.chia/chiadog/logs/chiadog.log 2>&1 & diff --git a/scripts/chiadog_notifier.sh b/scripts/chiadog_notifier.sh index 1389dbff..c5666f18 100644 --- a/scripts/chiadog_notifier.sh +++ b/scripts/chiadog_notifier.sh @@ -6,8 +6,12 @@ event_priority_name="$1" event_service_name="$2" event_message="$3" +hostname="$(hostname -s)" +now="$(date +'%Y-%m-%d %H:%M:%S')" +unique_id="${hostname}_chia_${now}" +echo "Creating alert ${hostname}_chia_${now} ${event_service_name} ${event_priority_name}: ${event_message}" -cd /root/.chia/chiadog/dbs -sqlite3 chiadog.db <{0}".format(response.content.decode('utf-8')), 'danger') def action_plots(action, plot_ids): plots_by_worker = group_plots_by_worker(plot_ids) app.logger.info("About to {0} plots: {1}".format(action, plots_by_worker)) error = False + error_message = "" for hostname in plots_by_worker.keys(): try: plotter = w.get_worker_by_hostname(hostname) plot_ids = plots_by_worker[hostname] - utils.send_post(plotter, "/actions/", debug=False, + response = utils.send_post(plotter, "/actions/", debug=False, payload={"service": "plotting","action": action, "plot_ids": plot_ids} ) + if response.status_code != 200: + error_message += response.content.decode('utf-8') + "\n" except: error = True app.logger.info(traceback.format_exc()) if error: flash('Failed to action all plots!', 'danger') - flash('Please see the log file(s) on your plotter(s).', 'warning') + flash('
{0}
'.format(error_message), 'warning') else: flash('Plotman was able to {0} the selected plots successfully.'.format( action), 'success') @@ -90,35 +96,44 @@ def group_plots_by_worker(plot_ids): def stop_plotman(plotter): app.logger.info("Stopping Plotman run...") try: - utils.send_post(plotter, "/actions/", payload={"service": "plotting","action": "stop"}, debug=False) + response = utils.send_post(plotter, "/actions/", payload={"service": "plotting","action": "stop"}, debug=False) except: app.logger.info(traceback.format_exc()) flash('Failed to stop Plotman plotting run!', 'danger') flash('Please see /root/.chia/plotman/logs/plotman.log', 'warning') else: - flash('Plotman stopped successfully. No new plots will be started, but existing ones will continue on.', 'success') + if response.status_code == 200: + flash('Plotman stopped successfully. No new plots will be started, but existing ones will continue on.', 'success') + else: + flash("
{0}
".format(response.content.decode('utf-8')), 'danger') def start_archiving(plotter): app.logger.info("Starting Archiver....") try: - utils.send_post(plotter, "/actions/", {"service": "archiving","action": "start"}, debug=False) + response = utils.send_post(plotter, "/actions/", {"service": "archiving","action": "start"}, debug=False) except: app.logger.info(traceback.format_exc()) flash('Failed to start Plotman archiver!', 'danger') flash('Please see log files.', 'warning') else: - flash('Archiver started successfully.', 'success') + if response.status_code == 200: + flash('Archiver started successfully.', 'success') + else: + flash("
{0}
".format(response.content.decode('utf-8')), 'danger') def stop_archiving(plotter): app.logger.info("Stopping Archiver run....") try: - utils.send_post(plotter, "/actions/", payload={"service": "archiving","action": "stop"}, debug=False) + response = utils.send_post(plotter, "/actions/", payload={"service": "archiving","action": "stop"}, debug=False) except: app.logger.info(traceback.format_exc()) flash('Failed to stop Plotman archiver', 'danger') flash('Please see /root/.chia/plotman/logs/archiver.log', 'warning') else: - flash('Archiver stopped successfully.', 'success') + if response.status_code == 200: + flash('Archiver stopped successfully.', 'success') + else: + flash("
{0}
".format(response.content.decode('utf-8')), 'danger') def load_key_pk(type): keys = c.load_keys_show() @@ -129,25 +144,28 @@ def load_key_pk(type): def load_pool_contract_address(): plotnfts = c.load_plotnfts() - m = re.search('P2 singleton address .*: (\w+)'.format(type), plotnfts.rows[0]['details']) - if m: - return m.group(1) + if len(plotnfts.rows) == 1: + m = re.search('P2 singleton address .*: (\w+)'.format(type), plotnfts.rows[0]['details']) + if m: + return m.group(1) + elif len(plotnfts.rows) > 1: + app.logger.info("Did not find a unique P2 singleton address as multiple plotnfts exist. Not replacing in plotman.yaml.") return None def load_config_replacements(): replacements = [] farmer_pk = load_key_pk('Farmer') if farmer_pk: - app.logger.info("FARMER_PK: {0}".format(farmer_pk)) - replacements.append([ 'farmer_pk:.*$', 'farmer_pk: '+ farmer_pk]) + #app.logger.info("FARMER_PK: {0}".format(farmer_pk)) + replacements.append([ 'farmer_pk:\s+REPLACE_WITH_THE_REAL_VALUE.*$', 'farmer_pk: '+ farmer_pk]) pool_pk = load_key_pk('Pool') if pool_pk: - app.logger.info("POOL_PK: {0}".format(pool_pk)) - replacements.append([ 'pool_pk:.*$', 'pool_pk: '+ pool_pk]) + #app.logger.info("POOL_PK: {0}".format(pool_pk)) + replacements.append([ 'pool_pk:\s+REPLACE_WITH_THE_REAL_VALUE.*$', 'pool_pk: '+ pool_pk]) pool_contract_address = load_pool_contract_address() if pool_contract_address: - app.logger.info("POOL_CONTRACT_ADDRESS: {0}".format(pool_contract_address)) - replacements.append([ 'pool_contract_address:.*$', 'pool_contract_address: '+ pool_contract_address]) + #app.logger.info("POOL_CONTRACT_ADDRESS: {0}".format(pool_contract_address)) + replacements.append([ 'pool_contract_address:\s+REPLACE_WITH_THE_REAL_VALUE.*$', 'pool_contract_address: '+ pool_contract_address]) return replacements def load_config(plotter): @@ -159,11 +177,18 @@ def load_config(plotter): app.logger.info(traceback.format_exc()) lines = [] config = utils.send_get(plotter, "/configs/plotting", debug=False).content.decode('utf-8') + replaces = 0 for line in config.splitlines(): for replacement in replacements: - line = re.sub(replacement[0], replacement[1], line) + (line, num_replaces) = re.subn(replacement[0], replacement[1], line) + replaces += num_replaces lines.append(line) - return '\n'.join(lines) + if replaces > 0: + #app.logger.info("Return true for replaced.") + return [ True, '\n'.join(lines) ] + else: + #app.logger.info("Return false for replaced.") + return [ False, '\n'.join(lines) ] def save_config(plotter, config): try: # Validate the YAML first @@ -173,12 +198,15 @@ def save_config(plotter, config): flash('Updated plotman.yaml failed validation! Fix and save or refresh page.', 'danger') flash(str(ex), 'warning') try: - utils.send_put(plotter, "/configs/plotting", config, debug=False) + response = utils.send_put(plotter, "/configs/plotting", config, debug=False) except Exception as ex: flash('Failed to save config to plotter. Please check log files.', 'danger') flash(str(ex), 'warning') else: - flash('Nice! Plotman\'s plotman.yaml validated and saved successfully.', 'success') + if response.status_code == 200: + flash('Nice! Plotman\'s plotman.yaml validated and saved successfully.', 'success') + else: + flash("
{0}
".format(response.content.decode('utf-8')), 'danger') def analyze(plot_file, plotters): # Don't know which plotter might have the plot result so try them in-turn @@ -199,4 +227,16 @@ def analyze(plot_file, plotters): app.logger.info("Plotter on {0} returned an unexpected error: {1}".format(plotter.hostname, response.status_code)) except: app.logger.info(traceback.format_exc()) - return make_response("Sorry, not plotting job log found. Perhaps plot was made elsewhere?", 200) \ No newline at end of file + return make_response("Sorry, not plotting job log found. Perhaps plot was made elsewhere?", 200) + +def load_plotting_keys(): + farmer_pk = load_key_pk('Farmer') + pool_pk = load_key_pk('Pool') + pool_contract_address = load_pool_contract_address() + if not farmer_pk: + farmer_pk = None if os.environ['farmer_pk'] == 'null' else os.environ['farmer_pk'] + if not pool_pk: + pool_pk = None if os.environ['pool_pk'] == 'null' else os.environ['pool_pk'] + if not pool_contract_address: + pool_contract_address = None if os.environ['pool_contract_address'] == 'null' else os.environ['pool_contract_address'] + return [farmer_pk, pool_pk, pool_contract_address] diff --git a/web/models/chia.py b/web/models/chia.py index 6a34b47b..16cdb304 100644 --- a/web/models/chia.py +++ b/web/models/chia.py @@ -30,11 +30,11 @@ def __init__(self, farms): if farm.mode == "fullnode": self.status = farm.status fullnode_plots_size = farm.plots_size - self.total_chia = '0.0' if not farm.total_chia else round(farm.total_chia, 3) + self.total_chia = '0.0' if not farm.total_chia else round(farm.total_chia, 6) self.netspace_display_size = '?' if not farm.netspace_size else converters.gib_to_fmt(farm.netspace_size) self.netspace_size = farm.netspace_size self.expected_time_to_win = farm.expected_time_to_win - self.total_flax = '0.0' if not farm.total_flax else round(farm.total_flax, 3) + self.total_flax = '0.0' if not farm.total_flax else round(farm.total_flax, 6) self.flax_netspace_display_size = '?' if not farm.flax_netspace_size else converters.gib_to_fmt(farm.flax_netspace_size) self.flax_netspace_size = farm.flax_netspace_size self.flax_expected_time_to_win = farm.flax_expected_time_to_win @@ -66,14 +66,21 @@ class FarmPlots: def __init__(self, plots): self.columns = ['worker', 'plot_id', 'dir', 'plot', 'create_date', 'size'] self.rows = [] + plots_by_id = {} for plot in plots: - self.rows.append({ \ - 'worker': plot.hostname, \ - 'plot_id': plot.plot_id, \ - 'dir': plot.dir, \ - 'plot': plot.file, \ - 'create_date': plot.created_at, \ - 'size': plot.size }) + if plot.plot_id in plots_by_id: + other_plot = plots_by_id[plot.plot_id] + app.logger.info("Skipping listing of plot on {0} at {1}/{2} because same plot_id found on {3} at {4}/{5}".format( + plot.hostname, plot.dir, plot.file, other_plot.hostname, other_plot.dir, other_plot.file)) + else: # No conflict so add it to plots list + plots_by_id[plot.plot_id] = plot + self.rows.append({ \ + 'worker': plot.hostname, \ + 'plot_id': plot.plot_id, \ + 'dir': plot.dir, \ + 'plot': plot.file, \ + 'create_date': plot.created_at, \ + 'size': plot.size }) class BlockchainChallenges: diff --git a/web/routes.py b/web/routes.py index d4fd954b..2a1711b2 100644 --- a/web/routes.py +++ b/web/routes.py @@ -113,11 +113,14 @@ def plots_check(): def alerts(): gc = globals.load() if request.method == 'POST': - w = worker.get_worker_by_hostname(request.form.get('hostname')) if request.form.get('action') == 'start': + w = worker.get_worker_by_hostname(request.form.get('hostname')) chiadog.start_chiadog(w) elif request.form.get('action') == 'stop': + w = worker.get_worker_by_hostname(request.form.get('hostname')) chiadog.stop_chiadog(w) + elif request.form.get('action') == 'remove': + chiadog.remove_alerts(request.form.getlist('unique_id')) else: app.logger.info("Unknown alerts form: {0}".format(request.form)) return redirect(url_for('alerts')) # Force a redirect to allow time to update status @@ -185,8 +188,6 @@ def settings_plotting(): if request.method == 'POST': selected_worker_hostname = request.form.get('worker') plotman.save_config(worker.get_worker_by_hostname(selected_worker_hostname), request.form.get("config")) - else: - flash('Automatically set your public key values below. Please review and save the config at least once!', 'message') workers_summary = worker.load_worker_summary() selected_worker = find_selected_worker(workers_summary, selected_worker_hostname) return render_template('settings/plotting.html', @@ -245,7 +246,9 @@ def views_settings_config(path): elif config_type == "farming": response = make_response(chia.load_config(w, request.args.get('blockchain')), 200) elif config_type == "plotting": - response = make_response(plotman.load_config(w), 200) + [replaced, config] = plotman.load_config(w) + response = make_response(config, 200) + response.headers.set('ConfigReplacementsOccurred', replaced) else: abort("Unsupported config type: {0}".format(config_type), 400) response.mimetype = "application/x-yaml" @@ -266,7 +269,13 @@ def logfile(): else: abort(500, "Unsupported log type: {0}".format(log_type)) +@app.route('/worker_launch') +def worker_launch(): + [farmer_pk, pool_pk, pool_contract_address] = plotman.load_plotting_keys() + return render_template('worker_launch.html', farmer_pk=farmer_pk, + pool_pk=pool_pk, pool_contract_address=pool_contract_address) + @app.route('/favicon.ico') def favicon(): return send_from_directory(os.path.join(app.root_path, 'static'), - 'favicon.ico', mimetype='image/vnd.microsoft.icon') \ No newline at end of file + 'favicon.ico', mimetype='image/vnd.microsoft.icon') diff --git a/web/static/styles.css b/web/static/styles.css index fd993639..33306e04 100644 --- a/web/static/styles.css +++ b/web/static/styles.css @@ -83,3 +83,13 @@ body { ::-webkit-scrollbar-corner { background: transparent; } + +/** datatables.js pagination footer */ +.page-item.active .page-link { + background-color: lightgrey !important; + border: 1px solid black; +} +.page-link { + color: black !important; +} + diff --git a/web/templates/alerts.html b/web/templates/alerts.html index 5a25c58a..117ca23b 100644 --- a/web/templates/alerts.html +++ b/web/templates/alerts.html @@ -2,27 +2,6 @@ {% block content %} - -
Alerts: Recent Notifications
@@ -54,14 +33,15 @@