Merge pull request #201 from guydavis/integration

v0.5.1

.github/workflows/develop.yaml

@@ -39,7 +39,7 @@ jobs:
           platforms: linux/amd64
           push: true
           build-args: |
-            "CHIA_BRANCH=1.2.0"
+            "CHIA_BRANCH=1.2.2"
             "FLAX_BRANCH=main"
           tags: |
             ${{ secrets.DOCKERHUB_USERNAME }}/machinaris:develop

.github/workflows/main.yaml

@@ -40,7 +40,7 @@ jobs:
           platforms: linux/amd64
           push: true
           build-args: |
-            "CHIA_BRANCH=1.2.0"
+            "CHIA_BRANCH=1.2.2"
             "FLAX_BRANCH=main"
           tags: |
             ${{ secrets.DOCKERHUB_USERNAME }}/machinaris:latest

.github/workflows/test.yaml

@@ -40,7 +40,7 @@ jobs:
           platforms: linux/amd64
           push: true
           build-args: |
-            "CHIA_BRANCH=1.2.0"
+            "CHIA_BRANCH=1.2.2"
             "FLAX_BRANCH=main"
           tags: |
             ${{ secrets.DOCKERHUB_USERNAME }}/machinaris:test

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.5.1] - 2021-07-22
+
+- Wizard on Workers page to create a Docker run/compose based on your settings. [Issue #97](https://github.com/guydavis/machinaris/issues/97)
+- Update to patch release of Chia 1.2.2, including a fix for harvester cache updates.  See their [changelog for details](https://github.com/Chia-Network/chia-blockchain/releases/tag/1.2.2).
+- Latest Madmax plotter with support for n_buckets3 and n_rmulti2 settings in Plotman.
+
 ## [0.5.0] - 2021-07-09
 
 - Support for [official Chia pools](https://github.com/guydavis/machinaris/issues/131). Chia and Madmax plotters can create portable plots.

CREDITS.md

@@ -18,6 +18,7 @@ A huge thank-you to the great teams/devs behind these projects, being used by Ma
 * [Other Icons](https://www.shareicon.net): Images and icons for web apps.
 * [Boostrap Sidebars](https://dev.to/codeply/bootstrap-5-sidebar-examples-38pb): Used for sidebar menu layout.
 * [Toplevel](https://github.com/and-semakin/marshmallow-toplevel): Used for list of objects sent to REST API.
+* [DataTables.js](https://datatables.net/): Filter/search/pagination of dynamic tables, as per [this tutorial](https://blog.miguelgrinberg.com/post/beautiful-interactive-tables-for-your-flask-templates).
 
 ## Blockchain Forks
 * [Flax](https://github.com/Flax-Network/flax-blockchain)

VERSION

@@ -1 +1 @@
-0.5.0
+0.5.1

api/commands/chiadog_cli.py

@@ -15,7 +15,6 @@
 from flask import Flask, jsonify, abort, request, flash, g
 from subprocess import Popen, TimeoutExpired, PIPE
 
-from api.models import chiadog
 from api import app
 
 def load_config(blockchain):
@@ -46,10 +45,6 @@ def get_chiadog_pid(blockchain):
             return proc.info['pid']
     return None
 
-def get_notifications(since):
-    return chiadog.Notification.query.filter(chiadog.Notification.created_at >= since). \
-        order_by(chiadog.Notification.created_at.desc()).limit(20).all()
-
 def dispatch_action(job):
     service = job['service']
     if service != 'monitoring':
@@ -72,6 +67,9 @@ def start_chiadog():
     for blockchain in blockchains:
         try:
             workdir = "/{0}dog".format(blockchain)
+            offset_file = "{0}/debug.log.offset".format(workdir)
+            if os.path.exists(offset_file):
+                os.remove(offset_file)
             configfile = "/root/.chia/{0}dog/config.yaml".format(blockchain)
             logfile = "/root/.chia/{0}dog/logs/{0}dog.log".format(blockchain)
             proc = Popen("nohup /{0}-blockchain/venv/bin/python3 -u main.py --config {1} >> {2} 2>&1 &".format(blockchain, configfile, logfile), \

api/commands/log_parser.py

@@ -39,7 +39,7 @@ def recent_challenges(blockchain):
         app.logger.debug(
             "Skipping challenges parsing as no such log file: {0}".format(log_file))
         return []
-    proc = Popen("grep -i eligible {0} | tail -n {1}".format(log_file, CHALLENGES_TO_LOAD),
+    proc = Popen("grep --text -i eligible {0} | tail -n {1}".format(log_file, CHALLENGES_TO_LOAD),
                  stdout=PIPE, stderr=PIPE, shell=True)
     try:
         outs, errs = proc.communicate(timeout=90)

api/commands/plotman_cli.py

@@ -41,10 +41,9 @@ def load_plotting_summary():
     except TimeoutExpired:
         proc.kill()
         proc.communicate()
-        abort(500, description="The timeout is expired!")
+        raise Exception("The timeout expired during plotman status.")
     if errs:
-        app.logger.error(errs.decode('utf-8'))
-        abort(500, description=errs.decode('utf-8'))
+        raise Exception("Errors during plotman status:\n {0}".format(errs.decode('utf-8')))
     cli_stdout = outs.decode('utf-8')
     return plotman.PlottingSummary(cli_stdout.splitlines(), get_plotman_pid())
 
@@ -76,14 +75,16 @@ def action_plots(job):
     #app.logger.info("About to {0} plots: {1}".format(action, plot_ids))
     for plot_id in plot_ids:
         try:
-            prefix = ""
+            param = ""
             if action == "kill":
-                prefix = "printf 'y\n' |"
+                param = "--force"
             logfile = "/root/.chia/plotman/logs/plotman.log"
             log_fd = os.open(logfile, os.O_RDWR | os.O_CREAT)
             log_fo = os.fdopen(log_fd, "a+")
-            proc = Popen("{0} {1} {2} {3}".format(prefix, PLOTMAN_SCRIPT, action, plot_id),
+            proc = Popen("{0} {1} {2} {3}".format(PLOTMAN_SCRIPT, action, param, plot_id),
                          shell=True, universal_newlines=True, stdout=log_fo, stderr=log_fo)
+            # Plotman regressed on cleaning temp after kill so do it here:
+            clean_tmp_dirs_after_kill(plot_id)
         except:
             app.logger.info('Failed to {0} selected plot {1}.'.format(action, plot_id))
             app.logger.info(traceback.format_exc())
@@ -98,36 +99,52 @@ def get_plotman_pid():
 def start_plotman():
     app.logger.info("Starting Plotman run...")
     check_config()
-    try:
-        if len(load_plotting_summary().rows) == 0:  # No plots running
-            clean_tmp_dirs_before_run()  
-        logfile = "/root/.chia/plotman/logs/plotman.log"
-        proc = Popen("nohup {0} {1} < /dev/tty >> {2} 2>&1 &".format(PLOTMAN_SCRIPT, 'plot', logfile),
-                     shell=True, stdin=DEVNULL, stdout=None, stderr=None, close_fds=True)
-        app.logger.info("Completed launch of plotman.")
-    except:
-        app.logger.info('Failed to start Plotman plotting run!')
-        app.logger.info(traceback.format_exc())
+    if len(load_plotting_summary().rows) == 0:  # No plots running
+        clean_tmp_dirs_before_run()  
+    logfile = "/root/.chia/plotman/logs/plotman.log"
+    proc = Popen("nohup {0} {1} < /dev/tty >> {2} 2>&1 &".format(PLOTMAN_SCRIPT, 'plot', logfile),
+                    shell=True, stdin=DEVNULL, stdout=None, stderr=None, close_fds=True)
+    app.logger.info("Completed launch of plotman.")
 
 def clean_tmp_dirs_before_run():
     try:
         with open("/root/.chia/plotman/plotman.yaml") as f:
             config = yaml.safe_load(f)
-            for tmp_dir in config['directories']['tmp']:
-                app.logger.info("No running plot jobs found so deleting {0}/*.tmp before starting plotman.".format(tmp_dir))
-                for p in pathlib.Path(tmp_dir).glob("*.tmp"):
-                    p.unlink()
+            if 'directories' in config:
+                if 'tmp' in config['directories']:
+                    for tmp_dir in config['directories']['tmp']:
+                        app.logger.info("No running plot jobs found so deleting {0}/*.tmp before starting plotman.".format(tmp_dir))
+                        for p in pathlib.Path(tmp_dir).glob("*.tmp"):
+                            p.unlink()
+                if 'tmp2' in config['directories']:
+                    tmp_dir = config['directories']['tmp2']
+                    app.logger.info("No running plot jobs found so deleting {0}/*.tmp before starting plotman.".format(tmp_dir))
+                    for p in pathlib.Path(tmp_dir).glob("*.tmp"):
+                        p.unlink()
     except Exception as ex:
-        app.logger.info(traceback.format_exc())
-        raise Exception('Updated plotman.yaml failed validation!\n' + str(ex))
+        app.logger.info("Skipping deletion of temp files due to {0}.".format(traceback.format_exc()))
+
+def clean_tmp_dirs_after_kill(plot_id):
+    try:
+        with open("/root/.chia/plotman/plotman.yaml") as f:
+            config = yaml.safe_load(f)
+            if 'directories' in config:
+                if 'tmp' in config['directories']:
+                    for tmp_dir in config['directories']['tmp']:
+                        for p in pathlib.Path(tmp_dir).glob("*{0}*.tmp".format(plot_id)):
+                            app.logger.info("After kill, deleting stale tmp file: {0}".format(p))
+                            p.unlink()
+                if 'tmp2' in config['directories']:
+                    tmp_dir = config['directories']['tmp2']
+                    for p in pathlib.Path(tmp_dir).glob("*{0}*.tmp".format(plot_id)):
+                        app.logger.info("After kill, deleting stale tmp file: {0}".format(p))
+                        p.unlink()
+    except Exception as ex:
+        app.logger.info("Skipping deletion of temp files due to {0}.".format(traceback.format_exc()))
 
 def stop_plotman():
     app.logger.info("Stopping Plotman run...")
-    try:
-        os.kill(get_plotman_pid(), signal.SIGTERM)
-    except:
-        app.logger.info('Failed to stop Plotman plotting run!')
-        app.logger.info(traceback.format_exc())
+    os.kill(get_plotman_pid(), signal.SIGTERM)
 
 def get_archiver_pid():
     for proc in psutil.process_iter(['pid', 'name', 'cmdline']):
@@ -138,49 +155,39 @@ def get_archiver_pid():
 def start_archiver():
     app.logger.info("Starting archiver run...")
     check_config()
-    try:
-        logfile = "/root/.chia/plotman/logs/archiver.log"
-        app.logger.info("About to start archiver...")
-        proc = Popen("nohup {0} {1} < /dev/tty >> {2} 2>&1 &".format(PLOTMAN_SCRIPT, 'archive', logfile),
-                     shell=True, stdin=DEVNULL, stdout=None, stderr=None, close_fds=True)
-        app.logger.info("Completed launch of archiver.")
-    except:
-        app.logger.info('Failed to start Plotman archiving run!')
-        app.logger.info(traceback.format_exc())
+    logfile = "/root/.chia/plotman/logs/archiver.log"
+    app.logger.info("About to start archiver...")
+    proc = Popen("nohup {0} {1} < /dev/tty >> {2} 2>&1 &".format(PLOTMAN_SCRIPT, 'archive', logfile),
+                    shell=True, stdin=DEVNULL, stdout=None, stderr=None, close_fds=True)
+    app.logger.info("Completed launch of archiver.")
 
 def stop_archiver():
     app.logger.info("Stopping Archiver run...")
-    try:
-        os.kill(get_archiver_pid(), signal.SIGTERM)
-    except:
-        app.logger.info('Failed to stop Plotman archiving run!')
-        app.logger.info(traceback.format_exc())
+    os.kill(get_archiver_pid(), signal.SIGTERM)
 
 def load_config():
     return open('/root/.chia/plotman/plotman.yaml','r').read()
 
 def save_config(config):
-    try:
-        # Validate the YAML first
-        yaml.safe_load(config)
-        # Save a copy of the old config file
-        src = "/root/.chia/plotman/plotman.yaml"
-        dst = "/root/.chia/plotman/plotman." + \
-            time.strftime("%Y%m%d-%H%M%S")+".yaml"
-        shutil.copy(src, dst)
-        # Now save the new contents to main config file
-        with open(src, 'w') as writer:
-            writer.write(config)
-    except Exception as ex:
-        app.logger.info(traceback.format_exc())
-        raise Exception('Updated plotman.yaml failed validation!\n' + str(ex))
-    else: # Restart services if running
-        if get_plotman_pid():
-            stop_plotman()
-            start_plotman()
-        if get_archiver_pid():
-            stop_archiver()
-            start_archiver()
+    # Validate the YAML first
+    yaml.safe_load(config)
+    # Save a copy of the old config file
+    src = "/root/.chia/plotman/plotman.yaml"
+    dst = "/root/.chia/plotman/plotman." + \
+        time.strftime("%Y%m%d-%H%M%S")+".yaml"
+    shutil.copy(src, dst)
+    # Now save the new contents to main config file
+    with open(src, 'w') as writer:
+        writer.write(config)
+    # Now try to validate config by calling plotman status
+    load_plotting_summary()
+    # Finally restart plotman and archiver if they are running
+    if get_plotman_pid():
+        stop_plotman()
+        start_plotman()
+    if get_archiver_pid():
+        stop_archiver()
+        start_archiver()
 
 def find_plotting_job_log(plot_id):
     dir_path = '/root/.chia/plotman/logs'

api/default_settings.py

@@ -13,8 +13,7 @@ class DefaultConfig:
     SQLALCHEMY_TRACK_MODIFICATIONS = False
     SQLALCHEMY_DATABASE_URI = 'sqlite:////root/.chia/machinaris/dbs/machinaris.db'
     SQLALCHEMY_BINDS = {
-        'stats':      'sqlite:////root/.chia/machinaris/dbs/stats.db',
-        'chiadog':    'sqlite:////root/.chia/chiadog/dbs/chiadog.db',
+        'stats':      'sqlite:////root/.chia/machinaris/dbs/stats.db'
     }
     SQLALCHEMY_ECHO = True if 'FLASK_ENV' in os.environ and os.environ['FLASK_ENV'] == "development" else False
     ETAG_DISABLED = True # https://flask-smorest.readthedocs.io/en/latest/etag.html

api/migrations/versions/bdfe8db75307_.py

@@ -0,0 +1,85 @@
+"""empty message
+
+Revision ID: bdfe8db75307
+Revises: 51f75e96b994
+Create Date: 2021-07-14 16:11:24.451771
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.sql import func
+
+
+# revision identifiers, used by Alembic.
+revision = 'bdfe8db75307'
+down_revision = '51f75e96b994'
+branch_labels = None
+depends_on = None
+
+
+def upgrade(engine_name):
+    globals()["upgrade_%s" % engine_name]()
+
+
+def downgrade(engine_name):
+    globals()["downgrade_%s" % engine_name]()
+
+
+
+
+
+def upgrade_():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('plots')
+    op.create_table('plots',
+        sa.Column('hostname', sa.String(length=255), primary_key=True),
+        sa.Column('plot_id', sa.String(length=16), primary_key=True),
+        sa.Column('dir', sa.String(length=255), nullable=False),
+        sa.Column('file', sa.String(length=255), nullable=False),
+        sa.Column('size', sa.Integer, nullable=False),
+        sa.Column('created_at', sa.String(length=64), nullable=False),
+        sa.Column('updated_at', sa.DateTime(), onupdate=func.now()),
+        sa.PrimaryKeyConstraint('hostname', 'plot_id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade_():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('plots')
+    op.create_table('plots',
+        sa.Column('plot_id', sa.String(length=16), primary_key=True),
+        sa.Column('hostname', sa.String(length=255), nullable=False),
+        sa.Column('dir', sa.String(length=255), nullable=False),
+        sa.Column('file', sa.String(length=255), nullable=False),
+        sa.Column('size', sa.Integer, nullable=False),
+        sa.Column('created_at', sa.String(length=64), nullable=False),
+        sa.Column('updated_at', sa.DateTime(), onupdate=func.now()),
+        sa.PrimaryKeyConstraint('plot_id')
+    )
+    # ### end Alembic commands ###
+
+
+def upgrade_stats():
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade_stats():
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def upgrade_chiadog():
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade_chiadog():
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+

api/models/chia.py

@@ -15,6 +15,7 @@ class FarmSummary:
     def __init__(self, cli_stdout=None, farm_plots=None):
         if cli_stdout:
             next_line_local_harvester = False
+            self.plot_count = 0
             self.plots_size = None
             for line in cli_stdout:
                 if next_line_local_harvester: