From d3f7cefa33526b0e91fbfbc5648fdd428dd441a2 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Fri, 23 Jul 2021 17:07:06 -0600 Subject: [PATCH 01/19] Summary - Daily Diffs --- common/utils/converters.py | 2 +- config/plotman.sample.yaml | 2 +- dockerfile | 1 - scripts/dev/start-api.sh | 11 ++- scripts/dev/start-web.sh | 9 ++- scripts/setup_databases.sh | 14 +--- scripts/stop_machinaris.sh | 10 +++ web/actions/chia.py | 137 ++++++++++++++++++++----------------- web/actions/stats.py | 108 +++++++++++++++++++++++++++++ web/default_settings.py | 3 + web/models/chia.py | 10 +-- web/routes.py | 6 +- web/templates/index.html | 116 +++++++++++++++++++------------ 13 files changed, 298 insertions(+), 131 deletions(-) create mode 100644 scripts/stop_machinaris.sh create mode 100644 web/actions/stats.py diff --git a/common/utils/converters.py b/common/utils/converters.py index 6dcaa046..5e63ac54 100644 --- a/common/utils/converters.py +++ b/common/utils/converters.py @@ -9,7 +9,7 @@ def sizeof_fmt(num, suffix='B'): for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']: if abs(num) < 1024.0: - return "%3.3f %s%s" % (num, unit, suffix) + return "%2.1f %s%s" % (num, unit, suffix) num /= 1024.0 return "%.3f %s%s" % (num, 'Yi', suffix) diff --git a/config/plotman.sample.yaml b/config/plotman.sample.yaml index adc98c3d..0308f953 100644 --- a/config/plotman.sample.yaml +++ b/config/plotman.sample.yaml @@ -106,7 +106,7 @@ plotting: chia: # The stock plotter: https://github.com/Chia-Network/chia-blockchain k: 32 # k-size of plot, leave at 32 most of the time - e: False # Use -e plotting option + e: False # Disable bitfield back sorting (default is True) n_threads: 2 # Threads per job n_buckets: 128 # Number of buckets to split data into job_buffer: 3389 # Per job memory diff --git a/dockerfile b/dockerfile index b0ea979b..f02b30e2 100644 --- a/dockerfile +++ b/dockerfile @@ -104,7 +104,6 @@ ENV controller_api_port=8927 ENV PATH="${PATH}:/chia-blockchain/venv/bin:/flax-blockchain/venv/bin" ENV TZ=Etc/UTC ENV FLASK_ENV=production -ENV FLASK_APP=/machinaris/main.py ENV XDG_CONFIG_HOME=/root/.chia ENV AUTO_PLOT=false diff --git a/scripts/dev/start-api.sh b/scripts/dev/start-api.sh index 73678dd1..7e222f16 100644 --- a/scripts/dev/start-api.sh +++ b/scripts/dev/start-api.sh @@ -7,14 +7,19 @@ echo 'Starting Machinaris...' mkdir -p /root/.chia/machinaris/logs cd /code/machinaris -LOG_LEVEL='info' -RELOAD='--reload' + +if [ $FLASK_ENV == "development" ]; +then + LOG_LEVEL='debug' +else + LOG_LEVEL='info' +fi # To enable SSL, use the Chia self-signed cert #--certfile=/root/.chia/mainnet/config/ssl/ca/chia_ca.crt \ #--keyfile=/root/.chia/mainnet/config/ssl/ca/chia_ca.key \ -/chia-blockchain/venv/bin/gunicorn ${RELOAD} \ +/chia-blockchain/venv/bin/gunicorn --reload \ --bind 0.0.0.0:8927 --timeout 90 \ --log-level=${LOG_LEVEL} \ --workers=2 \ diff --git a/scripts/dev/start-web.sh b/scripts/dev/start-web.sh index 1cbc9402..b1f0b099 100644 --- a/scripts/dev/start-web.sh +++ b/scripts/dev/start-web.sh @@ -8,10 +8,17 @@ echo 'Starting Machinaris...' mkdir -p /root/.chia/machinaris/logs cd /code/machinaris +if [ $FLASK_ENV == "development" ]; +then + LOG_LEVEL='debug' +else + LOG_LEVEL='info' +fi + /chia-blockchain/venv/bin/gunicorn \ --reload \ --bind 0.0.0.0:8926 \ --timeout 90 \ - --log-level=info \ + --log-level=$LOG_LEVEL \ --workers=2 \ web:app diff --git a/scripts/setup_databases.sh b/scripts/setup_databases.sh index 86c8c5c2..e46cf35b 100644 --- a/scripts/setup_databases.sh +++ b/scripts/setup_databases.sh @@ -11,7 +11,6 @@ mkdir -p /root/.chia/chiadog/dbs if [[ $1 == "reset" ]]; then mv /root/.chia/machinaris/dbs/machinaris.db /root/.chia/machinaris/dbs/machinaris.db.bak mv /root/.chia/machinaris/dbs/stats.db /root/.chia/machinaris/dbs/stats.db.bak - mv /root/.chia/chiadog/dbs/chiadog.db /root/.chia/chiadog/dbs/chiadog.db.bak fi # If old databases not managed by flask-migrate yet @@ -20,10 +19,6 @@ if [ ! -f /root/.chia/machinaris/dbs/.managed ] && [ -f /root/.chia/machinaris/d rm -f machinaris.db mv stats.db stats.db.old fi -if [ ! -f /root/.chia/chiadog/dbs/.managed ] && [ -f /root/.chia/chiadog/dbs/chiadog.db ]; then - cd /root/.chia/chiadog/dbs - mv chiadog.db chiadog.db.old -fi # Perform database migration, if any cd /machinaris/api @@ -48,10 +43,5 @@ EOF fi touch /root/.chia/machinaris/dbs/.managed -if [ ! -f /root/.chia/chiadog/dbs/.managed ] && [ -f /root/.chia/chiadog/dbs/chiadog.db.old ]; then - sqlite3 /root/.chia/chiadog/dbs/chiadog.db.old < Executed at: {0}\n".format(time.strftime("%Y%m%d-%H%M%S"))) @@ -342,6 +343,11 @@ def process_pool_leave(choice, wallet_id): flash('Error while leaving Chia pool.', 'danger') flash(line, 'warning') return False + except Exception as ex: + app.logger.info(traceback.format_exc()) + print(str(child)) + flash(str(ex), 'danger') + return False time.sleep(15) try: # Trigger a status update requests.get("http://localhost:8927/plotnfts/", timeout=5) @@ -356,6 +362,8 @@ def process_pool_join(choice, pool_url, pool_wallet_id): try: if not pool_url.strip(): raise Exception("Empty pool URL provided.") + if not pool_url.startswith('https://') and not pool_url.startswith('http://'): + pool_url = "https://" + pool_url result = urllib.parse.urlparse(pool_url) if result.scheme != 'https': raise Exception("Non-HTTPS scheme provided.") @@ -367,37 +375,42 @@ def process_pool_join(choice, pool_url, pool_wallet_id): return False if pool_wallet_id: # Just joining a pool with existing NFT cmd = "{0} plotnft join -y -u {1} -i {2}".format(CHIA_BINARY, pool_url, pool_wallet_id) + wallet_index = pool_wallet_id else: # Both creating NFT and joining pool in one setp cmd = "{0} plotnft create -y -u {1} -s pool".format(CHIA_BINARY, pool_url) + wallet_index = 1 app.logger.info("Executing: {0}".format(cmd)) - proc = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) + result = "" try: - outs, errs = proc.communicate(timeout=90) - except TimeoutExpired: - proc.kill() - proc.communicate() + child = pexpect.spawn(cmd) + child.logfile = sys.stdout.buffer + while True: + i = child.expect(["Choose wallet key:.*\r\n", pexpect.EOF]) + if i == 0: + app.logger.info("plotnft got index prompt so selecting #{0}".format(wallet_index)) + child.sendline("{0}".format(wallet_index)) + elif i==1: + app.logger.info("plotnft end of output...") + result += child.before.decode("utf-8") + child.read().decode("utf-8") + break + if result: # Chia outputs their errors to stdout, not stderr, so must check. + stdout_lines = result.splitlines() + out_file = '/root/.chia/mainnet/log/plotnft.log' + with open(out_file, 'a') as f: + f.write("\n{0} --> Executed at: {1}\n".format(cmd, time.strftime("%Y%m%d-%H%M%S"))) + for line in stdout_lines: + f.write(line) + f.write("\n**********************************************************************\n") + for line in stdout_lines: + if "Error" in line: + flash('Error while joining Chia pool. Please double-check pool URL: {0}'.format(pool_url), 'danger') + flash(line, 'warning') + return False + except Exception as ex: app.logger.info(traceback.format_exc()) - flash('Timed out while joining Chia pool!', 'danger') - flash(str(ex), 'warning') - return False - if errs: - app.logger.info("{0}".format(errs.decode('utf-8'))) - flash('Error while joining Chia pool. Please double-check pool URL: {0}'.format(pool_url), 'danger') - flash(errs.decode('utf-8'), 'warning') + print(str(child)) + flash(str(ex), 'danger') return False - if outs: # Chia outputs their errors to stdout, not stderr, so must check. - stdout_lines = outs.decode('utf-8').splitlines() - out_file = '/root/.chia/mainnet/log/plotnft.log' - with open(out_file, 'a') as f: - f.write("\n{0} --> Executed at: {1}\n".format(cmd, time.strftime("%Y%m%d-%H%M%S"))) - for line in stdout_lines: - f.write(line) - f.write("\n**********************************************************************\n") - for line in stdout_lines: - if "Error" in line: - flash('Error while joining Chia pool. Please double-check pool URL: {0}'.format(pool_url), 'danger') - flash(line, 'warning') - return False time.sleep(15) try: # Trigger a status update requests.get("http://localhost:8927/plotnfts/", timeout=5) @@ -408,26 +421,23 @@ def process_pool_join(choice, pool_url, pool_wallet_id): return True def process_self_pool(): - app.logger.info("Attempting to create NFT for self-pooling.") cmd = "{0} plotnft create -y -s local".format(CHIA_BINARY) - app.logger.info("Executing: {0}".format(cmd)) - proc = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) + app.logger.info("Attempting to create NFT for self-pooling. {0}".format(cmd)) + result = "" try: - outs, errs = proc.communicate(timeout=90) - except TimeoutExpired: - proc.kill() - proc.communicate() - app.logger.info(traceback.format_exc()) - flash('Timed out while creating NFT!', 'danger') - flash(str(ex), 'warning') - return False - if errs: - app.logger.info("{0}".format(errs.decode('utf-8'))) - flash('Error while creating NFT.', 'danger') - flash(errs.decode('utf-8'), 'warning') - return False - if outs: # Chia outputs their errors to stdout, not stderr, so must check. - stdout_lines = outs.decode('utf-8').splitlines() + child = pexpect.spawn(cmd) + child.logfile = sys.stdout.buffer + while True: + i = child.expect(["Choose wallet key:.*\r\n", pexpect.EOF]) + if i == 0: + app.logger.info("plotnft got index prompt so selecting #{0}".format(wallet_index)) + child.sendline("{0}".format(wallet_index)) + elif i==1: + app.logger.info("plotnft end of output...") + result += child.before.decode("utf-8") + child.read().decode("utf-8") + break + if result: # Chia outputs their errors to stdout, not stderr, so must check. + stdout_lines = result.splitlines() out_file = '/root/.chia/mainnet/log/plotnft.log' with open(out_file, 'a') as f: f.write("\n{0} --> Executed at: {1}\n".format(cmd, time.strftime("%Y%m%d-%H%M%S"))) @@ -439,6 +449,11 @@ def process_self_pool(): flash('Error while creating self-pooling NFT', 'danger') flash(line, 'warning') return False + except Exception as ex: + app.logger.info(traceback.format_exc()) + print(str(child)) + flash(str(ex), 'danger') + return False time.sleep(15) try: # Trigger a status update requests.get("http://localhost:8927/plotnfts/", timeout=5) diff --git a/web/actions/stats.py b/web/actions/stats.py new file mode 100644 index 00000000..7dcb9191 --- /dev/null +++ b/web/actions/stats.py @@ -0,0 +1,108 @@ +# +# Access to statistics and calculated values. +# + +import datetime + +from common.utils import converters +from common.models.alerts import Alert +from common.models.stats import StatPlotCount, StatPlotsSize, StatTotalChia, StatNetspaceSize, StatTimeToWin, \ + StatPlotsTotalUsed, StatPlotsDiskUsed, StatPlotsDiskFree, StatPlottingTotalUsed, \ + StatPlottingDiskUsed, StatPlottingDiskFree +from web import app, db, utils + +def load_daily_diff(): + summary = {} + # initialize defaults + since = (datetime.datetime.now() - datetime.timedelta(hours=24)).strftime("%Y%m%d%H%M%S") + summary['plot_count'] = plot_count_diff(since) + summary['plots_size'] = plots_size_diff(since) + summary['total_chia'] = total_coin_diff(since, 'chia') + summary['total_flax'] = total_coin_diff(since, 'flax') + summary['netspace_chia'] = netspace_size_diff(since, 'chia') + summary['netspace_flax'] = netspace_size_diff(since, 'flax') + summary['daily_summary_chia'] = daily_notification(since, 'chia') + summary['daily_summary_flax'] = daily_notification(since, 'flax') + #app.logger.info(summary) + return summary + +def plot_count_diff(since): + result = '-' + try: + latest = db.session.query(StatPlotCount).order_by(StatPlotCount.created_at.desc()).limit(1).first() + #app.logger.info(latest.value) + before = db.session.query(StatPlotCount).filter(StatPlotCount.created_at <= since).order_by(StatPlotCount.created_at.desc()).limit(1).first() + #app.logger.info(before.value) + result = "%+0g" % (latest.value - before.value) + except Exception as ex: + app.logger.info("Failed to query for day diff of plot_count because {0}".format(str(ex))) + #app.logger.info("Result is: {0}".format(result)) + return result + +def plots_size_diff(since): + result = '-' + try: + latest = db.session.query(StatPlotsSize).order_by(StatPlotsSize.created_at.desc()).limit(1).first() + #app.logger.info(latest.value) + before = db.session.query(StatPlotsSize).filter(StatPlotsSize.created_at <= since).order_by(StatPlotsSize.created_at.desc()).limit(1).first() + #app.logger.info(before.value) + gibs = (latest.value - before.value) + fmtted = converters.gib_to_fmt(gibs) + if fmtted == "0.0 B": + result = "+0" + elif not fmtted.startswith('-'): + result = "+{0}".format(fmtted) + else: + result = fmtted + except Exception as ex: + app.logger.info("Failed to query for day diff of plots_size because {0}".format(str(ex))) + #app.logger.info("Result is: {0}".format(result)) + return result + +def total_coin_diff(since, blockchain): + result = '-' + try: + latest = db.session.query(StatTotalChia).filter(StatTotalChia.blockchain==blockchain).order_by(StatTotalChia.created_at.desc()).limit(1).first() + #app.logger.info(latest.value) + before = db.session.query(StatTotalChia).filter(StatTotalChia.blockchain==blockchain, StatTotalChia.created_at <= since).order_by(StatTotalChia.created_at.desc()).limit(1).first() + #app.logger.info(before.value) + result = "%+6g" % (latest.value - before.value) + except Exception as ex: + app.logger.info("Failed to query for day diff of total_chia because {0}".format(str(ex))) + #app.logger.info("Result is: {0}".format(result)) + return result + +def netspace_size_diff(since, blockchain): + result = '-' + try: + latest = db.session.query(StatNetspaceSize).filter(StatNetspaceSize.blockchain==blockchain).order_by(StatNetspaceSize.created_at.desc()).limit(1).first() + #app.logger.info(latest.value) + before = db.session.query(StatNetspaceSize).filter(StatNetspaceSize.blockchain==blockchain, StatNetspaceSize.created_at <= since).order_by(StatNetspaceSize.created_at.desc()).limit(1).first() + #app.logger.info(before.value) + gibs = (latest.value - before.value) + fmtted = converters.gib_to_fmt(gibs) + if fmtted == "0.000 B": + result = "+0" + elif not fmtted.startswith('-'): + result = "+{0}".format(fmtted) + else: + result = fmtted + except Exception as ex: + app.logger.info("Failed to query for day diff of netspace_size because {0}".format(str(ex))) + #app.logger.info("Result is: {0}".format(result)) + return result + +def daily_notification(since, blockchain): + result = '-' + try: + result = db.session.query(Alert).filter( + Alert.blockchain==blockchain, + Alert.created_at >= since, + Alert.priority == "LOW", + Alert.service == "DAILY", + Alert.hostname == "aragorn" + ).order_by(Alert.created_at.desc()).limit(1).first().message + except Exception as ex: + app.logger.info("Failed to query for latest daily summary because {0}".format(str(ex))) + app.logger.info("Result is: {0}".format(result)) + return result \ No newline at end of file diff --git a/web/default_settings.py b/web/default_settings.py index 8b3e3535..9f19d0d4 100644 --- a/web/default_settings.py +++ b/web/default_settings.py @@ -4,6 +4,9 @@ class DefaultConfig: API_TITLE = "Machinaris WEB" SQLALCHEMY_TRACK_MODIFICATIONS = False SQLALCHEMY_DATABASE_URI = 'sqlite:////root/.chia/machinaris/dbs/machinaris.db' + SQLALCHEMY_BINDS = { + 'stats': 'sqlite:////root/.chia/machinaris/dbs/stats.db' + } SQLALCHEMY_ECHO = True if 'FLASK_ENV' in os.environ and os.environ['FLASK_ENV'] == "development" else False CONTROLLER_SCHEME = 'http' CONTROLLER_HOST = os.environ['controller_host'] if 'controller_host' in os.environ else 'localhost' diff --git a/web/models/chia.py b/web/models/chia.py index 16cdb304..7e549abd 100644 --- a/web/models/chia.py +++ b/web/models/chia.py @@ -19,10 +19,10 @@ def __init__(self, farms): self.total_flax = 0 self.netspace_size = 0 self.flax_netspace_size = 0 - self.netspace_display_size = "?" - self.flax_netspace_display_size = "?" - self.expected_time_to_win = "Unknown" - self.flax_expected_time_to_win = "Unknown" + self.netspace_display_size = "-" + self.flax_netspace_display_size = "-" + self.expected_time_to_win = "-" + self.flax_expected_time_to_win = "-" fullnode_plots_size = 0 for farm in farms: self.plot_count += farm.plot_count @@ -59,7 +59,7 @@ def calc_entire_farm_flax_etw(self, fullnode_plots_size, expected_time_to_win, t self.flax_expected_time_to_win = converters.format_minutes(int(total_farm_etw_mins)) except: app.logger.debug("Failed to calculate ETW for entire farm due to: {0}".format(traceback.format_exc())) - self.expected_time_to_win = "Unknown" + self.expected_time_to_win = "-" class FarmPlots: diff --git a/web/routes.py b/web/routes.py index 2a1711b2..ade24c85 100644 --- a/web/routes.py +++ b/web/routes.py @@ -9,7 +9,7 @@ from common.config import globals from web import app, utils -from web.actions import chia, plotman, chiadog, worker, log_handler +from web.actions import chia, plotman, chiadog, worker, log_handler, stats @app.route('/') def landing(): @@ -29,8 +29,10 @@ def index(): farming = chia.load_farm_summary() plotting = plotman.load_plotting_summary() challenges = chia.recent_challenges() + daily_diff = stats.load_daily_diff() return render_template('index.html', reload_seconds=60, farming=farming.__dict__, \ - plotting=plotting.__dict__, challenges=challenges, workers=workers, global_config=gc) + plotting=plotting.__dict__, challenges=challenges, workers=workers, + daily_diff=daily_diff, global_config=gc) @app.route('/views/challenges') def views_challenges(): diff --git a/web/templates/index.html b/web/templates/index.html index 07b37006..c3896093 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -19,7 +19,6 @@
- {% if global_config.plotting_enabled %}
Plotting: @@ -29,8 +28,6 @@
Plotting:
- {% endif %} - {% if global_config.farming_enabled %}
Farming: @@ -40,84 +37,115 @@
Farming:
- {% endif %}
-
+

{{ farming.plot_count }}

Total Plots
- {% if global_config.farming_enabled %} -
+
+
+

{{ daily_diff.plot_count }}

+
In Last Day
+
+
+

{{ farming.plots_display_size }}

Total Plots Size
- {% endif %} -
- - {% if farming.expected_time_to_win != 'Unknown' %} -
-
-
-

Chia - Expected Time to Win: - {{ farming.expected_time_to_win }} -

+
+
+

{{ daily_diff.plots_size }}

+
In Last Day
- {% endif %}
-
+

{{ farming.total_chia }}

-
Chia Total Farmed
+
Chia Farmed
- {% if global_config.farming_enabled %} -
+
-

{{ farming.netspace_display_size }}

-
Chia Netspace Size
+

{{ daily_diff.total_chia }}

+
In Last Day
- {% endif %} -
- - {% if global_config.flax_enabled %} - {% if farming.flax_expected_time_to_win != 'Unknown' %} -
-
-
-

Flax - Expected Time to Win: - {{ farming.flax_expected_time_to_win }} -

+ + {% if global_config.flax_enabled %} +
+
+

{{ farming.total_flax }}

+
Flax Farmed
+
+
+

{{ daily_diff.total_flax }}

+
In Last Day
+
+
+ {% endif %}
- {% endif %} +
-
+
-

{{ farming.total_flax }}

-
Flax Total Farmed
+
{{ daily_diff.daily_summary_chia }}
- {% if global_config.farming_enabled %} -
+
+
+
+
+

{{ farming.netspace_display_size }}

+
Chia Netspace
+
+
+
+
+
+
+

{{ daily_diff.netspace_chia }}

+
In Last Day
+
+
+
+
+ {% if global_config.flax_enabled %} +
-

{{ farming.flax_netspace_display_size }}

-
Flax Netspace Size
+
{{ daily_diff.daily_summary_flax }}
+
+
+
+
+
+
+

{{ farming.flax_netspace_display_size }}

+
Flax Netspace
+
+
+
+
+
+
+

{{ daily_diff.netspace_flax }}

+
In Last Day
+
+
{% endif %}
- {% endif %} {% if global_config.farming_enabled %}
From ad56c2384b4082ce253c8d4c64d5c04c2d456484 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Sat, 24 Jul 2021 09:41:45 -0600 Subject: [PATCH 02/19] Try to build with arm64. --- .github/workflows/develop.yaml | 2 +- web/actions/stats.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/develop.yaml b/.github/workflows/develop.yaml index 1a729a5c..730ae135 100644 --- a/.github/workflows/develop.yaml +++ b/.github/workflows/develop.yaml @@ -36,7 +36,7 @@ jobs: uses: docker/build-push-action@v2 with: context: . - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64 push: true build-args: | "CHIA_BRANCH=1.2.2" diff --git a/web/actions/stats.py b/web/actions/stats.py index 7dcb9191..06d53004 100644 --- a/web/actions/stats.py +++ b/web/actions/stats.py @@ -92,6 +92,7 @@ def netspace_size_diff(since, blockchain): #app.logger.info("Result is: {0}".format(result)) return result +### TODO - Pull all unique hosts and then parse and combine into one summary def daily_notification(since, blockchain): result = '-' try: From e3dab9ea2225fa43fdb4a5447df916a8d6bf26a5 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Tue, 27 Jul 2021 11:01:02 -0600 Subject: [PATCH 03/19] Move daily summary to Farming page. --- .github/workflows/develop.yaml | 2 +- .github/workflows/main.yaml | 4 +- .github/workflows/test.yaml | 13 ++-- CHANGELOG.md | 9 ++- CREDITS.md | 1 + api/schedules/stats_disk.py | 3 + dockerfile | 2 - scripts/chiadog_install.sh | 10 +-- scripts/patch_chiapos.sh | 18 ----- web/actions/stats.py | 45 +++++++----- web/models/chia.py | 3 +- web/routes.py | 3 +- web/templates/farming.html | 74 ++++++++++++++++++- web/templates/index.html | 126 +++++++++++++-------------------- 14 files changed, 174 insertions(+), 139 deletions(-) delete mode 100644 scripts/patch_chiapos.sh diff --git a/.github/workflows/develop.yaml b/.github/workflows/develop.yaml index 730ae135..e82e4d9d 100644 --- a/.github/workflows/develop.yaml +++ b/.github/workflows/develop.yaml @@ -39,7 +39,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - "CHIA_BRANCH=1.2.2" + "CHIA_BRANCH=1.2.3" "FLAX_BRANCH=main" tags: | ${{ secrets.DOCKERHUB_USERNAME }}/machinaris:develop diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 5bf38741..8e2b05d3 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -37,10 +37,10 @@ jobs: uses: docker/build-push-action@v2 with: context: . - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64 push: true build-args: | - "CHIA_BRANCH=1.2.2" + "CHIA_BRANCH=1.2.3" "FLAX_BRANCH=main" tags: | ${{ secrets.DOCKERHUB_USERNAME }}/machinaris:latest diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 537e8a00..0e597a99 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,10 +1,9 @@ name: test -on: - workflow_dispatch: - inputs: - version: - description: 'Test Version' +on: + push: + branches: + - 'integration' jobs: docker: @@ -37,10 +36,10 @@ jobs: uses: docker/build-push-action@v2 with: context: . - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64 push: true build-args: | - "CHIA_BRANCH=1.2.2" + "CHIA_BRANCH=1.2.3" "FLAX_BRANCH=main" tags: | ${{ secrets.DOCKERHUB_USERNAME }}/machinaris:test diff --git a/CHANGELOG.md b/CHANGELOG.md index 76736ef6..45f175a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,18 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.5.2] - 2021-07-? + +- Machinaris - Daily Farming Summary now available on Farming page for both Chia and Flax. +- Machinaris - Docker images now available for [Apple M1](https://github.com/guydavis/machinaris/issues/43) and [Raspberry Pi OS](https://github.com/guydavis/machinaris/issues/155) architectures. +- Chiadog - Update to dev branch to add [support for parsing partials and solo blocks](https://github.com/martomi/chiadog/pull/268). +- Chia - Update to patch release of 1.2.3. See their [changelog for details](https://github.com/Chia-Network/chia-blockchain/releases/tag/1.2.3). + ## [0.5.1] - 2021-07-22 - Wizard on Workers page to create a Docker run/compose based on your settings. [Issue #97](https://github.com/guydavis/machinaris/issues/97) -- Update to patch release of Chia 1.2.2, including a fix for harvester cache updates. See their [changelog for details](https://github.com/Chia-Network/chia-blockchain/releases/tag/1.2.2). - Latest Madmax plotter with support for n_buckets3 and n_rmulti2 settings in Plotman. +- Update to patch release of Chia 1.2.2, including a fix for harvester cache updates. See their [changelog for details](https://github.com/Chia-Network/chia-blockchain/releases/tag/1.2.2). ## [0.5.0] - 2021-07-09 diff --git a/CREDITS.md b/CREDITS.md index 80f911e7..c7d53e12 100644 --- a/CREDITS.md +++ b/CREDITS.md @@ -52,6 +52,7 @@ A big thanks to all that contributed with dev and test including: * barkcollar * dartec * elexx +* danicraftscz ## Trademark Notice diff --git a/api/schedules/stats_disk.py b/api/schedules/stats_disk.py index 2e6f06b2..9424576e 100644 --- a/api/schedules/stats_disk.py +++ b/api/schedules/stats_disk.py @@ -53,6 +53,9 @@ def store_disk_stats(db, current_datetime, disk_type): cur = db.cursor() disks = globals.get_disks(disk_type) for disk in disks: + if not os.path.exists(disk): + app.logger.info("Skipping disk stat collection for non-existant path: {0}".format(disk)) + continue try: total, used, free = shutil.disk_usage(disk) cur.execute("INSERT INTO stat_{0}_disk_used (hostname, path, value, created_at) VALUES (?,?,?,?)".format(disk_type), diff --git a/dockerfile b/dockerfile index f02b30e2..2ee0503d 100644 --- a/dockerfile +++ b/dockerfile @@ -52,7 +52,6 @@ RUN \ FROM package_stage # Base install of official Chia binaries at the given branch ARG CHIA_BRANCH -ARG PATCH_CHIAPOS ARG FLAX_BRANCH # copy local files @@ -64,7 +63,6 @@ WORKDIR /chia-blockchain # Install Chia, Plotman, Chiadog, Madmax, Flax, Machinaris, etc RUN \ /usr/bin/bash /machinaris/scripts/chia_install.sh ${CHIA_BRANCH} \ - && /usr/bin/bash /machinaris/scripts/patch_chiapos.sh ${PATCH_CHIAPOS} \ && /usr/bin/bash /machinaris/scripts/chiadog_install.sh \ && /usr/bin/bash /machinaris/scripts/plotman_install.sh \ && /usr/bin/bash /machinaris/scripts/madmax_install.sh \ diff --git a/scripts/chiadog_install.sh b/scripts/chiadog_install.sh index 114f8b0d..e40ebcbb 100644 --- a/scripts/chiadog_install.sh +++ b/scripts/chiadog_install.sh @@ -3,17 +3,13 @@ # Installs Chiadog for log monitoring and alerting # +CHIADOG_BRANCH=dev + echo 'Installing Chiadog...' cd / -git clone https://github.com/martomi/chiadog.git - -# Temporary patch for spam about partial proofs -# https://github.com/martomi/chiadog/issues/252#issuecomment-877416135 - -cd /chiadog/src/chia_log/handlers/ -sed -i 's/FoundProofs(),//g' harvester_activity_handler.py +git clone --branch ${CHIADOG_BRANCH} https://github.com/martomi/chiadog.git cd /chia-blockchain/ diff --git a/scripts/patch_chiapos.sh b/scripts/patch_chiapos.sh deleted file mode 100644 index cfc4781c..00000000 --- a/scripts/patch_chiapos.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/env bash -# -# Only if PATCH_CHIAPOS is set, when building the image tagged with ':chiapos' -# Patch for faster plotting, on some systems with newer/fast CPUs -# See https://github.com/guydavis/machinaris/wiki/Releases#chiapos -# - -PATCH_CHIAPOS=$1 -echo "PATCH_CHIAPOS=${PATCH_CHIAPOS}" -if [[ $PATCH_CHIAPOS = 'true' ]]; then - echo 'Patching with Chiapos...' - cd /chia-blockchain - curl -o install_multithreaded_chiapos.sh https://gist.githubusercontent.com/SippieCup/8420c831ffcd74f4c4c3c756d1bda912/raw/4be54e136f3f7c070f320e935e883e5ef4c7141d/install_multithreaded_chiapos.sh - chmod a+x install_multithreaded_chiapos.sh - ./install_multithreaded_chiapos.sh /chia-blockchain -else - echo 'Not patching for chiapos. Leaving Chia binaries as stock for main release.' -fi \ No newline at end of file diff --git a/web/actions/stats.py b/web/actions/stats.py index 06d53004..896ed025 100644 --- a/web/actions/stats.py +++ b/web/actions/stats.py @@ -14,16 +14,14 @@ def load_daily_diff(): summary = {} # initialize defaults - since = (datetime.datetime.now() - datetime.timedelta(hours=24)).strftime("%Y%m%d%H%M%S") - summary['plot_count'] = plot_count_diff(since) - summary['plots_size'] = plots_size_diff(since) - summary['total_chia'] = total_coin_diff(since, 'chia') - summary['total_flax'] = total_coin_diff(since, 'flax') - summary['netspace_chia'] = netspace_size_diff(since, 'chia') - summary['netspace_flax'] = netspace_size_diff(since, 'flax') - summary['daily_summary_chia'] = daily_notification(since, 'chia') - summary['daily_summary_flax'] = daily_notification(since, 'flax') - #app.logger.info(summary) + since_date = datetime.datetime.now() - datetime.timedelta(hours=24) + since_str = since_date.strftime("%Y%m%d%H%M%S") + summary['plot_count'] = plot_count_diff(since_str) + summary['plots_size'] = plots_size_diff(since_str) + summary['total_chia'] = total_coin_diff(since_str, 'chia') + summary['total_flax'] = total_coin_diff(since_str, 'flax') + summary['netspace_chia'] = netspace_size_diff(since_str, 'chia') + summary['netspace_flax'] = netspace_size_diff(since_str, 'flax') return summary def plot_count_diff(since): @@ -92,18 +90,29 @@ def netspace_size_diff(since, blockchain): #app.logger.info("Result is: {0}".format(result)) return result -### TODO - Pull all unique hosts and then parse and combine into one summary -def daily_notification(since, blockchain): - result = '-' +def load_daily_notifications(): + summary = {} + # initialize defaults + since_date = datetime.datetime.now() - datetime.timedelta(hours=24) + summary['daily_summary_chia'] = daily_notifications(since_date, 'chia') + summary['daily_summary_flax'] = daily_notifications(since_date, 'flax') + #app.logger.info(summary) + return summary + +def daily_notifications(since, blockchain): + result = [] try: - result = db.session.query(Alert).filter( + #app.logger.info(since) + dailys = db.session.query(Alert).filter( Alert.blockchain==blockchain, Alert.created_at >= since, Alert.priority == "LOW", - Alert.service == "DAILY", - Alert.hostname == "aragorn" - ).order_by(Alert.created_at.desc()).limit(1).first().message + Alert.service == "DAILY" + ).order_by(Alert.created_at.desc()).all() + for daily in dailys: + #app.logger.info("{0} at {1}".format(daily.hostname, daily.created_at)) + result.append(daily) except Exception as ex: app.logger.info("Failed to query for latest daily summary because {0}".format(str(ex))) - app.logger.info("Result is: {0}".format(result)) + result.sort(key=lambda daily: daily.hostname, reverse=False) return result \ No newline at end of file diff --git a/web/models/chia.py b/web/models/chia.py index 7e549abd..50bffa4e 100644 --- a/web/models/chia.py +++ b/web/models/chia.py @@ -38,6 +38,7 @@ def __init__(self, farms): self.flax_netspace_display_size = '?' if not farm.flax_netspace_size else converters.gib_to_fmt(farm.flax_netspace_size) self.flax_netspace_size = farm.flax_netspace_size self.flax_expected_time_to_win = farm.flax_expected_time_to_win + app.logger.debug("ETW: {0}".format(self.expected_time_to_win)) self.plots_display_size = converters.gib_to_fmt(self.plots_size) self.calc_status(self.status) @@ -59,7 +60,7 @@ def calc_entire_farm_flax_etw(self, fullnode_plots_size, expected_time_to_win, t self.flax_expected_time_to_win = converters.format_minutes(int(total_farm_etw_mins)) except: app.logger.debug("Failed to calculate ETW for entire farm due to: {0}".format(traceback.format_exc())) - self.expected_time_to_win = "-" + self.flax_expected_time_to_win = "-" class FarmPlots: diff --git a/web/routes.py b/web/routes.py index ade24c85..f24302f9 100644 --- a/web/routes.py +++ b/web/routes.py @@ -104,8 +104,9 @@ def farming(): farmers = chia.load_farmers() farming = chia.load_farm_summary() plots = chia.load_plots_farming() + daily_notifications = stats.load_daily_notifications() return render_template('farming.html', farming=farming, plots=plots, - farmers=farmers, global_config=gc) + farmers=farmers, daily_notifications=daily_notifications, global_config=gc) @app.route('/plots_check') def plots_check(): diff --git a/web/templates/farming.html b/web/templates/farming.html index 039f5ce4..006b0f7e 100644 --- a/web/templates/farming.html +++ b/web/templates/farming.html @@ -9,7 +9,7 @@ {% if global_config.flax_enabled %}
Flax ETW: {{ farming.flax_expected_time_to_win }}
{% endif %} - +
+ + + {% if (daily_notifications.daily_summary_chia|length > 0) or (daily_notifications.daily_summary_flax|length > 0) %} +
+ +
+

+ +

+
+
+
+ {% for daily in daily_notifications.daily_summary_chia %} +
+
+
{{ daily.hostname }}
+
{{ + daily.message }}
+
+
+ {% endfor %} +
+
+
+
+ + {% if global_config.flax_enabled %} +
+

+ +

+
+
+
+ {% for daily in daily_notifications.daily_summary_flax %} +
+
+
{{ daily.hostname }}
+
{{ + daily.message + }}
+
+
+ {% endfor %} +
+
+
+
+ {% endif %} + +
+ {% endif %} +
@@ -150,8 +218,8 @@ } $(document).ready(function () { $('#data').DataTable({ - "order": [[ 4, "desc" ]], - "columnDefs": [ { "orderable": false, targets: [6] } ], + "order": [[4, "desc"]], + "columnDefs": [{ "orderable": false, targets: [6] }], }); }); diff --git a/web/templates/index.html b/web/templates/index.html index c3896093..b8b8d4a0 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -40,112 +40,82 @@
Farming:
-
+
-

{{ farming.plot_count }}

+

{{ + farming.plot_count }}

Total Plots
-
-
-

{{ daily_diff.plot_count }}

-
In Last Day
-
-
-
+
-

{{ farming.plots_display_size }}

+

{{ + farming.plots_display_size }}

Total Plots Size
-
-
-

{{ daily_diff.plots_size }}

-
In Last Day
+
+ + {% if farming.expected_time_to_win != '-' %} +
+
+
+

Chia - Expected Time to Win: + {{ farming.expected_time_to_win }} +

+ {% endif %}
-
-
-

{{ farming.total_chia }}

-
Chia Farmed
-
-
-
-
-

{{ daily_diff.total_chia }}

-
In Last Day
-
-
- - {% if global_config.flax_enabled %} -
+
-

{{ farming.total_flax }}

-
Flax Farmed
+

{{ + farming.total_chia }}

+
Chia Farmed
-
-
-

{{ daily_diff.total_flax }}

-
In Last Day
+
+
+

{{ + farming.netspace_display_size }}

+
Chia Netspace
- {% endif %}
-
-
-
-
{{ daily_diff.daily_summary_chia }}
-
-
-
-
-
-
-

{{ farming.netspace_display_size }}

-
Chia Netspace
-
-
-
-
-
-
-

{{ daily_diff.netspace_chia }}

-
In Last Day
-
-
+ {% if global_config.flax_enabled %} + {% if farming.flax_expected_time_to_win != 'Unknown' %} +
+
+
+

Flax - Expected Time to Win: + {{ farming.flax_expected_time_to_win }} +

- {% if global_config.flax_enabled %} -
+
+ {% endif %} +
+
-
{{ daily_diff.daily_summary_flax }}
+

{{ + farming.total_flax }}

+
Flax Farmed
-
-
-
-
-

{{ farming.flax_netspace_display_size }}

-
Flax Netspace
-
-
-
-
-
-
-

{{ daily_diff.netspace_flax }}

-
In Last Day
-
-
+
+
+

{{ + farming.flax_netspace_display_size }}

+
Flax Netspace
- {% endif %}
+ {% endif %} {% if global_config.farming_enabled %}
From 95ef4c2fa9f7b37ee673811d36429db0ce3e7bb6 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Tue, 27 Jul 2021 13:28:22 -0600 Subject: [PATCH 04/19] Cleanups. --- .github/workflows/develop.yaml | 2 +- CHANGELOG.md | 2 +- common/utils/converters.py | 2 +- web/models/chia.py | 2 +- web/templates/index.html | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/develop.yaml b/.github/workflows/develop.yaml index e82e4d9d..bf2ee73a 100644 --- a/.github/workflows/develop.yaml +++ b/.github/workflows/develop.yaml @@ -36,7 +36,7 @@ jobs: uses: docker/build-push-action@v2 with: context: . - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 push: true build-args: | "CHIA_BRANCH=1.2.3" diff --git a/CHANGELOG.md b/CHANGELOG.md index 45f175a1..bab02144 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ All notable changes to this project will be documented in this file. The format - Machinaris - Daily Farming Summary now available on Farming page for both Chia and Flax. - Machinaris - Docker images now available for [Apple M1](https://github.com/guydavis/machinaris/issues/43) and [Raspberry Pi OS](https://github.com/guydavis/machinaris/issues/155) architectures. -- Chiadog - Update to dev branch to add [support for parsing partials and solo blocks](https://github.com/martomi/chiadog/pull/268). +- Chiadog - Update to `dev` branch to add [support for parsing partials and solo blocks](https://github.com/martomi/chiadog/pull/268). - Chia - Update to patch release of 1.2.3. See their [changelog for details](https://github.com/Chia-Network/chia-blockchain/releases/tag/1.2.3). ## [0.5.1] - 2021-07-22 diff --git a/common/utils/converters.py b/common/utils/converters.py index 5e63ac54..6dcaa046 100644 --- a/common/utils/converters.py +++ b/common/utils/converters.py @@ -9,7 +9,7 @@ def sizeof_fmt(num, suffix='B'): for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']: if abs(num) < 1024.0: - return "%2.1f %s%s" % (num, unit, suffix) + return "%3.3f %s%s" % (num, unit, suffix) num /= 1024.0 return "%.3f %s%s" % (num, 'Yi', suffix) diff --git a/web/models/chia.py b/web/models/chia.py index 50bffa4e..93425814 100644 --- a/web/models/chia.py +++ b/web/models/chia.py @@ -12,7 +12,7 @@ class FarmSummary: def __init__(self, farms): - self.status = "Unknown" + self.status = "-" self.plot_count = 0 self.plots_size = 0 self.total_chia = 0 diff --git a/web/templates/index.html b/web/templates/index.html index b8b8d4a0..d19c67bf 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -87,7 +87,7 @@
Chia Netspace
{% if global_config.flax_enabled %} - {% if farming.flax_expected_time_to_win != 'Unknown' %} + {% if farming.flax_expected_time_to_win != '-' %}
From 0d902ca02394ef41e93ecaf8e00f15b6bd7438a6 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Tue, 27 Jul 2021 21:14:52 -0600 Subject: [PATCH 05/19] Display versions for each worker. --- VERSION | 2 +- api/schedules/stats_disk.py | 1 + api/schedules/status_worker.py | 4 ++- common/config/globals.py | 8 ++++-- web/actions/chia.py | 4 +-- web/actions/stats.py | 26 +++++++++-------- web/models/worker.py | 23 +++++++++++++++ web/templates/base.html | 9 ++---- web/templates/index.html | 51 ++++++++++++++++++---------------- web/templates/workers.html | 6 +++- 10 files changed, 84 insertions(+), 50 deletions(-) diff --git a/VERSION b/VERSION index 5d4294b9..2411653a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.5.1 \ No newline at end of file +0.5.2 \ No newline at end of file diff --git a/api/schedules/stats_disk.py b/api/schedules/stats_disk.py index 9424576e..9b44e481 100644 --- a/api/schedules/stats_disk.py +++ b/api/schedules/stats_disk.py @@ -3,6 +3,7 @@ # import datetime +import os import shutil import sqlite3 import socket diff --git a/api/schedules/status_worker.py b/api/schedules/status_worker.py index 7c926fa8..9ab7760c 100644 --- a/api/schedules/status_worker.py +++ b/api/schedules/status_worker.py @@ -23,12 +23,14 @@ def update(): with app.app_context(): try: hostname = utils.get_hostname() + config = globals.load(); + del config['now'] payload = { "hostname": hostname, "mode": os.environ['mode'], "services": gather_services_status(), "url": utils.get_remote_url(), - "config": json.dumps(globals.load()), + "config": json.dumps(config), } utils.send_post('/workers/', payload, debug=False) except: diff --git a/common/config/globals.py b/common/config/globals.py index 21a63733..92391aa3 100644 --- a/common/config/globals.py +++ b/common/config/globals.py @@ -194,8 +194,8 @@ def load_plotman_version(): last_plotman_version = outs.decode('utf-8').strip() if last_plotman_version.startswith('plotman'): last_plotman_version = last_plotman_version[len('plotman'):].strip() - if last_plotman_version.endswith('+dev'): - last_plotman_version = last_plotman_version[:-len('+dev')].strip() + #if last_plotman_version.endswith('+dev'): + # last_plotman_version = last_plotman_version[:-len('+dev')].strip() last_plotman_version_load_time = datetime.datetime.now() return last_plotman_version @@ -223,6 +223,8 @@ def load_chiadog_version(): last_chiadog_version = outs.decode('utf-8').strip() if last_chiadog_version.startswith('v'): last_chiadog_version = last_chiadog_version[len('v'):].strip() + if '-' in last_chiadog_version: + last_chiadog_version = last_chiadog_version.split('-')[0] + '+dev' last_chiadog_version_load_time = datetime.datetime.now() return last_chiadog_version @@ -305,7 +307,7 @@ def load_flax_version(): if last_flax_version and last_flax_version_load_time >= \ (datetime.datetime.now() - datetime.timedelta(days=RELOAD_MINIMUM_DAYS)): return last_flax_version - proc = Popen("{0} version".format(CHIA_BINARY), + proc = Popen("{0} version".format(FLAX_BINARY), stdout=PIPE, stderr=PIPE, shell=True) try: outs, errs = proc.communicate(timeout=90) diff --git a/web/actions/chia.py b/web/actions/chia.py index 16161908..6c3177ce 100644 --- a/web/actions/chia.py +++ b/web/actions/chia.py @@ -42,8 +42,8 @@ def load_plots_farming(): return FarmPlots(plots) def recent_challenges(): - minute_ago = (datetime.datetime.now() - datetime.timedelta(seconds=80)).strftime("%Y-%m-%d %H:%M:%S.000") - challenges = db.session.query(c.Challenge).filter(c.Challenge.created_at >= minute_ago).order_by(c.Challenge.created_at.desc()) + five_minutes_ago = (datetime.datetime.now() - datetime.timedelta(minutes=5)).strftime("%Y-%m-%d %H:%M:%S.000") + challenges = db.session.query(c.Challenge).filter(c.Challenge.created_at >= five_minutes_ago).order_by(c.Challenge.created_at.desc()).limit(20) return BlockchainChallenges(challenges) def load_wallets(): diff --git a/web/actions/stats.py b/web/actions/stats.py index 896ed025..0aee2143 100644 --- a/web/actions/stats.py +++ b/web/actions/stats.py @@ -25,20 +25,21 @@ def load_daily_diff(): return summary def plot_count_diff(since): - result = '-' + result = '' try: latest = db.session.query(StatPlotCount).order_by(StatPlotCount.created_at.desc()).limit(1).first() #app.logger.info(latest.value) before = db.session.query(StatPlotCount).filter(StatPlotCount.created_at <= since).order_by(StatPlotCount.created_at.desc()).limit(1).first() #app.logger.info(before.value) - result = "%+0g" % (latest.value - before.value) + if (latest.value - before.value) != 0: + result = "%+0g in last day." % (latest.value - before.value) except Exception as ex: app.logger.info("Failed to query for day diff of plot_count because {0}".format(str(ex))) #app.logger.info("Result is: {0}".format(result)) return result def plots_size_diff(since): - result = '-' + result = '' try: latest = db.session.query(StatPlotsSize).order_by(StatPlotsSize.created_at.desc()).limit(1).first() #app.logger.info(latest.value) @@ -46,10 +47,10 @@ def plots_size_diff(since): #app.logger.info(before.value) gibs = (latest.value - before.value) fmtted = converters.gib_to_fmt(gibs) - if fmtted == "0.0 B": - result = "+0" + if fmtted == "0.000 B": + result = "" elif not fmtted.startswith('-'): - result = "+{0}".format(fmtted) + result = "+{0} in last day.".format(fmtted) else: result = fmtted except Exception as ex: @@ -58,20 +59,21 @@ def plots_size_diff(since): return result def total_coin_diff(since, blockchain): - result = '-' + result = '' try: latest = db.session.query(StatTotalChia).filter(StatTotalChia.blockchain==blockchain).order_by(StatTotalChia.created_at.desc()).limit(1).first() #app.logger.info(latest.value) before = db.session.query(StatTotalChia).filter(StatTotalChia.blockchain==blockchain, StatTotalChia.created_at <= since).order_by(StatTotalChia.created_at.desc()).limit(1).first() #app.logger.info(before.value) - result = "%+6g" % (latest.value - before.value) + if (latest.value - before.value) != 0: + result = "%+6g in last day." % (latest.value - before.value) except Exception as ex: app.logger.info("Failed to query for day diff of total_chia because {0}".format(str(ex))) #app.logger.info("Result is: {0}".format(result)) return result def netspace_size_diff(since, blockchain): - result = '-' + result = '' try: latest = db.session.query(StatNetspaceSize).filter(StatNetspaceSize.blockchain==blockchain).order_by(StatNetspaceSize.created_at.desc()).limit(1).first() #app.logger.info(latest.value) @@ -80,11 +82,11 @@ def netspace_size_diff(since, blockchain): gibs = (latest.value - before.value) fmtted = converters.gib_to_fmt(gibs) if fmtted == "0.000 B": - result = "+0" + result = "" elif not fmtted.startswith('-'): - result = "+{0}".format(fmtted) + result = "+{0} in last day.".format(fmtted) else: - result = fmtted + result = "{0} in last day.".format(fmtted) except Exception as ex: app.logger.info("Failed to query for day diff of netspace_size because {0}".format(str(ex))) #app.logger.info("Result is: {0}".format(result)) diff --git a/web/models/worker.py b/web/models/worker.py index 5057fa20..6ddd10ce 100644 --- a/web/models/worker.py +++ b/web/models/worker.py @@ -1,8 +1,11 @@ +import json import os import traceback from datetime import datetime +from common.config import globals + from web import app class WorkerSummary: @@ -25,6 +28,26 @@ def __init__(self, workers): self.farmers_harvesters.append(worker) if worker.mode == "fullnode": self.fullnodes.append(worker) + config = json.loads(worker.config) + worker.versions = {} + if 'machinaris_version' in config: + worker.versions['machinaris'] = config['machinaris_version'] + other_versions = "" + if 'chia_version' in config: + other_versions += "Chia: " + config['chia_version'] + "
" + if 'chiadog_version' in config: + other_versions += "Chiadog: " + config['chiadog_version'] + "
" + gc = globals.load() + if gc['flax_enabled']: + if 'flax_version' in config: + other_versions += "Flax: " + config['flax_version'] + "
" + if 'flaxdog_version' in config: + other_versions += "Flaxdog: " + config['flaxdog_version'] + "
" + if 'madmax_version' in config: + other_versions += "Madmax: " + config['madmax_version'] + "
" + if 'plotman_version' in config: + other_versions += "Plotman: " + config['plotman_version'] + worker.versions['components'] = other_versions def set_ping_response(self, response): self.ping_response = response diff --git a/web/templates/base.html b/web/templates/base.html index d804b956..32c4a232 100644 --- a/web/templates/base.html +++ b/web/templates/base.html @@ -4,9 +4,8 @@ - - + + Machinaris {% if reload_seconds %} @@ -160,9 +159,7 @@
- + diff --git a/web/templates/index.html b/web/templates/index.html index d19c67bf..530059e5 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -42,15 +42,15 @@
Farming:
-

{{ - farming.plot_count }}

+

{{farming.plot_count }}

Total Plots
-

{{ - farming.plots_display_size }}

+

{{farming.plots_display_size }}

Total Plots Size
@@ -71,16 +71,16 @@

Chia - Expected Time to Win:
-

{{ - farming.total_chia }}

+

{{farming.total_chia }}

Chia Farmed
-

{{ - farming.netspace_display_size }}

+

{{farming.netspace_display_size }}

Chia Netspace
@@ -101,16 +101,16 @@

Flax - Expected Time to Win:
-

{{ - farming.total_flax }}

+

{{farming.total_flax }}

Flax Farmed
-

{{ - farming.flax_netspace_display_size }}

+

{{farming.flax_netspace_display_size }}

Flax Netspace
@@ -130,18 +130,21 @@
Flax Netspace
{% block scripts %} {% if global_config.farming_enabled %} {% endif %} {% endblock %} \ No newline at end of file diff --git a/web/templates/workers.html b/web/templates/workers.html index 453e3fb7..7ccb602d 100644 --- a/web/templates/workers.html +++ b/web/templates/workers.html @@ -36,12 +36,13 @@ - + + @@ -53,6 +54,8 @@ + {% endfor %} @@ -107,6 +110,7 @@
For more, see the Machinaris '); $("#workers-form").submit(); }); + $('[data-toggle="tooltip"]').tooltip(); }) {% endblock %} \ No newline at end of file From a13e686f7df72a3a890d8f4fd3d584e9ce7cf45d Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Thu, 29 Jul 2021 20:55:35 -0600 Subject: [PATCH 06/19] Pools page for status. --- README.md | 4 +- api/commands/chia_cli.py | 1 + api/gunicorn.conf.py | 5 +- api/migrations/versions/5ea0ec1d8711_.py | 62 +++++++++++++++++++ api/rpc/chia.py | 53 +++++++++++++++++ api/schedules/status_points.py | 44 ++++++++++++++ api/schedules/status_pools.py | 53 +++++++++++++++++ api/schedules/status_worker.py | 4 +- api/utils.py | 3 + api/views/__init__.py | 2 + api/views/pools/__init__.py | 1 + api/views/pools/resources.py | 73 +++++++++++++++++++++++ api/views/pools/schemas.py | 27 +++++++++ api/views/workers/resources.py | 2 + common/models/__init__.py | 1 + common/models/pools.py | 18 ++++++ common/models/workers.py | 1 + config/plotman.sample.yaml | 2 +- scripts/chiadog_install.sh | 2 +- web/actions/chia.py | 11 +++- web/actions/plotman.py | 5 +- web/actions/worker.py | 2 + web/models/chia.py | 41 +++++++++++++ web/routes.py | 6 ++ web/templates/alerts.html | 2 +- web/templates/base.html | 6 ++ web/templates/farming.html | 2 +- web/templates/plotting.html | 2 +- web/templates/pools.html | 76 ++++++++++++++++++++++++ web/templates/worker_launch.html | 3 + web/templates/workers.html | 7 ++- 31 files changed, 507 insertions(+), 14 deletions(-) create mode 100644 api/migrations/versions/5ea0ec1d8711_.py create mode 100644 api/rpc/chia.py create mode 100644 api/schedules/status_points.py create mode 100644 api/schedules/status_pools.py create mode 100644 api/views/pools/__init__.py create mode 100644 api/views/pools/resources.py create mode 100644 api/views/pools/schemas.py create mode 100644 common/models/pools.py create mode 100644 web/templates/pools.html diff --git a/README.md b/README.md index e7541327..e71addd9 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # machinaris -A Docker image for plotting and farming the Chia™ cryptocurrency on [one computer](https://github.com/guydavis/machinaris/wiki/Docker) or across [many](https://github.com/guydavis/machinaris/wiki/Workers). Now with [official Pool support](https://github.com/guydavis/machinaris/wiki/Pooling)! +A Docker image for plotting and farming the Chia™ cryptocurrency on [one computer](https://github.com/guydavis/machinaris/wiki/Docker) or across [many](https://github.com/guydavis/machinaris/wiki/Workers). Try the easy install using [Launch Wizard](https://machinaris.app). ![Home](https://raw.githubusercontent.com/guydavis/machinaris-unraid/master/docs/img/machinaris_home.png) -To get started with Machinaris, follow an install guide for your platform: [Windows](https://github.com/guydavis/machinaris/wiki/Windows), [Linux](https://github.com/guydavis/machinaris/wiki/Linux), [Macintosh](https://github.com/guydavis/machinaris/wiki/MacOS), [Unraid](https://github.com/guydavis/machinaris/wiki/Unraid), and [others](https://github.com/guydavis/machinaris/wiki/Docker). +For details, see your particular platform: [Windows](https://github.com/guydavis/machinaris/wiki/Windows), [Linux](https://github.com/guydavis/machinaris/wiki/Linux), [Macintosh](https://github.com/guydavis/machinaris/wiki/MacOS), [Unraid](https://github.com/guydavis/machinaris/wiki/Unraid), and [others](https://github.com/guydavis/machinaris/wiki/Docker). ## Plotting View diff --git a/api/commands/chia_cli.py b/api/commands/chia_cli.py index 801213a6..c4f89a42 100644 --- a/api/commands/chia_cli.py +++ b/api/commands/chia_cli.py @@ -2,6 +2,7 @@ # CLI interactions with the chia binary. # +import asyncio import datetime import os import pexpect diff --git a/api/gunicorn.conf.py b/api/gunicorn.conf.py index f6819ddf..a1fc2449 100644 --- a/api/gunicorn.conf.py +++ b/api/gunicorn.conf.py @@ -9,7 +9,8 @@ def on_starting(server): from api import app from api.schedules import status_worker, status_farm, status_plotting, \ status_plots, status_challenges, status_wallets, status_blockchains, \ - status_connections, status_keys, status_alerts, status_controller, status_plotnfts + status_connections, status_keys, status_alerts, status_controller, \ + status_plotnfts, status_points, status_pools from api.schedules import stats_disk, stats_farm scheduler = BackgroundScheduler() @@ -35,6 +36,8 @@ def on_starting(server): scheduler.add_job(func=status_connections.update, trigger='interval', seconds=60, jitter=30) scheduler.add_job(func=status_keys.update, trigger='interval', seconds=60, jitter=30) scheduler.add_job(func=status_alerts.update, trigger='interval', seconds=60, jitter=30) + scheduler.add_job(func=status_pools.update, trigger='interval', seconds=60, jitter=30) + #scheduler.add_job(func=status_points.update, trigger='interval', seconds=10, jitter=0) app.logger.debug("Starting background scheduler...") scheduler.start() diff --git a/api/migrations/versions/5ea0ec1d8711_.py b/api/migrations/versions/5ea0ec1d8711_.py new file mode 100644 index 00000000..4d29c011 --- /dev/null +++ b/api/migrations/versions/5ea0ec1d8711_.py @@ -0,0 +1,62 @@ +"""empty message + +Revision ID: 5ea0ec1d8711 +Revises: bdfe8db75307 +Create Date: 2021-07-29 17:21:19.447778 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '5ea0ec1d8711' +down_revision = 'bdfe8db75307' +branch_labels = None +depends_on = None + + +def upgrade(engine_name): + globals()["upgrade_%s" % engine_name]() + + +def downgrade(engine_name): + globals()["downgrade_%s" % engine_name]() + + + + + +def upgrade_(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('pools', + sa.Column('unique_id', sa.String(length=255), nullable=False), + sa.Column('hostname', sa.String(length=255), nullable=False), + sa.Column('blockchain', sa.String(length=64), nullable=True), + sa.Column('launcher_id', sa.String(length=255), nullable=False), + sa.Column('pool_state', sa.String(), nullable=False), + sa.Column('updated_at', sa.String(length=64), nullable=False), + sa.PrimaryKeyConstraint('unique_id') + ) + op.add_column('workers', sa.Column('displayname', sa.String(length=255), nullable=True)) + # ### end Alembic commands ### + + +def downgrade_(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('workers', 'displayname') + op.drop_table('pools') + # ### end Alembic commands ### + + +def upgrade_stats(): + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade_stats(): + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + diff --git a/api/rpc/chia.py b/api/rpc/chia.py new file mode 100644 index 00000000..9aaf212a --- /dev/null +++ b/api/rpc/chia.py @@ -0,0 +1,53 @@ +# +# RPC interactions with Chia +# + +from chia.rpc.full_node_rpc_client import FullNodeRpcClient +from chia.rpc.farmer_rpc_client import FarmerRpcClient +from chia.util.default_root import DEFAULT_ROOT_PATH +from chia.util.ints import uint16 +from chia.util.config import load_config as load_chia_config + +from api import app + +async def get_signage_points(blockchain): + config = load_chia_config(DEFAULT_ROOT_PATH, 'config.yaml') + farmer_rpc_port = config["farmer"]["rpc_port"] + farmer = await FarmerRpcClient.create( + 'localhost', uint16(farmer_rpc_port), DEFAULT_ROOT_PATH, config + ) + points = await farmer.get_signage_points() + farmer.close() + await farmer.await_closed() + config = load_chia_config(DEFAULT_ROOT_PATH, 'config.yaml') + full_node_rpc_port = config["full_node"]["rpc_port"] + fullnode = await FullNodeRpcClient.create( + 'localhost', uint16(full_node_rpc_port), DEFAULT_ROOT_PATH, config + ) + for point in points: + sp = point['signage_point'] + signage_point = await fullnode.get_recent_signage_point_or_eos( + sp_hash=sp['challenge_chain_sp'], + challenge_hash=sp['challenge_hash']) + app.logger.info(signage_point) + fullnode.close() + await fullnode.await_closed() + return points + +async def get_pool_state(blockchain): + pools = [] + try: + config = load_chia_config(DEFAULT_ROOT_PATH, 'config.yaml') + farmer_rpc_port = config["farmer"]["rpc_port"] + farmer = await FarmerRpcClient.create( + 'localhost', uint16(farmer_rpc_port), DEFAULT_ROOT_PATH, config + ) + result = await farmer.get_pool_state() + farmer.close() + await farmer.await_closed() + if 'pool_state' in result: + for pool in result["pool_state"]: + pools.append(pool) + except Exception as ex: + app.logger.info("Error getting {0} blockchain pool states: {1}".format(blockchain, str(ex))) + return pools diff --git a/api/schedules/status_points.py b/api/schedules/status_points.py new file mode 100644 index 00000000..24ea1b48 --- /dev/null +++ b/api/schedules/status_points.py @@ -0,0 +1,44 @@ +# +# Performs a REST call to controller (possibly localhost) of latest points status. +# + +import asyncio +import datetime +import http +import json +import os +import requests +import socket +import sqlite3 +import traceback + +from flask import g + +from common.config import globals +from api.rpc import chia +from api import app +from api import utils + +def update(): + if not globals.farming_enabled(): + #app.logger.info("Skipping recent signage points collection on non-farming instance.") + return + with app.app_context(): + try: + blockchains = ['chia'] + # Flax doesn't support this yet. + #if globals.flax_enabled(): + # blockchains.append('flax') + for blockchain in blockchains: + hostname = utils.get_hostname() + points = asyncio.run(chia.get_signage_points(blockchain)) + payload = { + "hostname": hostname, + "blockchain": blockchain, + "details": points, + } + #app.logger.info(payload) + #utils.send_post('/plotnfts/', payload, debug=False) + except: + app.logger.info("Failed to load and send recent signage points.") + app.logger.info(traceback.format_exc()) diff --git a/api/schedules/status_pools.py b/api/schedules/status_pools.py new file mode 100644 index 00000000..de8350f9 --- /dev/null +++ b/api/schedules/status_pools.py @@ -0,0 +1,53 @@ +# +# Performs a REST call to controller (possibly localhost) of latest pools status. +# + +import asyncio +import datetime +import http +import json +import os +import requests +import socket +import sqlite3 +import traceback + +from flask import g + +from common.config import globals +from api.rpc import chia +from api import app +from api import utils + +def update(): + if not globals.farming_enabled(): + #app.logger.info("Skipping recent pools state collection on non-farming instance.") + return + with app.app_context(): + try: + blockchains = ['chia'] + # Flax doesn't support this yet. + #if globals.flax_enabled(): + # blockchains.append('flax') + for blockchain in blockchains: + payload = [] + hostname = utils.get_hostname() + pools = asyncio.run(chia.get_pool_state(blockchain)) + for pool in pools: + launcher_id = pool['pool_config']['launcher_id'] + if launcher_id.startswith('0x'): + launcher_id = launcher_id[2:] + payload.append({ + "unique_id": hostname + '_' + blockchain + '_' + launcher_id, + "hostname": hostname, + "blockchain": blockchain, + "launcher_id": launcher_id, + "pool_state": json.dumps(pool), + "updated_at": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + }) + #app.logger.info(payload) + response = utils.send_post('/pools/', payload, debug=False) + #app.logger.info(response.content) + except: + app.logger.info("Failed to load and send pools state.") + app.logger.info(traceback.format_exc()) diff --git a/api/schedules/status_worker.py b/api/schedules/status_worker.py index 9ab7760c..05dd564b 100644 --- a/api/schedules/status_worker.py +++ b/api/schedules/status_worker.py @@ -23,10 +23,12 @@ def update(): with app.app_context(): try: hostname = utils.get_hostname() - config = globals.load(); + displayname = utils.get_displayname() + config = globals.load() del config['now'] payload = { "hostname": hostname, + "displayname": displayname, "mode": os.environ['mode'], "services": gather_services_status(), "url": utils.get_remote_url(), diff --git a/api/utils.py b/api/utils.py index 3d3abaa9..a3b278a2 100644 --- a/api/utils.py +++ b/api/utils.py @@ -56,5 +56,8 @@ def get_hostname(): hostname = socket.gethostname() return hostname +def get_displayname(): + return socket.gethostname() + def is_controller(): return app.config['CONTROLLER_HOST'] == "localhost" diff --git a/api/views/__init__.py b/api/views/__init__.py index d62166c4..3424fe9c 100644 --- a/api/views/__init__.py +++ b/api/views/__init__.py @@ -12,6 +12,7 @@ from . import plotnfts from . import plots from . import plottings +from . import pools from . import wallets from . import workers @@ -31,6 +32,7 @@ plotnfts, plots, plottings, + pools, wallets, workers, ) diff --git a/api/views/pools/__init__.py b/api/views/pools/__init__.py new file mode 100644 index 00000000..172d2b79 --- /dev/null +++ b/api/views/pools/__init__.py @@ -0,0 +1 @@ +from .resources import blp # noqa \ No newline at end of file diff --git a/api/views/pools/resources.py b/api/views/pools/resources.py new file mode 100644 index 00000000..eb416ece --- /dev/null +++ b/api/views/pools/resources.py @@ -0,0 +1,73 @@ +import datetime as dt + +from flask.views import MethodView + +from api import app +from api.extensions.api import Blueprint, SQLCursorPage +from common.extensions.database import db +from common.models import Pool + +from .schemas import PoolSchema, PoolQueryArgsSchema, BatchOfPoolSchema, BatchOfPoolQueryArgsSchema + + +blp = Blueprint( + 'Pool', + __name__, + url_prefix='/pools', + description="Operations on all pools recorded on farmer" +) + + +@blp.route('/') +class Pools(MethodView): + + @blp.etag + @blp.arguments(BatchOfPoolQueryArgsSchema, location='query') + @blp.response(200, PoolSchema(many=True)) + @blp.paginate(SQLCursorPage) + def get(self, args): + ret = Pool.query.filter_by(**args) + return ret + + @blp.etag + @blp.arguments(BatchOfPoolSchema) + @blp.response(201, PoolSchema(many=True)) + def post(self, new_items): + if len(new_items) == 0: + return "No pools provided.", 400 + db.session.query(Pool).filter(Pool.hostname==new_items[0]['hostname']).delete() + items = [] + for new_item in new_items: + item = Pool(**new_item) + items.append(item) + db.session.add(item) + db.session.commit() + return items + + +@blp.route('//') +class PoolByHostname(MethodView): + + @blp.etag + @blp.response(200, PoolSchema) + def get(self, hostname, blockchain): + return db.session.query(Pool).filter(Pool.hostname==hostname, Pool.blockchain==blockchain) + + @blp.etag + @blp.arguments(BatchOfPoolSchema) + @blp.response(200, PoolSchema(many=True)) + def put(self, new_items, hostname, blockchain): + db.session.query(Pool).filter(Pool.hostname==hostname, Pool.blockchain==blockchain).delete() + items = [] + for new_item in new_items: + item = Pool(**new_item) + items.append(item) + db.session.add(item) + db.session.commit() + return items + + @blp.etag + @blp.response(204) + def delete(self, hostname, blockchain): + db.session.query(Pool).filter(Pool.hostname==hostname, Pool.blockchain==blockchain).delete() + db.session.commit() diff --git a/api/views/pools/schemas.py b/api/views/pools/schemas.py new file mode 100644 index 00000000..f58addbc --- /dev/null +++ b/api/views/pools/schemas.py @@ -0,0 +1,27 @@ +import marshmallow as ma +from marshmallow_sqlalchemy import field_for +from marshmallow_toplevel import TopLevelSchema + +from api.extensions.api import Schema, AutoSchema +from common.models.pools import Pool + +class PoolSchema(AutoSchema): + unique_id = field_for(Pool, "unique_id") + + class Meta(AutoSchema.Meta): + table = Pool.__table__ + + +class PoolQueryArgsSchema(Schema): + unique_id = ma.fields.Str() + hostname = ma.fields.Str() + +class BatchOfPoolSchema(TopLevelSchema): + _toplevel = ma.fields.Nested( + PoolSchema, + required=True, + many=True + ) + +class BatchOfPoolQueryArgsSchema(Schema): + hostname = ma.fields.Str() diff --git a/api/views/workers/resources.py b/api/views/workers/resources.py index 540ed323..bd80ddd9 100644 --- a/api/views/workers/resources.py +++ b/api/views/workers/resources.py @@ -33,6 +33,8 @@ def get(self, args): @blp.response(201, WorkerSchema) def post(self, new_item): item = Worker.query.get(new_item['hostname']) + if not 'displayname' in new_item: # Old clients use hostname + new_item['displayname'] = new_item['hostname'] if item: # update new_item['created_at'] = item.created_at new_item['updated_at'] = dt.datetime.now() diff --git a/common/models/__init__.py b/common/models/__init__.py index 8d5723ef..c8979d4e 100644 --- a/common/models/__init__.py +++ b/common/models/__init__.py @@ -7,6 +7,7 @@ from .plots import Plot from .plotnfts import Plotnft from .plottings import Plotting +from .pools import Pool from .stats import StatPlotCount, StatPlotsSize, StatTotalChia, StatNetspaceSize, StatTimeToWin, \ StatPlotsTotalUsed, StatPlotsDiskUsed, StatPlotsDiskFree, StatPlottingTotalUsed, \ StatPlottingDiskUsed, StatPlottingDiskFree diff --git a/common/models/pools.py b/common/models/pools.py new file mode 100644 index 00000000..6f45beb1 --- /dev/null +++ b/common/models/pools.py @@ -0,0 +1,18 @@ +import datetime as dt +import sqlalchemy as sa + +from sqlalchemy.sql import func +from sqlalchemy.orm import relationship, backref + +from common.extensions.database import db + +class Pool(db.Model): + __tablename__ = "pools" + + unique_id = sa.Column(sa.String(length=255), primary_key=True) + hostname = sa.Column(sa.String(length=255), nullable=False) + blockchain = sa.Column(sa.String(length=64), nullable=True) + launcher_id = sa.Column(sa.String(length=255), nullable=False) + pool_state = sa.Column(sa.String, nullable=False) + updated_at = sa.Column(sa.String(length=64), nullable=False) + \ No newline at end of file diff --git a/common/models/workers.py b/common/models/workers.py index bd426f71..15b7cf23 100644 --- a/common/models/workers.py +++ b/common/models/workers.py @@ -11,6 +11,7 @@ class Worker(db.Model): __tablename__ = "workers" hostname = sa.Column(sa.String(length=255), primary_key=True) + displayname = sa.Column(sa.String(length=255), nullable=True) mode = sa.Column(sa.String(length=64), nullable=False) services = sa.Column(sa.String, nullable=False) url = sa.Column(sa.String, nullable=False) diff --git a/config/plotman.sample.yaml b/config/plotman.sample.yaml index 0308f953..576d3971 100644 --- a/config/plotman.sample.yaml +++ b/config/plotman.sample.yaml @@ -96,7 +96,7 @@ plotting: farmer_pk: REPLACE_WITH_THE_REAL_VALUE # ONLY FOR OLD SOLO PLOTS, COMMENT OUT IF PORTABLE PLOTTING!!! pool_pk: REPLACE_WITH_THE_REAL_VALUE - # See 'Settings | Pools' page, for 'P2 singleton address' value, UNCOMMENT IF PORTABLE PLOTTING!!! + # See 'Settings | Pools' page, for 'Pool contract address' value, UNCOMMENT IF PORTABLE PLOTTING!!! #pool_contract_address: REPLACE_WITH_THE_REAL_VALUE # If you enable 'chia', plot in *parallel* with higher tmpdir_max_jobs and global_max_jobs diff --git a/scripts/chiadog_install.sh b/scripts/chiadog_install.sh index e40ebcbb..33ab466a 100644 --- a/scripts/chiadog_install.sh +++ b/scripts/chiadog_install.sh @@ -3,7 +3,7 @@ # Installs Chiadog for log monitoring and alerting # -CHIADOG_BRANCH=dev +CHIADOG_BRANCH=main echo 'Installing Chiadog...' diff --git a/web/actions/chia.py b/web/actions/chia.py index 6c3177ce..a3621190 100644 --- a/web/actions/chia.py +++ b/web/actions/chia.py @@ -24,10 +24,10 @@ from web import app, db, utils from common.models import farms as f, plots as p, challenges as c, wallets as w, \ - blockchains as b, connections as co, keys as k, plotnfts as pn + blockchains as b, connections as co, keys as k, plotnfts as pn, pools as po from common.config import globals from web.models.chia import FarmSummary, FarmPlots, BlockchainChallenges, Wallets, \ - Blockchains, Connections, Keys, Plotnfts + Blockchains, Connections, Keys, Plotnfts, Pools from . import worker as wk CHIA_BINARY = '/chia-blockchain/venv/bin/chia' @@ -66,6 +66,11 @@ def load_plotnfts(): plotnfts = db.session.query(pn.Plotnft).all() return Plotnfts(plotnfts) +def load_pools(): + plotnfts = db.session.query(pn.Plotnft).all() + pools = db.session.query(po.Pool).all() + return Pools(pools, plotnfts) + def load_farmers(): worker_summary = wk.load_worker_summary() farmers = [] @@ -73,11 +78,13 @@ def load_farmers(): if farmer in worker_summary.farmers: farmers.append({ 'hostname': farmer.hostname, + 'displayname': farmer.displayname, 'farming_status': farmer.farming_status().lower() }) elif farmer in worker_summary.harvesters: farmers.append({ 'hostname': farmer.hostname, + 'displayname': farmer.displayname, 'farming_status': 'harvesting' }) return farmers diff --git a/web/actions/plotman.py b/web/actions/plotman.py index aacb1486..7e03cf66 100644 --- a/web/actions/plotman.py +++ b/web/actions/plotman.py @@ -35,6 +35,7 @@ def load_plotters(): for plotter in w.load_worker_summary().plotters: plotters.append({ 'hostname': plotter.hostname, + 'displayname': plotter.displayname, 'plotting_status': plotter.plotting_status(), 'archiving_status': plotter.archiving_status(), 'archiving_enabled': plotter.archiving_enabled() @@ -145,11 +146,11 @@ def load_key_pk(type): def load_pool_contract_address(): plotnfts = c.load_plotnfts() if len(plotnfts.rows) == 1: - m = re.search('P2 singleton address .*: (\w+)'.format(type), plotnfts.rows[0]['details']) + m = re.search('Pool contract address .*: (\w+)'.format(type), plotnfts.rows[0]['details']) if m: return m.group(1) elif len(plotnfts.rows) > 1: - app.logger.info("Did not find a unique P2 singleton address as multiple plotnfts exist. Not replacing in plotman.yaml.") + app.logger.info("Did not find a unique Pool contract address as multiple plotnfts exist. Not replacing in plotman.yaml.") return None def load_config_replacements(): diff --git a/web/actions/worker.py b/web/actions/worker.py index d976f03f..02b7d1e2 100644 --- a/web/actions/worker.py +++ b/web/actions/worker.py @@ -27,8 +27,10 @@ 'connections', 'farms', 'keys', + 'plotnfts', 'plots', 'plottings', + 'pools', 'wallets', 'workers' ] diff --git a/web/models/chia.py b/web/models/chia.py index 93425814..3877ff65 100644 --- a/web/models/chia.py +++ b/web/models/chia.py @@ -1,3 +1,4 @@ +import json import os import traceback @@ -207,3 +208,43 @@ def get_current_pool_url(self): elif "Target state: SELF_POOLING" in line: return None # Switching back to self-pooling, no pool_url return pool_url + +class Pools: + + def __init__(self, pools, plotnfts): + self.columns = ['hostname', 'blockchain', 'pool_state', 'updated_at'] + self.rows = [] + for pool in pools: + launcher_id = pool.launcher_id + plotnft = self.find_plotnft(plotnfts, launcher_id) + updated_at = pool.updated_at or datetime.now() + pool_state = json.loads(pool.pool_state) + if plotnft: + status = self.extract_plotnft_value(plotnft, "Current state:") + points_successful_last_24h = self.extract_plotnft_value(plotnft, "Percent Successful Points (24h)") + else: + status = "-" + pool_errors_24h = len(pool_state['pool_errors_24h']) + points_found_24h = len(pool_state['points_found_24h']) + points_successful_last_24h = "%.2f"% ( (points_found_24h - pool_errors_24h) / points_found_24h * 100) + self.rows.append({ + 'hostname': pool.hostname, + 'launcher_id': pool.launcher_id, + 'blockchain': pool.blockchain, + 'pool_state': pool_state, + 'updated_at': pool.updated_at, + 'status': status, + 'points_successful_last_24h': points_successful_last_24h + }) + + def find_plotnft(self, plotnfts, launcher_id): + for plotnft in plotnfts: + if launcher_id in plotnft.details: + return plotnft + return None + + def extract_plotnft_value(self, plotnft, key): + for line in plotnft.details.splitlines(): + if line.startswith(key): + return line[line.index(':'):].strip() + return None \ No newline at end of file diff --git a/web/routes.py b/web/routes.py index f24302f9..b3a7f3fe 100644 --- a/web/routes.py +++ b/web/routes.py @@ -278,6 +278,12 @@ def worker_launch(): return render_template('worker_launch.html', farmer_pk=farmer_pk, pool_pk=pool_pk, pool_contract_address=pool_contract_address) +@app.route('/pools') +def pools(): + gc = globals.load() + pools = chia.load_pools() + return render_template('pools.html', pools=pools, global_config=gc) + @app.route('/favicon.ico') def favicon(): return send_from_directory(os.path.join(app.root_path, 'static'), diff --git a/web/templates/alerts.html b/web/templates/alerts.html index 117ca23b..40dc4da5 100644 --- a/web/templates/alerts.html +++ b/web/templates/alerts.html @@ -29,7 +29,7 @@
+ {% if worker.hostname == worker.displayname %} + + {% else %} + + {% endif %} From 731be76c5958ebf77f7d9d69dc4f82e12494e0e4 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Fri, 30 Jul 2021 14:02:49 -0600 Subject: [PATCH 07/19] Display recent partial proofs on Summary page. --- api/commands/log_parser.py | 31 ++++++++++ api/gunicorn.conf.py | 6 +- api/migrations/versions/f529fc25f0d6_.py | 61 ++++++++++++++++++++ api/models/log.py | 28 ++++++++- api/schedules/status_partials.py | 44 ++++++++++++++ api/views/__init__.py | 2 + api/views/partials/__init__.py | 1 + api/views/partials/resources.py | 73 ++++++++++++++++++++++++ api/views/partials/schemas.py | 27 +++++++++ common/models/__init__.py | 1 + common/models/partials.py | 19 ++++++ web/__init__.py | 8 ++- web/actions/chia.py | 9 ++- web/actions/chiadog.py | 1 + web/models/chia.py | 18 +++++- web/routes.py | 3 +- web/templates/index.html | 28 ++++++++- 17 files changed, 350 insertions(+), 10 deletions(-) create mode 100644 api/migrations/versions/f529fc25f0d6_.py create mode 100644 api/schedules/status_partials.py create mode 100644 api/views/partials/__init__.py create mode 100644 api/views/partials/resources.py create mode 100644 api/views/partials/schemas.py create mode 100644 common/models/partials.py diff --git a/api/commands/log_parser.py b/api/commands/log_parser.py index 22a96365..c9225b90 100644 --- a/api/commands/log_parser.py +++ b/api/commands/log_parser.py @@ -27,6 +27,9 @@ # Roughly 1 minutes worth of challenges CHALLENGES_TO_LOAD = 8 +# Most recent partial proofs, actually double as 2 log lines per partial +PARTIALS_TO_LOAD = 50 + # When reading tail of a log, only send this many lines MAX_LOG_LINES = 250 @@ -56,6 +59,34 @@ def recent_challenges(blockchain): # app.logger.debug(challenges) return challenges +def recent_partials(blockchain): + log_file = CHIA_LOG + if blockchain == 'flax': + log_file = FLAX_LOG + if not os.path.exists(log_file): + app.logger.debug( + "Skipping partials parsing as no such log file: {0}".format(log_file)) + return [] + rotated_log_file = '' + if os.path.exists(log_file + '.1'): + rotated_log_file = log_file + '.1' + proc = Popen("grep -h --text -C1 -i partial {0} {1} | tail -n {2}".format(rotated_log_file, log_file, PARTIALS_TO_LOAD), + stdout=PIPE, stderr=PIPE, shell=True) + try: + outs, errs = proc.communicate(timeout=90) + except TimeoutExpired: + proc.kill() + proc.communicate() + abort(500, description="The timeout is expired!") + if errs: + app.logger.error(errs.decode('utf-8')) + abort(500, description=errs.decode('utf-8')) + cli_stdout = outs.decode('utf-8') + #app.logger.debug("Partials grep: {0}".format(cli_stdout)) + partials = log.Partials(cli_stdout.splitlines()) + # app.logger.debug(partials) + return partials + def find_plotting_job_log(plot_id): dir_path = '/root/.chia/plotman/logs' diff --git a/api/gunicorn.conf.py b/api/gunicorn.conf.py index a1fc2449..487e085e 100644 --- a/api/gunicorn.conf.py +++ b/api/gunicorn.conf.py @@ -10,7 +10,7 @@ def on_starting(server): from api.schedules import status_worker, status_farm, status_plotting, \ status_plots, status_challenges, status_wallets, status_blockchains, \ status_connections, status_keys, status_alerts, status_controller, \ - status_plotnfts, status_points, status_pools + status_plotnfts, status_points, status_pools, status_partials from api.schedules import stats_disk, stats_farm scheduler = BackgroundScheduler() @@ -36,7 +36,9 @@ def on_starting(server): scheduler.add_job(func=status_connections.update, trigger='interval', seconds=60, jitter=30) scheduler.add_job(func=status_keys.update, trigger='interval', seconds=60, jitter=30) scheduler.add_job(func=status_alerts.update, trigger='interval', seconds=60, jitter=30) - scheduler.add_job(func=status_pools.update, trigger='interval', seconds=60, jitter=30) + scheduler.add_job(func=status_pools.update, trigger='interval', seconds=60, jitter=30) + scheduler.add_job(func=status_partials.update, trigger='interval', seconds=60, jitter=30) + #scheduler.add_job(func=status_points.update, trigger='interval', seconds=10, jitter=0) app.logger.debug("Starting background scheduler...") scheduler.start() diff --git a/api/migrations/versions/f529fc25f0d6_.py b/api/migrations/versions/f529fc25f0d6_.py new file mode 100644 index 00000000..581da917 --- /dev/null +++ b/api/migrations/versions/f529fc25f0d6_.py @@ -0,0 +1,61 @@ +"""empty message + +Revision ID: f529fc25f0d6 +Revises: 5ea0ec1d8711 +Create Date: 2021-07-30 13:38:56.345029 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'f529fc25f0d6' +down_revision = '5ea0ec1d8711' +branch_labels = None +depends_on = None + + +def upgrade(engine_name): + globals()["upgrade_%s" % engine_name]() + + +def downgrade(engine_name): + globals()["downgrade_%s" % engine_name]() + + + + + +def upgrade_(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('partials', + sa.Column('unique_id', sa.String(length=255), nullable=False), + sa.Column('hostname', sa.String(length=255), nullable=False), + sa.Column('blockchain', sa.String(length=64), nullable=False), + sa.Column('launcher_id', sa.String(length=255), nullable=False), + sa.Column('pool_url', sa.String(length=255), nullable=False), + sa.Column('pool_response', sa.String(), nullable=False), + sa.Column('created_at', sa.String(length=64), nullable=False), + sa.PrimaryKeyConstraint('unique_id') + ) + # ### end Alembic commands ### + + +def downgrade_(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('partials') + # ### end Alembic commands ### + + +def upgrade_stats(): + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade_stats(): + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + diff --git a/api/models/log.py b/api/models/log.py index 7ab6756e..d0abe622 100644 --- a/api/models/log.py +++ b/api/models/log.py @@ -10,7 +10,7 @@ class Challenges: - # Parse the provided most recent 5 lines of grepped output for challenges + # Parse the provided most recent lines of grepped output for challenges def __init__(self, cli_stdout): self.columns = [ 'challenge_id', 'plots_past_filter', 'proofs_found', 'time_taken', 'created_at'] self.rows = [] @@ -29,3 +29,29 @@ def __init__(self, cli_stdout): app.logger.info("Failed to parse challenge line: {0}".format(line)) app.logger.info(traceback.format_exc()) self.rows.reverse() + +class Partials: + + # Parse the provided most recent lines for partials. Grep grabs 2 lines (partial submit and response) per. + def __init__(self, cli_stdout): + self.columns = [ 'challenge_id', 'plots_past_filter', 'proofs_found', 'time_taken', 'created_at'] + self.rows = [] + for line in cli_stdout: + try: + if "Submitting partial" in line: + app.logger.debug(line) + created_at = line.split()[0].replace('T', ' ') + launcher_id = re.search('partial for (\w+) to', line, re.IGNORECASE).group(1) + pool_url = re.search('to (.*)$', line, re.IGNORECASE).group(1) + elif "Pool response" in line: + pool_response = line[line.index('{'):] + self.rows.append({ + 'launcher_id': launcher_id, + 'pool_url': pool_url.strip(), + 'pool_response': pool_response, + 'created_at': created_at + }) + except: + app.logger.info("Failed to parse partial line: {0}".format(line)) + app.logger.info(traceback.format_exc()) + self.rows.reverse() diff --git a/api/schedules/status_partials.py b/api/schedules/status_partials.py new file mode 100644 index 00000000..88fd199f --- /dev/null +++ b/api/schedules/status_partials.py @@ -0,0 +1,44 @@ +# +# Performs a REST call to controller (possibly localhost) of latest blockchain partials. +# + +import os +import traceback + +from flask import g + +from common.config import globals +from common.utils import converters +from api import app +from api.commands import log_parser +from api import utils + +def update(): + if not globals.farming_enabled() and not globals.harvesting_enabled(): + #app.logger.info("Skipping recent partials collection on plotting-only instance.") + return + with app.app_context(): + try: + hostname = utils.get_hostname() + blockchains = ['chia'] + if globals.flax_enabled(): + blockchains.append('flax') + payload = [] + for blockchain in blockchains: + recent_partials = log_parser.recent_partials(blockchain) + for partial in recent_partials.rows: + app.logger.debug(partial) + payload.append({ + "unique_id": hostname + '_' + partial['launcher_id'] + '_' + partial['created_at'], + "hostname": hostname, + "blockchain": blockchain, + "launcher_id": partial['launcher_id'], + "pool_url": partial['pool_url'], + "pool_response": partial['pool_response'], + "created_at": partial['created_at'], + }) + app.logger.debug(payload) + utils.send_post('/partials/', payload, debug=False) + except: + app.logger.info("Failed to load recent partials and send.") + app.logger.info(traceback.format_exc()) diff --git a/api/views/__init__.py b/api/views/__init__.py index 3424fe9c..5b368b0c 100644 --- a/api/views/__init__.py +++ b/api/views/__init__.py @@ -8,6 +8,7 @@ from . import farms from . import keys from . import logs +from . import partials from . import ping from . import plotnfts from . import plots @@ -28,6 +29,7 @@ farms, keys, logs, + partials, ping, plotnfts, plots, diff --git a/api/views/partials/__init__.py b/api/views/partials/__init__.py new file mode 100644 index 00000000..172d2b79 --- /dev/null +++ b/api/views/partials/__init__.py @@ -0,0 +1 @@ +from .resources import blp # noqa \ No newline at end of file diff --git a/api/views/partials/resources.py b/api/views/partials/resources.py new file mode 100644 index 00000000..804929a7 --- /dev/null +++ b/api/views/partials/resources.py @@ -0,0 +1,73 @@ +import datetime as dt + +from flask.views import MethodView + +from api import app +from api.extensions.api import Blueprint, SQLCursorPage +from common.extensions.database import db +from common.models import Partial + +from .schemas import PartialSchema, PartialQueryArgsSchema, BatchOfPartialSchema, BatchOfPartialQueryArgsSchema + + +blp = Blueprint( + 'Partial', + __name__, + url_prefix='/partials', + description="Operations on all partials recorded on farmer" +) + + +@blp.route('/') +class Partials(MethodView): + + @blp.etag + @blp.arguments(BatchOfPartialQueryArgsSchema, location='query') + @blp.response(200, PartialSchema(many=True)) + @blp.paginate(SQLCursorPage) + def get(self, args): + ret = Partial.query.filter_by(**args) + return ret + + @blp.etag + @blp.arguments(BatchOfPartialSchema) + @blp.response(201, PartialSchema(many=True)) + def post(self, new_items): + if len(new_items) == 0: + return "No partials provided.", 400 + db.session.query(Partial).filter(Partial.hostname==new_items[0]['hostname']).delete() + items = [] + for new_item in new_items: + item = Partial(**new_item) + items.append(item) + db.session.add(item) + db.session.commit() + return items + + +@blp.route('//') +class PartialByHostname(MethodView): + + @blp.etag + @blp.response(200, PartialSchema) + def get(self, hostname, blockchain): + return db.session.query(Partial).filter(Partial.hostname==hostname, Partial.blockchain==blockchain) + + @blp.etag + @blp.arguments(BatchOfPartialSchema) + @blp.response(200, PartialSchema(many=True)) + def put(self, new_items, hostname, blockchain): + db.session.query(Partial).filter(Partial.hostname==hostname, Partial.blockchain==blockchain).delete() + items = [] + for new_item in new_items: + item = Partial(**new_item) + items.append(item) + db.session.add(item) + db.session.commit() + return items + + @blp.etag + @blp.response(204) + def delete(self, hostname, blockchain): + db.session.query(Partial).filter(Partial.hostname==hostname, Partial.blockchain==blockchain).delete() + db.session.commit() diff --git a/api/views/partials/schemas.py b/api/views/partials/schemas.py new file mode 100644 index 00000000..ab7689b8 --- /dev/null +++ b/api/views/partials/schemas.py @@ -0,0 +1,27 @@ +import marshmallow as ma +from marshmallow_sqlalchemy import field_for +from marshmallow_toplevel import TopLevelSchema + +from api.extensions.api import Schema, AutoSchema +from common.models.partials import Partial + +class PartialSchema(AutoSchema): + unique_id = field_for(Partial, "unique_id") + + class Meta(AutoSchema.Meta): + table = Partial.__table__ + + +class PartialQueryArgsSchema(Schema): + unique_id = ma.fields.Str() + hostname = ma.fields.Str() + +class BatchOfPartialSchema(TopLevelSchema): + _toplevel = ma.fields.Nested( + PartialSchema, + required=True, + many=True + ) + +class BatchOfPartialQueryArgsSchema(Schema): + hostname = ma.fields.Str() diff --git a/common/models/__init__.py b/common/models/__init__.py index c8979d4e..b494b893 100644 --- a/common/models/__init__.py +++ b/common/models/__init__.py @@ -4,6 +4,7 @@ from .connections import Connection from .farms import Farm from .keys import Key +from .partials import Partial from .plots import Plot from .plotnfts import Plotnft from .plottings import Plotting diff --git a/common/models/partials.py b/common/models/partials.py new file mode 100644 index 00000000..d35700b6 --- /dev/null +++ b/common/models/partials.py @@ -0,0 +1,19 @@ +import datetime as dt +import sqlalchemy as sa + +from sqlalchemy.sql import func +from sqlalchemy.orm import relationship, backref + +from common.extensions.database import db + +class Partial(db.Model): + __tablename__ = "partials" + + unique_id = sa.Column(sa.String(length=255), primary_key=True) + hostname = sa.Column(sa.String(length=255), nullable=False) + blockchain = sa.Column(sa.String(length=64), nullable=False) + launcher_id = sa.Column(sa.String(length=255), nullable=False) + pool_url = sa.Column(sa.String(length=255), nullable=False) + pool_response = sa.Column(sa.String, nullable=False) + created_at = sa.Column(sa.String(length=64), nullable=False) + \ No newline at end of file diff --git a/web/__init__.py b/web/__init__.py index 9470d4a2..ba05713f 100644 --- a/web/__init__.py +++ b/web/__init__.py @@ -64,4 +64,10 @@ def plotnameshortener(value): match.group(7)[:20]) return value -app.jinja_env.filters['plotnameshortener'] = plotnameshortener \ No newline at end of file +app.jinja_env.filters['plotnameshortener'] = plotnameshortener + +def launcheridshortener(value): + #app.logger.info("Shorten: {0}".format(value)) + return value[:12] + '...' + +app.jinja_env.filters['launcheridshortener'] = launcheridshortener \ No newline at end of file diff --git a/web/actions/chia.py b/web/actions/chia.py index a3621190..976d6a74 100644 --- a/web/actions/chia.py +++ b/web/actions/chia.py @@ -24,10 +24,11 @@ from web import app, db, utils from common.models import farms as f, plots as p, challenges as c, wallets as w, \ - blockchains as b, connections as co, keys as k, plotnfts as pn, pools as po + blockchains as b, connections as co, keys as k, plotnfts as pn, pools as po, \ + partials as pr from common.config import globals from web.models.chia import FarmSummary, FarmPlots, BlockchainChallenges, Wallets, \ - Blockchains, Connections, Keys, Plotnfts, Pools + Blockchains, Connections, Keys, Plotnfts, Pools, Partials from . import worker as wk CHIA_BINARY = '/chia-blockchain/venv/bin/chia' @@ -46,6 +47,10 @@ def recent_challenges(): challenges = db.session.query(c.Challenge).filter(c.Challenge.created_at >= five_minutes_ago).order_by(c.Challenge.created_at.desc()).limit(20) return BlockchainChallenges(challenges) +def load_partials(): + partials = db.session.query(pr.Partial).order_by(pr.Partial.created_at.desc()).limit(10) + return Partials(partials) + def load_wallets(): wallets = db.session.query(w.Wallet).all() return Wallets(wallets) diff --git a/web/actions/chiadog.py b/web/actions/chiadog.py index 21f8f9ad..8a85215a 100644 --- a/web/actions/chiadog.py +++ b/web/actions/chiadog.py @@ -29,6 +29,7 @@ def load_farmers(): if (farmer in worker_summary.farmers) or (farmer in worker_summary.harvesters): farmers.append({ 'hostname': farmer.hostname, + 'displayname': farmer.displayname, 'monitoring_status': farmer.monitoring_status().lower() }) return farmers diff --git a/web/models/chia.py b/web/models/chia.py index 3877ff65..6aed5ec1 100644 --- a/web/models/chia.py +++ b/web/models/chia.py @@ -135,7 +135,7 @@ def __init__(self, keys): class Blockchains: def __init__(self, blockchains): - self.columns = ['hostname', 'details', 'updated_at'] + self.columns = ['hostname', 'blockchain', 'details', 'updated_at'] self.rows = [] for blockchain in blockchains: self.rows.append({ @@ -143,6 +143,20 @@ def __init__(self, blockchains): 'blockchain': blockchain.blockchain, 'details': blockchain.details, 'updated_at': blockchain.updated_at }) + +class Partials: + + def __init__(self, partials): + self.columns = ['hostname', 'blockchain', 'launcher_id', 'pool_url', 'pool_response', 'created_at'] + self.rows = [] + for partial in partials: + self.rows.append({ + 'hostname': partial.hostname, + 'blockchain': partial.blockchain, + 'launcher_id': partial.launcher_id, + 'pool_url': partial.pool_url, + 'pool_response': partial.pool_response, + 'created_at': partial.created_at }) class Connections: @@ -246,5 +260,5 @@ def find_plotnft(self, plotnfts, launcher_id): def extract_plotnft_value(self, plotnft, key): for line in plotnft.details.splitlines(): if line.startswith(key): - return line[line.index(':'):].strip() + return line[line.index(':')+1:].strip() return None \ No newline at end of file diff --git a/web/routes.py b/web/routes.py index b3a7f3fe..35a43feb 100644 --- a/web/routes.py +++ b/web/routes.py @@ -29,10 +29,11 @@ def index(): farming = chia.load_farm_summary() plotting = plotman.load_plotting_summary() challenges = chia.recent_challenges() + partials = chia.load_partials() daily_diff = stats.load_daily_diff() return render_template('index.html', reload_seconds=60, farming=farming.__dict__, \ plotting=plotting.__dict__, challenges=challenges, workers=workers, - daily_diff=daily_diff, global_config=gc) + daily_diff=daily_diff, partials=partials, global_config=gc) @app.route('/views/challenges') def views_challenges(): diff --git a/web/templates/index.html b/web/templates/index.html index 530059e5..ccfa1837 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -79,7 +79,7 @@
Chia Farmed
-

{{farming.netspace_display_size }}

Chia Netspace
@@ -125,6 +125,32 @@
Flax Netspace
{% endif %} + {% if partials.rows|length > 0 %} +
Hostname System Type Last Status Received At Last Ping to Worker Last Successful Ping to WorkerVersion
{{worker.updated_at | datetimefilter}} {{worker.latest_ping_result}} {{worker.ping_success_at | datetimefilter}}{{worker.versions.machinaris}}
- {{worker.hostname}}{{worker.displayname}}{{worker.displayname}}{{worker.mode}} {{worker.updated_at | datetimefilter}} {{worker.latest_ping_result}}
+ + + + + + + + + + + {% for partial in partials.rows %} + + + + + + + + {% endfor %} + +
BlockchainPoolResponseLauncherDate
{{partial.blockchain}}{{partial.pool_url}}{{partial.pool_response}}{{partial.launcher_id|launcheridshortener}}{{partial.created_at}}
+ + {% endif %} + {% endblock %} {% block scripts %} From 3a02a92e4aa314c4fc23b84a631ede3a246252da Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Sat, 7 Aug 2021 20:16:55 -0600 Subject: [PATCH 08/19] Fix hostname offset. --- web/templates/worker_launch.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/templates/worker_launch.html b/web/templates/worker_launch.html index ac53feb5..f9f91de9 100644 --- a/web/templates/worker_launch.html +++ b/web/templates/worker_launch.html @@ -250,10 +250,10 @@ cmd += " image: ghcr.io/guydavis/machinaris" + line_end; cmd += " container_name: machinaris" + line_end; if (document.getElementById("worker_hostname").value) { - cmd += ' hostname:' + document.getElementById("worker_hostname").value + line_end; + cmd += ' hostname: ' + document.getElementById("worker_hostname").value + line_end; } if (document.getElementById("dns_ip_addr").value) { - cmd += ' dns:' + document.getElementById("dns_ip_addr").value + line_end; + cmd += ' dns: ' + document.getElementById("dns_ip_addr").value + line_end; } cmd += ' restart: always' + line_end; cmd += ' tty: true' + line_end; From ef83b34f2b19eaa39c1cc09b4a03d44f71cd96a5 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Mon, 9 Aug 2021 21:33:37 -0600 Subject: [PATCH 09/19] Show the machine time on worker to expose TZ misconfigurations. Worker Prune fix for displayname. --- api/schedules/status_challenges.py | 2 +- api/schedules/status_controller.py | 4 ++-- api/schedules/status_worker.py | 1 - api/views/workers/resources.py | 2 ++ common/models/workers.py | 2 +- web/__init__.py | 3 +-- web/actions/worker.py | 4 +++- web/models/worker.py | 4 ++++ web/templates/workers.html | 3 ++- 9 files changed, 16 insertions(+), 9 deletions(-) diff --git a/api/schedules/status_challenges.py b/api/schedules/status_challenges.py index a78920df..20938449 100644 --- a/api/schedules/status_challenges.py +++ b/api/schedules/status_challenges.py @@ -19,7 +19,7 @@ def update(): return with app.app_context(): try: - hostname = utils.get_hostname() + hostname = utils.get_displayname() blockchains = ['chia'] if globals.flax_enabled(): blockchains.append('flax') diff --git a/api/schedules/status_controller.py b/api/schedules/status_controller.py index 2d264559..26b79936 100644 --- a/api/schedules/status_controller.py +++ b/api/schedules/status_controller.py @@ -35,13 +35,13 @@ def update(): app.logger.info("Failed to load and send worker status.") def ping_workers(workers): - tz = pytz.timezone('Etc/UTC') for worker in workers: try: #app.logger.info("Pinging worker api endpoint: {0}".format(worker.hostname)) utils.send_get(worker, "/ping/", timeout=3, debug=False) worker.latest_ping_result = "Responding" - worker.ping_success_at = datetime.datetime.now(tz=tz) + worker.updated_at = datetime.datetime.now() + worker.ping_success_at = datetime.datetime.now() except requests.exceptions.ConnectTimeout as ex: app.logger.info(str(ex)) worker.latest_ping_result = "Connection Timeout" diff --git a/api/schedules/status_worker.py b/api/schedules/status_worker.py index 05dd564b..aacc1a48 100644 --- a/api/schedules/status_worker.py +++ b/api/schedules/status_worker.py @@ -25,7 +25,6 @@ def update(): hostname = utils.get_hostname() displayname = utils.get_displayname() config = globals.load() - del config['now'] payload = { "hostname": hostname, "displayname": displayname, diff --git a/api/views/workers/resources.py b/api/views/workers/resources.py index bd80ddd9..dffd5f38 100644 --- a/api/views/workers/resources.py +++ b/api/views/workers/resources.py @@ -1,4 +1,6 @@ import datetime as dt +import pytz +import os from flask.views import MethodView diff --git a/common/models/workers.py b/common/models/workers.py index 15b7cf23..d4bbbf60 100644 --- a/common/models/workers.py +++ b/common/models/workers.py @@ -19,7 +19,7 @@ class Worker(db.Model): latest_ping_result = sa.Column(sa.String) # Holds status of most recent ping ping_success_at = sa.Column(sa.DateTime()) # Time of last successful ping created_at = sa.Column(sa.DateTime(), server_default=func.now()) - updated_at = sa.Column(sa.DateTime(), onupdate=func.now()) + updated_at = sa.Column(sa.DateTime()) def farming_status(self): return j.loads(self.services)['chia_farm_status'] diff --git a/web/__init__.py b/web/__init__.py index ba05713f..02a4f64d 100644 --- a/web/__init__.py +++ b/web/__init__.py @@ -46,9 +46,8 @@ def bytesfilter(num, suffix='B'): app.jinja_env.filters['bytesfilter'] = bytesfilter def datetimefilter(value, format="%Y-%m-%d %H:%M"): - tz = pytz.timezone(os.environ['TZ']) - utc = pytz.timezone('UTC') if value: + #app.logger.info("{0} => {1}".format(value, value.strftime(format))) return value.strftime(format) else: return "" diff --git a/web/actions/worker.py b/web/actions/worker.py index 02b7d1e2..f450afd1 100644 --- a/web/actions/worker.py +++ b/web/actions/worker.py @@ -46,6 +46,8 @@ def get_worker_by_hostname(hostname): def prune_workers_status(hostnames): for hostname in hostnames: + worker = get_worker_by_hostname(hostname) for table in ALL_TABLES_BY_HOSTNAME: - db.session.execute("DELETE FROM " + table + " WHERE hostname = :hostname", {"hostname":hostname}) + db.session.execute("DELETE FROM " + table + " WHERE hostname = :hostname OR hostname = :displayname", + {"hostname":hostname, "displayname":worker.displayname}) db.session.commit() diff --git a/web/models/worker.py b/web/models/worker.py index 6ddd10ce..69e79b8b 100644 --- a/web/models/worker.py +++ b/web/models/worker.py @@ -48,6 +48,10 @@ def __init__(self, workers): if 'plotman_version' in config: other_versions += "Plotman: " + config['plotman_version'] worker.versions['components'] = other_versions + if 'now' in config: + worker.time_on_worker = config['now'] + else: + worker.time_on_worker = '?' def set_ping_response(self, response): self.ping_response = response diff --git a/web/templates/workers.html b/web/templates/workers.html index d53edbde..4553af48 100644 --- a/web/templates/workers.html +++ b/web/templates/workers.html @@ -56,7 +56,8 @@ title="Pinged at: {{worker.hostname}}">{{worker.displayname}} {% endif %} {{worker.mode}} - {{worker.updated_at | datetimefilter}} + {{worker.updated_at | datetimefilter}} {{worker.latest_ping_result}} {{worker.ping_success_at | datetimefilter}} Date: Tue, 10 Aug 2021 17:24:45 -0600 Subject: [PATCH 10/19] Show displayname for Worker column on Alerts page. --- api/views/__init__.py | 2 ++ api/views/certificates/__init__.py | 1 + api/views/certificates/resources.py | 54 +++++++++++++++++++++++++++++ scripts/chia_launch.sh | 18 +++++++--- scripts/flax_launch.sh | 18 +++++++--- web/actions/chiadog.py | 4 ++- web/models/chiadog.py | 26 ++++++++++++++ web/routes.py | 3 ++ web/templates/alerts.html | 4 +-- 9 files changed, 117 insertions(+), 13 deletions(-) create mode 100644 api/views/certificates/__init__.py create mode 100644 api/views/certificates/resources.py create mode 100644 web/models/chiadog.py diff --git a/api/views/__init__.py b/api/views/__init__.py index 5b368b0c..3ccd68dd 100644 --- a/api/views/__init__.py +++ b/api/views/__init__.py @@ -2,6 +2,7 @@ from . import analysis from . import alerts from . import blockchains +from . import certificates from . import challenges from . import configs from . import connections @@ -24,6 +25,7 @@ alerts, blockchains, challenges, + certificates, configs, connections, farms, diff --git a/api/views/certificates/__init__.py b/api/views/certificates/__init__.py new file mode 100644 index 00000000..172d2b79 --- /dev/null +++ b/api/views/certificates/__init__.py @@ -0,0 +1 @@ +from .resources import blp # noqa \ No newline at end of file diff --git a/api/views/certificates/resources.py b/api/views/certificates/resources.py new file mode 100644 index 00000000..d6e8ace4 --- /dev/null +++ b/api/views/certificates/resources.py @@ -0,0 +1,54 @@ +import datetime +import json +import os +import re +import shutil +import time +import traceback + +from flask import request, Response, abort +from flask.views import MethodView + +from api import app +from api.extensions.api import Blueprint + +from api.commands import chia_cli, plotman_cli + +blp = Blueprint( + 'Certificates', + __name__, + url_prefix='/certificates', + description="Certificates to perform" +) + + +@blp.route('/') +class Certificates(MethodView): + + def get(self): + if not self.allow_download(): + abort(401) + blockchain = request.args.get('type') + dir = "/root/.{0}/mainnet/config/ssl/ca".format(blockchain) + zip = "/tmp/certs".format(blockchain) + zipname = "{0}.zip".format(zip) + try: + os.remove(zipname) + except: + pass + shutil.make_archive(zip, 'zip', dir) + with open(zipname, 'rb') as f: + data = f.readlines() + os.remove(zipname) + return Response(data, headers={ + 'Content-Type': 'application/zip', + 'Content-Disposition': 'attachment; filename=certs.zip;' + }) + + def allow_download(self): + worker_setup_marker = "/root/.chia/machinaris/tmp/worker_launch.tmp" + if os.path.exists(worker_setup_marker): + last_modified_date = datetime.datetime.fromtimestamp(os.path.getmtime(worker_setup_marker)) + fifteen_minutes_ago = datetime.datetime.now() - datetime.timedelta(minutes=15) + return last_modified_date >= fifteen_minutes_ago + return False \ No newline at end of file diff --git a/scripts/chia_launch.sh b/scripts/chia_launch.sh index f6d75625..456037fa 100644 --- a/scripts/chia_launch.sh +++ b/scripts/chia_launch.sh @@ -46,15 +46,23 @@ elif [[ ${mode} =~ ^harvester.* ]]; then echo "A farmer peer address and port are required." exit else - if [ -d /root/.chia/farmer_ca ]; then + if [ ! -d /root/.chia/farmer_ca ]; then + mkdir -p /root/.chia/farmer_ca + response=$(curl --write-out '%{http_code}' --silent http://${controller_host}:8927/certificates/?type=chia --output /tmp/certs.zip) + if [ $response == '200' ]; then + unzip /tmp/certs.zip -d /root/.chia/farmer_ca + fi + rm -f /tmp/certs.zip + fi + if [ -f /root/.chia/farmer_ca/chia_ca.crt ]; then chia init -c /root/.chia/farmer_ca 2>&1 > /root/.chia/mainnet/log/init.log + chia configure --set-farmer-peer ${farmer_address}:${farmer_port} + chia configure --enable-upnp false + chia start harvester -r else - echo "Did not find your farmer's ca folder at /root/.chia/farmer_ca." + echo "Did not find your farmer's certificates within /root/.chia/farmer_ca." echo "See: https://github.com/guydavis/machinaris/wiki/Workers#harvester" fi - chia configure --set-farmer-peer ${farmer_address}:${farmer_port} - chia configure --enable-upnp false - chia start harvester -r fi elif [[ ${mode} == 'plotter' ]]; then echo "Starting in Plotter-only mode. Run Plotman from either CLI or WebUI." diff --git a/scripts/flax_launch.sh b/scripts/flax_launch.sh index fe3bb064..aa94ddb8 100644 --- a/scripts/flax_launch.sh +++ b/scripts/flax_launch.sh @@ -56,15 +56,23 @@ elif [[ ${mode} =~ ^harvester.* ]]; then echo "A farmer peer address and port are required." exit else - if [ -d /root/.flax/farmer_ca ]; then + if [ ! -d /root/.flax/farmer_ca ]; then + mkdir -p /root/.flax/farmer_ca + response=$(curl --write-out '%{http_code}' --silent http://${controller_host}:8927/certificates/?type=flax --output /tmp/certs.zip) + if [ $response == '200' ]; then + unzip /tmp/certs.zip -d /root/.flax/farmer_ca + fi + rm -f /tmp/certs.zip + fi + if [ -f /root/.flax/farmer_ca/flax_ca.crt ]; then flax init -c /root/.flax/farmer_ca 2>&1 > /root/.flax/mainnet/log/init.log + flax configure --set-farmer-peer ${farmer_address}:${flax_farmer_port} + flax configure --enable-upnp false + flax start harvester -r else - echo "Did not find your farmer's ca folder at /root/.flax/farmer_ca." + echo "Did not find your farmer's certificates within /root/.flax/farmer_ca." echo "See: https://github.com/guydavis/machinaris/wiki/Workers#harvester" fi - flax configure --set-farmer-peer ${farmer_address}:${flax_farmer_port} - flax configure --enable-upnp false - flax start harvester -r fi elif [[ ${mode} == 'plotter' ]]; then echo "Starting in Plotter-only mode. Run Plotman from either CLI or WebUI." diff --git a/web/actions/chiadog.py b/web/actions/chiadog.py index 8a85215a..f5d621f3 100644 --- a/web/actions/chiadog.py +++ b/web/actions/chiadog.py @@ -17,6 +17,7 @@ from common.models import alerts as a from web import app, db, utils +from web.models.chiadog import Alerts from . import worker as wk def load_config(farmer, blockchain): @@ -50,7 +51,8 @@ def save_config(farmer, blockchain, config): flash('Nice! Chiadog\'s config.yaml validated and saved successfully.', 'success') def get_notifications(): - return db.session.query(a.Alert).order_by(a.Alert.created_at.desc()).all() + alerts = db.session.query(a.Alert).order_by(a.Alert.created_at.desc()).all() + return Alerts(alerts) def remove_alerts(unique_ids): app.logger.info("Removing {0} alerts: {1}".format(len(unique_ids), unique_ids)) diff --git a/web/models/chiadog.py b/web/models/chiadog.py new file mode 100644 index 00000000..baab0dbb --- /dev/null +++ b/web/models/chiadog.py @@ -0,0 +1,26 @@ +import traceback + +from web import app +from web.actions import worker + +class Alerts: + + def __init__(self, alerts): + self.rows = [] + for alert in alerts: + displayname = alert.hostname + try: + w = worker.get_worker_by_hostname(alert.hostname) + displayname = w.displayname + except: + app.logger.info("Failed to find worker for hostname: {0}".format(alert.hostname)) + self.rows.append({ + 'unique_id': alert.unique_id, + 'hostname': alert.hostname, + 'worker': displayname, + 'blockchain': alert.blockchain, + 'service': alert.service, + 'message': alert.message, + 'priority': alert.priority, + 'created_at': alert.created_at + }) \ No newline at end of file diff --git a/web/routes.py b/web/routes.py index 35a43feb..7c25e66c 100644 --- a/web/routes.py +++ b/web/routes.py @@ -1,4 +1,5 @@ +import pathlib import pytz import os import time @@ -276,6 +277,8 @@ def logfile(): @app.route('/worker_launch') def worker_launch(): [farmer_pk, pool_pk, pool_contract_address] = plotman.load_plotting_keys() + pathlib.Path('/root/.chia/machinaris/tmp/').mkdir(parents=True, exist_ok=True) + pathlib.Path('/root/.chia/machinaris/tmp/worker_launch.tmp').touch() return render_template('worker_launch.html', farmer_pk=farmer_pk, pool_pk=pool_pk, pool_contract_address=pool_contract_address) diff --git a/web/templates/alerts.html b/web/templates/alerts.html index 40dc4da5..200ecb58 100644 --- a/web/templates/alerts.html +++ b/web/templates/alerts.html @@ -69,7 +69,7 @@ - {% if notifications|length > 0 %} + {% if notifications.rows|length > 0 %}
@@ -87,7 +87,7 @@ - {% for notification in notifications %} + {% for notification in notifications.rows %} From 22401fe74bec6c95b4362b20e3e2f257d885cc54 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Tue, 10 Aug 2021 21:13:20 -0600 Subject: [PATCH 11/19] Handle missing crt files. --- scripts/chia_launch.sh | 2 +- scripts/flax_launch.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/chia_launch.sh b/scripts/chia_launch.sh index 456037fa..15a0aaf0 100644 --- a/scripts/chia_launch.sh +++ b/scripts/chia_launch.sh @@ -46,7 +46,7 @@ elif [[ ${mode} =~ ^harvester.* ]]; then echo "A farmer peer address and port are required." exit else - if [ ! -d /root/.chia/farmer_ca ]; then + if [ ! -f /root/.chia/farmer_ca/chia_ca.crt ]; then mkdir -p /root/.chia/farmer_ca response=$(curl --write-out '%{http_code}' --silent http://${controller_host}:8927/certificates/?type=chia --output /tmp/certs.zip) if [ $response == '200' ]; then diff --git a/scripts/flax_launch.sh b/scripts/flax_launch.sh index aa94ddb8..01f92b0c 100644 --- a/scripts/flax_launch.sh +++ b/scripts/flax_launch.sh @@ -56,7 +56,7 @@ elif [[ ${mode} =~ ^harvester.* ]]; then echo "A farmer peer address and port are required." exit else - if [ ! -d /root/.flax/farmer_ca ]; then + if [ ! -f /root/.flax/farmer_ca/flax_ca.crt ]; then mkdir -p /root/.flax/farmer_ca response=$(curl --write-out '%{http_code}' --silent http://${controller_host}:8927/certificates/?type=flax --output /tmp/certs.zip) if [ $response == '200' ]; then From fb38fc7ea0dd1f58094b6f4fe455a3ba77cc4807 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Wed, 11 Aug 2021 14:35:05 -0600 Subject: [PATCH 12/19] Require hostname for Worker launch. Links for Pool login and Port checker. --- api/commands/chia_cli.py | 8 ++++ api/migrations/README | 3 +- api/migrations/versions/ce840f016302_.py | 52 ++++++++++++++++++++++++ api/schedules/status_pools.py | 4 ++ common/models/pools.py | 1 + web/models/chia.py | 4 +- web/templates/index.html | 48 ++++++++++++---------- web/templates/network/connections.html | 5 +++ web/templates/pools.html | 8 +++- web/templates/worker_launch.html | 20 ++++----- 10 files changed, 116 insertions(+), 37 deletions(-) create mode 100644 api/migrations/versions/ce840f016302_.py diff --git a/api/commands/chia_cli.py b/api/commands/chia_cli.py index c4f89a42..9bb52171 100644 --- a/api/commands/chia_cli.py +++ b/api/commands/chia_cli.py @@ -314,3 +314,11 @@ def check_plots(first_load): ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') proc = Popen(['tail', '-n', str(MAX_LOG_LINES), output_file], stdout=PIPE) return class_escape.sub('', ansi_escape.sub('', proc.stdout.read().decode("utf-8"))) + +def get_pool_login_link(launcher_id): + try: + stream = os.popen("chia plotnft get_login_link -l {0}".format(launcher_id)) + return stream.read() + except Exception as ex: + app.logger.info("Failed to get_login_link: {0}".format(str(ex))) + return "" diff --git a/api/migrations/README b/api/migrations/README index 2d5a97b2..e39f6839 100644 --- a/api/migrations/README +++ b/api/migrations/README @@ -5,7 +5,8 @@ As developer, modify models, then run this to generate a new migration and commi ``` cd /code/machinaris/api -FLASK_APP=__init__.py flask db migrate +FLASK_APP=__init__.py flask db migrate +chown -R 1000.users . ``` This creates migration based on current model. diff --git a/api/migrations/versions/ce840f016302_.py b/api/migrations/versions/ce840f016302_.py new file mode 100644 index 00000000..e359795a --- /dev/null +++ b/api/migrations/versions/ce840f016302_.py @@ -0,0 +1,52 @@ +"""empty message + +Revision ID: ce840f016302 +Revises: f529fc25f0d6 +Create Date: 2021-08-11 10:15:42.883128 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'ce840f016302' +down_revision = 'f529fc25f0d6' +branch_labels = None +depends_on = None + + +def upgrade(engine_name): + globals()["upgrade_%s" % engine_name]() + + +def downgrade(engine_name): + globals()["downgrade_%s" % engine_name]() + + + + + +def upgrade_(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('pools', sa.Column('login_link', sa.String(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade_(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('pools', 'login_link') + # ### end Alembic commands ### + + +def upgrade_stats(): + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade_stats(): + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + diff --git a/api/schedules/status_pools.py b/api/schedules/status_pools.py index de8350f9..c1a016ef 100644 --- a/api/schedules/status_pools.py +++ b/api/schedules/status_pools.py @@ -15,6 +15,7 @@ from flask import g from common.config import globals +from api.commands import chia_cli from api.rpc import chia from api import app from api import utils @@ -35,6 +36,8 @@ def update(): pools = asyncio.run(chia.get_pool_state(blockchain)) for pool in pools: launcher_id = pool['pool_config']['launcher_id'] + login_link = chia_cli.get_pool_login_link(launcher_id) + app.logger.info("Pool login: {0}".format(login_link)) if launcher_id.startswith('0x'): launcher_id = launcher_id[2:] payload.append({ @@ -42,6 +45,7 @@ def update(): "hostname": hostname, "blockchain": blockchain, "launcher_id": launcher_id, + "login_link": login_link, "pool_state": json.dumps(pool), "updated_at": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") }) diff --git a/common/models/pools.py b/common/models/pools.py index 6f45beb1..b8920aba 100644 --- a/common/models/pools.py +++ b/common/models/pools.py @@ -13,6 +13,7 @@ class Pool(db.Model): hostname = sa.Column(sa.String(length=255), nullable=False) blockchain = sa.Column(sa.String(length=64), nullable=True) launcher_id = sa.Column(sa.String(length=255), nullable=False) + login_link = sa.Column(sa.String, nullable=True) pool_state = sa.Column(sa.String, nullable=False) updated_at = sa.Column(sa.String(length=64), nullable=False) \ No newline at end of file diff --git a/web/models/chia.py b/web/models/chia.py index 6aed5ec1..db783bc2 100644 --- a/web/models/chia.py +++ b/web/models/chia.py @@ -165,7 +165,8 @@ def __init__(self, connections): for connection in connections: self.rows.append({ 'hostname': connection.hostname, - 'blockchain': connection.blockchain, + 'blockchain': connection.blockchain, + 'protocol_port': '8444' if connection.blockchain == 'chia' else '6888', 'details': connection.details }) @@ -244,6 +245,7 @@ def __init__(self, pools, plotnfts): self.rows.append({ 'hostname': pool.hostname, 'launcher_id': pool.launcher_id, + 'login_link': pool.login_link, 'blockchain': pool.blockchain, 'pool_state': pool_state, 'updated_at': pool.updated_at, diff --git a/web/templates/index.html b/web/templates/index.html index ccfa1837..e1b77a87 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -126,28 +126,32 @@
Flax Netspace
{% endif %} {% if partials.rows|length > 0 %} -
- - - - - - - - - - - {% for partial in partials.rows %} - - - - - - - - {% endfor %} - -
BlockchainPoolResponseLauncherDate
{{partial.blockchain}}{{partial.pool_url}}{{partial.pool_response}}{{partial.launcher_id|launcheridshortener}}{{partial.created_at}}
+
+
+ + + + + + + + + + + + {% for partial in partials.rows %} + + + + + + + + {% endfor %} + +
BlockchainPoolResponseLauncherDate
{{partial.blockchain}}{{partial.pool_url}}{{partial.pool_response}}{{partial.launcher_id|launcheridshortener}}{{partial.created_at}}
+
+
{% endif %} diff --git a/web/templates/network/connections.html b/web/templates/network/connections.html index 7ddd8b90..74e6182f 100644 --- a/web/templates/network/connections.html +++ b/web/templates/network/connections.html @@ -48,6 +48,11 @@ {% if connections.rows|length > 0 %} {% for connection in connections.rows %}
+ + + Check Port Forwarding + +

{{connection.hostname}} - {{connection.blockchain}}

{{connection.details}}
diff --git a/web/templates/pools.html b/web/templates/pools.html index e6db9bd0..ab6483db 100644 --- a/web/templates/pools.html +++ b/web/templates/pools.html @@ -47,7 +47,13 @@
Pool:
  • Points Successful in Last 24 Hours: {{ pool.points_successful_last_24h }}
    • Launcher ID: - {{ pool.launcher_id }} + + {% if pool.login_link %} + {{ pool.launcher_id }} + {% else %} + {{ pool.launcher_id }} + {% endif %} +
    {% endfor %} diff --git a/web/templates/worker_launch.html b/web/templates/worker_launch.html index f9f91de9..dfb641e7 100644 --- a/web/templates/worker_launch.html +++ b/web/templates/worker_launch.html @@ -155,11 +155,7 @@ cmd += ' -h ' + document.getElementById("worker_hostname").value + line_end; } else { - if (os == 'windows') { - cmd += ' -h (hostname)' + line_end; - } else { - cmd += ' -h $(hostname -s)' + line_end; - } + errors.push("Missing worker hostname. Please also provide an IP address if the hostname won't resolve on your LAN."); } if (document.getElementById("dns_ip_addr").value) { cmd += ' --dns ' + document.getElementById("dns_ip_addr").value + line_end; @@ -312,14 +308,14 @@ cmd += ' - AUTO_PLOT=' + auto_plot + line_end; } cmd += ' ports:' + line_end; - cmd += ' - 8926' + line_end; - cmd += ' - 8927' + line_end; + cmd += ' - 8926:8926' + line_end; + cmd += ' - 8927:8927' + line_end; if (!mode) { // Defaults to fullnode - cmd += ' - 8444' + line_end; - cmd += ' - 8447' + line_end; + cmd += ' - 8444:8444' + line_end; + cmd += ' - 8447:8447' + line_end; if (document.getElementById("blockchain-flax").checked) { - cmd += ' - 6888' + line_end; - cmd += ' - 6885' + line_end; + cmd += ' - 6888:6888' + line_end; + cmd += ' - 6885:6885' + line_end; } } return cmd; @@ -467,7 +463,7 @@

    Machinaris Worker - Launch Config

    - +
    From 18c8ec2dcb328acb63f99fc144ebe1829146204e Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Wed, 11 Aug 2021 17:23:45 -0600 Subject: [PATCH 13/19] Display plot type on Farming page. --- CHANGELOG.md | 11 +++-- api/migrations/versions/fc3cc23ea246_.py | 52 ++++++++++++++++++++++++ api/rpc/chia.py | 32 +++++++++++++++ api/schedules/status_plots.py | 11 +++++ api/schedules/status_points.py | 3 +- api/schedules/status_pools.py | 1 - common/models/plots.py | 1 + web/models/chia.py | 5 ++- web/templates/farming.html | 4 +- 9 files changed, 111 insertions(+), 9 deletions(-) create mode 100644 api/migrations/versions/fc3cc23ea246_.py diff --git a/CHANGELOG.md b/CHANGELOG.md index bab02144..21a9b463 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.5.2] - 2021-07-? +## [0.5.2] - 2021-08-13 -- Machinaris - Daily Farming Summary now available on Farming page for both Chia and Flax. - Machinaris - Docker images now available for [Apple M1](https://github.com/guydavis/machinaris/issues/43) and [Raspberry Pi OS](https://github.com/guydavis/machinaris/issues/155) architectures. -- Chiadog - Update to `dev` branch to add [support for parsing partials and solo blocks](https://github.com/martomi/chiadog/pull/268). +- Chiadog - Update to new v0.7.0 to [support for parsing partials and solo blocks](https://github.com/martomi/chiadog/pull/268). - Chia - Update to patch release of 1.2.3. See their [changelog for details](https://github.com/Chia-Network/chia-blockchain/releases/tag/1.2.3). +- TrueNAS - Support for Machinaris deployment via helm chart. [Issue #78](https://github.com/guydavis/machinaris/issues/78) - Big thanks to @kmoore134 for this! +- Machinaris - Daily Farming Summary now available on Farming page for both Chia and Flax. Add new plot type column. +- Machinaris - Pools - Show each Pool's status including link to your pool provider. List pool point events on Summary page. +- Machinaris - Workers - Use hostname for Worker display name, even when using IP addresses behind the scenes. Also show versions. Automated harvester setup. +- Machinaris - Connections page has link to test your router port forward for farming. +- Machinaris - New [public website](http://www.machinaris.app) with launch Wizard for generating first Docker run/compose of Machinaris. ## [0.5.1] - 2021-07-22 diff --git a/api/migrations/versions/fc3cc23ea246_.py b/api/migrations/versions/fc3cc23ea246_.py new file mode 100644 index 00000000..fa2904b1 --- /dev/null +++ b/api/migrations/versions/fc3cc23ea246_.py @@ -0,0 +1,52 @@ +"""empty message + +Revision ID: fc3cc23ea246 +Revises: ce840f016302 +Create Date: 2021-08-11 16:39:10.975245 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'fc3cc23ea246' +down_revision = 'ce840f016302' +branch_labels = None +depends_on = None + + +def upgrade(engine_name): + globals()["upgrade_%s" % engine_name]() + + +def downgrade(engine_name): + globals()["downgrade_%s" % engine_name]() + + + + + +def upgrade_(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('plots', sa.Column('type', sa.String(length=32), nullable=True)) + # ### end Alembic commands ### + + +def downgrade_(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('plots', 'type') + # ### end Alembic commands ### + + +def upgrade_stats(): + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade_stats(): + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + diff --git a/api/rpc/chia.py b/api/rpc/chia.py index 9aaf212a..a5ad29c0 100644 --- a/api/rpc/chia.py +++ b/api/rpc/chia.py @@ -10,6 +10,7 @@ from api import app +# Unused as I am getting signage points from debug.log as this API returns no dates async def get_signage_points(blockchain): config = load_chia_config(DEFAULT_ROOT_PATH, 'config.yaml') farmer_rpc_port = config["farmer"]["rpc_port"] @@ -34,6 +35,7 @@ async def get_signage_points(blockchain): await fullnode.await_closed() return points +# Used on Pools page to display each pool's state async def get_pool_state(blockchain): pools = [] try: @@ -51,3 +53,33 @@ async def get_pool_state(blockchain): except Exception as ex: app.logger.info("Error getting {0} blockchain pool states: {1}".format(blockchain, str(ex))) return pools + +# Used to load plot type (solo or portable) via RPC +async def get_all_plots(): + all_plots = [] + try: + config = load_chia_config(DEFAULT_ROOT_PATH, 'config.yaml') + farmer_rpc_port = config["farmer"]["rpc_port"] + farmer = await FarmerRpcClient.create( + 'localhost', uint16(farmer_rpc_port), DEFAULT_ROOT_PATH, config + ) + result = await farmer.get_harvesters() + farmer.close() + await farmer.await_closed() + for harvester in result["harvesters"]: + host = harvester["connection"]["host"] + plots = harvester["plots"] + for plot in plots: + all_plots.append({ + "hostname": host, + "type": "solo" if (plot["pool_contract_puzzle_hash"] is None) else "portable", + "plot_id": plot['plot_id'], + "file_size": plot['file_size'], # bytes + "filename": plot['filename'], # full path and name + "plot_public_key": plot['plot_public_key'], + "pool_contract_puzzle_hash": plot['pool_contract_puzzle_hash'], + "pool_public_key": plot['pool_public_key'], + }) + except Exception as ex: + app.logger.info("Error getting plots via RPC: {0}".format(str(ex))) + return all_plots \ No newline at end of file diff --git a/api/schedules/status_plots.py b/api/schedules/status_plots.py index 5f93ed97..19bdd4a4 100644 --- a/api/schedules/status_plots.py +++ b/api/schedules/status_plots.py @@ -2,6 +2,7 @@ # Performs a REST call to controller (possibly localhost) of all plots. # +import asyncio import os import traceback @@ -11,6 +12,7 @@ from common.utils import converters from api import app from api.commands import chia_cli +from api.rpc import chia from api import utils def update(): @@ -21,6 +23,7 @@ def update(): try: hostname = utils.get_hostname() plots_farming = chia_cli.load_plots_farming() + plots_via_rpc = asyncio.run(chia.get_all_plots()) plots_by_id = {} payload = [] for plot in plots_farming.rows: @@ -31,6 +34,7 @@ def update(): plot['dir'], plot['file'], other_plot['dir'], other_plot['file'])) else: # No conflict so add it to plots list plots_by_id[plot_id] = plot + plot_rpc_obj = find_plot_obj_by_id(plots_via_rpc, plot_id) payload.append({ "plot_id": plot_id, "hostname": hostname, @@ -38,6 +42,7 @@ def update(): "file": plot['file'], "created_at": plot['created_at'], "size": plot['size'], + "type": "" if not 'type' in plot_rpc_obj else plot_rpc_obj['type'] }) if len(payload) > 0: utils.send_post('/plots/', payload, debug=False) @@ -46,3 +51,9 @@ def update(): except: app.logger.info("Failed to load plots farming and send.") app.logger.info(traceback.format_exc()) + +def find_plot_obj_by_id(plots_via_rpc, plot_id): + for plot_rpc in plots_via_rpc: + if plot_rpc['plot_id'].startswith("0x{0}".format(plot_id)): + return plot_rpc + return None \ No newline at end of file diff --git a/api/schedules/status_points.py b/api/schedules/status_points.py index 24ea1b48..25434212 100644 --- a/api/schedules/status_points.py +++ b/api/schedules/status_points.py @@ -37,7 +37,8 @@ def update(): "blockchain": blockchain, "details": points, } - #app.logger.info(payload) + for point in points: + app.logger.info(point) #utils.send_post('/plotnfts/', payload, debug=False) except: app.logger.info("Failed to load and send recent signage points.") diff --git a/api/schedules/status_pools.py b/api/schedules/status_pools.py index c1a016ef..7ae92132 100644 --- a/api/schedules/status_pools.py +++ b/api/schedules/status_pools.py @@ -37,7 +37,6 @@ def update(): for pool in pools: launcher_id = pool['pool_config']['launcher_id'] login_link = chia_cli.get_pool_login_link(launcher_id) - app.logger.info("Pool login: {0}".format(login_link)) if launcher_id.startswith('0x'): launcher_id = launcher_id[2:] payload.append({ diff --git a/common/models/plots.py b/common/models/plots.py index d1ddb8aa..28c90727 100644 --- a/common/models/plots.py +++ b/common/models/plots.py @@ -11,6 +11,7 @@ class Plot(db.Model): hostname = sa.Column(sa.String(length=255), primary_key=True) plot_id = sa.Column(sa.String(length=16), primary_key=True) + type = sa.Column(sa.String(length=32), nullable=True) dir = sa.Column(sa.String(length=255), nullable=False) file = sa.Column(sa.String(length=255), nullable=False) size = sa.Column(sa.Integer, nullable=False) diff --git a/web/models/chia.py b/web/models/chia.py index db783bc2..b307e887 100644 --- a/web/models/chia.py +++ b/web/models/chia.py @@ -66,7 +66,7 @@ def calc_entire_farm_flax_etw(self, fullnode_plots_size, expected_time_to_win, t class FarmPlots: def __init__(self, plots): - self.columns = ['worker', 'plot_id', 'dir', 'plot', 'create_date', 'size'] + self.columns = ['worker', 'plot_id', 'dir', 'plot', 'type', 'create_date', 'size' ] self.rows = [] plots_by_id = {} for plot in plots: @@ -82,7 +82,8 @@ def __init__(self, plots): 'dir': plot.dir, \ 'plot': plot.file, \ 'create_date': plot.created_at, \ - 'size': plot.size }) + 'size': plot.size, \ + 'type': plot.type if plot.type else "" }) class BlockchainChallenges: diff --git a/web/templates/farming.html b/web/templates/farming.html index 8a584dd1..f9229762 100644 --- a/web/templates/farming.html +++ b/web/templates/farming.html @@ -218,8 +218,8 @@
    {{ daily.hostname }}
    } $(document).ready(function () { $('#data').DataTable({ - "order": [[4, "desc"]], - "columnDefs": [{ "orderable": false, targets: [6] }], + "order": [[5, "desc"]], + "columnDefs": [{ "orderable": false, targets: [7] }], }); }); From 51b1d4fb93e7968305eedc156f27163c2f48b859 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Wed, 11 Aug 2021 19:56:21 -0600 Subject: [PATCH 14/19] Only controller/farmer can invoke RPC endpoints, so refactoring plot type lookup to API endpoint receiver. --- api/rpc/chia.py | 2 +- api/schedules/status_plots.py | 13 +------------ api/views/plots/resources.py | 11 +++++++++-- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/api/rpc/chia.py b/api/rpc/chia.py index a5ad29c0..e4407f7e 100644 --- a/api/rpc/chia.py +++ b/api/rpc/chia.py @@ -82,4 +82,4 @@ async def get_all_plots(): }) except Exception as ex: app.logger.info("Error getting plots via RPC: {0}".format(str(ex))) - return all_plots \ No newline at end of file + return all_plots diff --git a/api/schedules/status_plots.py b/api/schedules/status_plots.py index 19bdd4a4..c78e6974 100644 --- a/api/schedules/status_plots.py +++ b/api/schedules/status_plots.py @@ -2,7 +2,6 @@ # Performs a REST call to controller (possibly localhost) of all plots. # -import asyncio import os import traceback @@ -12,7 +11,6 @@ from common.utils import converters from api import app from api.commands import chia_cli -from api.rpc import chia from api import utils def update(): @@ -23,7 +21,6 @@ def update(): try: hostname = utils.get_hostname() plots_farming = chia_cli.load_plots_farming() - plots_via_rpc = asyncio.run(chia.get_all_plots()) plots_by_id = {} payload = [] for plot in plots_farming.rows: @@ -34,15 +31,13 @@ def update(): plot['dir'], plot['file'], other_plot['dir'], other_plot['file'])) else: # No conflict so add it to plots list plots_by_id[plot_id] = plot - plot_rpc_obj = find_plot_obj_by_id(plots_via_rpc, plot_id) payload.append({ "plot_id": plot_id, "hostname": hostname, "dir": plot['dir'], "file": plot['file'], "created_at": plot['created_at'], - "size": plot['size'], - "type": "" if not 'type' in plot_rpc_obj else plot_rpc_obj['type'] + "size": plot['size'] }) if len(payload) > 0: utils.send_post('/plots/', payload, debug=False) @@ -51,9 +46,3 @@ def update(): except: app.logger.info("Failed to load plots farming and send.") app.logger.info(traceback.format_exc()) - -def find_plot_obj_by_id(plots_via_rpc, plot_id): - for plot_rpc in plots_via_rpc: - if plot_rpc['plot_id'].startswith("0x{0}".format(plot_id)): - return plot_rpc - return None \ No newline at end of file diff --git a/api/views/plots/resources.py b/api/views/plots/resources.py index 6df021c6..5412bea5 100644 --- a/api/views/plots/resources.py +++ b/api/views/plots/resources.py @@ -1,15 +1,16 @@ +import asyncio import datetime as dt from flask.views import MethodView from api import app +from api.rpc import chia from api.extensions.api import Blueprint, SQLCursorPage from common.extensions.database import db from common.models import Plot from .schemas import PlotSchema, PlotQueryArgsSchema, BatchOfPlotSchema, BatchOfPlotQueryArgsSchema - blp = Blueprint( 'Plot', __name__, @@ -17,7 +18,6 @@ description="Operations on all plots on farmer" ) - @blp.route('/') class Plots(MethodView): @@ -33,11 +33,18 @@ def get(self, args): @blp.arguments(BatchOfPlotSchema) @blp.response(201, PlotSchema(many=True)) def post(self, new_items): + # Collect plot info via RPC to determine type: solo or portable + plots_via_rpc = asyncio.run(chia.get_all_plots()) # Now delete all old plots by hostname of first new plotting db.session.query(Plot).filter(Plot.hostname==new_items[0]['hostname']).delete() items = [] for new_item in new_items: item = Plot(**new_item) + item.type = "" + for plot_rpc in plots_via_rpc: + if plot_rpc['plot_id'].startswith("0x{0}".format(item.plot_id)) and 'type' in plot_rpc: + item.type = plot_rpc['type'] + #app.logger.info("Found type: {0}".format(item.type)) db.session.add(item) items.append(item) db.session.commit() From 70f372eb8a73e16c2fdc9c6b825e6c33180e3fd0 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Thu, 12 Aug 2021 10:50:26 -0600 Subject: [PATCH 15/19] Fixes and optimizations. --- api/gunicorn.conf.py | 34 +++++++++++++++++----------------- api/rpc/chia.py | 17 ++++++++++++++++- api/views/plots/resources.py | 4 ++-- scripts/chia_launch.sh | 6 +++--- scripts/flax_launch.sh | 6 +++--- 5 files changed, 41 insertions(+), 26 deletions(-) diff --git a/api/gunicorn.conf.py b/api/gunicorn.conf.py index 487e085e..25e968b4 100644 --- a/api/gunicorn.conf.py +++ b/api/gunicorn.conf.py @@ -16,30 +16,30 @@ def on_starting(server): scheduler = BackgroundScheduler() # Statistics gathering locally - scheduler.add_job(func=stats_farm.collect, trigger='cron', minute=0) # Hourly - scheduler.add_job(func=stats_disk.collect, trigger='cron', minute="*/5") # Every 5 minutes + scheduler.add_job(func=stats_farm.collect, name="stats_farm", trigger='cron', minute=0) # Hourly + scheduler.add_job(func=stats_disk.collect, name="stats_disk", trigger='cron', minute="*/5") # Every 5 minutes # Testing only #scheduler.add_job(func=stats_farm.collect, trigger='interval', seconds=10) # Test immediately #scheduler.add_job(func=stats_disk.collect, trigger='interval', seconds=10) # Test immediately # Status gathering - reported via API - scheduler.add_job(func=status_challenges.update, trigger='interval', seconds=5) - scheduler.add_job(func=status_worker.update, trigger='interval', seconds=60, jitter=30) - scheduler.add_job(func=status_controller.update, trigger='interval', seconds=60, jitter=30) - scheduler.add_job(func=status_farm.update, trigger='interval', seconds=60, jitter=30) - scheduler.add_job(func=status_plotting.update, trigger='interval', seconds=60, jitter=30) - scheduler.add_job(func=status_plots.update, trigger='interval', seconds=60, jitter=30) - scheduler.add_job(func=status_wallets.update, trigger='interval', seconds=60, jitter=30) - scheduler.add_job(func=status_plotnfts.update, trigger='interval', seconds=60, jitter=30) - scheduler.add_job(func=status_blockchains.update, trigger='interval', seconds=60, jitter=30) - scheduler.add_job(func=status_connections.update, trigger='interval', seconds=60, jitter=30) - scheduler.add_job(func=status_keys.update, trigger='interval', seconds=60, jitter=30) - scheduler.add_job(func=status_alerts.update, trigger='interval', seconds=60, jitter=30) - scheduler.add_job(func=status_pools.update, trigger='interval', seconds=60, jitter=30) - scheduler.add_job(func=status_partials.update, trigger='interval', seconds=60, jitter=30) + scheduler.add_job(func=status_challenges.update, name="challenges", trigger='interval', seconds=5) + scheduler.add_job(func=status_worker.update, name="workers", trigger='interval', seconds=60, jitter=30) + scheduler.add_job(func=status_controller.update, name="controller", trigger='interval', seconds=60, jitter=30) + scheduler.add_job(func=status_farm.update, name="farms", trigger='interval', seconds=60, jitter=30) + scheduler.add_job(func=status_plotting.update, name="plottings", trigger='interval', seconds=60, jitter=30) + scheduler.add_job(func=status_plots.update, name="plots", trigger='interval', seconds=60, jitter=30) + scheduler.add_job(func=status_wallets.update, name="wallets", trigger='interval', seconds=60, jitter=30) + scheduler.add_job(func=status_plotnfts.update, name="plotnfts", trigger='interval', seconds=60, jitter=30) + scheduler.add_job(func=status_blockchains.update, name="blockchains", trigger='interval', seconds=60, jitter=30) + scheduler.add_job(func=status_connections.update, name="connections", trigger='interval', seconds=60, jitter=30) + scheduler.add_job(func=status_keys.update, name="keys", trigger='interval', seconds=60, jitter=30) + scheduler.add_job(func=status_alerts.update, name="alerts", trigger='interval', seconds=60, jitter=30) + scheduler.add_job(func=status_pools.update, name="pools", trigger='interval', seconds=60, jitter=30) + scheduler.add_job(func=status_partials.update, name="partials", trigger='interval', seconds=60, jitter=30) - #scheduler.add_job(func=status_points.update, trigger='interval', seconds=10, jitter=0) + #scheduler.add_job(func=status_points.update, name="points", trigger='interval', seconds=10, jitter=0) app.logger.debug("Starting background scheduler...") scheduler.start() diff --git a/api/rpc/chia.py b/api/rpc/chia.py index e4407f7e..294a4790 100644 --- a/api/rpc/chia.py +++ b/api/rpc/chia.py @@ -2,6 +2,9 @@ # RPC interactions with Chia # +import asyncio +import datetime + from chia.rpc.full_node_rpc_client import FullNodeRpcClient from chia.rpc.farmer_rpc_client import FarmerRpcClient from chia.util.default_root import DEFAULT_ROOT_PATH @@ -55,7 +58,19 @@ async def get_pool_state(blockchain): return pools # Used to load plot type (solo or portable) via RPC -async def get_all_plots(): +plots_via_rpc = None +last_plots_via_rpc = None +def get_all_plots(): + global plots_via_rpc + global last_plots_via_rpc + if plots_via_rpc and last_plots_via_rpc >= (datetime.datetime.now() - datetime.timedelta(minutes=3)): + return plots_via_rpc + #app.logger.info("Reloading all plots on all harvesters via RPC") + plots_via_rpc = asyncio.run(load_all_plots()) + last_plots_via_rpc = datetime.datetime.now() + return plots_via_rpc + +async def load_all_plots(): all_plots = [] try: config = load_chia_config(DEFAULT_ROOT_PATH, 'config.yaml') diff --git a/api/views/plots/resources.py b/api/views/plots/resources.py index 5412bea5..13e0b39e 100644 --- a/api/views/plots/resources.py +++ b/api/views/plots/resources.py @@ -33,8 +33,8 @@ def get(self, args): @blp.arguments(BatchOfPlotSchema) @blp.response(201, PlotSchema(many=True)) def post(self, new_items): - # Collect plot info via RPC to determine type: solo or portable - plots_via_rpc = asyncio.run(chia.get_all_plots()) + # Get plot info via RPC to determine type: solo or portable + plots_via_rpc = chia.get_all_plots() # Now delete all old plots by hostname of first new plotting db.session.query(Plot).filter(Plot.hostname==new_items[0]['hostname']).delete() items = [] diff --git a/scripts/chia_launch.sh b/scripts/chia_launch.sh index 15a0aaf0..6295f4e8 100644 --- a/scripts/chia_launch.sh +++ b/scripts/chia_launch.sh @@ -56,13 +56,13 @@ elif [[ ${mode} =~ ^harvester.* ]]; then fi if [ -f /root/.chia/farmer_ca/chia_ca.crt ]; then chia init -c /root/.chia/farmer_ca 2>&1 > /root/.chia/mainnet/log/init.log - chia configure --set-farmer-peer ${farmer_address}:${farmer_port} - chia configure --enable-upnp false - chia start harvester -r else echo "Did not find your farmer's certificates within /root/.chia/farmer_ca." echo "See: https://github.com/guydavis/machinaris/wiki/Workers#harvester" fi + chia configure --set-farmer-peer ${farmer_address}:${farmer_port} + chia configure --enable-upnp false + chia start harvester -r fi elif [[ ${mode} == 'plotter' ]]; then echo "Starting in Plotter-only mode. Run Plotman from either CLI or WebUI." diff --git a/scripts/flax_launch.sh b/scripts/flax_launch.sh index 01f92b0c..168765c7 100644 --- a/scripts/flax_launch.sh +++ b/scripts/flax_launch.sh @@ -66,13 +66,13 @@ elif [[ ${mode} =~ ^harvester.* ]]; then fi if [ -f /root/.flax/farmer_ca/flax_ca.crt ]; then flax init -c /root/.flax/farmer_ca 2>&1 > /root/.flax/mainnet/log/init.log - flax configure --set-farmer-peer ${farmer_address}:${flax_farmer_port} - flax configure --enable-upnp false - flax start harvester -r else echo "Did not find your farmer's certificates within /root/.flax/farmer_ca." echo "See: https://github.com/guydavis/machinaris/wiki/Workers#harvester" fi + flax configure --set-farmer-peer ${farmer_address}:${flax_farmer_port} + flax configure --enable-upnp false + flax start harvester -r fi elif [[ ${mode} == 'plotter' ]]; then echo "Starting in Plotter-only mode. Run Plotman from either CLI or WebUI." From b225ef5cdc1e6c216455fa1189799b7825767744 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Thu, 12 Aug 2021 13:45:04 -0600 Subject: [PATCH 16/19] Try preload to avoid repeated Gunicorn worker timeouts. --- scripts/dev/start-api.sh | 2 +- scripts/start_machinaris.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/dev/start-api.sh b/scripts/dev/start-api.sh index 7e222f16..8780d8d1 100644 --- a/scripts/dev/start-api.sh +++ b/scripts/dev/start-api.sh @@ -19,7 +19,7 @@ fi #--certfile=/root/.chia/mainnet/config/ssl/ca/chia_ca.crt \ #--keyfile=/root/.chia/mainnet/config/ssl/ca/chia_ca.key \ -/chia-blockchain/venv/bin/gunicorn --reload \ +/chia-blockchain/venv/bin/gunicorn --preload --reload \ --bind 0.0.0.0:8927 --timeout 90 \ --log-level=${LOG_LEVEL} \ --workers=2 \ diff --git a/scripts/start_machinaris.sh b/scripts/start_machinaris.sh index 8257b864..4e47119d 100644 --- a/scripts/start_machinaris.sh +++ b/scripts/start_machinaris.sh @@ -62,7 +62,7 @@ if [ ! -z $api_pid ]; then kill $api_pid fi echo 'Starting Machinaris API server...' -/chia-blockchain/venv/bin/gunicorn ${RELOAD} \ +/chia-blockchain/venv/bin/gunicorn --preload ${RELOAD} \ --bind 0.0.0.0:8927 --timeout 90 \ --log-level=${LOG_LEVEL} \ --workers=2 \ From 54b43438b002060d1bbe50136546217af916f20a Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Thu, 12 Aug 2021 13:50:56 -0600 Subject: [PATCH 17/19] Improve preload/reload selection. --- scripts/dev/start-api.sh | 7 +++++-- scripts/dev/start-web.sh | 5 +++-- scripts/start_machinaris.sh | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/scripts/dev/start-api.sh b/scripts/dev/start-api.sh index 8780d8d1..db730e1f 100644 --- a/scripts/dev/start-api.sh +++ b/scripts/dev/start-api.sh @@ -11,16 +11,19 @@ cd /code/machinaris if [ $FLASK_ENV == "development" ]; then LOG_LEVEL='debug' + RELOAD='--reload' else LOG_LEVEL='info' + RELOAD='--preload' fi # To enable SSL, use the Chia self-signed cert #--certfile=/root/.chia/mainnet/config/ssl/ca/chia_ca.crt \ #--keyfile=/root/.chia/mainnet/config/ssl/ca/chia_ca.key \ -/chia-blockchain/venv/bin/gunicorn --preload --reload \ - --bind 0.0.0.0:8927 --timeout 90 \ +/chia-blockchain/venv/bin/gunicorn ${RELOAD} \ + --bind 0.0.0.0:8927 \ + --timeout 90 \ --log-level=${LOG_LEVEL} \ --workers=2 \ --config api/gunicorn.conf.py \ diff --git a/scripts/dev/start-web.sh b/scripts/dev/start-web.sh index b1f0b099..1d119688 100644 --- a/scripts/dev/start-web.sh +++ b/scripts/dev/start-web.sh @@ -11,12 +11,13 @@ cd /code/machinaris if [ $FLASK_ENV == "development" ]; then LOG_LEVEL='debug' + RELOAD='--reload' else LOG_LEVEL='info' + RELOAD='--preload' fi -/chia-blockchain/venv/bin/gunicorn \ - --reload \ +/chia-blockchain/venv/bin/gunicorn ${RELOAD} \ --bind 0.0.0.0:8926 \ --timeout 90 \ --log-level=$LOG_LEVEL \ diff --git a/scripts/start_machinaris.sh b/scripts/start_machinaris.sh index 4e47119d..38fc0bd2 100644 --- a/scripts/start_machinaris.sh +++ b/scripts/start_machinaris.sh @@ -53,7 +53,7 @@ then RELOAD='--reload' else LOG_LEVEL='info' - RELOAD='' + RELOAD='--preload' fi # Kill gunicorn if already running to allow restart @@ -62,7 +62,7 @@ if [ ! -z $api_pid ]; then kill $api_pid fi echo 'Starting Machinaris API server...' -/chia-blockchain/venv/bin/gunicorn --preload ${RELOAD} \ +/chia-blockchain/venv/bin/gunicorn ${RELOAD} \ --bind 0.0.0.0:8927 --timeout 90 \ --log-level=${LOG_LEVEL} \ --workers=2 \ From 938c5fc646107e803d029fb2173693fff7cb64c8 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Thu, 12 Aug 2021 14:31:43 -0600 Subject: [PATCH 18/19] Update changelog to --- CHANGELOG.md | 1 + scripts/flaxdog_install.sh | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21a9b463..84e769e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file. The format - Machinaris - Docker images now available for [Apple M1](https://github.com/guydavis/machinaris/issues/43) and [Raspberry Pi OS](https://github.com/guydavis/machinaris/issues/155) architectures. - Chiadog - Update to new v0.7.0 to [support for parsing partials and solo blocks](https://github.com/martomi/chiadog/pull/268). - Chia - Update to patch release of 1.2.3. See their [changelog for details](https://github.com/Chia-Network/chia-blockchain/releases/tag/1.2.3). +- Flax - Update to version 0.1.1. See their [changelog for details](https://github.com/Flax-Network/flax-blockchain/releases/tag/0.1.1). - TrueNAS - Support for Machinaris deployment via helm chart. [Issue #78](https://github.com/guydavis/machinaris/issues/78) - Big thanks to @kmoore134 for this! - Machinaris - Daily Farming Summary now available on Farming page for both Chia and Flax. Add new plot type column. - Machinaris - Pools - Show each Pool's status including link to your pool provider. List pool point events on Summary page. diff --git a/scripts/flaxdog_install.sh b/scripts/flaxdog_install.sh index 459b7a5e..b49d5aa5 100644 --- a/scripts/flaxdog_install.sh +++ b/scripts/flaxdog_install.sh @@ -9,6 +9,11 @@ cd / git clone https://github.com/langhorst/flaxdog.git +# Temporary patch for spam about partial proofs +# https://github.com/martomi/chiadog/issues/252#issuecomment-877416135 +#cd /flaxdog/src/flax_log/handlers/ +#sed -i 's/FoundProofs(),//g' harvester_activity_handler.py + cd /flax-blockchain/ # Chia-blockchain needs PyYAML=5.4.1 but Chiadog wants exactly 5.4 From 16d0a6120bfa60111bc6646e9c1184c62196ad10 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Thu, 12 Aug 2021 20:08:22 -0600 Subject: [PATCH 19/19] Support paths in Worker volume paths. --- web/templates/worker_launch.html | 4 ++-- web/templates/workers.html | 10 ++-------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/web/templates/worker_launch.html b/web/templates/worker_launch.html index dfb641e7..bfd7ddeb 100644 --- a/web/templates/worker_launch.html +++ b/web/templates/worker_launch.html @@ -181,7 +181,7 @@ } else if (!volume_container_path) { errors.push("Empty container path provided for volume at '" + volume_host_path + "'. Please provide a container path."); } - cmd += ' -v ' + volume_host_path + ':' + volume_container_path + line_end; + cmd += ' -v "' + volume_host_path + ':' + volume_container_path + '"' + line_end; if ($('#volume_type' + index).val() == 'plots') { plots_dir.push(volume_container_path) } @@ -260,7 +260,7 @@ $('#volumes').children().each((index, element) => { volume_host_path = document.getElementById("volume_host_path" + index).value; volume_container_path = document.getElementById("volume_container_path" + index).value; - cmd += ' - ' + volume_host_path + ':' + volume_container_path + line_end; + cmd += ' - "' + volume_host_path + ':' + volume_container_path + '"' + line_end; if ($('#volume_type' + index).val() == 'plots') { plots_dir.push(volume_container_path) } diff --git a/web/templates/workers.html b/web/templates/workers.html index 4553af48..b7aa3dcd 100644 --- a/web/templates/workers.html +++ b/web/templates/workers.html @@ -68,16 +68,10 @@
    -
    -
    -
    -
    - -
    -
    +