More on distributed workers.

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2021 Guy Davis
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
@@ -199,3 +199,5 @@
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
+
+NOTE: See the 'licenses' folder for licenses of bundled tools/libraries.

api/default_settings.py

@@ -1,8 +1,6 @@
-"""Default application settings"""
-
+import os
 
 class DefaultConfig:
-    """Default configuration"""
     API_TITLE = "Machinaris API"
     API_VERSION = 0.1
     OPENAPI_VERSION = '3.0.2'
@@ -16,6 +14,6 @@ class DefaultConfig:
     SQLALCHEMY_DATABASE_URI = 'sqlite:////root/.chia/machinaris/dbs/machinaris.db'
     SQLALCHEMY_ECHO = True
     ETAG_DISABLED = True # https://flask-smorest.readthedocs.io/en/latest/etag.html
-    CONTROLLER_PROTO = 'http'
-    CONTROLLER_HOST = 'localhost'
-    CONTROLLER_PORT = 8927
+    CONTROLLER_SCHEME = 'http'
+    CONTROLLER_HOST = os.environ['controller_host'] if 'controller_host' in os.environ else 'localhost'
+    CONTROLLER_PORT = os.environ['controller_api_port'] if 'controller_api_port' in os.environ else '8927'

config/Machinaris.postman_collection.json → api/machinaris.postman_collection.json

@@ -169,7 +169,7 @@
 						"header": [],
 						"body": {
 							"mode": "raw",
-							"raw": "{\r\n    \"hostname\": \"aragorn\",\r\n    \"plot_count\": 174,\r\n    \"plot_size\": 17332.224,\r\n    \"netspace_size\": 15957229.568,\r\n    \"expected_time_to_win\": 262800.0\r\n}\r\n",
+							"raw": "{\r\n    \"hostname\": \"aragorn\",\r\n    \"mode\": \"fullnode\",\r\n    \"status\": \"Farming\",\r\n    \"plot_count\": 174,\r\n    \"plots_size\": 17332.224,\r\n    \"netspace_size\": 15957229.568,\r\n    \"expected_time_to_win\": 262800.0\r\n}\r\n",
 							"options": {
 								"raw": {
 									"language": "json"
@@ -197,14 +197,15 @@
 						"method": "GET",
 						"header": [],
 						"url": {
-							"raw": "{{proto}}://{{hostname}}:{{port}}/farms",
+							"raw": "{{proto}}://{{hostname}}:{{port}}/farms/aragorn",
 							"protocol": "{{proto}}",
 							"host": [
 								"{{hostname}}"
 							],
 							"port": "{{port}}",
 							"path": [
-								"farms"
+								"farms",
+								"aragorn"
 							]
 						},
 						"description": "Get a farm by hostname."
@@ -302,7 +303,7 @@
 						"header": [],
 						"body": {
 							"mode": "raw",
-							"raw": "{\r\n    \"plot_id\": \"39932db\",\r\n    \"hostname\":\"aragorn\",\r\n    \"dir\": \"/plots4\",\r\n    \"file\": \"plot-k32-2021-05-31-19-49-39932db6aac9678f69a66c7e13ae3df3c6387842994644ab1f545dc1a5b33831.plot\",\r\n    \"size\": 101.1\r\n}",
+							"raw": "[\r\n    {\r\n        \"plot_id\": \"39932db\",\r\n        \"hostname\": \"aragorn\",\r\n        \"dir\": \"/plots4\",\r\n        \"file\": \"plot-k32-2021-05-31-19-49-39932db6aac9678f69a66c7e13ae3df3c6387842994644ab1f545dc1a5b33831.plot\",\r\n        \"size\": 101.1,\r\n        \"created_at\": \"2021-06-01 12:00:00\"\r\n    },\r\n    {\r\n        \"plot_id\": \"abcd1234\",\r\n        \"hostname\": \"aragorn\",\r\n        \"dir\": \"/plots3\",\r\n        \"file\": \"plot-k32-2021-06-05-08-11-6babcdab98751845807084f9316e1ca4204d78cb05a1110415b9b36602f9bad1.plot\",\r\n        \"size\": 101.1,\r\n        \"created_at\": \"2021-06-02 23:23:12\"\r\n    }\r\n]",
 							"options": {
 								"raw": {
 									"language": "json"
@@ -324,48 +325,6 @@
 					},
 					"response": []
 				},
-				{
-					"name": "Plot",
-					"request": {
-						"method": "GET",
-						"header": [],
-						"url": {
-							"raw": "{{proto}}://{{hostname}}:{{port}}/plots/39932db",
-							"protocol": "{{proto}}",
-							"host": [
-								"{{hostname}}"
-							],
-							"port": "{{port}}",
-							"path": [
-								"plots",
-								"39932db"
-							]
-						},
-						"description": "Get a completed plot by plot_id (8-char prefix)"
-					},
-					"response": []
-				},
-				{
-					"name": "Plot",
-					"request": {
-						"method": "PUT",
-						"header": [],
-						"url": {
-							"raw": "{{proto}}://{{hostname}}:{{port}}/plots/39932db",
-							"protocol": "{{proto}}",
-							"host": [
-								"{{hostname}}"
-							],
-							"port": "{{port}}",
-							"path": [
-								"plots",
-								"39932db"
-							]
-						},
-						"description": "Update a plot by plot_id (8-char prefix)"
-					},
-					"response": []
-				},
 				{
 					"name": "Plot",
 					"request": {
@@ -419,7 +378,7 @@
 						"header": [],
 						"body": {
 							"mode": "raw",
-							"raw": "{\r\n    \"plot_id\": \"20dfc942\",\r\n    \"hostname\": \"aragorn\",\r\n    \"k\": 32,\r\n    \"tmp\": \"/plotting\",\r\n    \"dst\": \"/plots3\",\r\n    \"wall\": \"5:51\",\r\n    \"phase\": \"2:4\",\r\n    \"size\": 209.0,\r\n    \"pid\": \"27181\",\r\n    \"stat\": \"RUN\",\r\n    \"mem\": \"2.0G\",\r\n    \"user\": \"10:20\",\r\n    \"sys\": \"1:07\",\r\n    \"io\": \"0:04\"\r\n}",
+							"raw": "[\r\n    {\r\n        \"plot_id\": \"20dfc942\",\r\n        \"hostname\": \"aragorn\",\r\n        \"k\": 32,\r\n        \"tmp\": \"/plotting\",\r\n        \"dst\": \"/plots3\",\r\n        \"wall\": \"5:51\",\r\n        \"phase\": \"2:4\",\r\n        \"size\": \"209.0\",\r\n        \"pid\": \"27181\",\r\n        \"stat\": \"RUN\",\r\n        \"mem\": \"2.0G\",\r\n        \"user\": \"10:20\",\r\n        \"sys\": \"1:07\",\r\n        \"io\": \"0:04\"\r\n    },\r\n    {\r\n        \"plot_id\": \"abcd1234\",\r\n        \"hostname\": \"legolas\",\r\n        \"k\": 32,\r\n        \"tmp\": \"/plotting\",\r\n        \"dst\": \"/plots4\",\r\n        \"wall\": \"5:51\",\r\n        \"phase\": \"2:4\",\r\n        \"size\": \"209.0\",\r\n        \"pid\": \"27181\",\r\n        \"stat\": \"RUN\",\r\n        \"mem\": \"2.0G\",\r\n        \"user\": \"10:20\",\r\n        \"sys\": \"1:07\",\r\n        \"io\": \"0:04\"\r\n    }\r\n]",
 							"options": {
 								"raw": {
 									"language": "json"
@@ -436,76 +395,26 @@
 							"path": [
 								"plottings"
 							]
-						}
-					},
-					"response": []
-				},
-				{
-					"name": "Plotting",
-					"request": {
-						"method": "GET",
-						"header": [],
-						"url": {
-							"raw": "{{proto}}://{{hostname}}:{{port}}/plottings/20dfc942",
-							"protocol": "{{proto}}",
-							"host": [
-								"{{hostname}}"
-							],
-							"port": "{{port}}",
-							"path": [
-								"plottings",
-								"20dfc942"
-							]
 						},
-						"description": "Get a plotting job by plot_id (8-char prefix)"
+						"description": "Creates the current plottings for the given hostname (of first plotting).  Deletes any old plottings."
 					},
 					"response": []
 				},
 				{
-					"name": "Plotting",
-					"request": {
-						"method": "PUT",
-						"header": [],
-						"body": {
-							"mode": "raw",
-							"raw": "{\r\n    \"plot_id\": \"20dfc942\",\r\n    \"hostname\": \"aragorn\",\r\n    \"k\": 32,\r\n    \"tmp\": \"/plotting\",\r\n    \"dst\": \"/plots3\",\r\n    \"wall\": \"5:51\",\r\n    \"phase\": \"2:4\",\r\n    \"size\": 209.0,\r\n    \"pid\": \"27181\",\r\n    \"stat\": \"RUN\",\r\n    \"mem\": \"2.0G\",\r\n    \"user\": \"10:20\",\r\n    \"sys\": \"1:07\",\r\n    \"io\": \"0:04\"\r\n}",
-							"options": {
-								"raw": {
-									"language": "json"
-								}
-							}
-						},
-						"url": {
-							"raw": "{{proto}}://{{hostname}}:{{port}}/plottings/20dfc942",
-							"protocol": "{{proto}}",
-							"host": [
-								"{{hostname}}"
-							],
-							"port": "{{port}}",
-							"path": [
-								"plottings",
-								"20dfc942"
-							]
-						},
-						"description": "Update a plotting job by plot_id (8-char prefix)"
-					},
-					"response": []
-				},
-				{
-					"name": "Plotting",
+					"name": "Plottings",
 					"request": {
 						"method": "DELETE",
 						"header": [],
 						"url": {
-							"raw": "{{proto}}://{{hostname}}:{{port}}/plottings/20dfc942",
+							"raw": "{{proto}}://{{hostname}}:{{port}}/plottings/aragorn",
 							"protocol": "{{proto}}",
 							"host": [
 								"{{hostname}}"
 							],
 							"port": "{{port}}",
 							"path": [
 								"plottings",
-								"20dfc942"
+								"aragorn"
 							]
 						},
 						"description": "Delete a plotting job's status."

api/schedules/status_farm.py

@@ -11,7 +11,7 @@
 from common.utils import converters
 from api import app
 from api.commands import chia_cli
-from api.schedules import common
+from api import utils
 
 def update():
     if not globals.farming_enabled() and not globals.harvesting_enabled():
@@ -20,7 +20,7 @@ def update():
         return
     with app.app_context():
         try:
-            hostname = common.get_hostname()
+            hostname = utils.get_hostname()
             farm_summary = chia_cli.load_farm_summary()
             payload = {
                 "hostname": hostname,
@@ -30,9 +30,9 @@ def update():
                 "plots_size": converters.str_to_gibs(farm_summary.plot_size),
                 "total_chia": farm_summary.total_chia,
                 "netspace_size": converters.str_to_gibs(farm_summary.netspace_size),
-                "expected_time_to_win": converters.etw_to_minutes(farm_summary.time_to_win)
+                "expected_time_to_win": farm_summary.time_to_win
             }
-            common.send_post('/farms', payload, debug=False)
+            utils.send_post('/farms', payload, debug=False)
         except:
             app.logger.info("Failed to load farm summary and send.")
             app.logger.info(traceback.format_exc())

api/schedules/status_plots.py

@@ -11,7 +11,7 @@
 from common.utils import converters
 from api import app
 from api.commands import chia_cli
-from api.schedules import common
+from api import utils
 
 def update():
     if not globals.farming_enabled() and not globals.harvesting_enabled():
@@ -20,7 +20,7 @@ def update():
         return
     with app.app_context():
         try:
-            hostname = common.get_hostname()
+            hostname = utils.get_hostname()
             plots_farming = chia_cli.load_plots_farming()
             payload = []
             for plot in plots_farming.rows:
@@ -32,7 +32,10 @@ def update():
                     "created_at": plot['created_at'],
                     "size": plot['size'],
                 })
-            common.send_post('/plots', payload, debug=False)
+            if len(payload) > 0:
+                utils.send_post('/plots', payload, debug=False)
+            else:
+                utils.send_delete('/plots', debug=False)
         except:
             app.logger.info("Failed to load plots farming and send.")
             app.logger.info(traceback.format_exc())

api/schedules/status_plotting.py

@@ -11,7 +11,7 @@
 from common.utils import converters
 from api import app
 from api.commands import plotman_cli
-from api.schedules import common
+from api import utils
 
 def update():
     if not globals.plotting_enabled():
@@ -20,7 +20,7 @@ def update():
         return
     with app.app_context():
         try:
-            hostname = common.get_hostname()
+            hostname = utils.get_hostname()
             plotting_summary = plotman_cli.load_plotting_summary()
             payload = []
             for plot in plotting_summary.rows:
@@ -40,7 +40,10 @@ def update():
                     "sys": plot['sys'],
                     "io": plot['io'],
                 })
-            common.send_post('/plottings', payload, debug=False)
+            if len(payload) > 0:
+                utils.send_post('/plottings', payload, debug=False)
+            else:
+                utils.send_delete('/plottings/{0}'.format(hostname), debug=True)
         except:
             app.logger.info("Failed to load plotting summary and send.")
             app.logger.info(traceback.format_exc())

api/schedules/status_worker.py

@@ -2,7 +2,7 @@
 # Performs a REST call to controller (possibly localhost) of latest farm status.
 #
 
-from api.schedules.common import get_controller_url
+
 import datetime
 import http
 import json
@@ -17,7 +17,7 @@
 from common.config import globals
 from api.commands import plotman_cli
 from api import app
-from api.schedules import common
+from api import utils
 
 def update():
     with app.app_context():
@@ -35,7 +35,7 @@ def update():
                 "mode": os.environ['mode'],
                 "plotting": plotting_status,
             }
-            common.send_post('/workers', payload, debug=False)
+            utils.send_post('/workers', payload, debug=False)
         except:
             app.logger.info("Failed to load send worker status.")
             app.logger.info(traceback.format_exc())

api/schedules/common.py → api/utils.py

@@ -1,9 +1,10 @@
 #
-# Common methods for schedules
+# Util methods for api
 #
 
 import http
 import json
+import os
 import requests
 import socket
 
@@ -18,15 +19,25 @@ def send_post(path, payload, debug=False):
     http.client.HTTPConnection.debuglevel = 0
     return response
 
+def send_delete(path, debug=False):
+    controller_url = get_controller_url()
+    headers = {'Content-type': 'application/json', 'Accept': 'application/json'}
+    if debug:
+        http.client.HTTPConnection.debuglevel = 1
+    response = requests.delete(controller_url + path, headers = headers)
+    http.client.HTTPConnection.debuglevel = 0
+    return response
+
 def get_controller_url():
     return "{0}://{1}:{2}".format(
-        app.config['CONTROLLER_PROTO'],
+        app.config['CONTROLLER_SCHEME'],
         app.config['CONTROLLER_HOST'],
         app.config['CONTROLLER_PORT']
     )
 
 def get_hostname():
-    hostname = socket.gethostname()
-    if 'MY_HOSTNAME' in app.config:
-        hostname = app.config['MY_HOSTAME']
+    if 'hostname' in os.environ:
+        hostname = os.environ['hostname']
+    else:
+        hostname = socket.gethostname()
     return hostname

common/models/farms.py

@@ -16,7 +16,7 @@ class Farm(db.Model):
     plots_size = sa.Column(sa.REAL)  # GiB
     total_chia = sa.Column(sa.REAL) 
     netspace_size = sa.Column(sa.REAL)  # GiB
-    expected_time_to_win = sa.Column(sa.Integer)  # Days
+    expected_time_to_win = sa.Column(sa.String(length=64))
     created_at = sa.Column(sa.DateTime(), server_default=func.now())
     updated_at = sa.Column(sa.DateTime(), onupdate=func.now())
 

common/utils/converters.py

@@ -34,7 +34,7 @@ def str_to_gibs(str):
 
 # Convert expected time to win back to minutes.  See https://github.com/Chia-Network/chia-blockchain/blob/9e21716965f6f6250f6fe4b3449a66f20794d3d9/chia/util/misc.py#L18
 def etw_to_minutes(etw):
-    #logging.info("ETW='{0}'".format(etw))
+    logging.info("ETW='{0}'".format(etw))
     etw_total_minutes = 0
     hour_minutes = 60
     day_minutes = 24 * hour_minutes