new update_ui script, and some support for release_archives

Pioreactor · Dec 5, 2023 · 2b4fee6 · 2b4fee6
1 parent c487868
commit 2b4fee6
Show file tree

Hide file tree

Showing 4 changed files with 43 additions and 90 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,6 @@
 ### Upcoming
 
- - Improvements to OD calibration and pump calibrations
+ - Improvements to OD calibration and pump calibrations. Both now have a `-f` option to provide a json file with calibration data, to skip rerunning data-gathering routines. For example: `pio run pump_calibration -f pump_data.json`.
 
 ### 23.11.29
 

diff --git a/pioreactor/actions/led_intensity.py b/pioreactor/actions/led_intensity.py
@@ -165,7 +165,7 @@ def led_intensity(
             try:
                 assert (
                     channel in ALL_LED_CHANNELS
-                ), f"saw incorrect channel {channel}, not in {ALL_LED_CHANNELS}"
+                ), f"Saw incorrect channel {channel}, not in {ALL_LED_CHANNELS}"
                 assert (
                     0.0 <= intensity <= 100.0
                 ), f"Channel {channel} intensity should be between 0 and 100, inclusive"

diff --git a/pioreactor/background_jobs/base.py b/pioreactor/background_jobs/base.py
@@ -635,9 +635,7 @@ def _on_mqtt_disconnect(self, client, rc: int) -> None:
         # Error codes are below, but don't always align
         # https://github.com/eclipse/paho.mqtt.python/blob/42f0b13001cb39aee97c2b60a3b4807314dfcb4d/src/paho/mqtt/client.py#L147
         elif rc == mqtt.MQTT_ERR_KEEPALIVE:
-            self.logger.warning(
-                "Lost contact with MQTT server. Is the leader Pioreactor still online?"
-            )
+            self.logger.warning("Lost contact with MQTT server. Is the leader Pioreactor still online?")
         else:
             self.logger.debug(f"Disconnected from MQTT with {rc=}: {mqtt.error_string(rc)}")
         return
@@ -814,9 +812,7 @@ def _clean_up_resources(self):
 
         self._clean = True
 
-    def _publish_properties_to_broker(
-        self, published_settings: dict[str, pt.PublishableSetting]
-    ) -> None:
+    def _publish_properties_to_broker(self, published_settings: dict[str, pt.PublishableSetting]) -> None:
         # this follows some of the Homie convention: https://homieiot.github.io/specification/
         self.publish(
             f"pioreactor/{self.unit}/{self.experiment}/{self.job_name}/$properties",
@@ -825,9 +821,7 @@ def _publish_properties_to_broker(
             retain=True,
         )
 
-    def _publish_settings_to_broker(
-        self, published_settings: dict[str, pt.PublishableSetting]
-    ) -> None:
+    def _publish_settings_to_broker(self, published_settings: dict[str, pt.PublishableSetting]) -> None:
         # this follows some of the Homie convention: https://homieiot.github.io/specification/
         for setting, props in published_settings.items():
             self.publish(
@@ -867,9 +861,7 @@ def get_attr_from_topic(topic: str) -> str:
             self.logger.debug(f"Unable to set `{attr}` in {self.job_name}.")
             return
         elif not self.published_settings[attr]["settable"]:
-            self.logger.warning(
-                f"Unable to set `{attr}` in {self.job_name}. `{attr}` is read-only."
-            )
+            self.logger.warning(f"Unable to set `{attr}` in {self.job_name}. `{attr}` is read-only.")
             return
 
         previous_value = getattr(self, attr)
@@ -1016,9 +1008,7 @@ def action_to_do_after_od_reading(self):
     def __init__(self, *args, source="app", **kwargs) -> None:
         super().__init__(*args, source=source, **kwargs)  # type: ignore
 
-        self.add_to_published_settings(
-            "enable_dodging_od", {"datatype": "boolean", "settable": True}
-        )
+        self.add_to_published_settings("enable_dodging_od", {"datatype": "boolean", "settable": True})
         self.set_enable_dodging_od(bool(self.get_from_config("enable_dodging_od", fallback=True)))
 
     def get_from_config(self, key, **get_kwargs):
@@ -1047,19 +1037,15 @@ def set_enable_dodging_od(self, value: bool) -> None:
                 self.action_to_do_after_od_reading()
             except Exception:
                 pass
-            self.sub_client.unsubscribe(
-                f"pioreactor/{self.unit}/{self.experiment}/od_reading/interval"
-            )
+            self.sub_client.unsubscribe(f"pioreactor/{self.unit}/{self.experiment}/od_reading/interval")
 
     def _setup_actions(self, msg: pt.MQTTMessage) -> None:
         if not msg.payload:
             # OD reading stopped: reset and exit
             if hasattr(self, "sneak_in_timer"):
                 self.sneak_in_timer.cancel()
             self.action_to_do_after_od_reading()
-            self.sub_client.unsubscribe(
-                f"pioreactor/{self.unit}/{self.experiment}/od_reading/interval"
-            )
+            self.sub_client.unsubscribe(f"pioreactor/{self.unit}/{self.experiment}/od_reading/interval")
             return
 
         # OD found - revert to paused state
@@ -1081,17 +1067,13 @@ def _setup_actions(self, msg: pt.MQTTMessage) -> None:
             pass
 
         post_delay = float(self.get_from_config("post_delay_duration", fallback=1.0))
-        pre_delay = float(self.get_from_config("pre_delay_duration", fallback=1.0))
+        pre_delay = float(self.get_from_config("pre_delay_duration", fallback=1.5))
 
         if post_delay <= 0.25:
-            self.logger.warning(
-                "For optimal OD readings, keep `post_delay_duration` more than 0.25 seconds."
-            )
+            self.logger.warning("For optimal OD readings, keep `post_delay_duration` more than 0.25 seconds.")
 
-        if pre_delay <= 0.1:
-            self.logger.warning(
-                "For optimal OD readings, keep `pre_delay_duration` more than 0.1 seconds."
-            )
+        if pre_delay <= 0.25:
+            self.logger.warning("For optimal OD readings, keep `pre_delay_duration` more than 0.25 seconds.")
 
         def sneak_in(ads_interval, post_delay, pre_delay) -> None:
             if self.state != self.READY:
@@ -1113,9 +1095,7 @@ def sneak_in(ads_interval, post_delay, pre_delay) -> None:
         else:
             return
 
-        ads_interval_msg = subscribe(
-            f"pioreactor/{self.unit}/{self.experiment}/od_reading/interval"
-        )
+        ads_interval_msg = subscribe(f"pioreactor/{self.unit}/{self.experiment}/od_reading/interval")
         if ads_interval_msg:
             ads_interval = float(ads_interval_msg.payload)
         else:

diff --git a/pioreactor/cli/pio.py b/pioreactor/cli/pio.py
@@ -292,9 +292,7 @@ def view_cache(cache: str) -> None:
     tmp_dir = tempfile.gettempdir()
 
     persistant_dir = (
-        "/home/pioreactor/.pioreactor/storage/"
-        if not whoami.is_testing_env()
-        else ".pioreactor/storage"
+        "/home/pioreactor/.pioreactor/storage/" if not whoami.is_testing_env() else ".pioreactor/storage"
     )
 
     # is it a temp cache or persistant cache?
@@ -316,19 +314,15 @@ def view_cache(cache: str) -> None:
 @pio.command(name="clear-cache", short_help="clear out the contents of a cache")
 @click.argument("cache")
 @click.argument("key")
-@click.option(
-    "--as-int", is_flag=True, help="evict after casting key to int, useful for gpio pins."
-)
+@click.option("--as-int", is_flag=True, help="evict after casting key to int, useful for gpio pins.")
 def clear_cache(cache: str, key: str, as_int: bool) -> None:
     import os.path
     import tempfile
 
     tmp_dir = tempfile.gettempdir()
 
     persistant_dir = (
-        "/home/pioreactor/.pioreactor/storage/"
-        if not whoami.is_testing_env()
-        else ".pioreactor/storage"
+        "/home/pioreactor/.pioreactor/storage/" if not whoami.is_testing_env() else ".pioreactor/storage"
     )
 
     # is it a temp cache or persistant cache?
@@ -376,9 +370,7 @@ def update_settings(ctx, job: str) -> None:
 
     for setting, value in extra_args.items():
         setting = setting.replace("-", "_")
-        pubsub.publish(
-            f"pioreactor/{unit}/{exp}/{job}/{setting}/set", value, qos=pubsub.QOS.EXACTLY_ONCE
-        )
+        pubsub.publish(f"pioreactor/{unit}/{exp}/{job}/{setting}/set", value, qos=pubsub.QOS.EXACTLY_ONCE)
 
 
 @pio.group()
@@ -472,11 +464,9 @@ def update_app(
     """
     Update the Pioreactor core software
     """
-    logger = create_logger(
-        "update-app", unit=whoami.get_unit_name(), experiment=whoami.UNIVERSAL_EXPERIMENT
-    )
+    logger = create_logger("update-app", unit=whoami.get_unit_name(), experiment=whoami.UNIVERSAL_EXPERIMENT)
 
-    commands_and_priority: list[tuple[str, int]] = []
+    commands_and_priority: list[tuple[str, float]] = []
 
     if source is not None:
         source = quote(source)
@@ -486,43 +476,32 @@ def update_app(
         if re.search("release_(.*).zip", source):
             version_installed = re.search("release_(.*).zip", source).groups()[0]  # type: ignore
             tmp_release_folder = f"/tmp/release_{version_installed}"
+            # fmt: off
             commands_and_priority.extend(
                 [
                     (f"rm -rf {tmp_release_folder}", -3),
                     (f"unzip {source} -d {tmp_release_folder}", -2),
-                    (
-                        f"unzip {tmp_release_folder}/wheels_{version_installed}.zip -d {tmp_release_folder}/wheels",
-                        0,
-                    ),
-                    (f"sudo bash {tmp_release_folder}/pre_update.sh || true", 1),
-                    (
-                        f"sudo pip install --force-reinstall --no-index --find-links={tmp_release_folder}/wheels/ {tmp_release_folder}/pioreactor-{version_installed}-py3-none-any.whl",
-                        2,
-                    ),
-                    (f"sudo bash {tmp_release_folder}/update.sh || true", 3),
-                    (
-                        f'sudo sqlite3 {config["storage"]["database"]} < {tmp_release_folder}/update.sql || true',
-                        4,
-                    ),
-                    (f"sudo bash {tmp_release_folder}/post_update.sh || true", 5),
+                    (f"unzip {tmp_release_folder}/wheels_{version_installed}.zip -d {tmp_release_folder}/wheels", 0),
+                    (f"mv {tmp_release_folder}/pioreactorui_*.tar.gz /tmp/pioreactorui_archive || :", 0.5),
+                    (f"sudo bash {tmp_release_folder}/pre_update.sh || :", 1),
+                    (f"sudo pip install --force-reinstall --no-index --find-links={tmp_release_folder}/wheels/ {tmp_release_folder}/pioreactor-{version_installed}-py3-none-any.whl", 2),
+                    (f"sudo bash {tmp_release_folder}/update.sh || :", 3),
+                    (f'sudo sqlite3 {config["storage"]["database"]} < {tmp_release_folder}/update.sql || :', 4),
+                    (f"sudo bash {tmp_release_folder}/post_update.sh || :", 5),
                 ]
             )
+            # fmt: on
 
         else:
             version_installed = source
-            commands_and_priority.append(
-                (f"sudo pip3 install --force-reinstall --no-index {source}", 1)
-            )
+            commands_and_priority.append((f"sudo pip3 install --force-reinstall --no-index {source}", 1))
 
     elif branch is not None:
         cleaned_branch = quote(branch)
         cleaned_repo = quote(repo)
         version_installed = cleaned_branch
         commands_and_priority.append(
-            (
-                f"sudo pip3 install -U --force-reinstall https://github.com/{cleaned_repo}/archive/{cleaned_branch}.zip",
-                1,
-            )
+            (f"sudo pip3 install -U --force-reinstall https://github.com/{cleaned_repo}/archive/{cleaned_branch}.zip", 1,)  # fmt: skip
         )
 
     else:
@@ -616,9 +595,7 @@ def update_firmware(version: Optional[str]) -> None:
     """
     Update the RP2040 firmware
     """
-    logger = create_logger(
-        "update-app", unit=whoami.get_unit_name(), experiment=whoami.UNIVERSAL_EXPERIMENT
-    )
+    logger = create_logger("update-app", unit=whoami.get_unit_name(), experiment=whoami.UNIVERSAL_EXPERIMENT)
     commands_and_priority: list[tuple[str, int]] = []
 
     if version is None:
@@ -713,9 +690,7 @@ def db() -> None:
     def mqtt(topic: str) -> None:
         import os
 
-        os.system(
-            f"""mosquitto_sub -v -t '{topic}' -F "%19.19I  |  %t   %p" -u pioreactor -P raspberry"""
-        )
+        os.system(f"""mosquitto_sub -v -t '{topic}' -F "%19.19I  |  %t   %p" -u pioreactor -P raspberry""")
 
     @pio.command(name="add-pioreactor", short_help="add a new Pioreactor to cluster")
     @click.argument("hostname")
@@ -829,15 +804,17 @@ def display_data_for(hostname_status: tuple[str, str]) -> bool:
 
             ip, state, reachable, versions = get_metadata(hostname)
 
-            statef = click.style(
-                f"{state:15s}", fg="green" if state in ("ready", "init") else "red"
-            )
+            statef = click.style(f"{state:15s}", fg="green" if state in ("ready", "init") else "red")
             ipf = f"{ip if (ip is not None) else 'unknown':20s}"
 
             is_leaderf = f"{('Y' if hostname==get_leader_hostname() else 'N'):15s}"
             hostnamef = f"{hostname:20s}"
-            reachablef = f"{(click.style('Y', fg='green') if reachable       else click.style('N', fg='red')):23s}"
-            statusf = f"{(click.style('Y', fg='green') if (status == '1') else click.style('N', fg='red')):23s}"
+            reachablef = (
+                f"{(click.style('Y', fg='green') if reachable       else click.style('N', fg='red')):23s}"
+            )
+            statusf = (
+                f"{(click.style('Y', fg='green') if (status == '1') else click.style('N', fg='red')):23s}"
+            )
             versionf = f"{versions['hat']:15s}"
 
             click.echo(f"{hostnamef} {is_leaderf} {ipf} {statef} {reachablef} {statusf} {versionf}")
@@ -869,9 +846,7 @@ def display_data_for(hostname_status: tuple[str, str]) -> bool:
     )
     @click.option("--source", help="use a tar.gz file")
     @click.option("-v", "--version", help="install a specific version")
-    def update_ui(
-        branch: Optional[str], repo: str, source: Optional[str], version: Optional[str]
-    ) -> None:
+    def update_ui(branch: Optional[str], repo: str, source: Optional[str], version: Optional[str]) -> None:
         """
         Update the PioreactorUI
 
@@ -890,8 +865,7 @@ def update_ui(
 
         if source is not None:
             source = quote(source)
-            assert version is not None, "version must be provided with the -v option"
-            version_installed = version
+            version_installed = source
 
         elif branch is not None:
             cleaned_branch = quote(branch)
@@ -911,8 +885,7 @@ def update_ui(
             commands.append(["wget", url, "-O", source])
 
         assert source is not None
-        assert version_installed is not None
-        commands.append(["bash", "/usr/local/bin/update_ui.sh", source, version_installed])
+        commands.append(["bash", "/usr/local/bin/update_ui.sh", source])
 
         for command in commands:
             logger.debug(" ".join(command))