adding new versions to monitor

Author: CamDavidsonPilon

CHANGELOG.md

@@ -1,5 +1,7 @@
 ### Upcoming
- - introducing some outlier protection in growth rate calculations
+ - fixed `datum` bug in the Overview that was crashing the UI. Sorry!
+ - introducing some outlier protection in growth rate calculations. This is tunable with the `ekf_outlier_std_threshold` value under `[growth_rate_calculating.config]`. The default value, 5.0, is pretty conservative, and will still allow some spikes to permeate to the growth rate and nOD. Try a value of 3.5 or 4.0 if you want to see more. Don't put it less than 3.0 - that's silly.
+
 
 ### 24.2.11
  - boot-up performance improvements

pioreactor/background_jobs/growth_rate_calculating.py

@@ -181,20 +181,25 @@ def initialize_extended_kalman_filter(
         ]
 
         self.logger.debug(f"{angles=}")
-        outlier_std_threshold = config.getfloat(
+        ekf_outlier_std_threshold = config.getfloat(
             "growth_rate_calculating.config",
-            "outlier_std_threshold",
+            "ekf_outlier_std_threshold",
             fallback=5.0,
         )
-        self.logger.debug(f"{outlier_std_threshold=}")
+        if ekf_outlier_std_threshold <= 2.0:
+            raise ValueError(
+                "outlier_std_threshold should not be less than 2.0 - that's eliminating too many data points."
+            )
+
+        self.logger.debug(f"{ekf_outlier_std_threshold=}")
 
         return CultureGrowthEKF(
             initial_state,
             initial_covariance,
             process_noise_covariance,
             observation_noise_covariance,
             angles,
-            outlier_std_threshold,
+            ekf_outlier_std_threshold,
         )
 
     def create_obs_noise_covariance(self, obs_std):  # typing: ignore
@@ -326,12 +331,10 @@ def get_filtered_od_from_cache_or_computed(self) -> float:
             allow_retained=True,  # maybe?
             timeout=10,
         )
-        print(f"pioreactor/{self.unit}/{self.experiment}/od_reading/ods")
         if msg is None:
             return 1.0  # default?
 
         od_readings = decode(msg.payload, type=structs.ODReadings)
-        print(od_readings)
         scaled_ods = self.scale_raw_observations(self._batched_raw_od_readings_to_dict(od_readings.ods))
         assert scaled_ods is not None
         return mean(scaled_ods.values())

pioreactor/background_jobs/monitor.py

@@ -88,7 +88,7 @@ def off(): # functions don't take any arguments, nothing is passed in
     published_settings = {
         "computer_statistics": {"datatype": "json", "settable": False},
         "button_down": {"datatype": "boolean", "settable": False},
-        "versions": {"datatype": "json", "settable": False},
+        "versions": {"datatype": "json", "settable": True},
         "voltage_on_pwm_rail": {"datatype": "Voltage", "settable": False},
         "ipv4": {"datatype": "string", "settable": False},
         "wlan_mac_address": {"datatype": "string", "settable": False},
@@ -109,20 +109,21 @@ def pretty_version(info: tuple) -> str:
         # Sol2: pio update app republishes this data, OR publishes an event that Monitor listens to.
         #
         self.versions = {
-            "software": pretty_version(version.software_version_info),
+            "app": pretty_version(version.software_version_info),
             "hat": pretty_version(version.hardware_version_info),
+            "firmware": pretty_version(version.get_firmware_version()),
             "hat_serial": version.serial_number,
+            "rpi_machine": version.rpi_version_info,
             "timestamp": current_utc_timestamp(),
         }
 
-        self.logger.debug(f"Pioreactor software version: {pretty_version(version.software_version_info)}")
+        self.logger.debug(f"Pioreactor software version: {self.versions['app']}")
+        self.logger.debug(f"Raspberry Pi: {self.versions['rpi_machine']}")
 
         if whoami.am_I_active_worker():
             self.logger.debug(f"Pioreactor HAT version: {self.versions['hat']}")
 
-            self.logger.debug(
-                f"Pioreactor firmware version: {pretty_version(version.get_firmware_version())}"
-            )
+            self.logger.debug(f"Pioreactor firmware version: {self.versions['firmware']}")
 
             self.logger.debug(f"Pioreactor HAT serial number: {self.versions['hat_serial']}")
 
@@ -716,6 +717,12 @@ def flicker_error_code_from_mqtt(self, message: MQTTMessage) -> None:
         error_code = int(message.payload)
         Thread(target=self.flicker_led_with_error_code, args=(error_code,), daemon=True).start()
 
+    def set_versions(self, data: dict):
+        # TODO: this can also be a dict merge
+        for key, value in data.items():
+            if key in self.versions:
+                self.versions[key] = value
+
     def start_passive_listeners(self) -> None:
         self.subscribe_and_callback(
             self.flicker_led_response_okay_and_publish_state,

pioreactor/cli/pio.py

@@ -36,6 +36,7 @@
 from pioreactor.utils.networking import add_local
 from pioreactor.utils.networking import is_using_local_access_point
 from pioreactor.utils.timing import catchtime
+from pioreactor.utils.timing import current_utc_timestamp
 
 
 JOBS_TO_SKIP_KILLING = [
@@ -605,6 +606,11 @@ def update_app(
             logger.debug(p.stdout)
 
     logger.notice(f"Updated {whoami.get_unit_name()} to version {version_installed}.")  # type: ignore
+    # everything work? Let's publish to MQTT. This is a terrible hack, as monitor should do this.
+    pubsub.publish(
+        f"pioreactor/{whoami.get_unit_name()}/{whoami.UNIVERSAL_EXPERIMENT}/monitor/versions/set",
+        {"app": version_installed, "timestamp": current_utc_timestamp()},
+    )
 
 
 @update.command(name="firmware")

pioreactor/utils/streaming_calculations.py

@@ -280,12 +280,10 @@ def update(self, observation_: list[float], dt: float):
             + self.state_[0] * self.observation_noise_covariance
         )
 
-        # print(abs(residual_state[0])/np.sqrt(residual_covariance[0, 0]))
         # check if outlier
         if self.ignore_outliers and (
             abs(residual_state[0]) > self.outlier_std_threshold * np.sqrt(residual_covariance[0, 0])
         ):
-            print("OUTLIER!")
             # don't update, but allow randomly
             if np.random.random() > 0.1:
                 return self.state_, self.covariance_