Reorganises how Odin data adapters are configured and updated

    Bug fixes preventing collecting offsets or data acquisition before system initialised
    Deletes RdmaUdp unit test, a relic from years gone by
    Removes legacy python 2 support from unit tests
    Some code structure, naming improvements
    Cuts number of flake8 warnings
    Amends unit tests.

Author: dw96

control/src/hexitec/HexitecDAQ.py

@@ -297,7 +297,7 @@ def calculate_average_occupancy(self):
             return return_value
 
     def prepare_odin(self):
-        """Ensure the odin data FP and FR are configured."""
+        """Ensure the odin data FP(s) and FR(s) are configured."""
         logging.debug("Setting up Acquisition")
         fr_status = self.get_adapter_status("fr")
         fp_status = self.get_adapter_status("fp")
@@ -513,7 +513,7 @@ def prepare_hdf_file(self):
             self.hdf_retry = 0
             self.in_progress = False
             self.daq_ready = True
-            self.parent.fem.flag_error("Error reopening HDF file: %s" % e)
+            self.parent.fem.flag_error("Reopening HDF file: %s" % e)
             self.parent.software_state = "Error"
             return
 
@@ -959,21 +959,21 @@ def _set_bin_end(self, bin_end):
         if bin_end < 1:
             raise ParameterTreeError("bin_end must be positive!")
         self.bin_end = bin_end
-        self.update_histogram_dimensions()
+        self.update_number_histograms()
 
     def _set_bin_start(self, bin_start):
         """Update bin_start and datasets' histograms' dimensions."""
         if bin_start < 0:
             raise ParameterTreeError("bin_start must be positive!")
         self.bin_start = bin_start
-        self.update_histogram_dimensions()
+        self.update_number_histograms()
 
     def _set_bin_width(self, bin_width):
         """Update bin_width and datasets' histograms' dimensions."""
         if bin_width <= 0:
             raise ParameterTreeError("bin_width must be positive!")
         self.bin_width = bin_width
-        self.update_histogram_dimensions()
+        self.update_number_histograms()
 
     def update_datasets_frame_dimensions(self):
         """Update frames' datasets' dimensions."""
@@ -983,69 +983,24 @@ def update_datasets_frame_dimensions(self):
             request = ApiAdapterRequest(str(payload), content_type="application/json")
             self.adapters["fp"].put(command, request)
 
-    def update_histogram_dimensions(self):
-        """Update histograms' dimensions in the relevant datasets."""
+    def update_number_histograms(self):
+        """Update number of histograms."""
         self.number_histograms = int((self.bin_end - self.bin_start) / self.bin_width)
-        self.gcf = GenerateConfigFiles(self.param_tree.get(''), self.number_histograms,
-                                       compression_type=self.compression_type,
-                                       master_dataset=self.master_dataset,
-                                       extra_datasets=self.extra_datasets,
-                                       live_view_selected=False,
-                                       odin_path=self.odin_path)
-        # spectra_bins dataset
-        payload = '{"dims": [%s], "chunks": [1, %s]}' % \
-            (self.number_histograms, self.number_histograms)
-        command = "config/hdf/dataset/" + "spectra_bins"
-        request = ApiAdapterRequest(str(payload), content_type="application/json")
-        self.adapters["fp"].put(command, request)
-
-        # pixel_spectra dataset
-        ps_params = self.gcf.generate_pixel_spectra_params()
-        payload = '{%s}' % (ps_params)
-        command = "config/hdf/dataset/" + "pixel_spectra"
-        request = ApiAdapterRequest(str(payload), content_type="application/json")
-        self.adapters["fp"].put(command, request)
-
-        # summed_spectra dataset
-        payload = '{"dims": [%s], "chunks": [1, %s]}' % \
-            (self.number_histograms, self.number_histograms)
-        command = "config/hdf/dataset/" + "summed_spectra"
-        request = ApiAdapterRequest(str(payload), content_type="application/json")
-        self.adapters["fp"].put(command, request)
-        del self.gcf
-        self.gcf = None
 
     def _set_max_frames_received(self, max_frames_received):
         self.max_frames_received = max_frames_received
 
     def _set_pass_processed(self, pass_processed=None):
+        """Toggle passing processed dataset on/off."""
         if pass_processed is not None:
             self.pass_processed = pass_processed
-        self.commit_config_before_acquire = True
-        command = "config/histogram"
-        formatted_string = ('{"pass_processed": %s}' % self.pass_processed).lower()
-        request = ApiAdapterRequest(formatted_string, content_type="application/json")
-
-        response = self.adapters["fp"].put(command, request)
-        status_code = response.status_code
-        if (status_code != 200):
-            error = "Error {} updating histogram's processed dataset".format(status_code)
-            self.parent.fem.flag_error(error)
+        self.commit_config_before_acquire = True 
 
     def _set_pass_raw(self, pass_raw=None):
-        """Change pass_raw if provided, then update FP setting."""
+        """Toggle passing raw dataset on/off."""
         if pass_raw is not None:
             self.pass_raw = pass_raw
         self.commit_config_before_acquire = True
-        command = "config/histogram"
-        formatted_string = ('{"pass_raw": %s}' % self.pass_raw).lower()
-        request = ApiAdapterRequest(formatted_string, content_type="application/json")
-
-        response = self.adapters["fp"].put(command, request)
-        status_code = response.status_code
-        if (status_code != 200):
-            error = "Error {} updating fp histogram's raw dataset".format(status_code)
-            self.parent.fem.flag_error(error)
 
     def _set_threshold_filename(self, threshold_filename):
         threshold_filename = self.data_config_path + threshold_filename
@@ -1102,7 +1057,7 @@ def _set_sensors_layout(self, layout):
 
         self.update_rows_columns_pixels()
         self.update_datasets_frame_dimensions()
-        self.update_histogram_dimensions()
+        self.update_number_histograms()
 
     def _get_compression_type(self):
         return self.compression_type
@@ -1152,6 +1107,7 @@ def commit_configuration(self):
         # Enable live view for first node only
         live_view_selected = True
         logging.debug("Sending configuration to %s FP(s)" % self.number_nodes)
+
         # Loop over node(s)
         for index in range(self.number_nodes):
             self.gcf = GenerateConfigFiles(parameter_tree, self.number_histograms,
@@ -1181,10 +1137,33 @@ def commit_configuration(self):
             if (status_code != 200):
                 error = "Error {} loading plugins config in fp adapter".format(status_code)
                 self.parent.fem.flag_error(error)
+            pixel_spectra_params = self.gcf.generate_pixel_spectra_params()
             # Delete GCF object before next iteration
             del self.gcf
             self.gcf = None
 
+        # Update dataset dimensions
+
+        # spectra_bins dataset
+        payload = '{"dims": [%s], "chunks": [1, %s]}' % \
+            (self.number_histograms, self.number_histograms)
+        command = "config/hdf/dataset/" + "spectra_bins"
+        request = ApiAdapterRequest(str(payload), content_type="application/json")
+        self.adapters["fp"].put(command, request)
+
+        # pixel_spectra dataset
+        payload = '{%s}' % (pixel_spectra_params)
+        command = "config/hdf/dataset/" + "pixel_spectra"
+        request = ApiAdapterRequest(str(payload), content_type="application/json")
+        self.adapters["fp"].put(command, request)
+
+        # summed_spectra dataset
+        payload = '{"dims": [%s], "chunks": [1, %s]}' % \
+            (self.number_histograms, self.number_histograms)
+        command = "config/hdf/dataset/" + "summed_spectra"
+        request = ApiAdapterRequest(str(payload), content_type="application/json")
+        self.adapters["fp"].put(command, request)
+
         # Allow FP time to process above PUT requests before configuring plugin settings
         IOLoop.instance().call_later(0.4, self.submit_configuration)
 
@@ -1214,6 +1193,26 @@ def submit_configuration(self):
         else:
             self.plugin = "histogram"
 
+        command = "config/histogram"
+        formatted_string = ('{"pass_processed": %s}' % self.pass_processed).lower()
+        request = ApiAdapterRequest(formatted_string, content_type="application/json")
+
+        response = self.adapters["fp"].put(command, request)
+        status_code = response.status_code
+        if (status_code != 200):
+            error = "Error {} updating histogram's processed dataset".format(status_code)
+            self.parent.fem.flag_error(error)
+
+        command = "config/histogram"
+        formatted_string = ('{"pass_raw": %s}' % self.pass_raw).lower()
+        request = ApiAdapterRequest(formatted_string, content_type="application/json")
+
+        response = self.adapters["fp"].put(command, request)
+        status_code = response.status_code
+        if (status_code != 200):
+            error = "Error {} updating fp histogram's raw dataset".format(status_code)
+            self.parent.fem.flag_error(error)
+
         # Update live histogram labelling according to calibration enabled (or not)
         command = ""
         payload = ('{"calibration_enable": %s}' % self.calibration_enable).lower()

control/src/hexitec/HexitecFem.py

@@ -323,10 +323,6 @@ def verify_farm_mode_parameters(self, iface):
         self.number_nodes = num_ips
         self.parent.set_number_nodes(self.number_nodes)
 
-    def prepare_hardware(self, bDebug=False):
-        """Prepare hardware connection."""
-        return self.prepare_farm_mode()
-
     def configure_camera_interfaces(self):
         """Configure IP, Mac and port parameters for detector's Control and Data interfaces."""
         Hex2x6CtrlRdma = RdmaUDP(local_ip=self.server_ctrl_ip, local_port=self.server_ctrl_port,
@@ -489,23 +485,35 @@ def connect(self):
             raise socket_error("Failed to setup Control connection: %s" % e)
         return
 
-    def check_hardware_status(self, action):
+    def check_hardware_ready(self, action):
         """Helper function checking hardware connected and not busy.
 
         Raise Exception if hardware not connected, or hardware busy.
         """
         if self.hardware_connected is not True:
-            raise ParameterTreeError(f"Error: Can't {action} without a connection")
+            raise ParameterTreeError(f"Can't {action} without a connection")
         if self.hardware_busy:
-            error = f"Error: Can't {action}, Hardware busy"
+            error = f"Can't {action}, Hardware busy"
+            self.flag_error(error, "")
+            raise ParameterTreeError(error)
+        else:
+            self._set_status_error("")
+
+    def check_system_initialised(self, action):
+        """Helper function checking system initialised.
+
+        Raise Exception if system not initialised.
+        """
+        if self.system_initialised is not True:
+            error = f"Can't {action}, system not initialised"
             self.flag_error(error, "")
             raise ParameterTreeError(error)
         else:
             self._set_status_error("")
 
     def environs(self, msg=None):
         """Readout environmental data if hardware connected and not busy."""
-        self.check_hardware_status("read sensors")
+        self.check_hardware_ready("read sensors")
         IOLoop.instance().add_callback(self.read_sensors)
 
     @run_on_executor(executor='thread_executor')
@@ -605,16 +613,16 @@ def connect_hardware(self, msg=None):
                 self.parent.software_state = "Cold"
                 return
             if self.hardware_connected:
-                raise ParameterTreeError("Error: Connection already established")
+                error = "Connection already established"
+                self.flag_error(error, "")
+                raise ParameterTreeError(error)
             else:
                 self._set_status_error("")
             self.hardware_busy = True
             self.hardware_connected = True
             # Configure control, data lines unless already configured
             # - Insist Control interface configured on every connect
             if self.parent.cold_start:
-                # Commit configuration (again) otherwise FP(s) shutdown prematurely
-                self.parent.daq.commit_configuration()  # TODO superfluous?
                 # Configure Control, Camera interfaces
                 self.configure_camera_interfaces()
                 self.parent.cold_start = True
@@ -679,7 +687,7 @@ def power_up_modules(self):
 
     def initialise_hardware(self, msg=None):
         """Initialise sensors, load enables, etc to initialise both VSR boards."""
-        self.check_hardware_status("initialise hardware")
+        self.check_hardware_ready("initialise hardware")
         try:
             self.hardware_busy = True
             self.parent.software_state = "Initialising"
@@ -692,10 +700,8 @@ def initialise_hardware(self, msg=None):
 
     def collect_data(self, msg=None):
         """Acquire data from camera."""
-        self.check_hardware_status("collect data")
         try:
             self.hardware_busy = True
-            # self.parent.software_state = "Acquiring"
             self._set_status_message("Acquiring data..")
             self.acquire_data()
         except Exception as e:
@@ -741,6 +747,7 @@ def cam_connect_completed(self):
         logging.debug("Modules Enabled")
         self._set_status_message("VSRs booted")
         self.hardware_busy = False
+        self.parent.daq.commit_configuration()
         self.parent.software_state = "Idle"
 
     def cam_disconnect(self):
@@ -751,14 +758,14 @@ def cam_disconnect(self):
             logging.debug("Modules Disabled")
             self.disconnect()
             logging.debug("Camera is Disconnected")
-            self.system_initialised = False
         except socket_error as e:
             self.flag_error("Unable to disconnect camera", str(e))
             raise HexitecFemError(e)
         except AttributeError as e:
             error = "Unable to disconnect camera: No active connection"
             self.flag_error(error, str(e))
             raise HexitecFemError("%s; %s" % (e, "No active connection"))
+        self.system_initialised = False
 
     def acquire_data(self):
         """Acquire data, poll fem for completion."""
@@ -868,7 +875,8 @@ def acquire_data_completed(self):
 
     def run_collect_offsets(self):
         """Run collect offsets sequence if connected and hardware not busy."""
-        self.check_hardware_status("collect offsets")
+        self.check_hardware_ready("collect offsets")
+        self.check_system_initialised("collect offsets")
         self.collect_offsets()
 
     @run_on_executor(executor='thread_executor')
@@ -1597,7 +1605,7 @@ def convert_bias_to_dac_values(self, hv):
 
     def hv_on(self):
         """Switch HV on."""
-        self.check_hardware_status("switch HV on")
+        self.check_hardware_ready("switch HV on")
         logging.debug("Going to set HV bias to -{} volts".format(self.bias_level))
         hv_msb, hv_lsb = self.convert_bias_to_dac_values(self.bias_level)
         # print(f" HV Bias (-{self.bias_level}) : {hv_msb[0]:X} {hv_msb[1]:X}",
@@ -1611,7 +1619,7 @@ def hv_on(self):
 
     def hv_off(self):
         """Switch HV off."""
-        self.check_hardware_status("switch HV off")
+        self.check_hardware_ready("switch HV off")
         logging.debug("Disable: [0xE2]")
         # Can call hv_off function on any VSR object
         self.vsr_list[0].hv_off()