Update documentation

sed/develop/_sources/sed/dataset.rst.txt

@@ -318,7 +318,7 @@ we can remove one instance
 Default datasets.json
 ---------------------
 
-.. literalinclude::  ../../sed/dataset/datasets.json
+.. literalinclude::  ../../src/sed/config/datasets.json
    :language: json
 
 API

sed/develop/_sources/tutorial/10_hextof_workflow_trXPS_bam_correction.ipynb.txt

@@ -96,7 +96,7 @@
    "outputs": [],
    "source": [
     "# pick the default configuration file for hextof@FLASH\n",
-    "config_file = Path('../sed/config/flash_example_config.yaml')\n",
+    "config_file = Path('../src/sed/config/flash_example_config.yaml')\n",
     "assert config_file.exists()"
    ]
   },
@@ -112,8 +112,8 @@
     "    \"core\": {\n",
     "        \"beamtime_id\": 11019101,\n",
     "        \"paths\": {\n",
-    "            \"data_raw_dir\": path,\n",
-    "            \"data_parquet_dir\": buffer_path\n",
+    "            \"raw\": path,\n",
+    "            \"processed\": buffer_path\n",
     "        },\n",
     "    },\n",
     "}"
@@ -129,26 +129,24 @@
     "energy_cal = {\n",
     "    \"energy\": {\n",
     "        \"calibration\": {\n",
-    "            \"E0\": -53.96145014592986,\n",
-    "            \"creation_date\": 1732056868.029444,\n",
-    "            \"d\": 0.8096677233434938,\n",
+    "            \"E0\": -132.47100427179566,\n",
+    "            \"creation_date\": '2024-11-30T20:47:03.305244',\n",
+    "            \"d\": 0.8096677238144319,\n",
     "            \"energy_scale\": \"kinetic\",\n",
-    "            \"t0\": 4.0148196718030886e-07,\n",
+    "            \"t0\": 4.0148196706891397e-07,\n",
     "        },\n",
     "        \"offsets\":{\n",
-    "            \"constant\": -77.5,\n",
-    "            \"creation_date\": 1732056874.060922,\n",
-    "            \"monochromatorPhotonEnergy\": {\n",
-    "                \"preserve_mean\": True,\n",
-    "                \"weight\": -1,\n",
-    "            },\n",
-    "            \"sampleBias\": {\n",
-    "                \"preserve_mean\": False,\n",
-    "                \"weight\": 1,\n",
-    "            },\n",
-    "            \"tofVoltage\": {\n",
-    "                \"preserve_mean\": True,\n",
-    "                \"weight\": -1,\n",
+    "            \"constant\": 1,\n",
+    "            \"creation_date\": '2024-11-30T21:17:07.762199',\n",
+    "            \"columns\": {\n",
+    "                \"monochromatorPhotonEnergy\": {\n",
+    "                    \"preserve_mean\": True,\n",
+    "                    \"weight\": -1,\n",
+    "                },\n",
+    "                \"tofVoltage\": {\n",
+    "                    \"preserve_mean\": True,\n",
+    "                    \"weight\": -1,\n",
+    "                },\n",
     "            },\n",
     "        },\n",
     "    },\n",
@@ -230,7 +228,7 @@
     "axes = ['energy', 'delayStage']\n",
     "ranges = [[-37.5,-27.5], [1446.75,1449.15]]\n",
     "bins = [200,40]\n",
-    "res = sp_44498.compute(bins=bins, axes=axes, ranges=ranges, normalize_to_acquisition_time=\"delayStage\")"
+    "res = sp_44498.compute(bins=bins, axes=axes, ranges=ranges)"
    ]
   },
   {
@@ -396,7 +394,7 @@
     "axes = ['energy', 'delayStage']\n",
     "ranges = [[-37.5,-27.5], [-1.5,1.5]]\n",
     "bins = [200,60]\n",
-    "res_corr = sp_44498.compute(bins=bins, axes=axes, ranges=ranges, normalize_to_acquisition_time=\"delayStage\")"
+    "res_corr = sp_44498.compute(bins=bins, axes=axes, ranges=ranges)"
    ]
   },
   {
@@ -535,11 +533,19 @@
    "metadata": {},
    "outputs": [],
    "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e2e6e852",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "python3",
+   "display_name": "sed-processor-7Jy-bAA8-py3.9",
    "language": "python",
    "name": "python3"
   },

sed/develop/_sources/tutorial/11_hextof_workflow_trXPS_energy_calibration_using_SB.ipynb.txt

@@ -97,7 +97,7 @@
    "outputs": [],
    "source": [
     "# pick the default configuration file for hextof@FLASH\n",
-    "config_file = Path('../sed/config/flash_example_config.yaml')\n",
+    "config_file = Path('../src/sed/config/flash_example_config.yaml')\n",
     "assert config_file.exists()"
    ]
   },
@@ -113,8 +113,8 @@
     "    \"core\": {\n",
     "        \"beamtime_id\": 11019101,\n",
     "        \"paths\": {\n",
-    "            \"data_raw_dir\": path,\n",
-    "            \"data_parquet_dir\": buffer_path\n",
+    "            \"raw\": path,\n",
+    "            \"processed\": buffer_path\n",
     "        },\n",
     "    },\n",
     "}"
@@ -175,14 +175,6 @@
     "sp_44455.find_bias_peaks(ranges=ranges, ref_id=ref_id, apply=True)"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "id": "d3e4f30d",
-   "metadata": {},
-   "source": [
-    "We offset the reference energy by the different in bias voltages between this run and run 44498"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -191,14 +183,12 @@
    "outputs": [],
    "source": [
     "sp_44455.calibrate_energy_axis(\n",
-    "    ref_id=0,\n",
-    "    ref_energy=-34.9,\n",
+    "    ref_energy=-31.4,\n",
     "    method=\"lmfit\",\n",
     "    energy_scale='kinetic',\n",
     "    d={'value':1.0,'min': .7, 'max':1.0, 'vary':True},\n",
     "    t0={'value':5e-7, 'min': 1e-7, 'max': 1e-6, 'vary':True},\n",
     "    E0={'value': 0., 'min': -200, 'max': 100, 'vary': True},\n",
-    "    verbose=True,\n",
     ")"
    ]
   },
@@ -225,7 +215,8 @@
    "id": "98266c62-ab48-4746-96c8-2d47cf92c0e9",
    "metadata": {},
    "source": [
-    "### Now we can use those parameters and load our trXPS data using the additional config file"
+    "### Now we can use those parameters and load our trXPS data using the additional config file\n",
+    "To obtain a correct energy axis, we offset the energy axis by the difference of photon energy between this run and the energy calibration runs"
    ]
   },
   {
@@ -238,7 +229,13 @@
     "run_number = 44498\n",
     "sp_44498 = SedProcessor(runs=[run_number], config=config_override, folder_config=\"reference_calib.yaml\", system_config=config_file, verbose=True)\n",
     "sp_44498.add_jitter()\n",
-    "sp_44498.append_energy_axis()"
+    "sp_44498.append_energy_axis()\n",
+    "sp_44498.add_energy_offset(\n",
+    "    constant=1,\n",
+    "    columns=['monochromatorPhotonEnergy','tofVoltage'],\n",
+    "    weights=[-1,-1],\n",
+    "    preserve_mean=[True, True],\n",
+    ")"
    ]
   },
   {
@@ -328,14 +325,14 @@
     "axes = ['dldTimeSteps', 'delayStage']\n",
     "ranges = [[3900,4200], [-1.5,1.5]]\n",
     "bins = [100,60]\n",
-    "res_corr = sp_44498.compute(bins=bins, axes=axes, ranges=ranges, normalize_to_acquisition_time=\"delayStage\")\n",
+    "res_corr = sp_44498.compute(bins=bins, axes=axes, ranges=ranges)\n",
     "\n",
     "fig,ax = plt.subplots(1,2,figsize=(8,3), layout='constrained')\n",
     "fig.suptitle(f\"Run {run_number}: W 4f, side bands\")\n",
-    "res_corr.plot(robust=True, ax=ax[0], cmap='terrain')\n",
+    "res_corr.plot(ax=ax[0], cmap='terrain')\n",
     "ax[0].set_title('raw')\n",
     "bg = res_corr.sel(delayStage=slice(-1.3,-1.0)).mean('delayStage')\n",
-    "(res_corr-bg).plot(robust=True, ax=ax[1])\n",
+    "(res_corr-bg).plot(ax=ax[1])\n",
     "ax[1].set_title('difference')"
    ]
   },
@@ -395,29 +392,27 @@
    "outputs": [],
    "source": [
     "### Kinetic energy of w4f peaks and their SB\n",
-    "ref_energy = -31.4\n",
-    "ref_id = 1\n",
+    "ref_energy = -30.2\n",
     "sp_44498.ec.biases = -1*np.array([-30.2,-31.4,-32.6,-33.6,-34.8])\n",
     "sp_44498.ec.peaks = np.expand_dims(data[peaks]['dldTimeSteps'].data,1)\n",
     "sp_44498.ec.tof = res_corr.dldTimeSteps.data\n",
     "\n",
     "sp_44498.calibrate_energy_axis(\n",
-    "    ref_id=ref_id,\n",
     "    ref_energy=ref_energy,\n",
     "    method=\"lmfit\",\n",
     "    d={'value':1.0,'min': .8, 'max':1.0, 'vary':True},\n",
     "    t0={'value':5e-7, 'min': 1e-7, 'max': 1e-6, 'vary':True},\n",
     "    E0={'value': -100., 'min': -200, 'max': 15, 'vary': True},\n",
-    "    labels=\"\",\n",
-    "    verbose=True)"
+    ")"
    ]
   },
   {
    "cell_type": "markdown",
    "id": "4052d629-1178-4248-a945-d60a6ff34bf3",
    "metadata": {},
    "source": [
-    "### Append energy axis into a data frame, bin and visualize data in the calibrated energy and corrected delay axis "
+    "### Append energy axis into a data frame, bin and visualize data in the calibrated energy and corrected delay axis\n",
+    "To get a correct energy axis, we undo the shifts imposed by the calibration function"
    ]
   },
   {
@@ -427,7 +422,13 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "sp_44498.append_energy_axis()"
+    "sp_44498.append_energy_axis()\n",
+    "sp_44498.add_energy_offset(\n",
+    "    constant=30.2,\n",
+    "    columns=['monochromatorPhotonEnergy','tofVoltage','sampleBias'],\n",
+    "    weights=[-1,-1,-1],\n",
+    "    preserve_mean=[True, True,False],\n",
+    ")"
    ]
   },
   {
@@ -440,14 +441,14 @@
     "axes = ['energy', 'delayStage']\n",
     "ranges = [[-37.5,-27.5], [-1.5,1.5]]\n",
     "bins = [200,60]\n",
-    "res_corr = sp_44498.compute(bins=bins, axes=axes, ranges=ranges, normalize_to_acquisition_time=\"delayStage\")\n",
+    "res_corr = sp_44498.compute(bins=bins, axes=axes, ranges=ranges)\n",
     "\n",
     "fig,ax = plt.subplots(1,2,figsize=(8,3), layout='constrained')\n",
     "fig.suptitle(f\"Run {run_number}: W 4f, side bands\")\n",
-    "res_corr.plot(robust=True, ax=ax[0], cmap='terrain')\n",
+    "res_corr.plot(ax=ax[0], cmap='terrain')\n",
     "ax[0].set_title('raw')\n",
     "bg = res_corr.sel(delayStage=slice(-1.3,-1.0)).mean('delayStage')\n",
-    "(res_corr-bg).plot(robust=True, ax=ax[1])\n",
+    "(res_corr-bg).plot(ax=ax[1])\n",
     "ax[1].set_title('difference')"
    ]
   },

sed/develop/_sources/tutorial/2_conversion_pipeline_for_example_time-resolved_ARPES_data.ipynb.txt

@@ -59,7 +59,7 @@
    "outputs": [],
    "source": [
     "# create sed processor using the config file:\n",
-    "sp = sed.SedProcessor(folder=scandir, config=\"../sed/config/mpes_example_config.yaml\", verbose=True)"
+    "sp = sed.SedProcessor(folder=scandir, config=\"../src/sed/config/mpes_example_config.yaml\", system_config={}, verbose=True)"
    ]
   },
   {
@@ -466,9 +466,9 @@
    "metadata": {},
    "source": [
     "#### 3. Step:\n",
-    "Next, the detected peak positions and bias voltages are used to determine the calibration function. This can be either done by fitting the functional form d^2/(t-t0)^2 via lmfit (\"lmfit\"), or using a polynomial approximation (\"lstsq\" or \"lsqr\"). Here, one can also define a reference id, and a reference energy. Those define the absolute energy position of the feature used for calibration in the \"reference\" trace, at the bias voltage where the final measurement has been performed. The energy scale can be either \"kinetic\" (decreasing energy with increasing TOF), or \"binding\" (increasing energy with increasing TOF).\n",
+    "Next, the detected peak positions and bias voltages are used to determine the calibration function. Essentially, the functional Energy(TOF) is being determined by either least-squares fitting of the functional form d^2/(t-t0)^2 via lmfit (``method``: \"lmfit\"), or by analytically obtaining a polynomial approximation (``method``: \"lstsq\" or \"lsqr\"). The parameter ``ref_energy`` is used to define the absolute energy position of the feature used for calibration in the calibrated energy scale. ``energy_scale`` can be either \"kinetic\" (decreasing energy with increasing TOF), or \"binding\" (increasing energy with increasing TOF).\n",
     "\n",
-    "After calculating the calibration, all traces corrected with the calibration are plotted ontop of each other, the calibration function together with the extracted features is plotted."
+    "After calculating the calibration, all traces corrected with the calibration are plotted ontop of each other, and the calibration function (Energy(TOF)) together with the extracted features is being plotted."
    ]
   },
   {
@@ -478,21 +478,17 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# use the refid of the bias that the measurement was taken at\n",
-    "# Eref can be used to set the absolute energy (kinetic energy, E-EF) of the feature used for energy calibration (if known)\n",
-    "refid=4\n",
-    "Eref=-0.5\n",
+    "# Eref can be used to set the absolute energy (kinetic energy, E-EF, etc.) of the feature used for energy calibration (if known)\n",
+    "Eref=-1.3\n",
     "# the lmfit method uses a fit of (d/(t-t0))**2 to determine the energy calibration\n",
     "# limits and starting values for the fitting parameters can be provided as dictionaries\n",
     "sp.calibrate_energy_axis(\n",
-    "    ref_id=refid,\n",
     "    ref_energy=Eref,\n",
     "    method=\"lmfit\",\n",
     "    energy_scale='kinetic',\n",
     "    d={'value':1.0,'min': .7, 'max':1.2, 'vary':True},\n",
     "    t0={'value':8e-7, 'min': 1e-7, 'max': 1e-6, 'vary':True},\n",
     "    E0={'value': 0., 'min': -100, 'max': 0, 'vary': True},\n",
-    "    verbose=True,\n",
     ")"
    ]
   },
@@ -523,7 +519,7 @@
    "metadata": {},
    "source": [
     "#### 4. Step:\n",
-    "Finally, the the energy axis is added to the dataframe."
+    "Finally, the the energy axis is added to the dataframe. Here, the applied bias voltages of the measurement is taken into account to provide the correct energy offset. If the bias cannot be read from the file, it can be provided manually."
    ]
   },
   {
@@ -533,7 +529,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "sp.append_energy_axis()"
+    "sp.append_energy_axis(bias_voltage=16.8)"
    ]
   },
   {
@@ -546,6 +542,16 @@
     "The delay axis is calculated from the ADC input column based on the provided delay range. ALternatively, the delay scan range can also be extracted from attributes inside a source file, if present."
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1619cbc6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sp.dataframe.head()"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -651,9 +657,6 @@
   }
  ],
  "metadata": {
-  "interpreter": {
-   "hash": "728003ee06929e5fa5ff815d1b96bf487266025e4b7440930c6bf4536d02d243"
-  },
   "kernelspec": {
    "display_name": "python3",
    "language": "python",
@@ -669,7 +672,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.12"
+   "version": "3.9.19"
   }
  },
  "nbformat": 4,