Skip to content

Commit

Permalink
reorganize sections and add TODOs
Browse files Browse the repository at this point in the history
  • Loading branch information
@jsdillon
jsdillon committed Jun 4, 2024
1 parent 58f80cd commit fb63c69
Showing 1 changed file with 146 additions and 115 deletions.
261 changes: 146 additions & 115 deletions hera_notebook_templates/notebooks/lststack.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -719,66 +719,46 @@
]
},
{
"cell_type": "markdown",
"id": "af3b55b2",
"cell_type": "code",
"execution_count": null,
"id": "84d270e0-fc06-475f-9f3b-9b939b6346fe",
"metadata": {},
"outputs": [],
"source": [
"## LST-bin the Autos"
"inpaint_bands = [(0, FM_low_freq), (FM_high_freq, np.inf)] # default below and above FM\n",
"\n",
"# Get slices for the inpaint bands\n",
"_inp = []\n",
"for _bnd in inpaint_bands:\n",
" idx = np.nonzero((stackconf.config.datameta.freq_array >= _bnd[0] * 1e6) & (stackconf.config.datameta.freq_array < _bnd[1]*1e6))[0]\n",
" _inp.append(slice(idx[0], idx[-1] + 1))\n",
"inpaint_bands = _inp\n",
"print(\"Using the following bands for inpainting (channels):\")\n",
"for bnd in inpaint_bands:\n",
" print(bnd)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f54691c7",
"cell_type": "markdown",
"id": "361a1e4d-42c6-4800-824f-1e190d114e2f",
"metadata": {
"tags": []
"execution": {
"iopub.execute_input": "2024-06-04T18:20:46.071514Z",
"iopub.status.busy": "2024-06-04T18:20:46.070413Z",
"iopub.status.idle": "2024-06-04T18:20:46.084305Z",
"shell.execute_reply": "2024-06-04T18:20:46.081600Z",
"shell.execute_reply.started": "2024-06-04T18:20:46.071441Z"
}
},
"outputs": [],
"source": [
"def make_auto_plot(auto_stacks: list[UVData], lstbin: list[dict]):\n",
" \n",
" fig, ax = plt.subplots(\n",
" len(stackconf.autopairs)*len(stackconf.pols), len(auto_stacks), \n",
" sharex=True, sharey=True, squeeze=False, constrained_layout=True,\n",
" figsize=(12, 6)\n",
" )\n",
"\n",
" for i, (stack, avg) in enumerate(zip(auto_stacks, lstbin)):\n",
" for j, autopair in enumerate(stackconf.autopairs):\n",
" for p, pol in enumerate(stackconf.pols):\n",
" axx = ax[j*len(stackconf.pols) + p, i]\n",
" \n",
" for k, t in enumerate(stack.time_array[::stack.Nbls]):\n",
" flg = stack.get_flags(autopair + (pol,))[k]\n",
" d = stack.get_data(autopair+(pol,))[k]\n",
" \n",
" axx.plot(\n",
" stack.freq_array / 1e6,\n",
" np.where(flg, np.nan, d.real),\n",
" label=f\"{int(t)}\" if not p else None,\n",
" **styles[int(t)]\n",
" )\n",
" axx.set_yscale('log')\n",
" axx.set_title(f\"Pair {autopair}, pol={pol}, LST {stackconf.lst_grid[i]*12/np.pi:.3f} hr\")\n",
"\n",
" # plot the mean\n",
" axx.plot(\n",
" stack.freq_array / 1e6,\n",
" np.where(avg['flags'][j, :, p], np.nan, avg['data'][j, :, p].real),\n",
" label='LSTBIN',\n",
" color='k', lw=2\n",
" )\n",
" \n",
" ax[0,0].legend(ncols=3)"
"## Perform Initial Stacking of Autos and Crosses"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "29c9e7b2",
"metadata": {
"tags": []
},
"id": "759dfcac-3eb9-41d6-a7ed-ca640bbb9047",
"metadata": {},
"outputs": [],
"source": [
"auto_stacks, autos_lstavg = stack_blchunk('autos') # Auto-stacks\n",
Expand All @@ -787,19 +767,40 @@
},
{
"cell_type": "markdown",
"id": "3cc4d037",
"id": "0e1501b7-2715-4821-8775-0a9b87e38fd5",
"metadata": {},
"source": [
"### LST-Bin Calibration"
"## LST-Bin Calibration"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "434945d9",
"metadata": {},
"outputs": [],
"execution_count": 2,
"id": "efe77bc8-8ffe-4d6c-9a46-32941b6fd4c0",
"metadata": {
"execution": {
"iopub.execute_input": "2024-06-04T18:25:38.129897Z",
"iopub.status.busy": "2024-06-04T18:25:38.129316Z",
"iopub.status.idle": "2024-06-04T18:25:38.308596Z",
"shell.execute_reply": "2024-06-04T18:25:38.308021Z",
"shell.execute_reply.started": "2024-06-04T18:25:38.129864Z"
}
},
"outputs": [
{
"ename": "NameError",
"evalue": "name 'np' is not defined",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[2], line 7\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# TODO: graduate this code to hera_cal, unittest it, and remove it from this notebook\u001b[39;00m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mlstbin_calibration\u001b[39m(\n\u001b[1;32m 3\u001b[0m stack,\n\u001b[1;32m 4\u001b[0m model, \n\u001b[1;32m 5\u001b[0m all_reds,\n\u001b[1;32m 6\u001b[0m inpaint_bands,\n\u001b[0;32m----> 7\u001b[0m auto_stack: \u001b[43mnp\u001b[49m\u001b[38;5;241m.\u001b[39mndarray\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 8\u001b[0m run_amplitude_cal: \u001b[38;5;28mbool\u001b[39m\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, \n\u001b[1;32m 9\u001b[0m run_phase_cal: \u001b[38;5;28mbool\u001b[39m\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, \n\u001b[1;32m 10\u001b[0m smoothing_scale: \u001b[38;5;28mfloat\u001b[39m\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m10e6\u001b[39m,\n\u001b[1;32m 11\u001b[0m calibrate_inplace: \u001b[38;5;28mbool\u001b[39m\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 12\u001b[0m return_gains: \u001b[38;5;28mbool\u001b[39m\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m 13\u001b[0m ):\n\u001b[1;32m 14\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 15\u001b[0m \u001b[38;5;124;03m \u001b[39;00m\n\u001b[1;32m 16\u001b[0m \u001b[38;5;124;03m Parameters:\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[38;5;124;03m \u001b[39;00m\n\u001b[1;32m 25\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[1;32m 26\u001b[0m \u001b[38;5;66;03m# Assert some calibration done\u001b[39;00m\n",
"\u001b[0;31mNameError\u001b[0m: name 'np' is not defined"
]
}
],
"source": [
"# TODO: graduate this code to hera_cal, unittest it, and remove it from this notebook\n",
"def lstbin_calibration(\n",
" stack,\n",
" model, \n",
Expand Down Expand Up @@ -1038,27 +1039,7 @@
{
"cell_type": "code",
"execution_count": null,
"id": "5ac541a5",
"metadata": {},
"outputs": [],
"source": [
"inpaint_bands = [(0, FM_low_freq), (FM_high_freq, np.inf)] # default below and above FM\n",
"\n",
"# Get slices for the inpaint bands\n",
"_inp = []\n",
"for _bnd in inpaint_bands:\n",
" idx = np.nonzero((stackconf.config.datameta.freq_array >= _bnd[0] * 1e6) & (stackconf.config.datameta.freq_array < _bnd[1]*1e6))[0]\n",
" _inp.append(slice(idx[0], idx[-1] + 1))\n",
"inpaint_bands = _inp\n",
"print(\"Using the following bands for inpainting (channels):\")\n",
"for bnd in inpaint_bands:\n",
" print(bnd)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9834e703",
"id": "08c855c9-0869-4d91-bd45-9dd7e21a6599",
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -1116,40 +1097,27 @@
},
{
"cell_type": "markdown",
"id": "c646623f",
"id": "6ed1a12d-c274-46e5-b106-f373f2ee5de9",
"metadata": {},
"source": [
"### Compute Stats for Autos"
"## Define New Simutaneous Inpainter And Averager \n",
"TODO: graduate this code to a unittested repo and remove it from this notebook"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d48853eb-611a-463c-966f-6435da5312aa",
"execution_count": 1,
"id": "4c40956f-79e2-4144-916f-e32723c3b417",
"metadata": {
"tags": []
"execution": {
"iopub.execute_input": "2024-06-04T18:24:53.613655Z",
"iopub.status.busy": "2024-06-04T18:24:53.612450Z",
"iopub.status.idle": "2024-06-04T18:24:53.647345Z",
"shell.execute_reply": "2024-06-04T18:24:53.646946Z",
"shell.execute_reply.started": "2024-06-04T18:24:53.613585Z"
}
},
"outputs": [],
"source": [
"auto_stats = [\n",
" lstmet.LSTBinStats.from_reduced_data(rdc=rdc, antpairs=stackconf.autopairs, pols=stackconf.pols, reds=reds_with_pols) for rdc in autos_lstavg\n",
"]"
]
},
{
"cell_type": "markdown",
"id": "ea584858",
"metadata": {},
"source": [
"### Inpaint Autos"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7c4af378",
"metadata": {},
"outputs": [],
"source": [
"def average_and_inpaint_simultaneously(\n",
" stack, \n",
Expand Down Expand Up @@ -1352,24 +1320,42 @@
" return lstavg, all_models"
]
},
{
"cell_type": "markdown",
"id": "af3b55b2",
"metadata": {},
"source": [
"## Autos"
]
},
{
"cell_type": "markdown",
"id": "c646623f",
"metadata": {},
"source": [
"### Compute Stats for Autos"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "20fc85fe",
"metadata": {},
"id": "d48853eb-611a-463c-966f-6435da5312aa",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"inpaint_bands = [(0, 87.5), (108.0, 250.0)] # default below and above FM\n",
"\n",
"# Get slices for the inpaint bands\n",
"_inp = []\n",
"for _bnd in inpaint_bands:\n",
" idx = np.nonzero((stackconf.config.datameta.freq_array >= _bnd[0] * 1e6) & (stackconf.config.datameta.freq_array < _bnd[1]*1e6))[0]\n",
" _inp.append(slice(idx[0], idx[-1] + 1))\n",
"inpaint_bands = _inp\n",
"print(\"Using the following bands for inpainting (channels):\")\n",
"for bnd in inpaint_bands:\n",
" print(bnd)"
"auto_stats = [\n",
" lstmet.LSTBinStats.from_reduced_data(rdc=rdc, antpairs=stackconf.autopairs, pols=stackconf.pols, reds=reds_with_pols) for rdc in autos_lstavg\n",
"]"
]
},
{
"cell_type": "markdown",
"id": "ea584858",
"metadata": {},
"source": [
"### Inpaint Autos"
]
},
{
Expand Down Expand Up @@ -1411,6 +1397,50 @@
"### Plot"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1fccd669-53d7-4888-8382-831e028966a3",
"metadata": {},
"outputs": [],
"source": [
"def make_auto_plot(auto_stacks: list[UVData], lstbin: list[dict]):\n",
" \n",
" fig, ax = plt.subplots(\n",
" len(stackconf.autopairs)*len(stackconf.pols), len(auto_stacks), \n",
" sharex=True, sharey=True, squeeze=False, constrained_layout=True,\n",
" figsize=(12, 6)\n",
" )\n",
"\n",
" for i, (stack, avg) in enumerate(zip(auto_stacks, lstbin)):\n",
" for j, autopair in enumerate(stackconf.autopairs):\n",
" for p, pol in enumerate(stackconf.pols):\n",
" axx = ax[j*len(stackconf.pols) + p, i]\n",
" \n",
" for k, t in enumerate(stack.time_array[::stack.Nbls]):\n",
" flg = stack.get_flags(autopair + (pol,))[k]\n",
" d = stack.get_data(autopair+(pol,))[k]\n",
" \n",
" axx.plot(\n",
" stack.freq_array / 1e6,\n",
" np.where(flg, np.nan, d.real),\n",
" label=f\"{int(t)}\" if not p else None,\n",
" **styles[int(t)]\n",
" )\n",
" axx.set_yscale('log')\n",
" axx.set_title(f\"Pair {autopair}, pol={pol}, LST {stackconf.lst_grid[i]*12/np.pi:.3f} hr\")\n",
"\n",
" # plot the mean\n",
" axx.plot(\n",
" stack.freq_array / 1e6,\n",
" np.where(avg['flags'][j, :, p], np.nan, avg['data'][j, :, p].real),\n",
" label='LSTBIN',\n",
" color='k', lw=2\n",
" )\n",
" \n",
" ax[0,0].legend(ncols=3)"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down Expand Up @@ -1567,6 +1597,7 @@
"id": "461a78dc-d227-4479-8ebb-546e6ac5568e",
"metadata": {},
"source": [
"TODO: update this text \n",
"We simultaneously inpaint and average the data with the flags we're given. We don't try and inpaint the stack itself (i.e. on a nightly basis), since we have only one solution per LST bin (per freq and baseline), and there's no useful information gained by doing so (one might think that the newly-inpainted nightly solution might be useful in terms of getting a new $Z^2$ score for the statistics, but the $Z^2$ for the inpainted data is not well-defined: it has zero nsamples). "
]
},
Expand Down Expand Up @@ -3213,9 +3244,9 @@
],
"metadata": {
"kernelspec": {
"display_name": "hera_dev_kernel",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "hera_dev_kernel"
"name": "python3"
},
"language_info": {
"codemirror_mode": {
Expand All @@ -3227,7 +3258,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.3"
"version": "3.10.12"
},
"toc": {
"base_numbering": 1,
Expand Down

0 comments on commit fb63c69

Please sign in to comment.