Skip to content
This repository was archived by the owner on Sep 25, 2023. It is now read-only.

Commit 4de4081

Browse files
charlesblucacharlesbluca
authored
Add option for smaller dataset in IO notebook (#473)
Adds the option to use a smaller ~300MB SigMF dataset for the IO examples, as well as code to automatically download / extract the dataset of choice. Note that there are some failures in the notebook - raised #474 to cover this. Closes #472 Authors: - Charles Blackmon-Luca (https://github.com/charlesbluca) Approvers: - Adam Thompson (https://github.com/awthomp) URL: #473
1 parent 1dacf7d commit 4de4081

File tree

1 file changed

+136
-31
lines changed

1 file changed

+136
-31
lines changed

‎notebooks/api_guide/io_examples.ipynb

+136-31
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
"cell_type": "markdown",
3131
"metadata": {},
3232
"source": [
33-
"We are using the Northeastern University Oracle SigMF recordings found [here](http://www.genesys-lab.org/oracle) (Dataset #2)"
33+
"We are using the [Northeastern University Oracle RF Fingerprinting Dataset](http://www.genesys-lab.org/oracle):"
3434
]
3535
},
3636
{
@@ -39,8 +39,106 @@
3939
"metadata": {},
4040
"outputs": [],
4141
"source": [
42-
"meta_file = '/data/oracle/KRI-16Devices-RawData/2ft/WiFi_air_X310_3123D7B_2ft_run1.sigmf-meta'\n",
43-
"data_file = '/data/oracle/KRI-16Devices-RawData/2ft/WiFi_air_X310_3123D7B_2ft_run1.sigmf-data'"
42+
"# set input SigMF dataset - note that this is ~28gb\n",
43+
"base_url = 'https://repository.library.northeastern.edu/downloads/neu:m044q5210?datastream_id=content'\n",
44+
"path = 'KRI-16Devices-RawData'\n",
45+
"file = '2ft/WiFi_air_X310_3123D7B_2ft_run1'\n",
46+
"\n",
47+
"# uncomment to use smaller ~300mb dataset\n",
48+
"# base_url = 'https://repository.library.northeastern.edu/downloads/neu:m044q523j?datastream_id=content'\n",
49+
"# path = 'KRI-16IQImbalances-DemodulatedData'\n",
50+
"# file = 'Demod_WiFi_cable_X310_3123D76_IQ#1_run1'\n",
51+
"\n",
52+
"comp = '.zip'"
53+
]
54+
},
55+
{
56+
"cell_type": "code",
57+
"execution_count": 3,
58+
"metadata": {},
59+
"outputs": [
60+
{
61+
"name": "stdout",
62+
"output_type": "stream",
63+
"text": [
64+
"Downloading https://repository.library.northeastern.edu/downloads/neu:m044q523j?datastream_id=content to ../data/KRI-16IQImbalances-DemodulatedData.zip\n"
65+
]
66+
},
67+
{
68+
"name": "stderr",
69+
"output_type": "stream",
70+
"text": [
71+
" % Total % Received % Xferd Average Speed Time Time Time Current\n",
72+
" Dload Upload Total Spent Left Speed\n",
73+
"100 295M 100 295M 0 0 798k 0 0:06:18 0:06:18 --:--:-- 822k\n"
74+
]
75+
},
76+
{
77+
"name": "stdout",
78+
"output_type": "stream",
79+
"text": [
80+
"Decompressing ../data/KRI-16IQImbalances-DemodulatedData.zip...\n",
81+
"Archive: ../data/KRI-16IQImbalances-DemodulatedData.zip\n",
82+
" creating: ../data/KRI-16IQImbalances-DemodulatedData/\n",
83+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#13_run1.sigmf-data \n",
84+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#13_run1.sigmf-meta \n",
85+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#17_run1.sigmf-data \n",
86+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#17_run1.sigmf-meta \n",
87+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#5_run1.sigmf-data \n",
88+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#5_run1.sigmf-meta \n",
89+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#15_run1.sigmf-data \n",
90+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#15_run1.sigmf-meta \n",
91+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#14_run1.sigmf-data \n",
92+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#14_run1.sigmf-meta \n",
93+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#26_run1.sigmf-data \n",
94+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#26_run1.sigmf-meta \n",
95+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#25_run1.sigmf-data \n",
96+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#25_run1.sigmf-meta \n",
97+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#3_run1.sigmf-data \n",
98+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#3_run1.sigmf-meta \n",
99+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#2_run1.sigmf-data \n",
100+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#2_run1.sigmf-meta \n",
101+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#4_run1.sigmf-data \n",
102+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#4_run1.sigmf-meta \n",
103+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#9_run1.sigmf-data \n",
104+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#9_run1.sigmf-meta \n",
105+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#1_run1.sigmf-data \n",
106+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#1_run1.sigmf-meta \n",
107+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#19_run1.sigmf-data \n",
108+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#19_run1.sigmf-meta \n",
109+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#18_run1.sigmf-data \n",
110+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#18_run1.sigmf-meta \n",
111+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#7_run1.sigmf-data \n",
112+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#7_run1.sigmf-meta \n",
113+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#32_run1.sigmf-data \n",
114+
" inflating: ../data/KRI-16IQImbalances-DemodulatedData/Demod_WiFi_cable_X310_3123D76_IQ#32_run1.sigmf-meta \n",
115+
"../data/KRI-16IQImbalances-DemodulatedData.zip decompressed\n"
116+
]
117+
}
118+
],
119+
"source": [
120+
"import os\n",
121+
"\n",
122+
"# create data directory\n",
123+
"data_dir = '../data/'\n",
124+
"if not os.path.exists(data_dir):\n",
125+
" print('creating data directory')\n",
126+
" os.system('mkdir ../data')\n",
127+
"\n",
128+
"# download dataset\n",
129+
"if not os.path.isdir(data_dir + path):\n",
130+
" if not os.path.isfile(data_dir + path + comp):\n",
131+
" print(f'Downloading {base_url} to {data_dir + path + comp}')\n",
132+
" os.system(f'curl -o {data_dir + path + comp} {base_url}')\n",
133+
" print(f'Decompressing {data_dir + path + comp}...')\n",
134+
" os.system(f'unzip {data_dir + path + comp} -d {data_dir}')\n",
135+
" print(f'{data_dir + path + comp} decompressed')\n",
136+
"else:\n",
137+
" print(f'Dataset already exists under {data_dir + path}')\n",
138+
" \n",
139+
"# set input SigMF files\n",
140+
"meta_file = os.path.join(data_dir, path, file + '.sigmf-meta')\n",
141+
"data_file = os.path.join(data_dir, path, file + '.sigmf-data')"
44142
]
45143
},
46144
{
@@ -54,7 +152,7 @@
54152
"cell_type": "markdown",
55153
"metadata": {},
56154
"source": [
57-
"For our purposes here, [SigMF](https://github.com/gnuradio/SigMF) data is treated as a JSON header and processed on CPU, while the *binary* payload file is mmaped to GPU and cuSignal uses a CUDA kernel to parse the file. While we've focused on SigMF here, you can use the underlying `cusignal.read_bin` and `cusignal.parse_bin` (and corresponding write functions) for your own datasets."
155+
"For our purposes here, [SigMF](https://github.com/gnuradio/SigMF) data is treated as a JSON header and processed on CPU, while the *binary* payload file is mapped to GPU and cuSignal uses a CUDA kernel to parse the file. While we've focused on SigMF here, you can use the underlying `cusignal.read_bin` and `cusignal.parse_bin` (and corresponding write functions) for your own datasets."
58156
]
59157
},
60158
{
@@ -66,7 +164,7 @@
66164
},
67165
{
68166
"cell_type": "code",
69-
"execution_count": 3,
167+
"execution_count": 4,
70168
"metadata": {},
71169
"outputs": [],
72170
"source": [
@@ -79,14 +177,14 @@
79177
},
80178
{
81179
"cell_type": "code",
82-
"execution_count": 4,
180+
"execution_count": 5,
83181
"metadata": {},
84182
"outputs": [
85183
{
86184
"name": "stdout",
87185
"output_type": "stream",
88186
"text": [
89-
"128 ms ± 147 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
187+
"13.5 ms ± 12.4 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
90188
]
91189
}
92190
],
@@ -104,14 +202,14 @@
104202
},
105203
{
106204
"cell_type": "code",
107-
"execution_count": 5,
205+
"execution_count": 6,
108206
"metadata": {},
109207
"outputs": [
110208
{
111209
"name": "stdout",
112210
"output_type": "stream",
113211
"text": [
114-
"188 ms ± 341 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
212+
"15.5 ms ± 430 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
115213
]
116214
}
117215
],
@@ -137,14 +235,14 @@
137235
},
138236
{
139237
"cell_type": "code",
140-
"execution_count": 6,
238+
"execution_count": 7,
141239
"metadata": {},
142240
"outputs": [
143241
{
144242
"name": "stdout",
145243
"output_type": "stream",
146244
"text": [
147-
"82.2 ms ± 172 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
245+
"7.38 ms ± 28.2 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
148246
]
149247
}
150248
],
@@ -170,7 +268,7 @@
170268
},
171269
{
172270
"cell_type": "code",
173-
"execution_count": 7,
271+
"execution_count": 8,
174272
"metadata": {},
175273
"outputs": [],
176274
"source": [
@@ -180,14 +278,14 @@
180278
},
181279
{
182280
"cell_type": "code",
183-
"execution_count": 8,
281+
"execution_count": 9,
184282
"metadata": {},
185283
"outputs": [
186284
{
187285
"name": "stdout",
188286
"output_type": "stream",
189287
"text": [
190-
"82.2 ms ± 50.1 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
288+
"7.37 ms ± 32.2 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
191289
]
192290
}
193291
],
@@ -213,7 +311,7 @@
213311
},
214312
{
215313
"cell_type": "code",
216-
"execution_count": 9,
314+
"execution_count": 10,
217315
"metadata": {},
218316
"outputs": [],
219317
"source": [
@@ -223,14 +321,14 @@
223321
},
224322
{
225323
"cell_type": "code",
226-
"execution_count": 10,
324+
"execution_count": 11,
227325
"metadata": {},
228326
"outputs": [
229327
{
230328
"name": "stdout",
231329
"output_type": "stream",
232330
"text": [
233-
"82.8 ms ± 866 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
331+
"7.39 ms ± 22.9 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
234332
]
235333
}
236334
],
@@ -249,7 +347,7 @@
249347
},
250348
{
251349
"cell_type": "code",
252-
"execution_count": 11,
350+
"execution_count": 12,
253351
"metadata": {},
254352
"outputs": [],
255353
"source": [
@@ -271,14 +369,14 @@
271369
},
272370
{
273371
"cell_type": "code",
274-
"execution_count": 12,
372+
"execution_count": 13,
275373
"metadata": {},
276374
"outputs": [
277375
{
278376
"name": "stdout",
279377
"output_type": "stream",
280378
"text": [
281-
"338 ms ± 837 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
379+
"41.8 ms ± 576 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
282380
]
283381
}
284382
],
@@ -297,14 +395,14 @@
297395
},
298396
{
299397
"cell_type": "code",
300-
"execution_count": 13,
398+
"execution_count": 14,
301399
"metadata": {},
302400
"outputs": [
303401
{
304402
"name": "stdout",
305403
"output_type": "stream",
306404
"text": [
307-
"341 ms ± 830 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
405+
"43 ms ± 1.4 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
308406
]
309407
}
310408
],
@@ -323,7 +421,7 @@
323421
},
324422
{
325423
"cell_type": "code",
326-
"execution_count": 14,
424+
"execution_count": 15,
327425
"metadata": {},
328426
"outputs": [],
329427
"source": [
@@ -333,14 +431,14 @@
333431
},
334432
{
335433
"cell_type": "code",
336-
"execution_count": 15,
434+
"execution_count": 16,
337435
"metadata": {},
338436
"outputs": [
339437
{
340438
"name": "stdout",
341439
"output_type": "stream",
342440
"text": [
343-
"253 ms ± 1.22 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
441+
"36 ms ± 2.24 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
344442
]
345443
}
346444
],
@@ -359,7 +457,7 @@
359457
},
360458
{
361459
"cell_type": "code",
362-
"execution_count": 16,
460+
"execution_count": 17,
363461
"metadata": {},
364462
"outputs": [],
365463
"source": [
@@ -369,14 +467,14 @@
369467
},
370468
{
371469
"cell_type": "code",
372-
"execution_count": 17,
470+
"execution_count": 18,
373471
"metadata": {},
374472
"outputs": [
375473
{
376474
"name": "stdout",
377475
"output_type": "stream",
378476
"text": [
379-
"253 ms ± 950 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
477+
"36.3 ms ± 419 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
380478
]
381479
}
382480
],
@@ -385,11 +483,18 @@
385483
"cusignal.write_sigmf(test_file_ext, sigmf, buffer, append=False)\n",
386484
"cp.cuda.runtime.deviceSynchronize()"
387485
]
486+
},
487+
{
488+
"cell_type": "code",
489+
"execution_count": null,
490+
"metadata": {},
491+
"outputs": [],
492+
"source": []
388493
}
389494
],
390495
"metadata": {
391496
"kernelspec": {
392-
"display_name": "Python 3",
497+
"display_name": "Python 3 (ipykernel)",
393498
"language": "python",
394499
"name": "python3"
395500
},
@@ -403,9 +508,9 @@
403508
"name": "python",
404509
"nbconvert_exporter": "python",
405510
"pygments_lexer": "ipython3",
406-
"version": "3.8.2"
511+
"version": "3.9.7"
407512
}
408513
},
409514
"nbformat": 4,
410515
"nbformat_minor": 4
411-
}
516+
}

0 commit comments

Comments
 (0)