-
Notifications
You must be signed in to change notification settings - Fork 0
WSN Server
ArcticSnow edited this page Aug 7, 2019
·
6 revisions
Description of the server run at UiO that hosts the final database where all data from WSNs are stored.
Administrators:
- John Burkhart
- J. David Ibanez
- Simon Filhol
- Norbert Pirk
The database is accessible with login and password to the admins via the url: https://wsn.latice.eu/admin/login/ This links brings you to an HTML version of the DB written in Python Django (see this Github repo)
The Django App code is available at: https://github.com/spectraphilic/wsn_server
To access data from the DB, it is possible to use Python 3 for query. You will need to copy this python code into a file called db_query.py:
import datetime
import os, sys
import pprint
import requests
from pandas.io.json import json_normalize
import pandas as pd
URL = 'https://wsn.latice.eu/api/query/v2/'
#URL = 'http://localhost:8000/wsn/api/query/v2/'
#TOKEN = os.getenv('WSN_TOKEN')
TOKEN = os.getenv('WSN_TOKEN', 'dcff0c629050b5492362ec28173fa3e051648cb1')
path = os.getcwd()
def query(
limit=100, # Pagination
fields=None, # Fields to return (all by default)
tags=None, # Tags to return (all by default)
interval=None, # If given will return the average in the interval (seconds)
debug=False, # Not sent to the API
# Filters
time__gte=None, time__lte=None, # Time is special
**kw):
# Parameters
if time__gte:
time__gte = time__gte.timestamp()
if time__lte:
time__lte = time__lte.timestamp()
params = {
'limit': limit, # Pagination
'time__gte': time__gte, 'time__lte': time__lte, # Time filter
'fields': fields,
'tags': tags,
'interval': interval,
}
# Filter inside json
for key, value in kw.items():
if value is None:
params[key] = None
continue
if type(value) is datetime.datetime:
value = int(value.timestamp())
if isinstance(value, int):
key += ':int'
params[key] = value
# Query
headers = {'Authorization': 'Token %s' % TOKEN}
response = requests.get(URL, headers=headers, params=params)
response.raise_for_status()
json = response.json()
# Debug
if debug:
pprint.pprint(params)
pprint.pprint(json)
print()
return json
def get_token():
try:
token = os.environ['WSN_TOKEN']
return token
except KeyError:
print("Please set the environment variable WSN_TOKEN in .bashrc as follow: \n\t export WSN_TOKEN=xxxxxxxxxxxxxxxxx ")
sys.exit(1)
def query_df(
limit=100, # Pagination
fields=None, # Fields to return (all by default)
tags=None, # Tags to return (all by default)
interval=None, # If given will return the average in the interval (seconds)
debug=False, # Not sent to the API
# Filters
time__gte=None, time__lte=None, # Time is special
**kw):
# Parameters
if time__gte:
time__gte = time__gte.timestamp()
if time__lte:
time__lte = time__lte.timestamp()
params = {
'limit': limit, # Pagination
'time__gte': time__gte, 'time__lte': time__lte, # Time filter
'fields': fields,
'tags': tags,
'interval': interval,
}
# Filter inside json
for key, value in kw.items():
if value is None:
params[key] = None
continue
if type(value) is datetime.datetime:
value = int(value.timestamp())
if isinstance(value, int):
key += ':int'
params[key] = value
# Query
headers = {'Authorization': 'Token %s' % TOKEN}
response = requests.get(URL, headers=headers, params=params)
response.raise_for_status()
json = response.json()
# Debug
if debug:
pprint.pprint(params)
pprint.pprint(json)
print()
df = json_normalize(json['results']) # convert json object to pandas dataframe
try:
df.time = pd.to_datetime(df.time, unit='s')
except:
print('WARNING: no timestamp')
return df
def biomet_metadata():
meta = pd.read_csv(path + '/FINSE-stationary_variables_biomet.csv', sep=';')
return meta
if __name__ == '__main__':
# We need an authentication token
TOKEN = os.getenv('WSN_TOKEN', 'dcff0c629050b5492362ec28173fa3e051648cb1')
# Number of elements to return in every query
limit = 100
# Example 1: Get all the fields and tags of a given mote from a given time.
# This is good to explore the data, but bad on performance.
response = query(limit=limit,
serial=0x1F566F057C105487,
time__gte=datetime.datetime(2017, 11, 15),
debug=True,
)
# Example 2: Get the RSSI of an Xbee module identified by its address
print('==============================================')
response = query(limit=limit,
source_addr_long=0x0013A2004105D4B6,
fields=['rssi'],
debug=True,
)
# Example 3: Get the battery and internal temperature from all motes,
# include the serial tag to tell them apart.
# Frames that don't have at least one of the fields we ask for will not be
# included.
print('==============================================')
response = query(limit=limit,
fields=['bat', 'in_temp'],
tags=['serial'],
debug=True,
)
# Example 4: Get the time the frame was received by the Pi
print('==============================================')
response = query(limit=limit,
serial=408520806,
fields=['received'],
debug=True,
)
# Example 5: Get the battery once every hour
response = query(limit=10,
serial=0x1F566F057C105487,
fields=['bat'],
interval=3600,
debug=True,
)For instance, from there you can follow this example to download data from one weather station:
import pandas as pd
import datetime
import db_query as db
# enter here the serial number of the station you want to download in decimal format
serial = 3900520821626524703
# Here you set how many days you want data from now
ndays = 120
start = datetime.datetime.now() - datetime.timedelta(days=ndays)
end = datetime.datetime.now()
# Line that queries the DB, and return a Pandas Dataframe
df = db.query_df(serial=serial, time__gte=start, time__lte=end, limit=10000000000)
# Set the timestamp as index in the dataframe
df.set_index(pd.to_datetime(df.time), inplace=True)
# Have fun!!!List of Available variable at the Stationnary Flux station in the biomet table:
CR6_serials={'finse_stationnary':3668,
'finse_mobile':744}
CR6_biomet_perm = {'BEC_99_99_3_1_1': 'bec',
'CS650PERIOD_99_99_3_1_1': 'cs650period',
'CS650VRATIO_99_99_3_1_1': 'cs650vratio',
'FC1DRIFTmax_99_99_1_1_1': 'fc1drift_max',
'FC1DRIFTmean_99_99_1_1_1': 'fc1drift_mean',
'FC1DRIFTmin_99_99_1_1_1': 'fc1drift_min',
'FC1DRIFTstd_99_99_1_1_1': 'fc1drift_std',
'FC1DRIFTsum_99_99_1_1_1': 'fc1drift_sum',
'FC1WSmax_16_99_1_1_1': 'fc1ws_max',
'FC1WSmean_16_99_1_1_1': 'fc1ws_mean',
'FC1WSmin_16_99_1_1_1': 'fc1ws_min',
'FC2DRIFTmax_99_99_1_1_1': 'fc2drift_max',
'FC2DRIFTmean_99_99_1_1_1': 'fc2drift_mean',
'FC2DRIFTmin_99_99_1_1_1': 'fc2drift_min',
'FC2DRIFTstd_99_99_1_1_1': 'fc2drift_std',
'FC2DRIFTsum_99_99_1_1_1': 'fc2drift_sum',
'FC2WSmax_16_99_1_1_1': 'fc2ws_max',
'FC2WSmean_16_99_1_1_1': 'fc2ws_mean',
'FC2WSmin_16_99_1_1_1': 'fc2ws_min',
'LWIN_6_14_1_1_1': 'lwin',
'LWOUT_6_15_1_1_1': 'lwout',
'METNORA_99_99_1_1_1': 'metno_ra',
'METNORR_99_99_1_1_1': 'metno_rr',
'METNOR_99_99_1_1_1': 'metno_r',
'METNOS_99_99_1_1_1': 'metno_s',
'PA_4_2_1_1_1': 'pa',
'PERMITTIVITY_99_99_3_1_1': 'permittivty',
'RECORD': 'record',
'RH_19_3_1_1_1': 'RH_19_3_1_1_1', # Check what is the difference between the two RH
'RH_19_3_1_2_1': 'RH_19_3_1_2_1',
'SHF_6_37_1_1_1': 'SHF_6_37_1_1_1',
'SHF_6_37_2_1_1': 'SHF_6_37_2_1_1',
'SHF_99_37_1_1_1': 'SHF_99_37_1_1_1',
'SHF_99_37_1_1_2': 'SHF_99_37_1_1_2',
'SHF_99_37_2_1_1': 'SHF_99_37_2_1_1',
'SHF_99_37_2_1_2': 'SHF_99_37_2_1_2',
'SWC_12_36_3_1_1': 'SWC_12_36_3_1_1',
'SWIN_6_10_1_1_1': 'swin',
'SWOUT_6_11_1_1_1': 'swout',
'TA_2_1_1_1_1': 'ta',
'TA_2_1_1_2_1': 'ta',
'TSS_2_99_1_1_1': 'tss',
'TS_2_38_1_1_1': 'ts_1',
'TS_2_38_2_1_1': 'ts_2',
'TS_2_38_3_1_1': 'ts_3',
'VIN_18_39_1_1_1': 'vin',
'WD_20_35_1_1_1': 'wd',
'WS_16_33_1_1_1': 'ws',
'model': 'model',
'name': 'name',
'os_version': 'os_version',
'prog_name': 'prog_name',
'prog_signature': 'prog_signature',
'serial': 'serial',
'table_name': 'table_name',
'time': 'time',
}
CR6_biomet_perm_unit = {}
CR6_biomet_perm_description = {}
CR6_biomet_mobile = {
'BEC 99 99 3 1 1': '__',
'CS650PERIOD 99 99 3 1 1': '__',
'CS650VRATIO 99 99 3 1 1': '__',
'LWIN 6 14 1 1 1': 'lwin',
'LWOUT 6 15 1 1 1': 'lwout',
'PA 4 2 1 1 1': 'pa',
'PERMITTIVITY 99 99 3 1 1': 'perm',
'P RAIN 8 19 1 1 1': 'precip',
'RECORD': 'record',
'RH 19 3 1 1 1': 'rh',
'SHF 6 37 1 1 1': 'sh1',
'SHF 6 37 2 1 1': 'sh2',
'SHF 99 37 1 1 2': 'sh3',
'SHF 99 37 2 1 2': 'sh4',
'SR50DISTANCE 9 99 1 1 1': 's50_dist',
'SR50QUALITY 99 99 1 1 1': 'sr50_qu',
'SURFACETEMP 2 99 1 1 1': 'tsurf',
'SWC 12 36 3 1 1': 'swc',
'SWIN 6 10 1 1 1': 'swin',
'SWOUT 6 11 1 1 1': 'swout',
'TA 2 1 1 1 1': 'ta',
'TS 2 38 2 1 1': 'ts',
'TS 2 38 3 1 1': 'ts',
'VIN 18 39 1 1 1': 'vin',
}