Skip to content

Commit fad2834

Browse files
committed
Initial commit
0 parents  commit fad2834

6 files changed

+283
-0
lines changed

.env.tmpl

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
WEB_PORTAL_USERNAME=
2+
WEB_PORTAL_PASSWORD=
3+
4+
# date in the format YYYY-MM-DD where to start looking for consumption records
5+
MEASURE_START_DATE=
6+
7+
# file path where the json files should be stored
8+
STORAGE_PATH=
9+
10+
USER_AGENT=Mozilla/5.0 (X11; Linux x86_64; rv:99.0) Gecko/20100101 Firefox/99.0

.gitignore

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
__pycache__
2+
.env
3+
.venv
4+
.vscode

import_sqlite.py

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
import datetime
2+
import json
3+
import sqlite3
4+
5+
import settings
6+
7+
_settings = settings.Settings()
8+
9+
DB_FILENAME = "consumption_data.db"
10+
11+
CREATE_TABLE_SQL = """
12+
CREATE TABLE consumption_data
13+
(id INTEGER PRIMARY KEY AUTOINCREMENT,
14+
timestamp DATETIME,
15+
metered REAL,
16+
estimated REAL,
17+
metered_peak REAL,
18+
estimated_peak REAL,
19+
mean_profile REAL)
20+
"""
21+
CREATE_UNIQUE_INDEX_SQL = """
22+
CREATE UNIQUE INDEX consumption_timestamp_idx
23+
ON consumption_data (timestamp)
24+
"""
25+
INSERT_SQL = """
26+
INSERT INTO consumption_data
27+
(timestamp, metered, estimated, metered_peak, estimated_peak, mean_profile)
28+
VALUES
29+
(?, ?, ?, ?, ?, ?)
30+
ON CONFLICT(timestamp) DO NOTHING
31+
"""
32+
33+
34+
def _check_database(con):
35+
try:
36+
con.execute("SELECT 1 FROM consumption_data WHERE false")
37+
except sqlite3.OperationalError as except_inst:
38+
print(except_inst)
39+
con.execute(CREATE_TABLE_SQL)
40+
con.execute(CREATE_UNIQUE_INDEX_SQL)
41+
42+
43+
def import_data():
44+
for account_path in _settings.storage_path.iterdir():
45+
# account_id = account_path.name
46+
47+
for energy_meter_path in account_path.iterdir():
48+
# energy_meter = energy_meter_path.name
49+
50+
db_file = energy_meter_path / DB_FILENAME
51+
con = sqlite3.connect(str(db_file))
52+
_check_database(con)
53+
54+
for day_path in sorted(energy_meter_path.glob("*.json")):
55+
# day = datetime.datetime.strptime(day_path.stem, "%Y-%m-%d").date()
56+
with day_path.open() as fobj:
57+
json_data = json.loads(fobj.read())
58+
59+
data = list(
60+
zip(
61+
[
62+
datetime.datetime.fromisoformat(d)
63+
for d in json_data["peakDemandTimes"]
64+
],
65+
json_data["meteredValues"],
66+
json_data["estimatedValues"],
67+
json_data["meteredPeakDemands"],
68+
json_data["estimatedPeakDemands"],
69+
json_data["meanProfile"],
70+
)
71+
)
72+
73+
with con:
74+
con.executemany(INSERT_SQL, data)
75+
76+
con.close()
77+
78+
79+
if __name__ == "__main__":
80+
import_data()

requirements.txt

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pydantic[dotenv]==1.9.0
2+
requests==2.27.1
3+
4+
# development
5+
black==22.3.0
6+
ipython==8.2.0

settings.py

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
import datetime
2+
import pathlib
3+
4+
from pydantic import BaseSettings, Field
5+
6+
7+
class Settings(BaseSettings):
8+
username: str = Field(..., env="WEB_PORTAL_USERNAME")
9+
password: str = Field(..., env="WEB_PORTAL_PASSWORD")
10+
measure_start_date: datetime.date
11+
storage_path: pathlib.Path
12+
user_agent: str
13+
14+
class Config:
15+
env_file = ".env"
16+
env_file_encoding = "utf-8"

smartmeter.py

+167
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
import datetime
2+
import json
3+
import logging
4+
import pathlib
5+
from typing import Union
6+
7+
import requests
8+
9+
import settings
10+
11+
BASE_URL = "https://smartmeter.netz-noe.at/orchestration"
12+
13+
_logger = logging.getLogger(__name__)
14+
_settings = settings.Settings()
15+
16+
17+
class SmartMeter:
18+
def __init__(self, username: str, password: str) -> None:
19+
self.session = requests.Session()
20+
self.session.headers.update({"User-Agent": _settings.user_agent})
21+
22+
self._login(username, password)
23+
24+
def _login(self, username: str, password: str) -> None:
25+
"""Log into the smartmeter portal from Netz NÖ."""
26+
url = f"{BASE_URL}/Authentication/Login"
27+
response = self.session.post(url, json={"user": username, "pwd": password})
28+
response.raise_for_status()
29+
30+
def _get_account_ids(self) -> list[str]:
31+
"""
32+
Retrieve account ids linked to the login.
33+
34+
:return: list of account ids
35+
"""
36+
url = f"{BASE_URL}/User/GetAccountIdByBussinespartnerId"
37+
38+
response = self.session.get(url)
39+
response.raise_for_status()
40+
_logger.debug("Response for '%s' was: %s", url, json.dumps(response.json()))
41+
42+
return [account["accountId"] for account in response.json()]
43+
44+
def _get_meters(self, account_id: str) -> list[str]:
45+
"""
46+
Retrieve metering points for the given account id.
47+
48+
:param account_id: account id
49+
:return: list of metering points
50+
"""
51+
url = f"{BASE_URL}/User/GetMeteringPointByAccountId"
52+
53+
response = self.session.get(url, params={"accountId": account_id})
54+
response.raise_for_status()
55+
_logger.debug("Response for '%s' was: %s", url, json.dumps(response.json()))
56+
57+
return [entry["meteringPointId"] for entry in response.json()]
58+
59+
def get_account_meters(self) -> dict[str, list[str]]:
60+
"""
61+
Retrieves all metering points and account-ids linked to the login.
62+
63+
:return: Dictionary with
64+
key: account-id
65+
value: list of metering points for the account
66+
"""
67+
return {
68+
account_id: self._get_meters(account_id)
69+
for account_id in self._get_account_ids()
70+
}
71+
72+
def _get_mean_profile_for_day(
73+
self, metering_point: str, day: datetime.date
74+
) -> dict[str, list[Union[None, str, float]]]:
75+
"""Retrieve mean profile for the given day."""
76+
url = f"{BASE_URL}/ConsumptionRecord/MeanProfileDay"
77+
78+
response = self.session.get(
79+
url, params={"meterId": metering_point, "day": day.isoformat()}
80+
)
81+
response.raise_for_status()
82+
_logger.debug("Response for '%s' was: %s", url, json.dumps(response.json()))
83+
84+
return response.json()
85+
86+
def get_consumption_records_for_day(
87+
self,
88+
metering_point: str,
89+
day: datetime.date,
90+
*,
91+
include_mean_profile: bool = True,
92+
) -> dict[str, list[Union[None, str, float]]]:
93+
"""
94+
Retrieve the consumption records for the given day and the given metering point.
95+
96+
:param metering_point: metering point
97+
:param day: day
98+
:param include_mean_profile: indicates if mean profile of the day should be included
99+
"""
100+
url = f"{BASE_URL}/ConsumptionRecord/Day"
101+
102+
response = self.session.get(
103+
url, params={"meterId": metering_point, "day": day.isoformat()}
104+
)
105+
response.raise_for_status()
106+
_logger.debug("Response for '%s' was: %s", url, json.dumps(response.json()))
107+
108+
if not include_mean_profile:
109+
return response.json()
110+
111+
mean_profile = self._get_mean_profile_for_day(metering_point, day)
112+
return response.json() | {"meanProfile": mean_profile}
113+
114+
115+
def download_consumptions_for_meter(
116+
smartmeter: SmartMeter,
117+
storage_path: pathlib.Path,
118+
energy_meter: str,
119+
start_date: datetime.date,
120+
):
121+
"""
122+
Download consumption records starting from start-date for given engery-meter.
123+
"""
124+
number_since_measure_start = (datetime.date.today() - start_date).days
125+
for day in [
126+
start_date + datetime.timedelta(days=i)
127+
for i in range(number_since_measure_start)
128+
]:
129+
json_file = storage_path / f"{day.isoformat()}.json"
130+
if json_file.exists():
131+
_logger.debug(
132+
"Consumption records for '%s' and '%s' already exists.",
133+
energy_meter,
134+
day.isoformat(),
135+
)
136+
continue
137+
138+
json_file.write_text(
139+
json.dumps(
140+
smartmeter.get_consumption_records_for_day(energy_meter, day),
141+
indent=4,
142+
)
143+
)
144+
145+
146+
def main():
147+
smartmeter = SmartMeter(_settings.username, _settings.password)
148+
acount_meters = smartmeter.get_account_meters()
149+
for account_id, energy_meters in acount_meters.items():
150+
account_dir = _settings.storage_path / account_id
151+
if not account_dir.exists():
152+
account_dir.mkdir()
153+
154+
for energy_meter in energy_meters:
155+
energy_meter_dir = account_dir / energy_meter
156+
if not energy_meter_dir.exists():
157+
energy_meter_dir.mkdir()
158+
159+
download_consumptions_for_meter(
160+
smartmeter, energy_meter_dir, energy_meter, _settings.measure_start_date
161+
)
162+
163+
164+
if __name__ == "__main__":
165+
logging.basicConfig(level=logging.DEBUG)
166+
167+
main()

0 commit comments

Comments
 (0)