Skip to content

Commit 4d6bdc3

Browse files
erikamundsonerikamundson
committed
add timeseries schema
1 parent 09499b0 commit 4d6bdc3

14 files changed

+327
-7
lines changed

‎.gitignore

+160
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
# Byte-compiled / optimized / DLL files
2+
__pycache__/
3+
*.py[cod]
4+
*$py.class
5+
6+
# C extensions
7+
*.so
8+
9+
# Distribution / packaging
10+
.Python
11+
build/
12+
develop-eggs/
13+
dist/
14+
downloads/
15+
eggs/
16+
.eggs/
17+
lib/
18+
lib64/
19+
parts/
20+
sdist/
21+
var/
22+
wheels/
23+
share/python-wheels/
24+
*.egg-info/
25+
.installed.cfg
26+
*.egg
27+
MANIFEST
28+
29+
# PyInstaller
30+
# Usually these files are written by a python script from a template
31+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
32+
*.manifest
33+
*.spec
34+
35+
# Installer logs
36+
pip-log.txt
37+
pip-delete-this-directory.txt
38+
39+
# Unit test / coverage reports
40+
htmlcov/
41+
.tox/
42+
.nox/
43+
.coverage
44+
.coverage.*
45+
.cache
46+
nosetests.xml
47+
coverage.xml
48+
*.cover
49+
*.py,cover
50+
.hypothesis/
51+
.pytest_cache/
52+
cover/
53+
54+
# Translations
55+
*.mo
56+
*.pot
57+
58+
# Django stuff:
59+
*.log
60+
local_settings.py
61+
db.sqlite3
62+
db.sqlite3-journal
63+
64+
# Flask stuff:
65+
instance/
66+
.webassets-cache
67+
68+
# Scrapy stuff:
69+
.scrapy
70+
71+
# Sphinx documentation
72+
docs/_build/
73+
74+
# PyBuilder
75+
.pybuilder/
76+
target/
77+
78+
# Jupyter Notebook
79+
.ipynb_checkpoints
80+
81+
# IPython
82+
profile_default/
83+
ipython_config.py
84+
85+
# pyenv
86+
# For a library or package, you might want to ignore these files since the code is
87+
# intended to run in multiple environments; otherwise, check them in:
88+
# .python-version
89+
90+
# pipenv
91+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
93+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
94+
# install all needed dependencies.
95+
#Pipfile.lock
96+
97+
# poetry
98+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
99+
# This is especially recommended for binary packages to ensure reproducibility, and is more
100+
# commonly ignored for libraries.
101+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
102+
#poetry.lock
103+
104+
# pdm
105+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
106+
#pdm.lock
107+
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
108+
# in version control.
109+
# https://pdm.fming.dev/#use-with-ide
110+
.pdm.toml
111+
112+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
113+
__pypackages__/
114+
115+
# Celery stuff
116+
celerybeat-schedule
117+
celerybeat.pid
118+
119+
# SageMath parsed files
120+
*.sage.py
121+
122+
# Environments
123+
.env
124+
.venv
125+
env/
126+
venv/
127+
ENV/
128+
env.bak/
129+
venv.bak/
130+
131+
# Spyder project settings
132+
.spyderproject
133+
.spyproject
134+
135+
# Rope project settings
136+
.ropeproject
137+
138+
# mkdocs documentation
139+
/site
140+
141+
# mypy
142+
.mypy_cache/
143+
.dmypy.json
144+
dmypy.json
145+
146+
# Pyre type checker
147+
.pyre/
148+
149+
# pytype static type analyzer
150+
.pytype/
151+
152+
# Cython debug symbols
153+
cython_debug/
154+
155+
# PyCharm
156+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
157+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
158+
# and can be added to the global gitignore or merged into this file. For a more nuclear
159+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
160+
#.idea/

‎Makefile

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
.PHONY: install
2+
install:
3+
python -m pip install --upgrade pip
4+
pip install -e . --upgrade --upgrade-strategy eager
5+
6+
.PHONY: dev_install
7+
dev_install:
8+
python -m pip install --upgrade pip
9+
pip install -e .[dev] --upgrade --upgrade-strategy eager
10+
11+
.PHONY: format
12+
format:
13+
ruff format .
14+
ruff check . --fix
15+
mypy . --install-types --ignore-missing-imports --non-interactive
16+
17+
.PHONY: test_format
18+
test_format:
19+
ruff format . --check
20+
ruff check .
21+
mypy . --install-types --ignore-missing-imports --non-interactive

‎pyproject.toml

+121
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
[project]
2+
name = "pulse-telemetry"
3+
description = "Spark applications for transforming raw incoming data into a set of schemas for analysis."
4+
readme = "README.md"
5+
authors = [
6+
{name = "Maxwell Dylla", email = "[email protected]"},
7+
{name = "Erik Amundson", email = "[email protected]"}
8+
]
9+
dynamic = ["version"]
10+
dependencies = [
11+
"pyspark==3.5.3",
12+
]
13+
14+
[project.optional-dependencies]
15+
dev = [
16+
"mypy==1.10.1",
17+
"pytest==8.2.2",
18+
"pytest-cov==5.0.0",
19+
"ruff==0.5.2",
20+
]
21+
22+
[tool.setuptools.dynamic]
23+
version = {attr = "pulse_telemetry.__version__"}
24+
25+
[tool.ruff]
26+
include = ["*.py", "*.pyi", "**/pyproject.toml", "*.ipynb"]
27+
line-length = 120
28+
target-version = "py310"
29+
30+
[tool.ruff.lint]
31+
select = [
32+
# pyflakes
33+
"F",
34+
# pycodestyle
35+
"E",
36+
"W",
37+
# flake8-2020
38+
"YTT",
39+
# flake8-bugbear
40+
"B",
41+
# flake8-commas
42+
"COM",
43+
# flake8-datetimez
44+
"DTZ",
45+
# flake8-quotes
46+
"Q",
47+
# pylint
48+
"PLE", "PLR", "PLW",
49+
# misc lints
50+
"PIE",
51+
# flake8-pyi
52+
"PYI",
53+
# tidy imports
54+
"TID",
55+
# implicit string concatenation
56+
"ISC",
57+
# type-checking imports
58+
"TCH",
59+
# isort
60+
"I",
61+
# comprehensions
62+
"C4",
63+
# pygrep-hooks
64+
"PGH",
65+
# Ruff-specific rules
66+
"RUF",
67+
# Upgrade syntax
68+
"UP",
69+
]
70+
ignore = [
71+
# module level import not at top of file
72+
"E402",
73+
# too many branches
74+
"PLR0912",
75+
# too many arguments to function call
76+
"PLR0913",
77+
# too many statements in function
78+
"PLR0915",
79+
# magic value used in comparison,
80+
"PLR2004",
81+
# do not use mutable data structures for argument defaults
82+
"B006",
83+
# recommended by Ruff to disable to avoid issues with formatter
84+
"COM812", "ISC001",
85+
]
86+
per-file-ignores = {"__init__.py" = ["F401"]}
87+
88+
[tool.ruff.lint.flake8-type-checking]
89+
quote-annotations = true
90+
91+
[tool.mypy]
92+
disable_error_code = "type-var"
93+
94+
[tool.coverage.report]
95+
# Regexes for lines to exclude from consideration
96+
exclude_also = [
97+
# Don't complain about missing debug-only code:
98+
"def __repr__",
99+
"if self\\.debug",
100+
101+
# Don't complain if tests don't hit defensive assertion code:
102+
"raise AssertionError",
103+
"raise NotImplementedError",
104+
105+
# Don't complain if non-runnable code isn't run:
106+
"if 0:",
107+
"if __name__ == .__main__.:",
108+
109+
# Don't complain about abstract methods, they aren't run:
110+
"@(abc\\.)?abstractmethod",
111+
112+
# Ignore type checking
113+
"if TYPE_CHECKING:",
114+
]
115+
ignore_errors = true
116+
117+
[tool.coverage.run]
118+
omit = [
119+
# Workflows are not tested
120+
"*/workflows/*"
121+
]

‎src/pulse_telemetry/__init__.py

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
__version__ = "0.0.1"
2+
__all__ = ["__version__"]
3+
4+
if __name__ == "__main__":
5+
# Note: needed to get version for Docker build, etc.
6+
print(__version__)

‎src/pulse_telemetry/apps/timeseries.py

-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
2-
31
def main(variable: str):
42
spark = ( # noqa: F841
53
)

‎src/pulse_telemetry/sparklib/spark_session.py

-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,2 @@
1-
2-
31
def spark_session(with_delta: bool = True):
42
pass

‎src/pulse_telemetry/sparklib/transformations/timeseries.py

+19-3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,20 @@
1+
import pyspark.sql.types as T
12

2-
class Series:
3-
id: str
4-
device_id: str
3+
timeseries_schema = T.StructType(
4+
[
5+
T.StructField("device_id", dataType=T.StringType(), nullable=False),
6+
T.StructField("test_id", dataType=T.LongType(), nullable=True),
7+
T.StructField("sequence_number", dataType=T.LongType(), nullable=False),
8+
T.StructField("step_number", dataType=T.LongType(), nullable=False),
9+
T.StructField("step_id", dataType=T.IntegerType(), nullable=True),
10+
T.StructField("cycle_number", dataType=T.IntegerType(), nullable=False),
11+
T.StructField("timestamp", dataType=T.TimestampType(), nullable=False),
12+
T.StructField("current__A", dataType=T.DoubleType(), nullable=False),
13+
T.StructField("voltage__V", dataType=T.DoubleType(), nullable=False),
14+
T.StructField("power__W", dataType=T.DoubleType(), nullable=False),
15+
T.StructField("capacity__Ah", dataType=T.DoubleType(), nullable=False),
16+
T.StructField("energy__Wh", dataType=T.DoubleType(), nullable=False),
17+
T.StructField("auxiliary", dataType=T.MapType(T.StringType(), T.DoubleType()), nullable=True),
18+
T.StructField("update_ts", dataType=T.TimestampType(), nullable=False),
19+
]
20+
)

‎src/requirements-dev.txt

Whitespace-only changes.

‎src/requirements.txt

Whitespace-only changes.

‎src/setup.cfg

Whitespace-only changes.

‎src/setup.py

Whitespace-only changes.

‎tests/integration/conftest.py ‎tests/conftest.py

File renamed without changes.

‎tests/system/conftest.py

Whitespace-only changes.

‎tests/unit/conftest.py

Whitespace-only changes.

0 commit comments

Comments
 (0)