From 90aa416528902443f5634b8ce07d968ecd23ab6a Mon Sep 17 00:00:00 2001 From: ghiggi Date: Sat, 6 Apr 2024 11:51:09 +0200 Subject: [PATCH] Update code format --- tstore/__init__.py | 7 +++++++ tstore/archive/__init__.py | 1 - tstore/archive/attributes/__init__.py | 1 - tstore/archive/attributes/dask.py | 1 - tstore/archive/attributes/geopandas.py | 1 - tstore/archive/attributes/pandas.py | 1 - tstore/archive/attributes/polars.py | 1 - tstore/archive/attributes/pyarrow.py | 1 - tstore/archive/checks.py | 17 +++++++---------- tstore/archive/io.py | 1 + tstore/archive/readers.py | 3 +-- tstore/archive/ts/__init__.py | 1 - tstore/archive/ts/dask.py | 5 ++--- tstore/archive/ts/filtering.py | 1 - tstore/archive/ts/partitioning.py | 1 - tstore/archive/ts/pyarrow.py | 1 - tstore/archive/writers.py | 1 + tstore/tsdf/__init__.py | 7 ++++++- tstore/tsdf/extensions/array.py | 9 ++++++--- tstore/tsdf/extensions/ts_dtype.py | 3 +-- tstore/tsdf/reader.py | 6 +----- tstore/tsdf/ts_class.py | 5 ++--- tstore/tsdf/writer.py | 19 ++++++++++++------- tstore/tsdt/__init__.py | 1 - tstore/tslong/__init__.py | 1 - tstore/tslong/dask.py | 1 - tstore/tslong/lance.py | 1 - tstore/tslong/pandas.py | 1 - tstore/tslong/polars.py | 1 - tstore/tslong/pyarrow.py | 5 +++-- tstore/tswide/__init__.py | 1 - tutorials/00_example_ts.py | 1 - tutorials/01_example_tsdf.py | 3 +-- tutorials/02_example_tslong.py | 1 - 34 files changed, 51 insertions(+), 60 deletions(-) diff --git a/tstore/__init__.py b/tstore/__init__.py index 7762763..c5983df 100644 --- a/tstore/__init__.py +++ b/tstore/__init__.py @@ -1,4 +1,7 @@ """tstore.""" +import contextlib +from importlib.metadata import PackageNotFoundError, version + from tstore.tsdf import TSDF from tstore.tsdf.extensions.array import TSArray from tstore.tsdf.extensions.ts_dtype import TSDtype @@ -12,3 +15,7 @@ "TS", "TSDF", ] + +# Get version +with contextlib.suppress(PackageNotFoundError): + __version__ = version("tstore") \ No newline at end of file diff --git a/tstore/archive/__init__.py b/tstore/archive/__init__.py index 4c8b5c5..765cd09 100644 --- a/tstore/archive/__init__.py +++ b/tstore/archive/__init__.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Mon Jun 12 22:23:06 2023. diff --git a/tstore/archive/attributes/__init__.py b/tstore/archive/attributes/__init__.py index 2f163ba..8afca86 100644 --- a/tstore/archive/attributes/__init__.py +++ b/tstore/archive/attributes/__init__.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Mon Jun 12 23:21:19 2023. diff --git a/tstore/archive/attributes/dask.py b/tstore/archive/attributes/dask.py index 7f95c5c..c196b00 100644 --- a/tstore/archive/attributes/dask.py +++ b/tstore/archive/attributes/dask.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Mon Jun 12 23:22:06 2023. diff --git a/tstore/archive/attributes/geopandas.py b/tstore/archive/attributes/geopandas.py index ca95c3d..2545399 100644 --- a/tstore/archive/attributes/geopandas.py +++ b/tstore/archive/attributes/geopandas.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Mon Jun 12 23:22:11 2023. diff --git a/tstore/archive/attributes/pandas.py b/tstore/archive/attributes/pandas.py index 350e55f..858af41 100644 --- a/tstore/archive/attributes/pandas.py +++ b/tstore/archive/attributes/pandas.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Mon Jun 12 23:22:02 2023. diff --git a/tstore/archive/attributes/polars.py b/tstore/archive/attributes/polars.py index d3a76e7..e84911a 100644 --- a/tstore/archive/attributes/polars.py +++ b/tstore/archive/attributes/polars.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Mon Jun 12 23:22:22 2023. diff --git a/tstore/archive/attributes/pyarrow.py b/tstore/archive/attributes/pyarrow.py index 4400e52..a66f345 100644 --- a/tstore/archive/attributes/pyarrow.py +++ b/tstore/archive/attributes/pyarrow.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Mon Jun 12 23:22:16 2023. diff --git a/tstore/archive/checks.py b/tstore/archive/checks.py index fbfddf8..03dc84d 100644 --- a/tstore/archive/checks.py +++ b/tstore/archive/checks.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Mon Jun 12 21:32:37 2023. @@ -36,14 +35,12 @@ def check_ts_variables(ts_variables, base_dir): if isinstance(ts_variables, type(None)): ts_variables = available_ts_variables ts_variables = np.array(ts_variables) - unvalid_ts_variables = ts_variables[ - np.isin(ts_variables, available_ts_variables, invert=True) - ] + unvalid_ts_variables = ts_variables[np.isin(ts_variables, available_ts_variables, invert=True)] ts_variables = ts_variables.tolist() unvalid_ts_variables = unvalid_ts_variables.tolist() if len(unvalid_ts_variables) > 0: raise ValueError( - f"Valid ts_variables are {ts_variables}. Unvalid: {unvalid_ts_variables}." + f"Valid ts_variables are {ts_variables}. Invalid: {unvalid_ts_variables}.", ) return ts_variables @@ -76,13 +73,13 @@ def check_time(time): datetime.datetime object. """ - # TODO: adapt to return the more approriate object !!! + # TODO: adapt to return the more appropriate object !!! if not isinstance( - time, (datetime.datetime, datetime.date, np.datetime64, np.ndarray, str) + time, + (datetime.datetime, datetime.date, np.datetime64, np.ndarray, str), ): raise TypeError( - "Specify time with datetime.datetime objects or a " - "string of format 'YYYY-MM-DD hh:mm:ss'." + "Specify time with datetime.datetime objects or a " "string of format 'YYYY-MM-DD hh:mm:ss'.", ) # If numpy array with datetime64 (and size=1) if isinstance(time, np.ndarray): @@ -117,5 +114,5 @@ def check_start_end_time(start_time, end_time): # Check start_time and end_time are chronological if start_time is not None and end_time is not None: if start_time > end_time: - raise ValueError("Provide start_time occuring before of end_time") + raise ValueError("Provide start_time occurring before of end_time") return (start_time, end_time) diff --git a/tstore/archive/io.py b/tstore/archive/io.py index 019cf31..5e69910 100644 --- a/tstore/archive/io.py +++ b/tstore/archive/io.py @@ -1,4 +1,5 @@ """TStore I/O tools.""" + import glob import os import shutil diff --git a/tstore/archive/readers.py b/tstore/archive/readers.py index 1323618..134bf1c 100644 --- a/tstore/archive/readers.py +++ b/tstore/archive/readers.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Mon Jun 12 15:35:06 2023. @@ -12,7 +11,7 @@ def _read_yaml_metadata(fpath): """Read metadata YAML file.""" - with open(fpath, "r") as file: + with open(fpath) as file: metadata = yaml.safe_load(file) return metadata diff --git a/tstore/archive/ts/__init__.py b/tstore/archive/ts/__init__.py index 7569c08..43e032a 100644 --- a/tstore/archive/ts/__init__.py +++ b/tstore/archive/ts/__init__.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Mon Jun 12 22:22:27 2023. diff --git a/tstore/archive/ts/dask.py b/tstore/archive/ts/dask.py index 6d28af4..a2f73af 100644 --- a/tstore/archive/ts/dask.py +++ b/tstore/archive/ts/dask.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Mon Jun 12 22:24:07 2023. @@ -29,14 +28,14 @@ def open_ts( # Define Apache Arrow settings arrow_to_pandas = { - "zero_copy_only": False, # Default is False. If True, raise error if doing copys + "zero_copy_only": False, # Default is False. If True, raise error if doing copies "strings_to_categorical": False, "date_as_object": False, # Default is True. If False convert to datetime64[ns] "timestamp_as_object": False, # Default is True. If False convert to np.datetime64[ns] "use_threads": True, # parallelize the conversion using multiple threads. "safe": True, "split_blocks": False, - "ignore_metadata": False, # Default False. If False, use the ‘pandas’ metadata to get the Index + "ignore_metadata": False, # Default False. If False, use the 'pandas' metadata to get the Index "types_mapper": pd.ArrowDtype, # Ensure pandas is created with Arrow dtype } diff --git a/tstore/archive/ts/filtering.py b/tstore/archive/ts/filtering.py index 5fb931d..8ad1c2b 100644 --- a/tstore/archive/ts/filtering.py +++ b/tstore/archive/ts/filtering.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Mon Jun 12 22:27:07 2023. diff --git a/tstore/archive/ts/partitioning.py b/tstore/archive/ts/partitioning.py index 918016f..0dd5e01 100644 --- a/tstore/archive/ts/partitioning.py +++ b/tstore/archive/ts/partitioning.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Mon Jun 12 22:21:47 2023. diff --git a/tstore/archive/ts/pyarrow.py b/tstore/archive/ts/pyarrow.py index eee0038..d6cb228 100644 --- a/tstore/archive/ts/pyarrow.py +++ b/tstore/archive/ts/pyarrow.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Mon Jun 12 22:19:51 2023. diff --git a/tstore/archive/writers.py b/tstore/archive/writers.py index bec934a..d81b48b 100644 --- a/tstore/archive/writers.py +++ b/tstore/archive/writers.py @@ -1,4 +1,5 @@ """TStore writers.""" + import yaml from tstore.archive.io import ( diff --git a/tstore/tsdf/__init__.py b/tstore/tsdf/__init__.py index af341c7..c042cfa 100644 --- a/tstore/tsdf/__init__.py +++ b/tstore/tsdf/__init__.py @@ -1,4 +1,5 @@ """TSDF.""" + import pandas as pd @@ -9,7 +10,11 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) def to_tstore( - self, base_dir, partition_str=None, tstore_structure="id-var", overwrite=True + self, + base_dir, + partition_str=None, + tstore_structure="id-var", + overwrite=True, ): """Write TStore from TSDF object.""" from tstore.tsdf.writer import write_tstore diff --git a/tstore/tsdf/extensions/array.py b/tstore/tsdf/extensions/array.py index 46fb763..5885e38 100644 --- a/tstore/tsdf/extensions/array.py +++ b/tstore/tsdf/extensions/array.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Sun Jun 11 22:47:54 2023. @@ -28,6 +27,7 @@ # https://itnext.io/guide-to-pandas-extension-types-and-how-to-create-your-own-3b213d689c86 + ####--------------------------------------------------------------------------. def get_tabular_object_type(obj): """Get inner class of the TS object.""" @@ -73,7 +73,7 @@ def __str__(self): return str(self._data) def __repr__(self): - """repr representation.""" + """Repr representation.""" n = len(self._data) return f"TSArray composed of {n} TS objects." @@ -173,6 +173,9 @@ def take(self, indices, allow_fill=False, fill_value=None): fill_value = self.dtype.na_value result = pd.core.algorithms.take( - self._data, indices, allow_fill=allow_fill, fill_value=fill_value + self._data, + indices, + allow_fill=allow_fill, + fill_value=fill_value, ) return self._from_sequence(result) diff --git a/tstore/tsdf/extensions/ts_dtype.py b/tstore/tsdf/extensions/ts_dtype.py index ea735c5..5ec3ce9 100644 --- a/tstore/tsdf/extensions/ts_dtype.py +++ b/tstore/tsdf/extensions/ts_dtype.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Mon Jun 12 16:23:54 2023. @@ -91,7 +90,7 @@ def construct_from_string(cls, string: str): Example: ------- - >>> TSDtype.construct_from_string('TS[pandas]') + >>> TSDtype.construct_from_string("TS[pandas]") TS['pandas'] """ if not isinstance(string, str): diff --git a/tstore/tsdf/reader.py b/tstore/tsdf/reader.py index 9f5453c..4434d34 100644 --- a/tstore/tsdf/reader.py +++ b/tstore/tsdf/reader.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Mon Jun 12 15:48:39 2023. @@ -31,10 +30,7 @@ def _read_tsarray(base_dir, ts_variable): def _read_tsarrays(base_dir, metadata): """Read list of TSArrays.""" ts_variables = metadata["ts_variables"] - list_ts_series = [ - _read_tsarray(base_dir=base_dir, ts_variable=ts_variable) - for ts_variable in ts_variables - ] + list_ts_series = [_read_tsarray(base_dir=base_dir, ts_variable=ts_variable) for ts_variable in ts_variables] return list_ts_series diff --git a/tstore/tsdf/ts_class.py b/tstore/tsdf/ts_class.py index 35d2e8c..4418456 100644 --- a/tstore/tsdf/ts_class.py +++ b/tstore/tsdf/ts_class.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Mon Jun 12 22:17:59 2023. @@ -19,8 +18,8 @@ def add_partitioning_columns(df, partitioning_str): """Add partitioning columns to the dataframe.""" # TODO: as function of TS_partitioning_string (YYYY/MM/DD) or (YY/DOY/HH) # dayofyear, dayofweek, hour, minute, ... - df["month"] = df.index.dt.month.values - df["year"] = df.index.dt.year.values + df["month"] = df.index.dt.month.to_numpy() + df["year"] = df.index.dt.year.to_numpy() partition_on = ["year", "month"] return df, partition_on diff --git a/tstore/tsdf/writer.py b/tstore/tsdf/writer.py index a109f44..16c094c 100644 --- a/tstore/tsdf/writer.py +++ b/tstore/tsdf/writer.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Mon Jun 12 15:49:54 2023. @@ -18,9 +17,7 @@ def _get_ts_variables(df): """Get list of TSArray columns.""" columns = np.array(list(df.columns)) - ts_variables = columns[ - [isinstance(df[column].dtype, TSDtype) for column in columns] - ].tolist() + ts_variables = columns[[isinstance(df[column].dtype, TSDtype) for column in columns]].tolist() return ts_variables @@ -62,7 +59,9 @@ def _write_tsarrays(df, base_dir, tstore_structure): tsarray_columns = _get_ts_variables(df) for column in tsarray_columns: _write_ts_series( - ts_series=df[column], base_dir=base_dir, tstore_structure=tstore_structure + ts_series=df[column], + base_dir=base_dir, + tstore_structure=tstore_structure, ) @@ -72,12 +71,18 @@ def _write_metadata(df, base_dir, tstore_structure): ts_variables = _get_ts_variables(df) write_metadata( - base_dir=base_dir, ts_variables=ts_variables, tstore_structure=tstore_structure + base_dir=base_dir, + ts_variables=ts_variables, + tstore_structure=tstore_structure, ) def write_tstore( - df, base_dir, partition_str=None, tstore_structure="id-var", overwrite=True + df, + base_dir, + partition_str=None, + tstore_structure="id-var", + overwrite=True, ): """Write TStore from TSDF object.""" # Checks diff --git a/tstore/tsdt/__init__.py b/tstore/tsdt/__init__.py index 67363cd..4ff3acb 100644 --- a/tstore/tsdt/__init__.py +++ b/tstore/tsdt/__init__.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Mon Jun 12 16:36:54 2023. diff --git a/tstore/tslong/__init__.py b/tstore/tslong/__init__.py index 48f8fc9..ff1d27c 100644 --- a/tstore/tslong/__init__.py +++ b/tstore/tslong/__init__.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Mon Jun 12 16:37:13 2023. diff --git a/tstore/tslong/dask.py b/tstore/tslong/dask.py index 77cb9b4..fe1b774 100644 --- a/tstore/tslong/dask.py +++ b/tstore/tslong/dask.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Mon Jun 12 17:57:53 2023. diff --git a/tstore/tslong/lance.py b/tstore/tslong/lance.py index 66427f8..85cdf23 100644 --- a/tstore/tslong/lance.py +++ b/tstore/tslong/lance.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Mon Jun 12 17:58:15 2023. diff --git a/tstore/tslong/pandas.py b/tstore/tslong/pandas.py index 46fe749..4b42e93 100644 --- a/tstore/tslong/pandas.py +++ b/tstore/tslong/pandas.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Mon Jun 12 17:57:40 2023. diff --git a/tstore/tslong/polars.py b/tstore/tslong/polars.py index d297eea..e82647b 100644 --- a/tstore/tslong/polars.py +++ b/tstore/tslong/polars.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Mon Jun 12 17:58:05 2023. diff --git a/tstore/tslong/pyarrow.py b/tstore/tslong/pyarrow.py index 2ccec18..11f5130 100644 --- a/tstore/tslong/pyarrow.py +++ b/tstore/tslong/pyarrow.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Mon Jun 12 17:57:14 2023. @@ -81,7 +80,9 @@ def _join_tables(left_table, right_table): """Joining functions of pyarrow tables.""" # TODO: update keys to 'time' return left_table.join( - right_table, keys=["timestamp", "tstore_id"], join_type="full outer" + right_table, + keys=["timestamp", "tstore_id"], + join_type="full outer", ) diff --git a/tstore/tswide/__init__.py b/tstore/tswide/__init__.py index 1f804a7..6cba8fd 100644 --- a/tstore/tswide/__init__.py +++ b/tstore/tswide/__init__.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Mon Jun 12 16:37:25 2023. diff --git a/tutorials/00_example_ts.py b/tutorials/00_example_ts.py index 811dd7f..fbf45cd 100644 --- a/tutorials/00_example_ts.py +++ b/tutorials/00_example_ts.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Sun Jun 11 20:56:07 2023. diff --git a/tutorials/01_example_tsdf.py b/tutorials/01_example_tsdf.py index ba905b6..bafcc8d 100644 --- a/tutorials/01_example_tsdf.py +++ b/tutorials/01_example_tsdf.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Sun Jun 11 22:21:38 2023. @@ -50,7 +49,7 @@ df_series = pd.Series(ts_arr, index=tstore_ids) df_series # dtype: TS[dask.DataFrame] ... pandas/polars/pyarrow df_series.dtype # TSDtype -df_series.values # TSArray +df_series.values # TSArray # noqa df_series.array # TSArray # zero-copy reference to the data ! df_series.to_numpy() # object dtype diff --git a/tutorials/02_example_tslong.py b/tutorials/02_example_tslong.py index 5e08164..979b796 100644 --- a/tutorials/02_example_tslong.py +++ b/tutorials/02_example_tslong.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ Created on Tue Jun 13 00:01:04 2023.