Skip to content

Commit 2fa84df

Browse files
committed
Reorder functions in run_timestamp backfill migration
1 parent 97df92d commit 2fa84df

File tree

1 file changed

+28
-28
lines changed

1 file changed

+28
-28
lines changed

migrations/001_2025_05_30_backfill_run_timestamp_column.py

Lines changed: 28 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -79,34 +79,6 @@ def backfill_dataset(location: str, *, dry_run: bool = False) -> None:
7979
)
8080

8181

82-
def get_s3_object_creation_date(file_path: str, filesystem: fs.FileSystem) -> datetime:
83-
"""Get the creation date of an S3 object.
84-
85-
Args:
86-
file_path: Path to the S3 object
87-
filesystem: PyArrow S3 filesystem instance
88-
89-
Returns:
90-
datetime: Creation date of the S3 object in UTC
91-
"""
92-
try:
93-
# Get creation date of S3 object
94-
file_info = filesystem.get_file_info(file_path)
95-
creation_date: datetime = file_info.mtime # type: ignore[assignment]
96-
97-
# Ensure it's timezone-aware and in UTC
98-
if creation_date.tzinfo is None:
99-
creation_date = creation_date.replace(tzinfo=UTC)
100-
elif creation_date.tzinfo != UTC:
101-
creation_date = creation_date.astimezone(UTC)
102-
103-
return creation_date
104-
105-
except Exception as e:
106-
logger.error(f"Error getting S3 object creation date for {file_path}: {e}")
107-
raise
108-
109-
11082
def backfill_parquet_file(
11183
parquet_filepath: str,
11284
dataset: ds.Dataset,
@@ -183,6 +155,34 @@ def backfill_parquet_file(
183155
}
184156

185157

158+
def get_s3_object_creation_date(file_path: str, filesystem: fs.FileSystem) -> datetime:
159+
"""Get the creation date of an S3 object.
160+
161+
Args:
162+
file_path: Path to the S3 object
163+
filesystem: PyArrow S3 filesystem instance
164+
165+
Returns:
166+
datetime: Creation date of the S3 object in UTC
167+
"""
168+
try:
169+
# Get creation date of S3 object
170+
file_info = filesystem.get_file_info(file_path)
171+
creation_date: datetime = file_info.mtime # type: ignore[assignment]
172+
173+
# Ensure it's timezone-aware and in UTC
174+
if creation_date.tzinfo is None:
175+
creation_date = creation_date.replace(tzinfo=UTC)
176+
elif creation_date.tzinfo != UTC:
177+
creation_date = creation_date.astimezone(UTC)
178+
179+
return creation_date
180+
181+
except Exception as e:
182+
logger.error(f"Error getting S3 object creation date for {file_path}: {e}")
183+
raise
184+
185+
186186
if __name__ == "__main__":
187187
parser = argparse.ArgumentParser(
188188
description=(

0 commit comments

Comments
 (0)