@@ -79,34 +79,6 @@ def backfill_dataset(location: str, *, dry_run: bool = False) -> None:
79
79
)
80
80
81
81
82
- def get_s3_object_creation_date (file_path : str , filesystem : fs .FileSystem ) -> datetime :
83
- """Get the creation date of an S3 object.
84
-
85
- Args:
86
- file_path: Path to the S3 object
87
- filesystem: PyArrow S3 filesystem instance
88
-
89
- Returns:
90
- datetime: Creation date of the S3 object in UTC
91
- """
92
- try :
93
- # Get creation date of S3 object
94
- file_info = filesystem .get_file_info (file_path )
95
- creation_date : datetime = file_info .mtime # type: ignore[assignment]
96
-
97
- # Ensure it's timezone-aware and in UTC
98
- if creation_date .tzinfo is None :
99
- creation_date = creation_date .replace (tzinfo = UTC )
100
- elif creation_date .tzinfo != UTC :
101
- creation_date = creation_date .astimezone (UTC )
102
-
103
- return creation_date
104
-
105
- except Exception as e :
106
- logger .error (f"Error getting S3 object creation date for { file_path } : { e } " )
107
- raise
108
-
109
-
110
82
def backfill_parquet_file (
111
83
parquet_filepath : str ,
112
84
dataset : ds .Dataset ,
@@ -183,6 +155,34 @@ def backfill_parquet_file(
183
155
}
184
156
185
157
158
+ def get_s3_object_creation_date (file_path : str , filesystem : fs .FileSystem ) -> datetime :
159
+ """Get the creation date of an S3 object.
160
+
161
+ Args:
162
+ file_path: Path to the S3 object
163
+ filesystem: PyArrow S3 filesystem instance
164
+
165
+ Returns:
166
+ datetime: Creation date of the S3 object in UTC
167
+ """
168
+ try :
169
+ # Get creation date of S3 object
170
+ file_info = filesystem .get_file_info (file_path )
171
+ creation_date : datetime = file_info .mtime # type: ignore[assignment]
172
+
173
+ # Ensure it's timezone-aware and in UTC
174
+ if creation_date .tzinfo is None :
175
+ creation_date = creation_date .replace (tzinfo = UTC )
176
+ elif creation_date .tzinfo != UTC :
177
+ creation_date = creation_date .astimezone (UTC )
178
+
179
+ return creation_date
180
+
181
+ except Exception as e :
182
+ logger .error (f"Error getting S3 object creation date for { file_path } : { e } " )
183
+ raise
184
+
185
+
186
186
if __name__ == "__main__" :
187
187
parser = argparse .ArgumentParser (
188
188
description = (
0 commit comments