diff --git a/test/service/test_manifest.py b/test/service/test_manifest.py index 7bdae5e620..697f6c5333 100644 --- a/test/service/test_manifest.py +++ b/test/service/test_manifest.py @@ -181,77 +181,6 @@ def _assert_or_create_json_can(self, expected: Path, actual: str): with open(expected, 'w') as f: f.write(actual) - -class ManifestTestCase(WebServiceTestCase, - StorageServiceTestCase, - CannedManifestTestCase, - metaclass=ABCMeta): - - def setUp(self): - super().setUp() - self.addPatch(patch.object(PagedManifestGenerator, 'page_size', 1)) - self._setup_indices() - self._setup_git_commit() - - def tearDown(self): - self._teardown_indices() - self._teardown_git_commit() - super().tearDown() - - def _filters(self, filters: FiltersJSON) -> Filters: - return Filters(explicit=filters, source_ids={self.source.id}) - - def _setup_git_commit(self): - """ - Set git variables required to derive the manifest object key - """ - assert 'azul_git_commit' not in os.environ - assert 'azul_git_dirty' not in os.environ - os.environ['azul_git_commit'] = '9347432ab0da43c73409ac7fd3edfe29cf3ae678' - os.environ['azul_git_dirty'] = 'False' - - def _teardown_git_commit(self): - os.environ.pop('azul_git_commit') - os.environ.pop('azul_git_dirty') - - @property - def _service(self): - return ManifestService(self.storage_service, self.app_module.app.file_url) - - def _get_manifest(self, - format: ManifestFormat, - filters: FiltersJSON, - stream=False - ) -> Response: - manifest, num_partitions = self._get_manifest_object(format, filters) - self.assertEqual(1, num_partitions) - url = furl(self._service.get_manifest_url(manifest)) - response = requests.get(str(url), stream=stream) - # Moto doesn't support signed S3 URLs with Content-Disposition baked in, - # so we'll retroactively inject it into the response header. - content_disposition = url.args.get('response-content-disposition') - if content_disposition is not None: - response.headers['content-disposition'] = content_disposition - return response - - def _get_manifest_object(self, - format: ManifestFormat, - filters: JSON - ) -> tuple[Manifest, int]: - filters = self._filters(filters) - partition = ManifestPartition.first() - num_partitions = 1 - while True: - partition = self._service.get_manifest(format=format, - catalog=self.catalog, - filters=filters, - partition=partition) - if isinstance(partition, Manifest): - return partition, num_partitions - # Emulate controller serializing the partition between steps - partition = ManifestPartition.from_json(partition.to_json()) - num_partitions += 1 - def _assert_tsv(self, expected: list[tuple[str, ...]], actual: Response): """ Assert that the body of the given response is the expected TSV, @@ -324,6 +253,77 @@ def _assert_pfb(self, self.assertEqual(expected_schema, schema) self.assertEqual(expected_entities, entities) + +class ManifestTestCase(WebServiceTestCase, + StorageServiceTestCase, + CannedManifestTestCase, + metaclass=ABCMeta): + + def setUp(self): + super().setUp() + self.addPatch(patch.object(PagedManifestGenerator, 'page_size', 1)) + self._setup_indices() + self._setup_git_commit() + + def tearDown(self): + self._teardown_indices() + self._teardown_git_commit() + super().tearDown() + + def _filters(self, filters: FiltersJSON) -> Filters: + return Filters(explicit=filters, source_ids={self.source.id}) + + def _setup_git_commit(self): + """ + Set git variables required to derive the manifest object key + """ + assert 'azul_git_commit' not in os.environ + assert 'azul_git_dirty' not in os.environ + os.environ['azul_git_commit'] = '9347432ab0da43c73409ac7fd3edfe29cf3ae678' + os.environ['azul_git_dirty'] = 'False' + + def _teardown_git_commit(self): + os.environ.pop('azul_git_commit') + os.environ.pop('azul_git_dirty') + + @property + def _service(self): + return ManifestService(self.storage_service, self.app_module.app.file_url) + + def _get_manifest(self, + format: ManifestFormat, + filters: FiltersJSON, + stream=False + ) -> Response: + manifest, num_partitions = self._get_manifest_object(format, filters) + self.assertEqual(1, num_partitions) + url = furl(self._service.get_manifest_url(manifest)) + response = requests.get(str(url), stream=stream) + # Moto doesn't support signed S3 URLs with Content-Disposition baked in, + # so we'll retroactively inject it into the response header. + content_disposition = url.args.get('response-content-disposition') + if content_disposition is not None: + response.headers['content-disposition'] = content_disposition + return response + + def _get_manifest_object(self, + format: ManifestFormat, + filters: JSON + ) -> tuple[Manifest, int]: + filters = self._filters(filters) + partition = ManifestPartition.first() + num_partitions = 1 + while True: + partition = self._service.get_manifest(format=format, + catalog=self.catalog, + filters=filters, + partition=partition) + if isinstance(partition, Manifest): + return partition, num_partitions + # Emulate controller serializing the partition between steps + partition = ManifestPartition.from_json(partition.to_json()) + num_partitions += 1 + def _file_url(self, file_id, version): return str(self.base_url.set(path='/repository/files/' + file_id, args=dict(catalog=self.catalog,