Skip to content

Commit

Permalink
Pull up methods for asserting manifest contents into parent class
Browse files Browse the repository at this point in the history
  • Loading branch information
nadove-ucsc committed Jan 29, 2025
1 parent 818bb0e commit c688659
Showing 1 changed file with 71 additions and 71 deletions.
142 changes: 71 additions & 71 deletions test/service/test_manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -181,77 +181,6 @@ def _assert_or_create_json_can(self, expected: Path, actual: str):
with open(expected, 'w') as f:
f.write(actual)


class ManifestTestCase(WebServiceTestCase,
StorageServiceTestCase,
CannedManifestTestCase,
metaclass=ABCMeta):

def setUp(self):
super().setUp()
self.addPatch(patch.object(PagedManifestGenerator, 'page_size', 1))
self._setup_indices()
self._setup_git_commit()

def tearDown(self):
self._teardown_indices()
self._teardown_git_commit()
super().tearDown()

def _filters(self, filters: FiltersJSON) -> Filters:
return Filters(explicit=filters, source_ids={self.source.id})

def _setup_git_commit(self):
"""
Set git variables required to derive the manifest object key
"""
assert 'azul_git_commit' not in os.environ
assert 'azul_git_dirty' not in os.environ
os.environ['azul_git_commit'] = '9347432ab0da43c73409ac7fd3edfe29cf3ae678'
os.environ['azul_git_dirty'] = 'False'

def _teardown_git_commit(self):
os.environ.pop('azul_git_commit')
os.environ.pop('azul_git_dirty')

@property
def _service(self):
return ManifestService(self.storage_service, self.app_module.app.file_url)

def _get_manifest(self,
format: ManifestFormat,
filters: FiltersJSON,
stream=False
) -> Response:
manifest, num_partitions = self._get_manifest_object(format, filters)
self.assertEqual(1, num_partitions)
url = furl(self._service.get_manifest_url(manifest))
response = requests.get(str(url), stream=stream)
# Moto doesn't support signed S3 URLs with Content-Disposition baked in,
# so we'll retroactively inject it into the response header.
content_disposition = url.args.get('response-content-disposition')
if content_disposition is not None:
response.headers['content-disposition'] = content_disposition
return response

def _get_manifest_object(self,
format: ManifestFormat,
filters: JSON
) -> tuple[Manifest, int]:
filters = self._filters(filters)
partition = ManifestPartition.first()
num_partitions = 1
while True:
partition = self._service.get_manifest(format=format,
catalog=self.catalog,
filters=filters,
partition=partition)
if isinstance(partition, Manifest):
return partition, num_partitions
# Emulate controller serializing the partition between steps
partition = ManifestPartition.from_json(partition.to_json())
num_partitions += 1

def _assert_tsv(self, expected: list[tuple[str, ...]], actual: Response):
"""
Assert that the body of the given response is the expected TSV,
Expand Down Expand Up @@ -324,6 +253,77 @@ def _assert_pfb(self,
self.assertEqual(expected_schema, schema)
self.assertEqual(expected_entities, entities)


class ManifestTestCase(WebServiceTestCase,
StorageServiceTestCase,
CannedManifestTestCase,
metaclass=ABCMeta):

def setUp(self):
super().setUp()
self.addPatch(patch.object(PagedManifestGenerator, 'page_size', 1))
self._setup_indices()
self._setup_git_commit()

def tearDown(self):
self._teardown_indices()
self._teardown_git_commit()
super().tearDown()

def _filters(self, filters: FiltersJSON) -> Filters:
return Filters(explicit=filters, source_ids={self.source.id})

def _setup_git_commit(self):
"""
Set git variables required to derive the manifest object key
"""
assert 'azul_git_commit' not in os.environ
assert 'azul_git_dirty' not in os.environ
os.environ['azul_git_commit'] = '9347432ab0da43c73409ac7fd3edfe29cf3ae678'
os.environ['azul_git_dirty'] = 'False'

def _teardown_git_commit(self):
os.environ.pop('azul_git_commit')
os.environ.pop('azul_git_dirty')

@property
def _service(self):
return ManifestService(self.storage_service, self.app_module.app.file_url)

def _get_manifest(self,
format: ManifestFormat,
filters: FiltersJSON,
stream=False
) -> Response:
manifest, num_partitions = self._get_manifest_object(format, filters)
self.assertEqual(1, num_partitions)
url = furl(self._service.get_manifest_url(manifest))
response = requests.get(str(url), stream=stream)
# Moto doesn't support signed S3 URLs with Content-Disposition baked in,
# so we'll retroactively inject it into the response header.
content_disposition = url.args.get('response-content-disposition')
if content_disposition is not None:
response.headers['content-disposition'] = content_disposition
return response

def _get_manifest_object(self,
format: ManifestFormat,
filters: JSON
) -> tuple[Manifest, int]:
filters = self._filters(filters)
partition = ManifestPartition.first()
num_partitions = 1
while True:
partition = self._service.get_manifest(format=format,
catalog=self.catalog,
filters=filters,
partition=partition)
if isinstance(partition, Manifest):
return partition, num_partitions
# Emulate controller serializing the partition between steps
partition = ManifestPartition.from_json(partition.to_json())
num_partitions += 1

def _file_url(self, file_id, version):
return str(self.base_url.set(path='/repository/files/' + file_id,
args=dict(catalog=self.catalog,
Expand Down

0 comments on commit c688659

Please sign in to comment.