#277: Added option to export slc to uncompressed tar (#278)

fixes #277

Author: tomuben

.github/workflows/checks.yml

@@ -47,6 +47,9 @@ jobs:
           python-version: ${{ matrix.python-version }}
           poetry-version: 2.1.2
 
+      - name: Run Unit tests
+        run: poetry run nox -s test:unit
+
       - name: Run lint
         run: poetry run nox -s lint:code
 

exasol/slc/api/deploy.py

@@ -23,6 +23,10 @@
 
 from exasol.slc.internal.tasks.upload.deploy_containers import DeployContainers
 from exasol.slc.internal.tasks.upload.deploy_info import toDeployResult
+from exasol.slc.models.compression_strategy import (
+    CompressionStrategy,
+    defaultCompressionStrategy,
+)
 from exasol.slc.models.deploy_result import DeployResult
 
 
@@ -61,6 +65,7 @@ def deploy(
     task_dependencies_dot_file: Optional[str] = None,
     log_level: Optional[str] = None,
     use_job_specific_log_file: bool = True,
+    compression_strategy: CompressionStrategy = defaultCompressionStrategy(),
 ) -> Dict[str, Dict[str, DeployResult]]:
     """
     This command uploads the whole script-language-container package of the flavor to the database.
@@ -118,6 +123,7 @@ def root_task_generator() -> DependencyLoggerBaseTask:
             bucketfs_name=bucketfs_name,
             ssl_cert_path=ssl_cert_path,
             use_ssl_cert_validation=use_ssl_cert_validation,
+            compression_strategy=compression_strategy,
         )
 
     deploy_infos = run_task(

exasol/slc/api/export.py

@@ -21,6 +21,10 @@
 )
 
 from exasol.slc.internal.tasks.export.export_containers import ExportContainers
+from exasol.slc.models.compression_strategy import (
+    CompressionStrategy,
+    defaultCompressionStrategy,
+)
 from exasol.slc.models.export_container_result import ExportContainerResult
 
 
@@ -51,6 +55,7 @@ def export(
     log_level: Optional[str] = None,
     use_job_specific_log_file: bool = True,
     cleanup_docker_images: bool = False,
+    compression_strategy: CompressionStrategy = defaultCompressionStrategy(),
 ) -> ExportContainerResult:
     """
     This command exports the whole script-language-container package of the flavor,
@@ -93,6 +98,7 @@ def root_task_generator() -> DependencyLoggerBaseTask:
             export_path=export_path,
             release_name=release_name,
             cleanup_docker_images=cleanup_docker_images,
+            compression_strategy=compression_strategy,
         )
 
     return run_task(

exasol/slc/api/run_db_tests.py

@@ -35,6 +35,10 @@
 from exasol.slc.internal.tasks.test.test_container_content import (
     build_test_container_content,
 )
+from exasol.slc.models.compression_strategy import (
+    CompressionStrategy,
+    defaultCompressionStrategy,
+)
 from exasol.slc.models.test_result import AllTestsResult
 
 
@@ -97,6 +101,7 @@ def run_db_test(
     task_dependencies_dot_file: Optional[str] = None,
     log_level: Optional[str] = None,
     use_job_specific_log_file: bool = True,
+    compression_strategy: CompressionStrategy = defaultCompressionStrategy(),
 ) -> AllTestsResult:
     """
     This command runs the integration tests in local docker-db.
@@ -146,6 +151,8 @@ def run_db_test(
             raise api_errors.MissingArgumentError("external_exasol_db_port")
         if external_exasol_bucketfs_port is None:
             raise api_errors.MissingArgumentError("external_exasol_bucketfs_port")
+        if external_exasol_ssh_port is None:
+            raise api_errors.MissingArgumentError("external_exasol_ssh_port")
 
     def root_task_generator() -> DependencyLoggerBaseTask:
         return generate_root_task(
@@ -190,6 +197,7 @@ def root_task_generator() -> DependencyLoggerBaseTask:
             create_certificates=create_certificates,
             additional_db_parameter=additional_db_parameter,
             test_container_content=build_test_container_content(test_container_folder),
+            compression_strategy=compression_strategy,
         )
 
     return run_task(

exasol/slc/api/upload.py

@@ -24,6 +24,10 @@
 )
 
 from exasol.slc.internal.tasks.upload.upload_containers import UploadContainers
+from exasol.slc.models.compression_strategy import (
+    CompressionStrategy,
+    defaultCompressionStrategy,
+)
 
 
 @cli_function
@@ -61,6 +65,7 @@ def upload(
     use_job_specific_log_file: bool = True,
     ssl_cert_path: str = "",
     use_ssl_cert_validation: bool = True,
+    compression_strategy: CompressionStrategy = defaultCompressionStrategy(),
 ) -> luigi.LocalTarget:
     warnings.warn(
         "The 'upload' function is deprecated, use 'deploy' instead", DeprecationWarning
@@ -113,6 +118,7 @@ def root_task_generator() -> DependencyLoggerBaseTask:
             bucketfs_name=bucketfs_name,
             ssl_cert_path=ssl_cert_path,
             use_ssl_cert_validation=use_ssl_cert_validation,
+            compression_strategy=compression_strategy,
         )
 
     return run_task(

exasol/slc/internal/tasks/export/export_container_base_task.py

@@ -26,6 +26,7 @@
 from exasol.slc.internal.tasks.export.remove_cached_export_file_task import (
     RemoveCachedExportTask,
 )
+from exasol.slc.models.compression_strategy import CompressionStrategy
 from exasol.slc.models.export_info import ExportInfo
 
 
@@ -112,6 +113,16 @@ def _create_export_info(
         )
         return export_info
 
+    def _get_export_file_extension(self) -> str:
+        if self.compression_strategy == CompressionStrategy.GZIP:
+            return ".tar.gz"
+        elif self.compression_strategy == CompressionStrategy.NONE:
+            return ".tar"
+        else:
+            raise ValueError(
+                f"Unsupported compression_strategy: {self.compression_strategy}"
+            )
+
     def _get_cache_file_path(
         self,
         image_info_of_release_image: ImageInfo,
@@ -120,5 +131,7 @@ def _get_cache_file_path(
         release_image_name = image_info_of_release_image.get_target_complete_name()
         export_path = Path(export_directory_future.get_output()).absolute()
         release_complete_name = f"""{image_info_of_release_image.target_tag}-{image_info_of_release_image.hash}"""
-        cache_file = Path(export_path, release_complete_name + ".tar.gz").absolute()
+        cache_file = Path(
+            export_path, release_complete_name + self._get_export_file_extension()
+        ).absolute()
         return str(cache_file), release_complete_name, release_image_name

exasol/slc/internal/tasks/export/export_container_parameters.py

@@ -1,12 +1,20 @@
 from typing import Optional, Tuple
 
 import luigi
-from luigi import Config
+
+from exasol.slc.models.compression_strategy import (
+    CompressionStrategy,
+    defaultCompressionStrategy,
+)
 
 CHECKSUM_ALGORITHM = "sha512sum"
 
 
-class ExportContainerParameterBase(Config):
+class ExportContainerOptionsParameter:
+    compression_strategy: CompressionStrategy = luigi.EnumParameter(enum=CompressionStrategy, default=defaultCompressionStrategy())  # type: ignore
+
+
+class ExportContainerParameterBase(ExportContainerOptionsParameter):
     export_path: Optional[str] = luigi.OptionalParameter(None)  # type: ignore
     release_name: Optional[str] = luigi.OptionalParameter(None)  # type: ignore
     cleanup_docker_images: bool = luigi.BoolParameter(False)  # type: ignore

exasol/slc/internal/tasks/export/export_container_to_cache_task.py

@@ -36,6 +36,7 @@
     CHECKSUM_ALGORITHM,
     ExportContainerParameter,
 )
+from exasol.slc.models.compression_strategy import CompressionStrategy
 
 
 class CheckCacheFileTask(DependencyLoggerBaseTask):
@@ -170,11 +171,17 @@ def _pack_release_file(
     ) -> None:
         self.logger.info("Pack container file %s", release_file)
         extract_content = " ".join(f"'{file}'" for file in os.listdir(extract_dir))
-        if not str(release_file).endswith("tar.gz"):
+        if str(release_file).endswith("tar.gz"):
+            tmp_release_file = release_file.with_suffix(
+                ""
+            )  # cut off ".gz" from ".tar.gz"
+        elif str(release_file).endswith("tar"):
+            tmp_release_file = release_file
+        else:
             raise ValueError(
-                f"Unexpected release file: '{release_file}'. Expected suffix 'tar.gz'."
+                f"Unexpected release file: '{release_file}'. Expected suffix: 'tar.gz' or 'tar'."
             )
-        tmp_release_file = release_file.with_suffix("")  # cut off ".gz" from ".tar.gz"
+
         command = (
             f"""tar -C '{extract_dir}' -vcf '{tmp_release_file}' {extract_content}"""
         )
@@ -195,12 +202,19 @@ def _pack_release_file(
             log_path.joinpath("pack_release_file.log"),
         )
         shutil.rmtree(extract_dir)
-        command = f"""gzip {tmp_release_file}"""
-        self.run_command(
-            command,
-            f"Creating '{release_file}'",
-            log_path.joinpath("pack_release_file.log"),
-        )
+        if self.compression_strategy == CompressionStrategy.GZIP:
+            command = f"""gzip {tmp_release_file}"""
+            self.run_command(
+                command,
+                f"Creating '{release_file}'",
+                log_path.joinpath("pack_release_file.log"),
+            )
+        elif self.compression_strategy == CompressionStrategy.NONE:
+            pass
+        else:
+            raise ValueError(
+                f"Unexpected compression_strategy: {self.compression_strategy}"
+            )
 
     @staticmethod
     def _modify_extracted_container(extract_dir: str) -> None:

exasol/slc/internal/tasks/export/export_container_to_file_task.py

@@ -20,6 +20,7 @@
 from exasol.slc.internal.tasks.export.export_container_to_cache_task import (
     ExportContainerToCacheTask,
 )
+from exasol.slc.internal.utils.file_utilities import detect_container_file_extension
 
 
 class ExportContainerToFileInfo(Info):
@@ -66,9 +67,8 @@ def _copy_cache_file_to_output_path(
                 suffix = f"""_{self.release_name}"""
             else:
                 suffix = ""
-            file_name = (
-                f"""{self.get_flavor_name()}_{self.release_goal}{suffix}.tar.gz"""
-            )
+            file_extension = detect_container_file_extension(cache_file.name)
+            file_name = f"""{self.get_flavor_name()}_{self.release_goal}{suffix}{file_extension}"""
             output_file = Path(str(self.export_path), file_name)
             output_checksum_file = Path(
                 str(self.export_path), file_name + "." + CHECKSUM_ALGORITHM

exasol/slc/internal/tasks/test/test_container.py

@@ -13,6 +13,9 @@
     SpawnTestEnvironmentParameter,
 )
 
+from exasol.slc.internal.tasks.export.export_container_parameters import (
+    ExportContainerOptionsParameter,
+)
 from exasol.slc.internal.tasks.test.run_db_tests_parameter import (
     GeneralRunDBTestParameter,
     RunDBTestsInTestConfigParameter,
@@ -30,7 +33,9 @@
 
 
 class TestContainerParameter(
-    RunDBTestsInTestConfigParameter, GeneralRunDBTestParameter
+    RunDBTestsInTestConfigParameter,
+    GeneralRunDBTestParameter,
+    ExportContainerOptionsParameter,
 ):
     release_goals: Tuple[str, ...] = luigi.ListParameter(["release"])  # type: ignore
     languages: Tuple[Optional[str], ...] = luigi.ListParameter([None])  # type: ignore