feat: test of adding Gitlab provider

snakedeploy/deploy.py

@@ -1,4 +1,3 @@
-import subprocess as sp
 import tempfile
 from pathlib import Path
 import os
@@ -11,43 +10,28 @@
 from snakedeploy.exceptions import UserError
 
 
-def deploy(
-    source_url: str, name: str, tag: str, branch: str, dest_path: Path, force=False
-):
-    """Deploy a given workflow to the local machine, using the Snakemake module system."""
-    provider = get_provider(source_url)
-    env = Environment(loader=PackageLoader("snakedeploy"))
-    template = env.get_template("use_module.smk.jinja")
-
-    snakefile = dest_path / "workflow/Snakefile"
-    if snakefile.exists() and not force:
-        raise UserError(
-            f"{snakefile} already exists, aborting (use --force to overwrite)"
-        )
-    dest_config = dest_path / "config"
-    if dest_config.exists() and not force:
-        raise UserError(
-            f"{dest_config} already exists, aborting (use --force to overwrite)"
-        )
+class WorkflowDeployer:
+    def __init__(self, source: str, dest: Path, force=False):
+        self.provider = get_provider(source)
+        self.env = Environment(loader=PackageLoader("snakedeploy"))
+        self.dest_path = dest
+        self.force = force
+
+    @property
+    def snakefile(self):
+        return self.dest_path / "workflow/Snakefile"
+
+    @property
+    def config(self):
+        return self.dest_path / "config"
+
+    def deploy_config(self, tmpdir: str):
+        """
+        Deploy the config directory, either using an existing or creating a dummy.
 
-    logger.info("Writing Snakefile with module definition...")
-    os.makedirs(dest_path / "workflow", exist_ok=True)
-    module_deployment = template.render(
-        name=name or provider.get_repo_name().replace("-", "_"),
-        snakefile=provider.get_source_file_declaration(
-            "workflow/Snakefile", tag, branch
-        ),
-        repo=source_url,
-    )
-    with open(snakefile, "w") as f:
-        print(module_deployment, file=f)
-
-    with tempfile.TemporaryDirectory() as tmpdir:
-        logger.info("Obtaining source repository...")
-        try:
-            sp.run(["git", "clone", source_url, "."], cwd=tmpdir, check=True)
-        except sp.CalledProcessError as e:
-            raise UserError("Failed to clone repository {}:\n{}", source_url, e)
+        returns a boolean "no_config" to indicate if there is not a config (True)
+        """
+        # Handle the config/
         config_dir = Path(tmpdir) / "config"
         no_config = not config_dir.exists()
         if no_config:
@@ -56,17 +40,98 @@ def deploy(
                 "Please check whether the source repository really does not "
                 "need or provide any configuration."
             )
-            os.makedirs(dest_path / "config", exist_ok=True)
-            dummy_config_file = dest_path / "config" / "config.yaml"
+
+            # This will fail running the workflow if left empty
+            os.makedirs(self.dest_path / "config", exist_ok=True)
+            dummy_config_file = self.dest_path / "config" / "config.yaml"
             if not dummy_config_file.exists():
                 with open(dummy_config_file, "w"):
                     pass
         else:
             logger.info("Writing template configuration...")
-            shutil.copytree(config_dir, dest_config, dirs_exist_ok=force)
+            shutil.copytree(config_dir, self.config, dirs_exist_ok=self.force)
+        return no_config
+
+    def deploy(self, name: str, tag: str, branch: str):
+        """
+        Deploy a source to a destination.
+        """
+        self.check()
+
+        # Create a temporary directory to grab config directory and snakefile
+        with tempfile.TemporaryDirectory() as tmpdir:
+            logger.info("Obtaining source repository...")
+
+            # Clone temporary directory to find assets
+            self.provider.clone(tmpdir)
 
-    logger.info(
-        env.get_template("post-instructions.txt.jinja").render(
-            no_config=no_config, dest_path=dest_path
+            # Either copy existing config or create a dummy config
+            no_config = self.deploy_config(tmpdir)
+
+            # Inspect repository to find existing snakefile
+            self.deploy_snakefile(tmpdir, name, tag, branch)
+
+        logger.info(
+            self.env.get_template("post-instructions.txt.jinja").render(
+                no_config=no_config, dest_path=self.dest_path
+            )
         )
-    )
+
+    def check(self):
+        """
+        Check to ensure we haven't already deployed to the destination.
+        """
+        if self.snakefile.exists() and not self.force:
+            raise UserError(
+                f"{self.snakefile} already exists, aborting (use --force to overwrite)"
+            )
+
+        if self.config.exists() and not self.force:
+            raise UserError(
+                f"{self.config} already exists, aborting (use --force to overwrite)"
+            )
+
+    def deploy_snakefile(self, tmpdir: str, name: str, tag: str, branch: str):
+        """
+        Deploy the Snakefile to workflow/Snakefile
+        """
+        # The name cannot have -
+        name = name or self.provider.get_repo_name()
+        name = name.replace("-", "_")
+
+        snakefile_path = Path(tmpdir) / "workflow" / "Snakefile"
+        snakefile = os.path.join("workflow", "Snakefile")
+        if not snakefile_path.exists():
+
+            # Either we allow this or fail workflow here if it's not possible
+            logger.warning(
+                "Snakefile path not found in traditional path %s, workflow may be error prone."
+                % snakefile_path
+            )
+            snakefile_path = Path(tmpdir) / "Snakefile"
+            snakefile = "Snakefile"
+            if not snakefile_path.exists():
+                raise UserError(
+                    "No Snakefile was found at root or in workflow directory."
+                )
+
+        template = self.env.get_template("use_module.smk.jinja")
+        logger.info("Writing Snakefile with module definition...")
+        os.makedirs(self.dest_path / "workflow", exist_ok=True)
+        module_deployment = template.render(
+            name=name,
+            snakefile=self.provider.get_source_file_declaration(snakefile, tag, branch),
+            repo=self.provider.source_url,
+        )
+        with open(self.snakefile, "w") as f:
+            print(module_deployment, file=f)
+
+
+def deploy(
+    source_url: str, name: str, tag: str, branch: str, dest_path: Path, force=False
+):
+    """
+    Deploy a given workflow to the local machine, using the Snakemake module system.
+    """
+    sd = WorkflowDeployer(source=source_url, dest=dest_path, force=force)
+    sd.deploy(name=name, tag=tag, branch=branch)

snakedeploy/providers.py

@@ -1,19 +1,27 @@
 from abc import abstractmethod, ABC
-
+from distutils.dir_util import copy_tree
 from snakedeploy.exceptions import UserError
+import subprocess as sp
+import os
 
 
 def get_provider(source_url):
     for provider in PROVIDERS:
         if provider.matches(source_url):
             return provider(source_url)
 
+    raise UserError("No matching provider for source url %s" % source_url)
+
 
 class Provider(ABC):
     def __init__(self, source_url):
-        if not (source_url.startswith("https://") or source_url.startswith("file:")):
+        if not (
+            source_url.startswith("https://")
+            or source_url.startswith("file:")
+            or os.path.exists(source_url)
+        ):
             raise UserError(
-                "Repository source URLs must be given as https:// or file://."
+                "Repository source URLs must be given as https:// or file://, or exist."
             )
         # TODO replace with removesuffix once Python 3.9 becomes the minimal version of snakedeploy
         if source_url.endswith(".git"):
@@ -25,6 +33,10 @@ def __init__(self, source_url):
     def matches(cls, source_url: str):
         pass
 
+    @abstractmethod
+    def clone(self, path: str):
+        pass
+
     @abstractmethod
     def get_raw_file(self, path: str, tag: str):
         pass
@@ -33,10 +45,46 @@ def get_repo_name(self):
         return self.source_url.split("/")[-1]
 
 
+class Local(Provider):
+    @classmethod
+    def matches(cls, source_url: str):
+        return os.path.exists(source_url)
+
+    def clone(self, tmpdir: str):
+        """
+        A local "clone" means moving files.
+        """
+        copy_tree(self.source_url, tmpdir)
+
+    def get_raw_file(self, path: str, tag: str):
+        if tag:
+            print(
+                "Warning: tag is not supported for a local repository - check out the branch you need."
+            )
+        return f"{self.source_url}/{path}"
+
+    def get_source_file_declaration(self, path: str, tag: str, branch: str):
+        relative_path = path.replace(self.source_url, "").strip(os.sep)
+        return f'"{relative_path}"'
+
+
 class Github(Provider):
     @classmethod
     def matches(cls, source_url: str):
-        return source_url.startswith("https://github.com")
+        return cls.__name__.lower() in source_url
+
+    @property
+    def name(self):
+        return self.__class__.__name__.lower()
+
+    def clone(self, tmpdir: str):
+        """
+        Clone the known source URL to a temporary directory
+        """
+        try:
+            sp.run(["git", "clone", self.source_url, "."], cwd=tmpdir, check=True)
+        except sp.CalledProcessError as e:
+            raise UserError("Failed to clone repository {}:\n{}", self.source_url, e)
 
     def get_raw_file(self, path: str, tag: str):
         return f"{self.source_url}/raw/{tag}/{path}"
@@ -45,8 +93,14 @@ def get_source_file_declaration(self, path: str, tag: str, branch: str):
         owner_repo = "/".join(self.source_url.split("/")[-2:])
         if not (tag or branch):
             raise UserError("Either tag or branch has to be specified for deployment.")
-        ref_arg = f'tag="{tag}"' if tag is not None else f"branch={branch}"
-        return f'github("{owner_repo}", path="{path}", {ref_arg})'
+        ref_arg = f'tag="{tag}"' if tag is not None else f'branch="{branch}"'
+        return f'{self.name}("{owner_repo}", path="{path}", {ref_arg})'
+
+
+class Gitlab(Github):
+    @classmethod
+    def get_raw_file(self, path: str, tag: str):
+        return f"{self.source_url}/-/raw/{tag}/{path}"
 
 
-PROVIDERS = [Github]
+PROVIDERS = [Github, Gitlab, Local]

tests/test_client.sh

@@ -29,18 +29,31 @@ echo "#### Testing snakedeploy --help"
 runTest 0 $output snakedeploy --help
 
 echo
-echo "#### Testing snakedeploy deployment"
+echo "#### Testing snakedeploy GitHub deployment"
 runTest 0 $output snakedeploy deploy-workflow "${repo}" "${dest}" --tag v1.0.0 --name dna-seq
 
 echo
-echo "#### Testing snakedeploy directory exists"
+echo "#### Testing snakedeploy workflow directory exists"
 runTest 1 $output snakedeploy deploy-workflow "${repo}" "${dest}" --tag v1.0.0
 
 echo
 echo "#### Testing snakedeploy directory exists but enforcing"
 runTest 0 $output snakedeploy deploy-workflow "${repo}" "${dest}" --tag v1.0.0 --force
 
 echo
+echo "#### Testing snakedeploy GitLab deployment"
+dest=$tmpdir/gitlab-testing
+repo="https://gitlab.com/nate-d-olson/snaketestworkflow"
+runTest 0 $output snakedeploy deploy-workflow "${repo}" "${dest}" --name snake-test --branch master
+
+echo
+echo "#### Testing snakedeploy local deployment"
+local=$tmpdir/rna-seq-star-deseq2
+repo="https://github.com/snakemake-workflows/rna-seq-star-deseq2"
+dest=${tmpdir}/use-workflow-as-module
+git clone ${repo} ${local}
+runTest 0 $output snakedeploy deploy-workflow "${local}" ${dest} --tag v1.2.0
+
 echo "#### Testing snakedeply update-conda-envs"
 cp tests/test-env.yaml $tmpdir
 runTest 0 $output snakedeploy update-conda-envs --conda-frontend conda $tmpdir/test-env.yaml