📚 copy from zip to zip file. resolves #307. ✨ write multiple files into a zip. fix #308"

Author: chfw

docs/README.rst

@@ -22,7 +22,9 @@ This section covers the use cases for moban. Please check them out individually.
 #. `Force template type from moban file`_
 #. `User defined template types`_
 #. `Select a group target to run`_
-
+#. `Template files in a zip or tar`_
+#. `Template copying from a zip to a zip`_
+   
 .. _Jinja2 command line: level-1-jinja2-cli
 .. _Template inheritance: level-2-template-inheritance
 .. _Data override: level-3-data-override
@@ -42,3 +44,5 @@ This section covers the use cases for moban. Please check them out individually.
 .. _Force template type from moban file: level-17-force-template-type-from-moban-file
 .. _User defined template types: level-18-user-defined-template-types
 .. _Select a group target to run: level-19-moban-a-sub-group-in-targets
+.. _Template files in a zip or tar: level-20-templates-configs-in-zip-or-tar
+.. _Template copying from a zip to a zip: level-21-copy-templates-into-an-alien-file-system

docs/index.rst

@@ -34,7 +34,8 @@ examples folder.
    level-17-force-template-type-from-moban-file/README.rst
    level-18-user-defined-template-types/README.rst
    level-19-moban-a-sub-group-in-targets/README.rst
-
+   level-20-templates-configs-in-zip-or-tar/README.rst
+   level-21-copy-templates-into-an-alien-file-system/README.rst
 
 For more complex use case, please look at `its usage in pyexcel project <http://pyexcel.readthedocs.io/en/latest/guide.html>`_
 

docs/level-21-copy-templates-into-an-alien-file-system/.moban.yml

@@ -0,0 +1,19 @@
+configuration:
+  template_dir:
+    - "zip://template-sources.zip"
+targets:
+  - output: "zip://my.zip/simple.file.copy"
+    template: file-in-template-sources-folder.txt
+    template_type: copy
+  - output: "zip://my.zip/target_without_template_type"
+    template: file_extension_will_trigger.copy
+  - "zip://my.zip/target_in_short_form": as_long_as_this_one_has.copy
+  - output: "zip://my.zip/misc-1-copying/can-create-folder/if-not-exists.txt"
+    template: file-in-template-sources-folder.txt
+    template_type: copy
+  - output: "zip://my.zip/test-dir"
+    template: dir-for-copying
+    template_type: copy
+  - output: "zip://my.zip/test-recursive-dir"
+    template: dir-for-recusive-copying/**
+    template_type: copy

docs/level-21-copy-templates-into-an-alien-file-system/README.rst

@@ -0,0 +1,83 @@
+Level 21: template copying from a zip to a zip
+================================================================================
+
+In level 15, with `.moban.yml`, you can copy templates to your destination. Now
+with similiar moban syntax, let me show how to create a new zip file where
+all templates are copied to.
+
+Explicit syntax::
+
+    targets:
+       - output: "zip://your.zip/explicit"
+         template: template_file
+         template_type: copy
+
+
+Implicit syntax::
+
+    targets:
+       - output: "zip://your.zip/implicit"
+         template: template_file.copy
+
+
+Shorthand syntax::
+
+    targets:
+       - "zip://your.zip/shorthand": template_file.copy
+
+
+No implicit nor short hand syntax for the following directory copying unless
+you take a look at `force-template-type`. When you read
+`level-17-force-template-type-from-moban-file/README.rst`, you will find
+out more.
+
+
+Directory copying syntax::
+
+ 
+    targets:
+      - output: "zip://your.zip/dest-dir"
+        template: source-dir
+        template_type: copy
+   
+
+Recursive directory copying syntax::
+
+
+    targets:
+      - output: "zip://your.zip/dest-dir"
+        template: source-dir/**
+        template_type: copy
+
+
+Evaluation
+--------------------------------------------------------------------------------        
+
+Here is example moban file for copying::
+  
+    configuration:
+      template_dir:
+        - "zip://template-sources.zip"
+    targets:
+      - output: "zip://my.zip/simple.file.copy"
+        template: file-in-template-sources-folder.txt
+        template_type: copy
+      - output: "zip://my.zip/target_without_template_type"
+        template: file_extension_will_trigger.copy
+      - "zip://my.zip/target_in_short_form": as_long_as_this_one_has.copy
+      - output: "zip://my.zip/misc-1-copying/can-create-folder/if-not-exists.txt"
+        template: file-in-template-sources-folder.txt
+        template_type: copy
+      - output: "zip://my.zip/test-dir"
+        template: dir-for-copying
+        template_type: copy
+      - output: "zip://my.zip/test-recursive-dir"
+        template: dir-for-recusive-copying/**
+        template_type: copy
+
+
+template copy does:
+
+
+#. copies any template inside pre-declared template directory to anywhere. moban will create directory if needed.
+#. copies any directory to anywhere. If "**" is followed, moban attempts to do recursive copying.

docs/level-21-copy-templates-into-an-alien-file-system/template-sources.tar

@@ -77,10 +77,19 @@ def read_unicode(path):
 @log_fs_failure
 def read_binary(path):
     path = to_unicode(path)
-    dir_name = fs.path.dirname(path)
-    the_file_name = fs.path.basename(path)
-    with fs.open_fs(dir_name) as fs_system:
-        return fs_system.readbytes(the_file_name)
+    if "zip://" in path:
+        zip_file, folder = path.split(".zip/")
+        with fs.open_fs(zip_file + ".zip") as the_fs:
+            return the_fs.readbytes(folder)
+    elif "tar://" in path:
+        tar_file, folder = path.split(".tar/")
+        with fs.open_fs(tar_file + ".tar") as the_fs:
+            return the_fs.readbytes(folder)
+    else:
+        dir_name = fs.path.dirname(path)
+        the_file_name = fs.path.basename(path)
+        with fs.open_fs(dir_name) as fs_system:
+            return fs_system.readbytes(the_file_name)
 
 
 @log_fs_failure
@@ -133,18 +142,39 @@ def is_file(path):
 @log_fs_failure
 def exists(path):
     path = to_unicode(path)
+
     if "zip://" in path:
-        try:
-            with fs.open_fs(path) as the_fs:
-                return True
-        except fs.errors.CreateFailed:
-            return False
+        if path.endswith(".zip"):
+            zip_file, folder = path, "/"
+            try:
+                with fs.open_fs(zip_file) as the_fs:
+                    return True
+            except fs.errors.CreateFailed:
+                return False
+        else:
+            zip_file, folder = path.split(".zip/")
+            try:
+                with fs.open_fs(zip_file + ".zip") as the_fs:
+                    return the_fs.exists(folder)
+            except fs.errors.CreateFailed:
+                return False
+
     if "tar://" in path:
-        try:
-            with fs.open_fs(path) as the_fs:
-                return True
-        except fs.errors.CreateFailed:
-            return False
+        if path.endswith(".tar"):
+            zip_file, folder = path, "/"
+            try:
+                with fs.open_fs(zip_file) as the_fs:
+                    return True
+            except fs.errors.CreateFailed:
+                return False
+        else:
+            zip_file, folder = path.split(".tar/")
+            try:
+                with fs.open_fs(zip_file + ".tar") as the_fs:
+                    return the_fs.exists(folder)
+            except fs.errors.CreateFailed:
+                return False
+
     dir_name = fs.path.dirname(path)
     the_file_name = fs.path.basename(path)
 

docs/level-21-copy-templates-into-an-alien-file-system/template-sources.zip

@@ -109,6 +109,7 @@ def _iterate_list_of_dicts(list_of_dict):
 
 def handle_targets(merged_options, targets):
     list_of_templating_parameters = parse_targets(merged_options, targets)
+
     jobs_for_each_engine = OrderedDict()
 
     for target in list_of_templating_parameters:

moban/file_system.py

@@ -40,6 +40,7 @@ def parse_targets(options, targets):
 
 
 def _handle_explicit_target(options, target):
+
     common_data_file = options[constants.LABEL_CONFIG]
     default_template_type = options[constants.LABEL_TEMPLATE_TYPE]
     template_file = target.get(

moban/mobanfile/__init__.py

@@ -8,14 +8,21 @@
 
 def handle_template(template_file, output, template_dirs):
     log.info("handling %s" % template_file)
+
     if template_file.endswith("**"):
         source_dir = template_file[:-3]
         src_path = find_file_in_template_dirs(source_dir, template_dirs)
         if src_path:
-            for a_triple in _listing_directory_files_recusively(
-                source_dir, src_path, output
-            ):
-                yield a_triple
+            if "zip://" in src_path:
+                for a_triple in _listing_zip_directory_files_recusively(
+                    source_dir, src_path, output
+                ):
+                    yield a_triple
+            else:
+                for a_triple in _listing_directory_files_recusively(
+                    source_dir, src_path, output
+                ):
+                    yield a_triple
         else:
             reporter.report_error_message(
                 "{0} cannot be found".format(template_file)
@@ -30,10 +37,16 @@ def handle_template(template_file, output, template_dirs):
                 "{0} cannot be found".format(template_file)
             )
         elif file_system.is_dir(template_file_on_disk):
-            for a_triple in _list_dir_files(
-                template_file, template_file_on_disk, output
-            ):
-                yield a_triple
+            if "zip://" in template_file_on_disk:
+                for a_triple in _list_zip_dir_files(
+                    template_file, template_file_on_disk, output
+                ):
+                    yield a_triple
+            else:
+                for a_triple in _list_dir_files(
+                    template_file, template_file_on_disk, output
+                ):
+                    yield a_triple
         else:
             template_type = _get_template_type(template_file)
             yield (template_file, output, template_type)
@@ -54,6 +67,24 @@ def _list_dir_files(source, actual_source_path, dest):
                 yield (src_file_under_dir, dest_file_under_dir, template_type)
 
 
+def _list_zip_dir_files(source, actual_source_path, dest):
+    zip_file, folder = actual_source_path.split(".zip/")
+    with file_system.open_fs(zip_file + ".zip") as fs_handle:
+        for file_name in fs_handle.listdir(file_system.to_unicode(folder)):
+            if fs_handle.isfile(
+                file_system.to_unicode(folder + "/" + file_name)
+            ):
+                # please note jinja2 does NOT like windows path
+                # hence the following statement looks like cross platform
+                #  src_file_under_dir = os.path.join(source, file_name)
+                # but actually it breaks windows instead.
+                src_file_under_dir = "%s/%s" % (source, file_name)
+
+                dest_file_under_dir = dest + "/" + file_name
+                template_type = _get_template_type(src_file_under_dir)
+                yield (src_file_under_dir, dest_file_under_dir, template_type)
+
+
 def _listing_directory_files_recusively(source, actual_source_path, dest):
     with file_system.open_fs(actual_source_path) as fs_handle:
         for file_name in fs_handle.listdir(u"."):
@@ -70,6 +101,29 @@ def _listing_directory_files_recusively(source, actual_source_path, dest):
                     yield a_triple
 
 
+def _listing_zip_directory_files_recusively(source, actual_source_path, dest):
+    zip_file, folder = actual_source_path.split(".zip/")
+    print(actual_source_path)
+
+    with file_system.open_fs(zip_file + ".zip") as fs_handle:
+        for file_name in fs_handle.listdir(file_system.to_unicode(folder)):
+            src_file_under_dir = source + "/" + file_name
+            dest_file_under_dir = dest + "/" + file_name
+            real_src_file = "%s/%s" % (actual_source_path, file_name)
+            if fs_handle.isfile(
+                file_system.to_unicode(folder + "/" + file_name)
+            ):
+                template_type = _get_template_type(src_file_under_dir)
+                yield (src_file_under_dir, dest_file_under_dir, template_type)
+            elif fs_handle.isdir(
+                file_system.to_unicode(folder + "/" + file_name)
+            ):
+                for a_triple in _listing_zip_directory_files_recusively(
+                    src_file_under_dir, real_src_file, dest_file_under_dir
+                ):
+                    yield a_triple
+
+
 def _get_template_type(template_file):
     _, extension = file_system.path_splitext(template_file)
     if extension:

moban/mobanfile/targets.py

@@ -6,6 +6,7 @@
 from lml.plugin import PluginManager
 from moban.hashstore import HASH_STORE
 from moban.deprecated import deprecated
+from moban.buffered_writer import BufferedWriter
 from moban.plugins.context import Context
 from moban.plugins.library import LIBRARIES
 from moban.plugins.strategy import Strategy
@@ -68,6 +69,7 @@ def __init__(self, template_dirs, context_dirs, engine):
         self.engine = engine
         self.templated_count = 0
         self.file_count = 0
+        self.buffered_writer = BufferedWriter()
 
     def report(self):
         if self.templated_count == 0:
@@ -96,6 +98,7 @@ def render_to_file(self, template_file, data_file, output_file):
         if flag:
             reporter.report_templating(template_file, output_file)
             self.templated_count += 1
+        self.buffered_writer.close()
 
     def render_string_to_file(
         self, template_in_string, data_file, output_file
@@ -110,6 +113,7 @@ def render_string_to_file(
         if flag:
             reporter.report_templating(template_abs_path, output_file)
             self.templated_count += 1
+        self.buffered_writer.close()
 
     def apply_template(self, template_abs_path, template, data, output_file):
         rendered_content = self.engine.apply_template(
@@ -124,21 +128,26 @@ def apply_template(self, template_abs_path, template, data, output_file):
                 output_file, rendered_content, template_abs_path
             )
             if flag:
-                utils.write_file_out(output_file, rendered_content)
-                utils.file_permissions_copy(template_abs_path, output_file)
+                self.buffered_writer.write_file_out(
+                    output_file, rendered_content
+                )
+                if "zip://" not in output_file and "tar://" not in output_file:
+                    utils.file_permissions_copy(template_abs_path, output_file)
             return flag
         except exceptions.FileNotFound:
-            utils.write_file_out(output_file, rendered_content)
+            self.buffered_writer.write_file_out(output_file, rendered_content)
             return True
 
     def render_to_files(self, array_of_template_targets):
         sta = Strategy(array_of_template_targets)
+
         sta.process()
         choice = sta.what_to_do()
         if choice == Strategy.DATA_FIRST:
             self._render_with_finding_data_first(sta.data_file_index)
         else:
             self._render_with_finding_template_first(sta.template_file_index)
+        self.buffered_writer.close()
 
     def _render_with_finding_template_first(self, template_file_index):
         for (template_file, data_output_pairs) in template_file_index.items():