Port project to Python3.9

.gitignore

@@ -7,3 +7,6 @@ docs/build/
 #compiled output
 *.pyc
 
+.pytest_cache/
+
+.venv/

assetman/S3UploadThread.py

@@ -1,12 +1,12 @@
 #!/bin/python
-from __future__ import with_statement
+
 import re
 import os
 import os.path
 import sys
 import threading
 import datetime
-import Queue
+import queue as Queue
 import mimetypes
 import logging
 import boto3
@@ -42,7 +42,7 @@ def run(self):
             file_name, file_path = self.queue.get()
             try:
                 self.start_upload_file(file_name, file_path)
-            except Exception, e:
+            except Exception as e:
                 logging.error('Error uploading %s: %s', file_name, e)
                 self.errors.append((sys.exc_info(), self))
             finally:
@@ -52,8 +52,8 @@ def start_upload_file(self, file_name, file_path):
         """Starts the procecss of uploading a file to S3. Each file will be
         uploaded twice (once for CDN and once for our local CDN proxy).
         """
-        assert isinstance(file_name, (str, unicode))
-        assert isinstance(file_path, (str, unicode))
+        assert isinstance(file_name, str)
+        assert isinstance(file_path, str)
         assert os.path.isfile(file_path)
 
         content_type, content_encoding = mimetypes.guess_type(file_name)
@@ -88,7 +88,7 @@ def exists(self, obj):
         # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.head_object
         try:
             self.client.head_object(Bucket=obj.bucket_name, Key=obj.key)
-        except Exception, e:
+        except Exception as e:
             logging.error('got %s', e)
             return False
         return True
@@ -143,7 +143,7 @@ def upload_assets_to_s3(manifest, settings, skip_s3_upload=False):
 
     # We know we want to upload each asset block (these correspond to the
     # assetman.include_* blocks in each template)
-    for depspec in manifest.blocks.itervalues():
+    for depspec in manifest.blocks.values():
         file_name = depspec['versioned_path']
         file_path = make_output_path(settings['compiled_asset_root'], file_name)
         assert os.path.isfile(file_path), 'Missing compiled asset %s' % file_path
@@ -154,7 +154,7 @@ def upload_assets_to_s3(manifest, settings, skip_s3_upload=False):
     # but we'll need to filter out other entries in the complete 'assets'
     # block of the manifest.
     should_skip = re.compile(r'\.(scss|less|css|js|html)$', re.I).search
-    for rel_path, depspec in manifest.assets.iteritems():
+    for rel_path, depspec in manifest.assets.items():
         if should_skip(rel_path):
             continue
         file_path = make_absolute_static_path(settings['static_dir'], rel_path)
@@ -170,11 +170,11 @@ def upload_assets_to_s3(manifest, settings, skip_s3_upload=False):
     # Upload assets to S3 using 5 threads
     queue = Queue.Queue()
     errors = []
-    for i in xrange(5):
+    for i in range(5):
         uploader = S3UploadThread(queue, errors, manifest, settings)
         uploader.setDaemon(True)
         uploader.start()
-    map(queue.put, to_upload)
+    list(map(queue.put, to_upload))
     queue.join()
     if errors:
         raise Exception(errors)

assetman/compile.py

@@ -1,6 +1,6 @@
 #!/bin/python
 
-from __future__ import with_statement
+
 
 import os
 import re
@@ -333,7 +333,7 @@ def build_manifest(tornado_paths, django_paths, settings):
     for compiler in compilers:
         new_paths = compiler.get_paths()
         if settings.get('verbose'):
-            print compiler, new_paths
+            print(compiler, new_paths)
         paths.update(new_paths)
     paths = list(paths)
 
@@ -405,12 +405,12 @@ def run(settings):
     cached_manifest = Manifest(settings).load()
     try:
         current_manifest, compilers = build_manifest(tornado_paths, django_paths, settings)
-    except ParseError, e:
+    except ParseError as e:
         src_path, msg = e.args
         logging.error('Error parsing template %s', src_path)
         logging.error(msg)
         raise Exception
-    except DependencyError, e:
+    except DependencyError as e:
         src_path, missing_deps = e.args
         logging.error('Dependency error in source %s!', src_path)
         logging.error('Missing paths: %s', missing_deps)
@@ -421,7 +421,7 @@ def run(settings):
     # compiler's source path figures into the dependency tracking. But we only
     # need to actually compile each block once.
     logging.debug('Found %d assetman block compilers', len(compilers))
-    compilers = dict((c.get_hash(), c) for c in compilers).values()
+    compilers = list(dict((c.get_hash(), c) for c in compilers).values())
     logging.debug('%d unique assetman block compilers', len(compilers))
 
     # update the manifest on each our compilers to reflect the new manifest,
@@ -436,7 +436,7 @@ def needs_compile(compiler):
     if settings['force_recompile']:
         to_compile = compilers
     else:
-        to_compile = filter(needs_compile, compilers)
+        to_compile = list(filter(needs_compile, compilers))
 
     if to_compile or cached_manifest.needs_recompile(current_manifest):
         # If we're only testing whether a compile is needed, we're done
@@ -448,7 +448,7 @@ def needs_compile(compiler):
             # See note above about bug in pool.map w/r/t KeyboardInterrupt.
             _compile_worker = CompileWorker(settings.get('skip_inline_images', False), current_manifest)
             pool.map_async(_compile_worker, to_compile).get(1e100)
-        except CompileError, e:
+        except CompileError as e:
             cmd, msg = e.args
             logging.error('Compile error!')
             logging.error('Command: %s', ' '.join(cmd))

assetman/compilers.py

@@ -1,4 +1,4 @@
-from __future__ import absolute_import, with_statement
+
 
 import base64
 from collections import defaultdict
@@ -114,7 +114,7 @@ def get_current_content_hash(self, manifest):
 
     def get_paths(self):
         """Returns a list of absolute paths to the assets contained in this manager."""
-        paths = map(functools.partial(make_absolute_static_path, self.settings['static_dir']), self.rel_urls)
+        paths = list(map(functools.partial(make_absolute_static_path, self.settings['static_dir']), self.rel_urls))
         try:
             assert all(map(os.path.isfile, paths))
         except AssertionError:
@@ -222,7 +222,7 @@ def replacer(match):
 
         result = re.sub(pattern, replacer, css_src)
 
-        for url, count in seen_assets.iteritems():
+        for url, count in seen_assets.items():
             if count > 1:
                 logging.warn('inline asset duplicated %dx: %s', count, url)
 

assetman/managers.py

@@ -1,4 +1,4 @@
-from __future__ import absolute_import, with_statement
+
 
 import os
 import logging
@@ -46,7 +46,7 @@ def __init__(self, rel_url_text, local=False, include_tag=True, src_path=None, s
         Any extra kwargs will be interpreted as extra HTML params to include
         on the rendered element.
         """
-        self.rel_urls = filter(None, _utf8(rel_url_text).split())
+        self.rel_urls = [_f for _f in _utf8(rel_url_text).split() if _f]
         self.local = local
         self.include_tag = include_tag
         self.src_path = src_path
@@ -101,7 +101,7 @@ def render_attrs(self):
         leading space.
         """
         attrs = ' '.join('%s=%r' % (attr, _utf8(val))
-                         for attr, val in self.attrs.iteritems())
+                         for attr, val in self.attrs.items())
         return ' ' + attrs if attrs else ''
 
     def render_asset(self, url):
@@ -123,7 +123,7 @@ def render(self):
         """
         try:
             if self.settings['enable_static_compilation']:
-                urls = map(self.make_asset_url, self.rel_urls)
+                urls = list(map(self.make_asset_url, self.rel_urls))
                 return '\n'.join(map(self.render_asset, urls))
             else:
                 compiled_name = self.get_compiled_name()

assetman/manifest.py

@@ -1,4 +1,4 @@
-from __future__ import absolute_import, with_statement
+
 
 import os
 import simplejson as json
@@ -51,7 +51,7 @@ def load(self, compiled_asset_path=None):
             self._manifest = json.load(open(filename))
             assert isinstance(self.assets, dict)
             assert isinstance(self.blocks, dict)
-        except (AssertionError, KeyError, IOError, json.JSONDecodeError), e:
+        except (AssertionError, KeyError, IOError, json.JSONDecodeError) as e:
             logging.warn('error opening manifest file: %s', e)
             self._manifest = self.make_empty_manifest()
 
@@ -76,21 +76,21 @@ def normalize(self):
         when the manifest is built) and then by ensuring that every dependency has
         its own entry and version in the top level of the manifest.
         """
-        for parent, depspec in self.assets.iteritems():
+        for parent, depspec in self.assets.items():
             depspec['deps'] = list(depspec['deps'])
             for dep in depspec['deps']:
                 assert dep in self.assets, (parent, dep)
                 assert depspec['version'], (parent, dep)
-        for name_hash, depspec in self.blocks.iteritems():
+        for name_hash, depspec in self.blocks.items():
             assert depspec['version'], name_hash
 
     def union(self, newer_manifest):
         # add an age entry to the old side so we can know how many merge
         # generations ago an entry was created
         def age(entry):
             entry['age'] = entry.get('age', 0) + 1
-        map(age, self.blocks.values())
-        map(age, self.assets.values())
+        list(map(age, list(self.blocks.values())))
+        list(map(age, list(self.assets.values())))
         self.blocks.update(newer_manifest.blocks)
         self.assets.update(newer_manifest.assets)
 

assetman/parsers/base.py

@@ -1,4 +1,4 @@
-from __future__ import absolute_import, with_statement
+
 from assetman.settings import Settings
 from assetman.compilers import JSCompiler, LessCompiler, CSSCompiler, SassCompiler
 

assetman/parsers/django_parser.py

@@ -1,4 +1,4 @@
-from __future__ import absolute_import, with_statement
+
 
 import django.template
 import django.template.loader
@@ -38,7 +38,7 @@ def get_compilers(self):
             compiler_cls = get_compiler_class(node)
             logging.debug('node %r %r', node, compiler_cls)
             if self.settings.get('verbose'):
-                print self.path, node, compiler_cls, node.get_all_text()
+                print(self.path, node, compiler_cls, node.get_all_text())
             yield compiler_cls(node.get_all_text(),
                                self.template_path,
                                src_path=self.template_path,

assetman/parsers/tornado_parser.py

@@ -1,4 +1,4 @@
-from __future__ import absolute_import, with_statement
+
 
 import os
 import tornado.template

assetman/settings.py

@@ -1,4 +1,4 @@
-from __future__ import absolute_import, with_statement
+
 
 import os
 try:

assetman/tests/test_compiler.py

@@ -1,5 +1,5 @@
-from __future__ import with_statement
-import test_shunt # pyflakes.ignore
+
+from . import test_shunt # pyflakes.ignore
 
 import contextlib
 
@@ -92,7 +92,7 @@ def temporarily_alter_contents(path, data):
     contents = open(path).read()
     open(path, 'a').write(data)
     yield
-    print 'undoing alter contents! %d' % len(contents)
+    print('undoing alter contents! %d' % len(contents))
     open(path, 'w').write(contents)
 
 # 

assetman/tests/test_django_templates.py

@@ -1,5 +1,5 @@
 import unittest
-import test_shunt # pyflakes.ignore
+from . import test_shunt # pyflakes.ignore
 
 import django.template
 from assetman.compilers import JSCompiler, LessCompiler, CSSCompiler

assetman/tests/test_manifest.py

@@ -1,4 +1,4 @@
-from __future__ import with_statement
+
 import os
 import unittest
 import simplejson as json
@@ -32,14 +32,14 @@ def test_can_get_manifest_path_from_asset_path(self):
     def test_can_load_manifest_from_asset_path(self):
         manifest = Manifest().load(self.TEST_MANIFEST_PATH)
         assert manifest
-        assert manifest.blocks.keys()
+        assert list(manifest.blocks.keys())
 
     def test_can_load_manifest_from_settings(self):
         settings = Settings(compiled_asset_root=self.TEST_MANIFEST_PATH)
 
         manifest = Manifest(settings).load()
         assert manifest
-        assert manifest.blocks.keys()
+        assert list(manifest.blocks.keys())
 
     def test_can_write_manifest_to_path(self):
         manifest = Manifest()
@@ -52,7 +52,7 @@ def test_can_write_manifest_to_path(self):
 
         with open(self.TEST_MANIFEST_PATH + '/manifest.json', 'r') as manifest_file:
             manifest_json = json.loads(manifest_file.read())
-            for k, v in manifest_json.items():
+            for k, v in list(manifest_json.items()):
                 assert manifest._manifest.get(k) == v
 
 if __name__ == "__main__":

assetman/tests/test_settings.py

@@ -1,4 +1,4 @@
-from __future__ import with_statement
+
 
 import unittest
 import simplejson as json
@@ -24,7 +24,7 @@ def test_settings_can_load_from_file(self):
         s = Settings.load(savepath)
 
         assert s is not None
-        for k, v in s.items():
+        for k, v in list(s.items()):
             assert settings_stub[k] == v
 
     def test_settings_can_write_to_file(self):
@@ -40,10 +40,10 @@ def test_settings_can_write_to_file(self):
                 assert saved_file, "file not found, save failed"
 
                 settings_dict = json.loads(saved_file.read())
-                for k, v in settings_dict.items():
+                for k, v in list(settings_dict.items()):
                     assert s[k] == v
 
-        except Exception, ex:
+        except Exception as ex:
             self.fail(str(ex))
 
 if __name__ == "__main__":

assetman/tools.py

@@ -1,4 +1,4 @@
-from __future__ import absolute_import, with_statement
+
 
 import os
 import re
@@ -20,7 +20,7 @@ def _crc(key):
 
 def _utf8(s):
     """encode a unicode string as utf-8"""
-    if isinstance(s, unicode):
+    if isinstance(s, str):
         return s.encode("utf-8")
     assert isinstance(s, str), "_utf8 expected a str, not %r" % type(s)
     return s
@@ -34,7 +34,7 @@ def iter_template_paths(template_dirs, template_ext):
 
     for template_dir in template_dirs:
         for root, dirs, files in os.walk(template_dir):
-            for f in itertools.ifilter(template_file_matcher, files):
+            for f in filter(template_file_matcher, files):
                 yield os.path.join(root, f)
 
 # Shortcuts for creating paths relative to some other path
@@ -66,10 +66,10 @@ def get_parser(template_path, template_type, settings):
     #TODO: dynamic import / return based on settings / config
     #avoids circular dep
     assert template_type in ["tornado_template", "django_template"]
-    assert isinstance(template_path, (str, unicode))
+    assert isinstance(template_path, str)
     logging.info('parsing for %s %s', template_type, template_path)
     if settings.get('verbose'):
-        print template_type, template_path
+        print(template_type, template_path)
     if template_type == "tornado_template":
         from assetman.parsers.tornado_parser import TornadoParser
         return TornadoParser(template_path, settings)

assetman/tornadoutils/static.py

@@ -1,4 +1,4 @@
-from __future__ import with_statement
+
 
 import calendar
 import datetime