diff --git a/.gitignore b/.gitignore index 82c7d1d..465c10a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ docs/build/ #compiled output *.pyc +.pytest_cache/ + +.venv/ diff --git a/assetman/S3UploadThread.py b/assetman/S3UploadThread.py index 910637f..41b079f 100755 --- a/assetman/S3UploadThread.py +++ b/assetman/S3UploadThread.py @@ -1,12 +1,12 @@ #!/bin/python -from __future__ import with_statement + import re import os import os.path import sys import threading import datetime -import Queue +import queue as Queue import mimetypes import logging import boto3 @@ -42,7 +42,7 @@ def run(self): file_name, file_path = self.queue.get() try: self.start_upload_file(file_name, file_path) - except Exception, e: + except Exception as e: logging.error('Error uploading %s: %s', file_name, e) self.errors.append((sys.exc_info(), self)) finally: @@ -52,8 +52,8 @@ def start_upload_file(self, file_name, file_path): """Starts the procecss of uploading a file to S3. Each file will be uploaded twice (once for CDN and once for our local CDN proxy). """ - assert isinstance(file_name, (str, unicode)) - assert isinstance(file_path, (str, unicode)) + assert isinstance(file_name, str) + assert isinstance(file_path, str) assert os.path.isfile(file_path) content_type, content_encoding = mimetypes.guess_type(file_name) @@ -88,7 +88,7 @@ def exists(self, obj): # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.head_object try: self.client.head_object(Bucket=obj.bucket_name, Key=obj.key) - except Exception, e: + except Exception as e: logging.error('got %s', e) return False return True @@ -143,7 +143,7 @@ def upload_assets_to_s3(manifest, settings, skip_s3_upload=False): # We know we want to upload each asset block (these correspond to the # assetman.include_* blocks in each template) - for depspec in manifest.blocks.itervalues(): + for depspec in manifest.blocks.values(): file_name = depspec['versioned_path'] file_path = make_output_path(settings['compiled_asset_root'], file_name) assert os.path.isfile(file_path), 'Missing compiled asset %s' % file_path @@ -154,7 +154,7 @@ def upload_assets_to_s3(manifest, settings, skip_s3_upload=False): # but we'll need to filter out other entries in the complete 'assets' # block of the manifest. should_skip = re.compile(r'\.(scss|less|css|js|html)$', re.I).search - for rel_path, depspec in manifest.assets.iteritems(): + for rel_path, depspec in manifest.assets.items(): if should_skip(rel_path): continue file_path = make_absolute_static_path(settings['static_dir'], rel_path) @@ -170,11 +170,11 @@ def upload_assets_to_s3(manifest, settings, skip_s3_upload=False): # Upload assets to S3 using 5 threads queue = Queue.Queue() errors = [] - for i in xrange(5): + for i in range(5): uploader = S3UploadThread(queue, errors, manifest, settings) uploader.setDaemon(True) uploader.start() - map(queue.put, to_upload) + list(map(queue.put, to_upload)) queue.join() if errors: raise Exception(errors) diff --git a/assetman/compile.py b/assetman/compile.py index 832a299..5dd621c 100755 --- a/assetman/compile.py +++ b/assetman/compile.py @@ -1,6 +1,6 @@ #!/bin/python -from __future__ import with_statement + import os import re @@ -333,7 +333,7 @@ def build_manifest(tornado_paths, django_paths, settings): for compiler in compilers: new_paths = compiler.get_paths() if settings.get('verbose'): - print compiler, new_paths + print(compiler, new_paths) paths.update(new_paths) paths = list(paths) @@ -405,12 +405,12 @@ def run(settings): cached_manifest = Manifest(settings).load() try: current_manifest, compilers = build_manifest(tornado_paths, django_paths, settings) - except ParseError, e: + except ParseError as e: src_path, msg = e.args logging.error('Error parsing template %s', src_path) logging.error(msg) raise Exception - except DependencyError, e: + except DependencyError as e: src_path, missing_deps = e.args logging.error('Dependency error in source %s!', src_path) logging.error('Missing paths: %s', missing_deps) @@ -421,7 +421,7 @@ def run(settings): # compiler's source path figures into the dependency tracking. But we only # need to actually compile each block once. logging.debug('Found %d assetman block compilers', len(compilers)) - compilers = dict((c.get_hash(), c) for c in compilers).values() + compilers = list(dict((c.get_hash(), c) for c in compilers).values()) logging.debug('%d unique assetman block compilers', len(compilers)) # update the manifest on each our compilers to reflect the new manifest, @@ -436,7 +436,7 @@ def needs_compile(compiler): if settings['force_recompile']: to_compile = compilers else: - to_compile = filter(needs_compile, compilers) + to_compile = list(filter(needs_compile, compilers)) if to_compile or cached_manifest.needs_recompile(current_manifest): # If we're only testing whether a compile is needed, we're done @@ -448,7 +448,7 @@ def needs_compile(compiler): # See note above about bug in pool.map w/r/t KeyboardInterrupt. _compile_worker = CompileWorker(settings.get('skip_inline_images', False), current_manifest) pool.map_async(_compile_worker, to_compile).get(1e100) - except CompileError, e: + except CompileError as e: cmd, msg = e.args logging.error('Compile error!') logging.error('Command: %s', ' '.join(cmd)) diff --git a/assetman/compilers.py b/assetman/compilers.py index d842e23..3bb618e 100644 --- a/assetman/compilers.py +++ b/assetman/compilers.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import, with_statement + import base64 from collections import defaultdict @@ -114,7 +114,7 @@ def get_current_content_hash(self, manifest): def get_paths(self): """Returns a list of absolute paths to the assets contained in this manager.""" - paths = map(functools.partial(make_absolute_static_path, self.settings['static_dir']), self.rel_urls) + paths = list(map(functools.partial(make_absolute_static_path, self.settings['static_dir']), self.rel_urls)) try: assert all(map(os.path.isfile, paths)) except AssertionError: @@ -222,7 +222,7 @@ def replacer(match): result = re.sub(pattern, replacer, css_src) - for url, count in seen_assets.iteritems(): + for url, count in seen_assets.items(): if count > 1: logging.warn('inline asset duplicated %dx: %s', count, url) diff --git a/assetman/managers.py b/assetman/managers.py index bca0b3b..53943ba 100644 --- a/assetman/managers.py +++ b/assetman/managers.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import, with_statement + import os import logging @@ -46,7 +46,7 @@ def __init__(self, rel_url_text, local=False, include_tag=True, src_path=None, s Any extra kwargs will be interpreted as extra HTML params to include on the rendered element. """ - self.rel_urls = filter(None, _utf8(rel_url_text).split()) + self.rel_urls = [_f for _f in _utf8(rel_url_text).split() if _f] self.local = local self.include_tag = include_tag self.src_path = src_path @@ -101,7 +101,7 @@ def render_attrs(self): leading space. """ attrs = ' '.join('%s=%r' % (attr, _utf8(val)) - for attr, val in self.attrs.iteritems()) + for attr, val in self.attrs.items()) return ' ' + attrs if attrs else '' def render_asset(self, url): @@ -123,7 +123,7 @@ def render(self): """ try: if self.settings['enable_static_compilation']: - urls = map(self.make_asset_url, self.rel_urls) + urls = list(map(self.make_asset_url, self.rel_urls)) return '\n'.join(map(self.render_asset, urls)) else: compiled_name = self.get_compiled_name() diff --git a/assetman/manifest.py b/assetman/manifest.py index 5bc0987..3b4f1ff 100644 --- a/assetman/manifest.py +++ b/assetman/manifest.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import, with_statement + import os import simplejson as json @@ -51,7 +51,7 @@ def load(self, compiled_asset_path=None): self._manifest = json.load(open(filename)) assert isinstance(self.assets, dict) assert isinstance(self.blocks, dict) - except (AssertionError, KeyError, IOError, json.JSONDecodeError), e: + except (AssertionError, KeyError, IOError, json.JSONDecodeError) as e: logging.warn('error opening manifest file: %s', e) self._manifest = self.make_empty_manifest() @@ -76,12 +76,12 @@ def normalize(self): when the manifest is built) and then by ensuring that every dependency has its own entry and version in the top level of the manifest. """ - for parent, depspec in self.assets.iteritems(): + for parent, depspec in self.assets.items(): depspec['deps'] = list(depspec['deps']) for dep in depspec['deps']: assert dep in self.assets, (parent, dep) assert depspec['version'], (parent, dep) - for name_hash, depspec in self.blocks.iteritems(): + for name_hash, depspec in self.blocks.items(): assert depspec['version'], name_hash def union(self, newer_manifest): @@ -89,8 +89,8 @@ def union(self, newer_manifest): # generations ago an entry was created def age(entry): entry['age'] = entry.get('age', 0) + 1 - map(age, self.blocks.values()) - map(age, self.assets.values()) + list(map(age, list(self.blocks.values()))) + list(map(age, list(self.assets.values()))) self.blocks.update(newer_manifest.blocks) self.assets.update(newer_manifest.assets) diff --git a/assetman/parsers/base.py b/assetman/parsers/base.py index 20f119f..55be297 100644 --- a/assetman/parsers/base.py +++ b/assetman/parsers/base.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import, with_statement + from assetman.settings import Settings from assetman.compilers import JSCompiler, LessCompiler, CSSCompiler, SassCompiler diff --git a/assetman/parsers/django_parser.py b/assetman/parsers/django_parser.py index ef8be8b..e173618 100644 --- a/assetman/parsers/django_parser.py +++ b/assetman/parsers/django_parser.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import, with_statement + import django.template import django.template.loader @@ -38,7 +38,7 @@ def get_compilers(self): compiler_cls = get_compiler_class(node) logging.debug('node %r %r', node, compiler_cls) if self.settings.get('verbose'): - print self.path, node, compiler_cls, node.get_all_text() + print(self.path, node, compiler_cls, node.get_all_text()) yield compiler_cls(node.get_all_text(), self.template_path, src_path=self.template_path, diff --git a/assetman/parsers/tornado_parser.py b/assetman/parsers/tornado_parser.py index 277da80..fafcfe0 100644 --- a/assetman/parsers/tornado_parser.py +++ b/assetman/parsers/tornado_parser.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import, with_statement + import os import tornado.template diff --git a/assetman/settings.py b/assetman/settings.py index 23cd58c..1c23a28 100644 --- a/assetman/settings.py +++ b/assetman/settings.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import, with_statement + import os try: diff --git a/assetman/tests/test_compiler.py b/assetman/tests/test_compiler.py index 83a77dc..b408a2d 100644 --- a/assetman/tests/test_compiler.py +++ b/assetman/tests/test_compiler.py @@ -1,5 +1,5 @@ -from __future__ import with_statement -import test_shunt # pyflakes.ignore + +from . import test_shunt # pyflakes.ignore import contextlib @@ -92,7 +92,7 @@ def temporarily_alter_contents(path, data): contents = open(path).read() open(path, 'a').write(data) yield - print 'undoing alter contents! %d' % len(contents) + print('undoing alter contents! %d' % len(contents)) open(path, 'w').write(contents) # diff --git a/assetman/tests/test_django_templates.py b/assetman/tests/test_django_templates.py index d453544..5078d83 100644 --- a/assetman/tests/test_django_templates.py +++ b/assetman/tests/test_django_templates.py @@ -1,5 +1,5 @@ import unittest -import test_shunt # pyflakes.ignore +from . import test_shunt # pyflakes.ignore import django.template from assetman.compilers import JSCompiler, LessCompiler, CSSCompiler diff --git a/assetman/tests/test_manifest.py b/assetman/tests/test_manifest.py index 03aec68..ddee8ee 100644 --- a/assetman/tests/test_manifest.py +++ b/assetman/tests/test_manifest.py @@ -1,4 +1,4 @@ -from __future__ import with_statement + import os import unittest import simplejson as json @@ -32,14 +32,14 @@ def test_can_get_manifest_path_from_asset_path(self): def test_can_load_manifest_from_asset_path(self): manifest = Manifest().load(self.TEST_MANIFEST_PATH) assert manifest - assert manifest.blocks.keys() + assert list(manifest.blocks.keys()) def test_can_load_manifest_from_settings(self): settings = Settings(compiled_asset_root=self.TEST_MANIFEST_PATH) manifest = Manifest(settings).load() assert manifest - assert manifest.blocks.keys() + assert list(manifest.blocks.keys()) def test_can_write_manifest_to_path(self): manifest = Manifest() @@ -52,7 +52,7 @@ def test_can_write_manifest_to_path(self): with open(self.TEST_MANIFEST_PATH + '/manifest.json', 'r') as manifest_file: manifest_json = json.loads(manifest_file.read()) - for k, v in manifest_json.items(): + for k, v in list(manifest_json.items()): assert manifest._manifest.get(k) == v if __name__ == "__main__": diff --git a/assetman/tests/test_settings.py b/assetman/tests/test_settings.py index 0c8fe32..e5b065b 100644 --- a/assetman/tests/test_settings.py +++ b/assetman/tests/test_settings.py @@ -1,4 +1,4 @@ -from __future__ import with_statement + import unittest import simplejson as json @@ -24,7 +24,7 @@ def test_settings_can_load_from_file(self): s = Settings.load(savepath) assert s is not None - for k, v in s.items(): + for k, v in list(s.items()): assert settings_stub[k] == v def test_settings_can_write_to_file(self): @@ -40,10 +40,10 @@ def test_settings_can_write_to_file(self): assert saved_file, "file not found, save failed" settings_dict = json.loads(saved_file.read()) - for k, v in settings_dict.items(): + for k, v in list(settings_dict.items()): assert s[k] == v - except Exception, ex: + except Exception as ex: self.fail(str(ex)) if __name__ == "__main__": diff --git a/assetman/tools.py b/assetman/tools.py index 3375ebb..b6b42e5 100644 --- a/assetman/tools.py +++ b/assetman/tools.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import, with_statement + import os import re @@ -20,7 +20,7 @@ def _crc(key): def _utf8(s): """encode a unicode string as utf-8""" - if isinstance(s, unicode): + if isinstance(s, str): return s.encode("utf-8") assert isinstance(s, str), "_utf8 expected a str, not %r" % type(s) return s @@ -34,7 +34,7 @@ def iter_template_paths(template_dirs, template_ext): for template_dir in template_dirs: for root, dirs, files in os.walk(template_dir): - for f in itertools.ifilter(template_file_matcher, files): + for f in filter(template_file_matcher, files): yield os.path.join(root, f) # Shortcuts for creating paths relative to some other path @@ -66,10 +66,10 @@ def get_parser(template_path, template_type, settings): #TODO: dynamic import / return based on settings / config #avoids circular dep assert template_type in ["tornado_template", "django_template"] - assert isinstance(template_path, (str, unicode)) + assert isinstance(template_path, str) logging.info('parsing for %s %s', template_type, template_path) if settings.get('verbose'): - print template_type, template_path + print(template_type, template_path) if template_type == "tornado_template": from assetman.parsers.tornado_parser import TornadoParser return TornadoParser(template_path, settings) diff --git a/assetman/tornadoutils/static.py b/assetman/tornadoutils/static.py index 33645bc..0930daf 100644 --- a/assetman/tornadoutils/static.py +++ b/assetman/tornadoutils/static.py @@ -1,4 +1,4 @@ -from __future__ import with_statement + import calendar import datetime diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..92ccdc6 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,19 @@ +asgiref==3.7.2 +boto3==1.34.15 +botocore==1.34.15 +Django==4.2.9 +exceptiongroup==1.2.0 +iniconfig==2.0.0 +jmespath==1.0.1 +packaging==23.2 +pluggy==1.3.0 +pytest==7.4.4 +python-dateutil==2.8.2 +s3transfer==0.10.0 +simplejson==3.19.2 +six==1.16.0 +sqlparse==0.4.4 +tomli==2.0.1 +tornado==5.1.1 +typing-extensions==4.9.0 +urllib3==1.26.18 diff --git a/scripts/invalidate_cloudfront_assets.py b/scripts/invalidate_cloudfront_assets.py index 47292dd..64f0df2 100755 --- a/scripts/invalidate_cloudfront_assets.py +++ b/scripts/invalidate_cloudfront_assets.py @@ -24,7 +24,7 @@ def invalidate_paths(paths): def main(urls): if not urls: - choice = raw_input('No URLs given. Invalidate all URLs? (y/n) ') + choice = input('No URLs given. Invalidate all URLs? (y/n) ') if choice.lower() != 'y': return 1 urls = ['/*'] @@ -34,7 +34,7 @@ def main(urls): what = 'all URLs' else: what = '%d URLs' % len(urls) - print 'May or may not have invalidated cache for %s on %s CloudFront distributions' % (what, len(reqs)) + print('May or may not have invalidated cache for %s on %s CloudFront distributions' % (what, len(reqs))) return 0 diff --git a/setup.py b/setup.py index 8c6d345..0234546 100755 --- a/setup.py +++ b/setup.py @@ -17,8 +17,8 @@ 'multiprocessing', ], classifiers = [ - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", ], scripts=['scripts/assetman_compile'] )