lots of bug fixes

README.md

@@ -28,7 +28,7 @@ If you use the built in `static_url` function, an error will be raised.
 
 ### The compiler
 
-The Assetman compiler can be called via commandline using the `compile`
+The Assetman compiler can be called via commandline using the `assetman_compile`
 script.
 
 The compiler starts by building a manifest file which describes the
@@ -102,16 +102,16 @@ Static assets are served by a special `StaticFileHandler`. It knows how to
 handle three types of URLs (assume `static_url_prefix` is `/s/beta/` for 
 these examples):
 
- 1. Unversioned URLs: `/s/beta/{path}`
+ 1. Unversioned URLs: `/s/{path}`
 
     This is used to serve unversioned, uncompiled assets during development.
 
- 2. Versioned URLs: `/s/beta/v:{version}/{path}`
+ 2. Versioned URLs: `/s/v:{version}/{path}`
 
     This is used to serve static assets referenced directly or via
     `assetman.static_url`, where no fallbacks are necessary.
 
- 3. Versioned URLs with fallbacks: `/s/beta/v:{version}/f:{a:b:c}/{hash}`
+ 3. Versioned URLs with fallbacks: `/s/v:{version}/f:{a:b:c}/{hash}`
 
     This is used to request compiled asset blocks while also providing the
     ability for the server to respond with as many of the constituent assets
@@ -121,18 +121,10 @@ these examples):
 Assetman takes care of generating the appropriate URLs at compile time (for
 the second kind of URL) or template render time (for the 3rd).
 
-### Running the Assetman compiler manually
+### Running the Assetman compiler
 
-If any errors are encountered during the Assetman compilation process, a
-deploy will fail.  Errors are logged in logatron under the `www` app, so
-you may be able to find and fix the error without running Assetman manually.
-
-If you do need or want to run the compiler manually, though, here's the
-command that is run by the deploy process (split over multiple lines for
-readability's sake):
-
-    python www/scripts/assetman_compiler.py \
+    assetman_compile \
         --output_dir=/data/assets \
-        --static_url_prefix=/s/beta/ \
+        --static_url_prefix=/s/ \
         --template_dir=templates \
-        --template_dir=../www/templates
+        --template_dir=../other/templates

assetman/S3UploadThread.py

@@ -1,3 +1,5 @@
+#!/bin/python
+from __future__ import with_statement
 import re
 import os
 import sys
@@ -8,8 +10,9 @@
 import Queue
 import mimetypes
 import logging
+import binascii
 from boto.s3.connection import S3Connection
-from assetman import make_output_path, make_static_path, get_static_pattern
+from assetman.tools import make_output_path, make_static_path, get_static_pattern
 
 class S3UploadThread(threading.Thread):
     """Thread that knows how to read asset file names from a queue and upload
@@ -23,13 +26,14 @@ class S3UploadThread(threading.Thread):
     at CloudFront instead of our local CDN proxy.
     """
 
-    def __init__(self, queue, errors, manifest):
+    def __init__(self, queue, errors, manifest, settings):
         threading.Thread.__init__(self)
         cx = S3Connection(settings.get('aws_access_key'), settings.get('aws_secret_key'))
         self.bucket = cx.get_bucket(settings.get('s3_assets_bucket'))
         self.queue = queue
         self.errors = errors
         self.manifest = manifest
+        self.settings = settings
 
     def run(self):
         while True:
@@ -69,7 +73,7 @@ def start_upload_file(self, file_name, file_path):
 
             # Next we will upload the same file with a prefixed key, to be
             # served by our "local CDN proxy".
-            key_prefix = settings.get('local_cdn_url_prefix').lstrip('/').rstrip('/')
+            key_prefix = self.settings.get('local_cdn_url_prefix').lstrip('/').rstrip('/')
             key = self.bucket.new_key(key_prefix + '/' + file_name)
             self.upload_file(key, file_data, headers, for_cdn=False)
 
@@ -87,14 +91,15 @@ def upload_file(self, key, file_data, headers, for_cdn):
             if re.search(r'\.(css|js)$', key.name):
                 if for_cdn:
                     logging.info('Rewriting URLs => CDN: %s', key.name)
-                    replacement_prefix = settings.get('cdn_url_prefix')
+                    replacement_prefix = self.settings.get('cdn_url_prefix')
                 else:
                     logging.info('Rewriting URLs => local proxy: %s', key.name)
-                    replacement_prefix = settings.get('local_cdn_url_prefix')
+                    replacement_prefix = self.settings.get('local_cdn_url_prefix')
                 file_data = sub_static_version(
                     file_data,
                     self.manifest,
-                    replacement_prefix)
+                    replacement_prefix,
+                    self.settings.get('static_url_prefix'))
             key.set_contents_from_string(file_data, headers, replace=False)
             logging.info('Uploaded %s', key.name)
             logging.debug('Headers: %r', headers)
@@ -112,7 +117,7 @@ def get_cache_control(self):
         return 'public, max-age=%s' % (86400 * 365 * 10)
 
 
-def upload_assets(manifest, skip=False):
+def upload_assets(manifest, settings, skip=False):
     """Uploads any assets that are in the given manifest and in our compiled
     output dir but missing from our static assets bucket to that bucket on S3.
     """
@@ -149,14 +154,32 @@ def upload_assets(manifest, skip=False):
     queue = Queue.Queue()
     errors = []
     for i in xrange(5):
-        uploader = S3UploadThread(queue, errors, manifest)
+        uploader = S3UploadThread(queue, errors, manifest, settings)
         uploader.setDaemon(True)
         uploader.start()
     map(queue.put, to_upload)
     queue.join()
     return len(errors) == 0
 
-def sub_static_version(src, manifest, replacement_prefix):
+def get_shard_from_list(settings_list, shard_id):
+    assert isinstance(settings_list, (list, tuple)), "must be a list not %r" % settings_list
+    shard_id = _crc(shard_id)
+    bucket = shard_id % len(settings_list)
+    return settings_list[bucket]
+
+def _crc(key):
+    """crc32 hash a string"""
+    return binascii.crc32(_utf8(key)) & 0xffffffff
+
+def _utf8(s):
+    """encode a unicode string as utf-8"""
+    if isinstance(s, unicode):
+        return s.encode("utf-8")
+    assert isinstance(s, str), "_utf8 expected a str, not %r" % type(s)
+    return s
+
+
+def sub_static_version(src, manifest, replacement_prefix, static_url_prefix):
     """Adjusts any static URLs in the given source to point to a different
     location.
 
@@ -171,11 +194,11 @@ def replacer(match):
         if path in manifest['assets']:
             versioned_path = manifest['assets'][path]['versioned_path']
             if isinstance(replacement_prefix, (list, tuple)):
-                prefix = settings.get_shard_from_list(replacement_prefix, versioned_path)
+                prefix = get_shard_from_list(replacement_prefix, versioned_path)
             else:
                 prefix = replacement_prefix
             return prefix.rstrip('/') + '/' + versioned_path.lstrip('/')
         logging.warn('Missing path %s in manifest, using %s', path, match.group(0))
         return match.group(0)
-    pattern = get_static_pattern(settings.get('static_url_prefix'))
+    pattern = get_static_pattern(static_url_prefix)
     return re.sub(pattern, replacer, src)

assetman/compile.py

@@ -56,14 +56,9 @@
     '-i', '--skip-inline-images', action="store_true",
     help='Do not sub data URIs for small images in CSS.')
 
-#TODO: maybe make a /plugins directory and script that optionally adds this stuff
-parser.add_option(
-    '-u', '--skip-upload', action="store_true",
-    help='Do not upload anything to S3.')
-
 parser.add_option(
     '--aws_username', type="string", 
-    help="AWS username, required for uploading to s3")
+    help="AWS username, required for uploading to s3 (upload skipped if missing)")
 
 parser.add_option(
     '--aws_access_key', type="string",
@@ -356,22 +351,21 @@ def _create_settings(options):
     return Settings(compiled_asset_root=options.output_dir,
                     static_dir=options.static_dir,
                     static_url_prefix=options.static_url_path,
-                    compiled_manifest_path=options.compiled_manifest_path,
                     template_dirs=options.template_dir,
                     template_extension=options.template_ext,
+                    test_needs_compile=options.test_needs_compile,
+                    force=options.force,
                     aws_username=options.aws_username,
                     aws_access_key=options.aws_access_key,
                     aws_secret_key=options.aws_secret_key,
                     s3_assets_bucket=options.s3_assets_bucket)
 
-def main(options):
-    settings = _create_settings(options) 
-
+def run(settings):
     if not re.match(r'^/.*?/$', settings.get('static_url_prefix')):
         logging.error('static_url_prefix setting must begin and end with a slash')
         sys.exit(1)
 
-    if not os.path.isdir(settings['compiled_asset_root']) and not options.test_needs_compile:
+    if not os.path.isdir(settings['compiled_asset_root']) and not settings.test_needs_compile:
         logging.info('Creating output directory: %s', settings['compiled_asset_root'])
         os.makedirs(settings['compiled_asset_root'])
 
@@ -398,12 +392,12 @@ def main(options):
         src_path, msg = e.args
         logging.error('Error parsing template %s', src_path)
         logging.error(msg)
-        return 1
+        raise Exception
     except DependencyError, e:
         src_path, missing_deps = e.args
         logging.error('Dependency error in source %s!', src_path)
         logging.error('Missing paths: %s', missing_deps)
-        return 1
+        raise Exception
 
     # Remove duplicates from our list of compilers. This de-duplication must
     # happen after the current manifest is built, because each non-unique
@@ -422,7 +416,7 @@ def main(options):
     def needs_compile(compiler):
         return compiler.needs_compile(cached_manifest, current_manifest)
 
-    if options.force:
+    if settings.force:
         to_compile = compilers
     else:
         to_compile = filter(needs_compile, compilers)
@@ -458,14 +452,11 @@ def assets_in_sync(asset):
             logging.error('Command: %s', ' '.join(cmd))
             logging.error('Error:   %s', msg)
             return 1
-        except KeyboardInterrupt:
-            logging.error('Interrupted by user, exiting...')
-            return 1
 
         #TODO: refactor to some chain of command for plugins
-        if not S3UploadThread.upload_assets(current_manifest, options.skip_upload):
-            logging.error('Error uploading assets')
-            return 1
+        if settings['aws_username']:
+            if not S3UploadThread.upload_assets(current_manifest, settings):
+                raise Exception('Error uploading assets')
 
         Manifest(settings).write(current_manifest)
 
@@ -474,4 +465,5 @@ def assets_in_sync(asset):
 if __name__ == '__main__':
     logging.getLogger().setLevel(logging.DEBUG)
     options, args = parser.parse_args()
-    sys.exit(main(options))
+    settings = _create_settings(options) 
+    sys.exit(run(settings))

assetman/compilers.py

@@ -9,17 +9,8 @@
 import functools
 import os
 import re
-import sys
 
-# Find our project root, assuming this file lives in ./scripts/. We add that
-# root dir to sys.path and use it as our working directory.
-project_root = os.path.realpath(
-    os.path.join(os.path.dirname(__file__), '..'))
-if project_root not in sys.path:
-    sys.path.insert(0, project_root)
-os.chdir(project_root)
-
-import assetman
+import assetman.managers
 from assetman.tools import make_static_path, get_static_pattern, make_output_path
 
 def run_proc(cmd, stdin=None):
@@ -139,7 +130,7 @@ def get_compiled_path(self):
         return make_output_path(self.settings['static_dir'], self.get_compiled_name())
 
 
-class JSCompiler(AssetCompiler, assetman.JSManager):
+class JSCompiler(AssetCompiler, assetman.managers.JSManager):
 
     include_expr = 'include_js'
 
@@ -156,7 +147,7 @@ def do_compile(self, **kwargs):
         return run_proc(cmd)
 
 
-class CSSCompiler(AssetCompiler, assetman.CSSManager):
+class CSSCompiler(AssetCompiler, assetman.managers.CSSManager):
 
     include_expr = 'include_css'
 
@@ -226,7 +217,7 @@ def replacer(match):
         return result
 
 
-class LessCompiler(CSSCompiler, assetman.LessManager):
+class LessCompiler(CSSCompiler, assetman.managers.LessManager):
 
     include_expr = 'include_less'
 
@@ -244,7 +235,7 @@ def do_compile(self, **kwargs):
         return super(LessCompiler, self).do_compile(css_input='\n'.join(outputs))
 
 
-class SassCompiler(CSSCompiler, assetman.SassManager):
+class SassCompiler(CSSCompiler, assetman.managers.SassManager):
 
     include_expr = 'include_sass'
 

assetman/parsers/base.py

@@ -1,10 +1,10 @@
 from __future__ import absolute_import, with_statement
 from assetman.settings import Settings
-from assetman.compilers import JSCompiler, LessCompiler, CSSCompiler
+from assetman.compilers import JSCompiler, LessCompiler, CSSCompiler, SassCompiler
 
 # Map from template-side assetman manager calls to the corresponding
 # compiler classes
-compiler_classes = [JSCompiler, LessCompiler, CSSCompiler]
+compiler_classes = [JSCompiler, LessCompiler, CSSCompiler, SassCompiler]
 compiler_map = dict((c.include_expr, c) for c in compiler_classes)
 
 

assetman/parsers/tornado_parser.py

@@ -3,7 +3,6 @@
 import os
 import tornado.template
 from assetman.tools import include_expr_matcher
-from assetman.compilers import JSCompiler, LessCompiler, CSSCompiler
 from assetman.parsers import base
 
 
@@ -12,7 +11,8 @@ class TornadoParser(base.TemplateParser):
 
     def load_template(self, path):
         """ loads a template from a full file path """
-        dirpath, template_file = os.path.split(path) 
+        dirpath, template_file = path.split(os.path.sep, 1)
+        # logging.debug('loading template %r %r', dirpath, template_file)
         loader = tornado.template.Loader(dirpath)
         self.template = loader.load(template_file)
 

setup.py

@@ -11,7 +11,7 @@
       author_email="[email protected]",
       maintainer="Anton Fritsch",
       maintainer_email="[email protected]",
-      packages=['assetman', 'assetman/tornadoutils', 'assetman/django_assetman'],
+      packages=['assetman', 'assetman/parsers', 'assetman/tornadoutils', 'assetman/django_assetman'],
       install_requires=['simplejson',
                         'multiprocessing',
                        ],