Add generic uploader

.env

@@ -0,0 +1,44 @@
+# Variables in this file will be substituted into docker-compose.yml
+# Save a copy of this file as .env and insert your own values.
+# Verify correct substitution with "docker-compose config"
+# If variables are newly added or enabled, please delete and rebuild the images to pull in changes:
+# docker-compose down
+# docker rmi -f docker_ckan docker_db
+# docker rmi $(docker images -f dangling=true -q)
+# docker-compose build
+# docker-compose up -d
+# docker-compose restart ckan # give the db service time to initialize the db cluster on first run
+
+# Image: ckan
+CKAN_SITE_ID=default
+#
+# On AWS, your CKAN_SITE_URL is the output of:
+# curl -s http://169.254.169.254/latest/meta-data/public-hostname
+# CKAN_SITE_URL=http://ec2-xxx-xxx-xxx-xxx.ap-southeast-2.compute.amazonaws.com
+# When running locally, CKAN_SITE_URL must contain the port
+CKAN_SITE_URL=http://localhost:5000
+#
+# CKAN_PORT must be available on the host: sudo netstat -na
+# To apply change: docker-compose down && docker rmi docker_ckan && docker-compose build ckan
+CKAN_PORT=5000
+#
+# Email settings
+CKAN_SMTP_SERVER=smtp.corporateict.domain:25
+CKAN_SMTP_STARTTLS=True
+CKAN_SMTP_USER=user
+CKAN_SMTP_PASSWORD=pass
+CKAN_SMTP_MAIL_FROM=ckan@localhost
+#
+# Image: db
+POSTGRES_PASSWORD=ckan
+#
+# POSTGRES_PORT must be available on the host: sudo netstat -na | grep 5432
+# To apply change: docker-compose down && docker rmi docker_db docker_ckan && docker-compose build
+POSTGRES_PORT=5432
+#
+# The datastore database will be created in the db container as docs
+# Readwrite user/pass will be ckan:POSTGRES_PASSWORD
+# Readonly user/pass will be datastore_ro:DATASTORE_READONLY_PASSWORD
+DATASTORE_READONLY_PASSWORD=datastore cloudstorage
+
+CKAN_SITE_TITLE='Testing subject'

.gitignore

@@ -4,7 +4,7 @@ syntax: glob
 *.swp
 *.swo
 .DS_Store
-ckan.egg-info/*
+*.egg-info/
 sandbox/*
 dist
 
@@ -18,3 +18,5 @@ fl_notes.txt
 *.ini
 .noseids
 *~
+.coverage
+cover/

README.md

@@ -29,6 +29,20 @@ For most drivers, this is all you need:
 
     ckanext.cloudstorage.driver_options = {"key": "<your public key>", "secret": "<your secret key>"}
 
+## Google Storage
+
+To use the Google Storage driver the following driver options are required:
+
+    {"key": "<service account ID>", "secret": "<path to SA private key>", "project": "<google project ID>"}
+
+**Note on secure URL's with Google Storage**
+With Google's lack of folder-level permissions the whole bucket will need to be made private when
+using secure urls. This will now affect generic file uploads as well. To still allow
+generic files to be public (`ckanext.cloudstorage.use_secure_urls_for_generics` is `False` by default)
+we will set the ACL of a newly uploaded object to `public-read` when only
+`ckanext.cloudstorage.use_secure_urls` is activated. If you later decide to make the generic files private
+you will have to manually update the ACL on the already uploaded objects to make them private.
+
 # Support
 
 Most libcloud-based providers should work out of the box, but only those listed
@@ -39,6 +53,7 @@ below have been tested:
 | Azure    | YES | YES | YES (if `azure-storage` is installed) |
 | AWS S3   | YES | YES | YES (if `boto` is installed) |
 | Rackspace | YES | YES | No |
+| Google Storage | YES | YES | YES (if `google-cloud-storage` and `pycrypto` are installed) |
 
 # What are "Secure URLs"?
 
@@ -50,8 +65,18 @@ the resource. This means that the normal CKAN-provided access restrictions can
 apply to resources with no further effort on your part, but still get all the
 benefits of your CDN/blob storage.
 
+    # applies to resources
     ckanext.cloudstorage.use_secure_urls = 1
 
+    # applies to generic uploads eg. group images, logo
+    ckanext.cloudstorage.use_secure_urls_for_generics = 1
+
+The access permissions on the storage container used will have to be set accordingly to reflect
+these settings (if using Google Storage, see note on Google Storage use).
+
+`use_secure_urls_for_generics` is recommended to be off, to allow for caching of assets
+such as the logo.
+
 This option also enables multipart uploads, but you need to create database tables
 first. Run next command from extension folder:
     `paster cloudstorage initdb -c /etc/ckan/default/production.ini `
@@ -77,8 +102,7 @@ cloudstorage will take care of the rest. Ex:
 
 1. You should disable public listing on the cloud service provider you're
    using, if supported.
-2. Currently, only resources are supported. This means that things like group
-   and organization images still use CKAN's local file storage.
+2. Currently, the migration tool only supports resources.
 
 # FAQ
 

ckanext/cloudstorage/controller.py

@@ -1,14 +1,18 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 import os.path
+import logging
 
 from pylons import c
 from pylons.i18n import _
-
+from webob.exc import status_map
 from ckan import logic, model
 from ckan.lib import base, uploader
+from ckan.common import is_flask_request
 import ckan.lib.helpers as h
 
+log = logging.getLogger(__name__)
+
 
 class StorageController(base.BaseController):
     def resource_download(self, id, resource_id, filename=None):
@@ -52,3 +56,29 @@ def resource_download(self, id, resource_id, filename=None):
             base.abort(404, _('No download is available'))
 
         h.redirect_to(uploaded_url)
+
+    def uploaded_file_redirect(self, upload_to, filename):
+        '''Redirect static file requests to their location on cloudstorage.'''
+        upload = uploader.get_uploader('notused')
+        file_path = upload.path_from_filename(filename)
+        uploaded_url = upload.get_url_from_path(file_path)
+
+        if upload.use_secure_urls:
+            h.redirect_to(uploaded_url)
+        else:
+            if is_flask_request():
+                raise NotImplementedError("Permanent redirect for flask \
+                    requests is not implemented yet")
+            else:
+                # We are manually performing a redirect for Pylons
+                # as this is the only way to set the caching headers
+                # to make a Permanently Moved cachable
+                # (see https://github.com/Pylons/pylons/blob/master/pylons/controllers/util.py#L218-L229)
+                exc = status_map[301]
+                raise exc(
+                    location=uploaded_url.encode('utf-8'),
+                    headers={
+                        "Cache-Control": "public, max-age=3600",
+                        "Pragma": "none"
+                    }
+                )

ckanext/cloudstorage/plugin.py

@@ -53,9 +53,8 @@ def get_resource_uploader(self, data_dict):
         return storage.ResourceCloudStorage(data_dict)
 
     def get_uploader(self, upload_to, old_filename=None):
-        # We don't provide misc-file storage (group images for example)
-        # Returning None here will use the default Uploader.
-        return None
+        # Custom uploader for generic file uploads
+        return storage.FileCloudStorage(upload_to, old_filename)
 
     def before_map(self, map):
         sm = SubMapper(
@@ -77,6 +76,12 @@ def before_map(self, map):
                 action='resource_download'
             )
 
+            sm.connect(
+                'uploaded_file',
+                '/uploads/{upload_to}/{filename}',
+                action='uploaded_file_redirect'
+            )
+
         return map
 
     # IActions