Upgrade to django 1.11, upgrade django pipeline, use whitenoise for static

Based on LeoVerto's work at #11
thanks Leo!

Running in docker means that using whitenoise for static files is easier
than serving static files with nginx or other.

Author: alastair

botbot/apps/bots/migrations/0001_initial.py

@@ -2,7 +2,7 @@
 from __future__ import unicode_literals
 
 from django.db import models, migrations
-import djorm_pgarray.fields
+import django.contrib.postgres.fields
 
 
 class Migration(migrations.Migration):
@@ -57,7 +57,7 @@ class Migration(migrations.Migration):
             fields=[
                 ('id', models.AutoField(verbose_name='ID', primary_key=True, serialize=False, help_text='', auto_created=True)),
                 ('dt', models.DateField(help_text='')),
-                ('counts', djorm_pgarray.fields.ArrayField(blank=True, null=True, default=None, help_text='')),
+                ('counts', django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(blank=True), blank=True, size=None)),
                 ('channel', models.ForeignKey(help_text='', to='bots.Channel')),
             ],
             options={

botbot/apps/bots/models.py

@@ -2,14 +2,14 @@
 import random
 import string
 import uuid
+from collections import OrderedDict
 
 from django.core.cache import cache
 from django.db import models
 from django.db.models import Max, Min
 from django.db.models.aggregates import Count
-from django.utils.datastructures import SortedDict
 from django.utils.text import slugify
-from djorm_pgarray.fields import ArrayField
+from django.contrib.postgres.fields.array import ArrayField
 
 from botbot.apps.plugins import models as plugins_models
 from botbot.apps.plugins.models import Plugin, ActivePlugin
@@ -259,7 +259,7 @@ def filtered_logs(self):
 
     def get_months_active(self):
         """
-        Creates a SortedDict of the format:
+        Creates a OrderedDict of the format:
         {
             ...
             '2010': {
@@ -277,14 +277,14 @@ def get_months_active(self):
                 last_log=Max("timestamp"),
                 first_log=Min("timestamp"))
             if not minmax_dict['first_log']:
-                return SortedDict()
+                return OrderedDict()
             # cache for 10 days
             cache.set(minmax_dict_key, minmax_dict, 864000)
         first_log = minmax_dict['first_log'].date()
         last_log = minmax_dict['last_log'].date()
         last_log = datetime.date(last_log.year, last_log.month, 1)
         current = datetime.date(first_log.year, first_log.month, 1)
-        months_active = SortedDict()
+        months_active = OrderedDict()
         while current <= last_log:
             months_active.setdefault(current.year, []).append(current)
             if current.month == 12:
@@ -337,7 +337,7 @@ class UserCount(models.Model):
 
     channel = models.ForeignKey(Channel)
     dt = models.DateField()
-    counts = ArrayField(dbtype="int")
+    counts = ArrayField(models.IntegerField(blank=True), blank=True)
 
     def __unicode__(self):
         return "{} on {}: {}".format(self.channel, self.dt, self.counts)

botbot/apps/bots/urls.py

@@ -1,8 +1,8 @@
-from django.conf.urls import patterns, url
+from django.conf.urls import url
 
 from . import views
 
-urlpatterns = patterns('',
+urlpatterns = [
     url(r'^manage/$', views.ManageChannel.as_view(), name='manage_channel'),
     url(r'^delete/$', views.DeleteChannel.as_view(), name='delete_channel'),
-)
+]

botbot/apps/logs/templatetags/logs_tags.py

@@ -1,15 +1,16 @@
 """Near duplicate of Django's `urlizetrunc` with support for image classes"""
 import urlparse
 
-from django.template.base import Library, Node
+from django.template.base import Node
+from django.template.library import Library
 from django.template.defaultfilters import stringfilter
 from django.utils.safestring import mark_safe, SafeData
 from django.utils.encoding import force_text
 from django.utils.functional import allow_lazy
 from django.utils import six
-from django.utils.html import (TRAILING_PUNCTUATION, WRAPPING_PUNCTUATION,
+from django.utils.html import (TRAILING_PUNCTUATION_CHARS, WRAPPING_PUNCTUATION,
                                word_split_re, simple_url_re, smart_urlquote,
-                               simple_url_2_re, simple_email_re, escape)
+                               simple_url_2_re, escape)
 import re
 
 
@@ -19,6 +20,8 @@
 IMAGE = 1
 YOUTUBE = 2
 
+email_re = re.compile(r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)")
+
 
 @register.filter(is_safe=True, needs_autoescape=True)
 @stringfilter
@@ -176,7 +179,7 @@ def urlize_impl(text, trim_url_limit=None, nofollow=False, autoescape=False):
             if '.' in word or '@' in word or ':' in word:
                 # Deal with punctuation.
                 lead, middle, trail = '', word, ''
-                for punctuation in TRAILING_PUNCTUATION:
+                for punctuation in TRAILING_PUNCTUATION_CHARS:
                     if middle.endswith(punctuation):
                         middle = middle[:-len(punctuation)]
                         trail = punctuation + trail

botbot/apps/logs/urls.py

@@ -1,8 +1,8 @@
-from django.conf.urls import patterns, url
+from django.conf.urls import url
 
 from . import views
 
-urlpatterns = patterns('',
+urlpatterns = [
     url(r'(?P<year>\d{4})-(?P<month>0[1-9]|1[0-2])-(?P<day>0[1-9]|[1-2][0-9]|3[0-1])/$',
         views.DayLogViewer.as_view(), name="log_day"),
     url(r'(?P<year>\d{4})-(?P<month>0[1-9]|1[0-2])-(?P<day>0[1-9]|[1-2][0-9]|3[0-1]).log$',
@@ -18,4 +18,4 @@
     url(r'^stream/$', views.LogStream.as_view(), name='log_stream'),
     url(r'^$', views.DayLogViewer.as_view(),
         name="log_current"),
-)
+]

botbot/apps/logs/views.py

@@ -171,7 +171,7 @@ def _timeline_context(timeline):
 
     # the last month in the timeline needs special treatment so it
     # doesn't get ordered ahead of the last/current weeks
-    last_month = timeline[timeline.keyOrder[-1]].pop()
+    last_month = timeline.popitem(last=True)[1][-1]
     if last_month >= last_week:
         last_month_adjusted = (last_week -
                                datetime.timedelta(days=1))

botbot/apps/plugins/migrations/0002_auto_20140912_1656.py

@@ -2,11 +2,11 @@
 from __future__ import unicode_literals
 
 from django.db import models, migrations
-from django.db.models.loading import get_model
+from django.apps import apps
 
 
 def initial_plugins(*args):
-    Plugin = get_model('plugins', 'Plugin')
+    Plugin = apps.get_model('plugins', 'Plugin')
 
     intial = [
         {

botbot/apps/plugins/models.py

@@ -1,7 +1,7 @@
 from django.core.cache import cache
 from django.contrib.admindocs.utils import trim_docstring
 from django.db import models
-from django.utils.importlib import import_module
+from importlib import import_module
 
 from botbot.core.fields import JSONField
 

botbot/apps/sitemap/urls.py

@@ -1,7 +1,7 @@
 """
 Site map URLs
 """
-from django.conf.urls import patterns, url
+from django.conf.urls import url
 from django.contrib import sitemaps
 from django.contrib.sitemaps.views import sitemap
 from django.core.urlresolvers import reverse
@@ -27,9 +27,9 @@ def location(self, item):
     'static': StaticSitemap,
 }
 
-urlpatterns = patterns('',
+urlpatterns = [
     url(r'^$', cache_page(86400)(sitemap), {'sitemaps': sitemaps},
         name='sitemap'),
-)
+]
 
 

botbot/core/fields.py

@@ -3,18 +3,10 @@
 from django.db import models
 from django.core.serializers.json import DjangoJSONEncoder
 
-from django.forms import fields
-from django.forms.util import ValidationError
-
-
-
 class JSONField(models.TextField):
     """JSONField is a generic textfield that neatly serializes/unserializes
     JSON objects seamlessly"""
 
-    # Used so to_python() is called
-    __metaclass__ = models.SubfieldBase
-
     def to_python(self, value):
         """Convert our string value to JSON after we load it from the DB"""
         if value == "":
@@ -42,9 +34,3 @@ def value_from_object(self, obj):
         if self.null and value is None:
             return None
         return json.dumps(value)
-
-try:
-    from south.modelsinspector import add_introspection_rules
-    add_introspection_rules([], ["^botbot\.core\.fields\.JSONField"])
-except ImportError:
-    pass

botbot/jinja2.py

@@ -21,7 +21,7 @@ def environment(**options):
         "jinja2.ext.autoescape",
         "jinja2.ext.with_",
         "jinja2.ext.i18n",
-        'pipeline.templatetags.ext.PipelineExtension',
+        'pipeline.jinja2.PipelineExtension',
         'django_jinja.builtins.extensions.CacheExtension',
     ]
     env = Environment(**options)

botbot/settings/_asset_pipeline.py

@@ -1,6 +1,3 @@
-PIPELINE_CSS_COMPRESSOR = ''
-PIPELINE_JS_COMPRESSOR = ''
-
 PIPELINE_CSS = {
     'screen': {
         'source_filenames': ('css/screen.css',),
@@ -47,3 +44,10 @@
         'output_filename': 'channel.js',
     },
 }
+
+PIPELINE = {
+    'CSS_COMPRESSOR': 'pipeline.compressors.NoopCompressor',
+    'JS_COMPRESSOR': 'pipeline.compressors.NoopCompressor',
+    'STYLESHEETS': PIPELINE_CSS,
+    'JAVASCRIPT': PIPELINE_JS,
+}

botbot/settings/base.py

@@ -35,6 +35,7 @@
     'botbot.core',
 
     'pipeline',
+    'whitenoise.runserver_nostatic',
 
     'django.contrib.auth',
     'django.contrib.contenttypes',
@@ -167,6 +168,7 @@
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
     'botbot.core.middleware.TimezoneMiddleware',
+    'whitenoise.middleware.WhiteNoiseMiddleware',
 ]
 
 #==============================================================================
@@ -199,7 +201,7 @@
     'handlers': {
         'null': {
             'level': 'DEBUG',
-            'class': 'django.utils.log.NullHandler',
+            'class': 'logging.NullHandler',
         },
         'console': {
             'level': 'DEBUG',

botbot/urls/admin.py

@@ -1,9 +1,9 @@
-from django.conf.urls import patterns, include
+from django.conf.urls import include, url
 from django.contrib import admin
 
 admin.autodiscover()
 
-urlpatterns = patterns('',
-    (r'^admin/doc/', include('django.contrib.admindocs.urls')),
-    (r'^admin/', include(admin.site.urls)),
-)
+urlpatterns = [
+    url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
+    url(r'^admin/', include(admin.site.urls)),
+]

botbot/urls/base.py

@@ -1,29 +1,30 @@
 from django.conf import settings
-from django.conf.urls import patterns, url, include
+from django.conf.urls import url, include
+from django.shortcuts import render
 
 from botbot.apps.bots.views import ChannelList
 from botbot.apps.preview.views import LandingPage
 
-channel_patterns = patterns('',
+channel_patterns = [
     url(r'', include('botbot.apps.logs.urls')),
-)
+]
 
-urlpatterns = patterns('',
-                        (r'^$', LandingPage.as_view()),
-                        url(r'^sitemap\.xml$', include('botbot.apps.sitemap.urls')),
+urlpatterns = [
+                        url(r'^$', LandingPage.as_view()),
+                        url(r'^sitemap\.xml', include('botbot.apps.sitemap.urls')),
 
                         url(r'^(?P<bot_slug>[\-\w\:\.]+(\@[\w]+)?)/(?P<channel_slug>[\-\w\.]+)/',
                             include(channel_patterns)),
                         url(r'^(?P<network_slug>[\-\w\.]+)/$', ChannelList.as_view())
-                        )
+                        ]
 
 if settings.INCLUDE_DJANGO_ADMIN:
     from .admin import urlpatterns as admin_urlpatterns
     # Prepend the admin urls.
     urlpatterns = admin_urlpatterns + urlpatterns
 
 if settings.DEBUG:
-    urlpatterns += patterns('django.shortcuts',
-        url(r'^404/$', 'render', {'template_name': '404.html'}),
-        url(r'^500/$', 'render', {'template_name': '500.html'}),
-    )
+    urlpatterns += [
+        url(r'^404/$', lambda r: render(r, template_name='404.html')),
+        url(r'^500/$', lambda r: render(r, template_name='500.html')),
+    ]

docker/services/uwsgi/consul-template-uwsgi.conf

@@ -4,7 +4,7 @@ template {
 }
 
 exec {
-    command = "env $(cat /srv/botbot-web/environment | grep -v ^# | xargs) uwsgi --die-on-term /etc/uwsgi/uwsgi.ini"
+    command = "env $(cat /srv/botbot-web/environment | grep -v ^# | xargs) python /srv/botbot-web/manage.py collectstatic --noinput && env $(cat /srv/botbot-web/environment | grep -v ^# | xargs) uwsgi --die-on-term /etc/uwsgi/uwsgi.ini"
     splay = "5s"
     reload_signal = "SIGHUP"
     kill_signal = "SIGTERM"

environment.ctmpl

@@ -36,4 +36,5 @@ DJANGO_SETTINGS_MODULE=botbot.settings
 ALLOWED_HOSTS=chatlogs.metabrainz.org,test-chatlogs.metabrainz.org
 PYTHONIOENCODING=utf8
 LANG=en_US.UTF-8
-DEBUG=False
+DEBUG=False
+VAR_ROOT=/data

requirements.txt

@@ -1,25 +1,25 @@
-django==1.8.4
+django==1.11.29
 
 pytz
 Jinja2==2.7.3
-django-jinja==1.4.1
+django-jinja==2.6.0
 psycopg2==2.7.3.2
 dj-database-url==0.2.2
 # TODO (yml): Remove hstore when we squash the migration
 django-hstore==1.2.5
 
 markdown==2.3.1
 redis==2.9.1
-django-pipeline==1.5.1
+django-pipeline==1.7.0
 honcho==0.5.0
 
-djorm-ext-pgfulltext==0.9.0
-djorm-ext-pgarray==0.9.0
+djorm-ext-pgfulltext==0.10
+djorm-pgarray==1.2
 
-django-allauth==0.20.0
-    requests==2.7.0
-    oauthlib==0.7.2
-    requests-oauthlib==0.5.0
+django-allauth==0.40.0
+    requests==2.23.0
+    oauthlib==3.1.0
+    requests-oauthlib==1.3.0
 django-bootstrap-toolkit==2.15.0
 
 geoip2==2.1.0
@@ -30,3 +30,6 @@ geoip2==2.1.0
 # For plugins
 -e git+https://github.com/metabrainz/brainzbot-plugins.git#egg=brainzbot-plugins
 
+# For production
+uwsgi==2.0.21
+whitenoise==4.1.4