made more generic "request_logging" app

Author: evgenyfadeev

README.rst

@@ -0,0 +1,46 @@
+Log django request data for the diagnostic purposes
+---------------------------------------------------
+
+All requests matching filter functions will be logged
+in the database.
+
+The following data is logged::
+* user (if authenticated)
+* IP address
+* request path (url)
+* request method
+* request parameters
+* session key
+* session snapshot
+* request META dictionary
+* COOKIES dictionary
+
+Filters are functions defined via settings parameter
+`REQUEST_LOGGING_FILTERS`. For example::
+
+    from myapp.request_logging_filters import post_from_some_ips
+
+    REQUEST_LOGGING_FILTERS = (
+        'myapp.request_logging_filters.authenticated_posts',
+        posts_from_some_ips
+    )
+
+Filters may be defined either as a callable function or
+as a python dotted path to such function.
+Filter functions must take `request` instance as argument
+and return either `True` or `False`, when the request
+matches the filter or not, correspondingly.
+
+.. note::
+    Keep in mind that `request_logging` will store data
+    in the database which will take storage space and
+    computing resources. In production use sparingly,
+    for diagnostinc purposes.
+
+Configuration settings
+======================
+
+The following `settings.py` entries are available::
+* `REQUEST_LOGGING_FILTERS` - a tuple of filter functions or dotted paths.
+* `REQUEST_LOGGING_HIDE_PARAMETERS` - a tuple of parameter names that will be hidden
+   (e.g. `('password', 'credit_card_number')`)

askbot_log/admin.py

@@ -0,0 +1,52 @@
+from django.contrib import admin
+from .models import RequestLog
+from .utils import format_as_table
+
+class RequestLogAdmin(admin.ModelAdmin):
+    fieldsets = (
+        (None, {
+            'fields': ('user', 'url', 'timestamp',
+                'method', 'parameters_disp', 'ip_addr')
+        }),
+        ('Session', {
+            'fields': ('session_key', 'session_data_disp'),
+            'classes': ('collapse',)
+        }),
+        ('Cookies', {
+            'fields': ('cookies_disp',),
+            'classes': ('collapse',)
+        }),
+        ('Request META', {
+            'fields': ('meta_disp',),
+            'classes': ('collapse',)
+        })
+    )
+    readonly_fields = (
+        'user', 'timestamp', 'ip_addr', 'method', 'url', 'parameters_disp',
+        'cookies_disp', 'meta_disp', 'session_key',
+        'session_data_disp'
+    )
+
+    list_display = ('user', 'url', 'timestamp')
+
+    def parameters_disp(self, instance):
+        return format_as_table(instance.parameters)
+    parameters_disp.allow_tags = True
+    parameters_disp.short_description = 'Parameters'
+
+    def cookies_disp(self, instance):
+        return format_as_table(instance.cookies)
+    cookies_disp.allow_tags = True
+    cookies_disp.short_description = 'Cookies'
+
+    def meta_disp(self, instance):
+        return format_as_table(instance.meta)
+    meta_disp.allow_tags = True
+    meta_disp.short_description = 'Meta'
+
+    def session_data_disp(self, instance):
+        return format_as_table(instance.session_data)
+    session_data_disp.allow_tags = True
+    session_data_disp.short_description = 'Session data'
+
+admin.site.register(RequestLog, RequestLogAdmin)

askbot_log/middleware/__init__.py

@@ -0,0 +1,9 @@
+from django.conf import settings
+from appconf import AppConf
+
+class RequestLoggingConf(AppConf):
+    FILTERS = ()
+    HIDE_PARAMETERS = ()
+
+    class Meta:
+        prefix = 'request_logging'

askbot_log/middleware/watch_users.py

@@ -0,0 +1,72 @@
+"""Middleware that creates log records
+for watched users"""
+from .models import RequestLog
+from .conf import settings
+from .utils import clean_dict, replace_dict_values, import_attribute
+from django.conf import settings as django_settings
+import inspect
+
+class RequestLoggingMiddleware(object):
+    def process_request(self, request):
+        for test_filter in self.get_filters():
+            callable_filter = self.get_callable_filter(test_filter)
+            filter_name = self.get_filter_name(test_filter)
+            if callable_filter(request):
+                self.log_request(request, filter_name)
+                return
+
+    @classmethod 
+    def get_filters(cls):
+        #todo: get filters configurable via django admin
+        return settings.REQUEST_LOGGING_FILTERS
+
+    @classmethod
+    def get_callable_filter(cls, path_or_filter):
+        if callable(path_or_filter):
+            return path_or_filter
+        try: 
+            return import_attribute(path_or_filter)
+        except ImportError:
+            ImproperlyConfigured(
+                '{} is not an importable python path'.format(path_or_filter)
+            )
+
+    @classmethod
+    def get_filter_name(cls, path_or_filter):
+        if isinstance(path_or_filter, basestring):
+            return path_or_filter
+        elif inspect.isfunction(path_or_filter):
+            funcname = path_or_filter.__name__
+            module = inspect.getmodule(path_or_filter)
+            if module:
+                modname = module.__name__
+                return '{}.{}'.format(modname, funcname)
+            return funcname
+        raise ImproperlyConfigured(
+            'Invalid filter {}, must be dotted path'
+            'or function'.format(unicode(path_or_filter))
+        )
+
+    @classmethod
+    def log_request(cls, request, filter_name):
+        """logs request data"""
+        log = RequestLog()
+        log.filter_name = filter_name
+        log.ip_addr = request.META['REMOTE_ADDR']
+        user = getattr(request, 'user', None)
+        if user.is_authenticated():
+            log.user = user
+        log.url = request.path
+        parameters = clean_dict(request.REQUEST)
+        replace_dict_values(
+            parameters,
+            settings.REQUEST_LOGGING_HIDE_PARAMETERS,
+            '********'
+        )
+        log.parameters = parameters
+        log.session_key = request.session.session_key
+        log.session_data = clean_dict(request.session)
+        log.method = request.method
+        log.meta = clean_dict(request.META)
+        log.cookies = clean_dict(request.COOKIES)
+        log.save()

askbot_log/models.py

@@ -0,0 +1,19 @@
+from django.db import models
+from jsonfield import JSONField
+from django.contrib.auth.models import User
+
+class RequestLog(models.Model):
+    user = models.ForeignKey(User, null=True, blank=True)
+    filter_name = models.CharField(max_length=255)
+    ip_addr = models.GenericIPAddressField()
+    url = models.TextField()
+    parameters = JSONField()
+    session_key = models.CharField(max_length=40)
+    session_data = JSONField()
+    method = models.CharField(max_length=16)
+    meta = JSONField()
+    cookies = JSONField()
+    timestamp = models.DateTimeField(auto_now_add=True)
+
+    def __unicode__(self):
+        return u'%s %s %s' % (self.user.username, self.ip_addr, self.url)

askbot_log/__init__.py request_logging/__init__.py

@@ -0,0 +1,49 @@
+"""Utilities for the django_request_logging module"""
+from django.utils.safestring import mark_safe
+
+def clean_dict(data):
+    """
+    converts all items to unicode, so that
+    it can de json serialized
+    """
+    data = dict(data)
+    for key, value in data.items():
+        data[key] = unicode(value)
+    return data
+
+def format_as_table(data):
+    """
+    formats dictionary as HTML table
+    """
+    output = ''
+    for key in data:
+        row = '<tr><td>%s</td><td>%s</td></tr>' % (key, data[key])
+        output += row
+    return mark_safe('<table>'+ output + '</table>')
+
+def replace_dict_values(the_dict, keys, replacement):
+    """
+    replaces values in dictionary for keys
+    if present in the dictionary
+    """
+    for key in keys:
+        if key in the_dict:
+            the_dict[key] = replacement
+
+
+def import_attribute(path):
+    """
+    given path x.y.z
+    does the same as statement
+    from x.y import z
+
+    Code copied from appconfig.utils.import_attribute and simplified
+    https://github.com/jezdez/django-appconf/blob/develop/appconf/utils.py
+    """
+    try:
+        from importlib import import_module
+    except ImportError:
+        from django.utils.importlib import import_module
+    module_name, object_name = path.rsplit('.', 1)
+    module = import_module(module_name)
+    return getattr(module, object_name)

request_logging/admin.py

@@ -4,9 +4,9 @@
 import sys
 
 setup(
-    name = "askbot_log",
-    version = '0.0.1',
-    description = 'Log post activity of watched users',
+    name = "request_logging",
+    version = '0.1.0',
+    description = 'Log requests in the database for the diagrostic purposes',
     packages = find_packages(),
     author = 'Evgeny.Fadeev',
     author_email = '[email protected]',