Sphinx utils for drf serializers and http request log

Author: lezeroq

README.md

@@ -257,6 +257,26 @@ Usage: execfile('path/to/file.py', globals(), locals())
 
 Returns: True if file exists and executed, False if file doesn't exist
 
+
+## tipsi_tools.doc_utils.tipsi_sphinx
+
+Sphinx extensions to generate documentation for django restframework serializers and examples for http requests.
+
+In order to use them specify dependency for package installation:
+```
+pip install tipsi_tools[doc_utils]
+```
+
+Usage:
+```
+# Add to Sphinx conf.py
+extensions = [
+    # ...
+    'tipsi_tools.doc_utils.tipsi_sphinx.dyn_serializer',
+    'tipsi_tools.doc_utils.tipsi_sphinx.http_log'
+]
+```
+
 ## Commands
 
 ### tipsi_env_yaml
@@ -300,3 +320,20 @@ tipsi_wait -t 30 nginx 80
 ```
 run_filebeat -e CHECKME=VALUE path_to_template
 ```
+
+
+### doc_serializer
+
+* output rst with list of serializers
+* generates documentation artifacts for serializers
+
+```
+usage: doc_serializer [-h] [--rst] [--artifacts]
+
+Parse serializers sources
+
+optional arguments:
+  -h, --help   show this help message and exit
+  --rst        Output rst with serializers
+  --artifacts  Write serializers artifacts
+```

setup.py

@@ -20,6 +20,13 @@
         'pyyaml>=3.12',
         'python-json-logger>=0.1.5',
     ],
+    extras_require={
+        'doc_utils': [
+            'rest_framework_dyn_serializer>=1.3.*',
+            'docutils',
+            'djangorestframework==3.7.*',
+        ],
+    },
     tests_require=[
         'pytest==3.1.3',
     ],
@@ -29,6 +36,7 @@
             'tipsi_ci_script=tipsi_tools.scripts.tipsi_ci_script:main',
             'tipsi_wait=tipsi_tools.scripts.tipsi_tools_wait:main',
             'run_filebeat=tipsi_tools.scripts.run_filebeat:main',
+            'doc_serializer=tipsi_tools.doc_utils.tipsi_sphinx.dyn:main',
         ],
     },
     classifiers=[

tipsi_tools/__init__.py

@@ -1 +1 @@
-__version__ = '1.15.0'
+__version__ = '1.16.0'

tipsi_tools/doc_utils/__init__.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+
+import argparse
+import inspect
+import json
+import os
+from functools import partial
+from importlib import import_module
+
+import sys
+from rest_framework_dyn_serializer import DynModelSerializer
+
+from tipsi_tools.doc_utils.tipsi_sphinx.dyn_json import serializer_doc_info
+
+
+def compose(*funs):
+    it = funs[-1]
+    funs = funs[:-1]
+
+    def inner(*args, **kwargs):
+        for res in it(*args, **kwargs):
+            for fun in funs:
+                res = fun(res)
+            if inspect.isgenerator(res):
+                for r in res:
+                    yield r
+            else:
+                yield res
+
+    return inner
+
+
+def get_modules(apps):
+    for app in apps:
+        for mod in ('serializers', 'forms'):
+            try:
+                yield import_module('{}.{}'.format(app, mod))
+            except ImportError:
+                pass
+            except AttributeError:
+                pass
+
+
+def get_dynserializers(module):
+    filters = (
+        lambda i: inspect.isclass(i),
+        lambda i: i != DynModelSerializer,
+        lambda i: not(hasattr(i, 'Meta') and getattr(i.Meta, 'abstract', False)),
+        lambda i: issubclass(i, DynModelSerializer) or getattr(i, '_write_docs', False),
+    )
+
+    for item in dir(module):
+        item = getattr(module, item)
+        if all(f(item) for f in filters):
+            yield item
+
+
+class ArtifactWriter:
+    _inst = None
+
+    def __init__(self):
+        path = os.environ.get('DOCS_ROOT', './.doc')
+        if not os.path.exists(path):
+            os.mkdir(path)
+        self.doc_root = path
+
+    @classmethod
+    def instance(cls):
+        if not cls._inst:
+            cls._inst = cls()
+        return cls._inst
+
+    def dump(self, data, path):
+        f_name = os.path.join(self.doc_root, '{}.json'.format(path))
+        with open(f_name, 'w') as f:
+            json.dump(data, f)
+
+
+def container_type():
+    container = os.environ.get('CONTAINER_TYPE')
+    if not container:
+        sys.exit('Error: CONTAINER_TYPE is not defined')
+    return container
+
+
+def process_item(item, flags):
+    path = '{}.{}'.format(item.__module__, item.__name__)
+    if flags.rst:
+        print('.. dyn_serializer:: {}/{}\n'.format(container_type(), path))
+    if flags.artifacts:
+        data = serializer_doc_info(item, path)
+        ArtifactWriter.instance().dump(data, path)
+
+
+def process(apps, fun):
+    f = compose(get_dynserializers, get_modules)(apps)
+    for s in f:
+        fun(s)
+
+
+parser = argparse.ArgumentParser(description='Parse serializers sources')
+
+parser.add_argument('--rst', action='store_true', default=False,
+                    help='Output rst with serializers')
+parser.add_argument('--artifacts', action='store_true', default=False,
+                    help='Write serializers artifacts')
+
+
+def main():
+    sys.path.append('.')
+    args = parser.parse_args()
+
+    if not any((args.rst, args.artifacts, )):
+        parser.print_help()
+
+    import django
+    django.setup()
+    from django.apps import apps
+
+    all_apps = (app.module.__name__ for app in apps.get_app_configs())
+
+    process(all_apps, partial(process_item, flags=args))
+
+
+if __name__ == '__main__':
+    main()

tipsi_tools/doc_utils/tipsi_sphinx/__init__.py

@@ -0,0 +1,88 @@
+from rest_framework import fields
+from rest_framework.serializers import ListSerializer
+from rest_framework_dyn_serializer import DynModelSerializer
+
+primitive_types = {
+    fields.IntegerField: 'int',
+    fields.BooleanField: 'bool',
+    fields.FloatField: 'float',
+    fields.CharField: 'str',
+    fields.ImageField: 'http-link',
+    fields.DateTimeField: 'datetime',
+    fields.DateField: 'date',
+    fields.JSONField: 'json/object',
+}
+
+
+def serializer_doc_info(serializer, path_info=''):
+    print('Run: {}'.format(path_info))
+
+    if not hasattr(serializer, 'Meta'):
+        serializer.Meta = type("Meta", tuple(), {})
+    if not hasattr(serializer.Meta, 'fields_param'):
+        serializer.Meta.fields_param = 'any'
+
+    serializer_obj = serializer()
+
+    return {
+        'class_name': serializer.__name__,
+        'class_module': serializer.__module__,
+        'field_param': serializer.Meta.fields_param,
+        'fields': [
+            process_fields(name, field, serializer_obj)
+            for name, field in sorted(serializer_obj.fields.items())
+        ]
+    }
+
+
+def process_fields(name, field, serializer):
+    cls = field.__class__
+
+    result = {
+        'name': name,
+        'readonly': bool(field.read_only),
+        'write_only': bool(field.write_only),
+        'required': bool(field.required),
+        'allow_null': bool(field.allow_null),
+        'is_dyn_serializer': issubclass(cls, DynModelSerializer),
+        '__field__': repr(field),
+        '__class__': repr(cls),
+    }
+
+    _type = {}
+
+    if cls in primitive_types:
+        _type['primitive_type'] = primitive_types[cls]
+    elif cls == fields.SerializerMethodField:
+        method_name = field.method_name or 'get_{}'.format(name)
+        method = getattr(serializer, method_name)
+        _type['method_type'] = {
+            'method_name': method_name,
+            'method_doc': method.__doc__ if method and method.__doc__ else None,
+        }
+    elif issubclass(cls, DynModelSerializer):
+        _type['dyn_field_type'] = {
+            'ref_name': cls.__name__,
+        }
+    elif (issubclass(cls, ListSerializer) and
+          isinstance(field.child, DynModelSerializer)):
+        _type['list_field_type'] = {
+            'ref_name': field.child.__class__.__name__,
+        }
+    else:
+        model_doc = None
+        if hasattr(serializer, 'Meta') and hasattr(serializer.Meta, 'model'):
+            # parse docstring in custom model methods
+            fun = getattr(serializer.Meta.model, name, {})
+            model_doc = fun.__doc__
+
+        serializer_doc = field.__doc__
+        if model_doc or serializer_doc:
+            _type['doc_type'] = {
+                'model_doc': model_doc,
+                'serializer_doc': serializer_doc,
+            }
+
+    result['type'] = _type
+    print('Process: <SerializerField: {}/>'.format(name))
+    return result