support for testing via pytest

Python source code generation via Sphynx

.gitignore

@@ -1,4 +1,5 @@
 .DS_Store
+.cache
 .idea/
 .virtual_env/
 virtual_env/
@@ -19,3 +20,8 @@ alembic.ini
 
 # webui2
 target/
+
+Session.vim
+pytest.output
+sphinx/source/apidoc-*/
+sphinx/pyramid_autodoc

README.md

@@ -88,7 +88,7 @@ If you are using less recent distro version, replace `xenial` with `trusty` or `
              * `effective_cache_size` - 1/8 of total RAM (e.g. 4096MB)
              * `wal_level = replica`
      * Restart postgres: `sudo service postgresql restart`
-     * Import the latest database backup (from-scratch creation look in `# from psql` section): `sudo -u posgres psql < lingvodoc.sql`
+     * Import the latest database backup (from-scratch creation look in `# from psql` section): `sudo -u postgres psql lingvodoc < lingvodoc.sql`
 2. Install Redis: `sudo apt-get install redis-server`.
 3. Install compilers, libraries and git: `sudo apt-get install build-essential python3-dev libssl-dev git`
 4. Install venv-creator: `sudo apt-get install python3-virtualenv`
@@ -164,3 +164,34 @@ API documentation
 /authors
 /sound
 /markup
+
+Testing
+-------
+
+Testing is performed via [pytest](http://doc.pytest.org). Installation of required packages:
+```
+pip install -r develop-requirements.txt
+```
+
+Running tests from the root project directory:
+```
+$VENV/bin/py.test
+```
+
+Documentation
+-------------
+
+Documentation for Python source code, including tests, can be generated via [Sphinx](http://www.sphinx-doc.org). Installation of required packages:
+```
+pip install -r develop-requirements.txt
+```
+
+Generating documentation from the root project directory:
+```
+make -f sphinx/Makefile html
+```
+
+Generated documentation can be accessed at `sphinx/build/html/index.html`. It includes all Python packages, modules, classes and functions of the project; descriptions of packages, modules, classes and functions are automatically produced from their docstrings.
+
+Documentation also includes an up-to-date list of Pyramid REST API routes provided by the application, extracted via [pyramid\_autodoc](https://github.com/SurveyMonkey/pyramid_autodoc) Sphinx extension.
+

alembictests.ini

@@ -7,7 +7,7 @@
 # path to migration scripts
 script_location = alembic
 
-sqlalchemy.url =  postgresql+psycopg2://postgres:@localhost/lingvodoc_testing
+sqlalchemy.url =  postgresql+psycopg2://postgres@/lingvodoc_testing
 ;sqlalchemy.url = sqlite:////home/student/projects/lingvodoc/lingvodoc_testing.sqlite
 ;sqlalchemy.url = sqlite:///lingvodoc_testing.sqlite
 
@@ -50,7 +50,7 @@ pyramid.includes =
     pyramid_debugtoolbar
     pyramid_tm
 
-sqlalchemy.url = postgresql+psycopg2://postgres:@localhost/lingvodoc_testing
+sqlalchemy.url = postgresql+psycopg2://postgres@/lingvodoc_testing
 ;sqlalchemy.url = sqlite:////home/student/projects/lingvodoc/lingvodoc_testing.sqlite
 ;sqlalchemy.url = sqlite:///lingvodoc_testing.sqlite
 
@@ -63,15 +63,15 @@ sqlalchemy.url = postgresql+psycopg2://postgres:@localhost/lingvodoc_testing
 ###
 
 [server:main]
-;use = egg:waitress#main
-use = egg:gunicorn#main
-workers = 3
-timeout = 3000
-proc_name = lingvodoc
-bind = "0.0.0.0:6543"
+use = egg:waitress#main
+;use = egg:gunicorn#main
+;workers = 3
+;timeout = 3000
+;proc_name = lingvodoc
+;bind = "0.0.0.0:6543"
 host = 0.0.0.0
 port = 6543
-max_request_body_size = 2147483648
+;max_request_body_size = 2147483648
 
 ###
 # logging configuration
@@ -154,17 +154,17 @@ no-orphans = true
 
 log-slow = true
 
-virtualenv = /Users/al/environments/lingvodocenv/
+#virtualenv = /Users/al/environments/lingvodocenv/
 
 callable = lingvodoc
 
 
-[cache:dogpile]
-backend = dogpile.cache.redis
+#[cache:dogpile]
+#backend = dogpile.cache.redis
 
-[cache:dogpile:args]
-host: localhost
-port: 6379
-db: 0
-redis_expiration_time: 60
-distributed_lock: True
+#[cache:dogpile:args]
+#host: localhost
+#port: 6379
+#db: 0
+#redis_expiration_time: 60
+#distributed_lock: True

develop-requirements.txt

@@ -0,0 +1,4 @@
+Sphinx
+pytest
+simplejson
+webtest

development.ini

@@ -6,6 +6,8 @@
 [app:main]
 use = egg:lingvodoc
 
+secret = 'secret string'
+
 pyramid.reload_templates = true
 pyramid.debug_authorization = false
 pyramid.debug_notfound = false
@@ -15,7 +17,7 @@ pyramid.includes =
     pyramid_debugtoolbar
     pyramid_tm
 
-sqlalchemy.url = sqlite:///%(here)s/lingvodoc.sqlite
+sqlalchemy.url = postgresql+psycopg2://postgres@/lingvodoc
 
 # By default, the toolbar only appears for clients from IP addresses
 # '127.0.0.1' and '::1'.
@@ -45,7 +47,7 @@ keys = console
 keys = generic
 
 [logger_root]
-level = INFO
+level = DEBUG
 handlers = console
 
 [logger_lingvodoc]
@@ -78,21 +80,37 @@ administrator_password = password
 
 [backend:storage]
 # disk or openstack
-storagetype = disk
-path = /tmp
-authurl = http://10.10.10.121:5000/v2.0
-store = http://adelaide.intra.ispras.ru/horizon/project/containers
-user = admin
-key = tester
-auth_version = 2.0
-tenant_name = admin
-
-[cache:dogpile]
-backend = dogpile.cache.redis
-
-[cache:dogpile:args]
-host: localhost
-port: 6379
-db: 0
-redis_expiration_time: 60
-distributed_lock: True
+type = disk
+path = /tmp/
+prefix = http://localhost:6543/
+static_route = objects/
+#authurl = http://10.10.10.121:5000/v2.0
+#store = http://adelaide.intra.ispras.ru/horizon/project/containers
+#user = admin
+#key = tester
+#auth_version = 2.0
+#tenant_name = admin
+
+[uwsgi]
+socket = 0.0.0.0:6543
+protocol = http
+master = true
+
+processes = 4
+
+harakiri = 60
+harakiri-verbose = true
+limit-post = 65536
+post-buffering = 8192
+
+listen = 256
+
+max-requests = 1000
+
+reload-on-as = 128
+reload-on-rss = 96
+no-orphans = true
+
+log-slow = true
+
+callable = lingvodoc

lingvodoc/__init__.py

@@ -24,6 +24,7 @@ def configure_routes(config):
     """
     This function registers views from .views for pyramid event loop. Actually all listed items here is our
     'site-map' for:
+
     1) web-views (html+javascript) - will be marked as 'web-view'
     2) REST API - will be marked as 'API'
     3) internal calls for frontend (in most cases it's API but not always). If it's not API part, will be marked
@@ -675,15 +676,18 @@ def main(global_config, **settings):
     config_file = global_config['__file__']
     parser = ConfigParser()
     parser.read(config_file)
+
     # TODO: DANGER
-    storage = dict()
-    for k, v in parser.items('backend:storage'):
-        storage[k] = v
-    settings['storage'] = storage
+
+    settings['storage'] = dict(parser.items(
+        'backend:storage' if parser.has_section('backend:storage') else
+            'storage'))
+
     if parser.has_section('app:desktop'):
         for k, v in parser.items('app:desktop'):
             storage[k] = v
         settings['desktop'] = storage
+
     config = Configurator(settings=settings)
     log = logging.getLogger(__name__)
 

lingvodoc/cache/caching.py

@@ -5,7 +5,9 @@
 from lingvodoc.cache.mock.cache import MockCache
 
 
-MEMOIZE = None
+# We initialize MEMOIZE to identity function so that if the cache is not initialized (e.g. when an
+# automatically extracted source code documentation is being compiled), it is still possible to use it.
+MEMOIZE = lambda x: x
 CACHE = None
 
 
@@ -52,3 +54,4 @@ def initialize_cache(args):
     region = make_region().configure(**args)
     MEMOIZE = cache_responses(region)
     CACHE = CommonCache(region)
+

lingvodoc/models.py

@@ -590,12 +590,14 @@ class Field(CompositeIdMixin,
             AdditionalMetadataMixin):
     """
     With this objects we specify allowed fields for dictionary perspective. This class is used for three purposes:
+
         1. To control final web-page view. With it we know which fields belong to perspective (and what we should
-          show on dictionary page.
+           show on dictionary page.
         2. Also we can know what entities should be grouped under the buttons (for example paradigms). Also we can
-          control connections between level-one and level-two entities. And we can control grouping entities (if we
-          want to have not only etymology connections).
+           control connections between level-one and level-two entities. And we can control grouping entities (if we
+           want to have not only etymology connections).
         3. With it we can restrict to use any entity types except listed here (security concerns).
+
     Parent: DictionaryPerspective.
     """
     marked_for_deletion = Column(Boolean, default=False, nullable=False)

lingvodoc/scripts/corpora_converter.py

@@ -130,12 +130,13 @@ def parse_word(self, word):
 
 #-----------------------------------------------------------------------
 
-element_tree = ElementTree.parse('corpus.xml')
-
-with open('corpus.json', 'w') as corpus_file:
-    simplejson.dump(
-        Corpus().parse_document(element_tree.getroot()),
-        corpus_file,
-        ensure_ascii=False, separators=(',', ':'), sort_keys = False)
+if __name__ == '__main__':
+    element_tree = ElementTree.parse('corpus.xml')
+
+    with open('corpus.json', 'w') as corpus_file:
+        simplejson.dump(
+            Corpus().parse_document(element_tree.getroot()),
+            corpus_file,
+            ensure_ascii=False, separators=(',', ':'), sort_keys = False)
 
 #-----------------------------------------------------------------------