approve outer & testing

alembic/env.py

@@ -65,7 +65,10 @@ def run_migrations_online():
         with context.begin_transaction():
             context.run_migrations()
 
-if context.is_offline_mode():
-    run_migrations_offline()
-else:
-    run_migrations_online()
+def upgrade():
+    if context.is_offline_mode():
+        run_migrations_offline()
+    else:
+        run_migrations_online()
+
+upgrade()

lingvodoc/__init__.py

@@ -437,6 +437,9 @@ def configure_routes(config):
     config.add_route(name='approve_all', pattern='/dictionary/{dictionary_client_id}/{dictionary_object_id}'
                                                     '/perspective/{perspective_client_id}/{perspective_id}/approve_all',
                      factory='lingvodoc.models.PerspectivePublishAcl')  # ready, not tested
+    config.add_route(name='approve_all_outer', pattern='/dictionary/{dictionary_client_id}/{dictionary_object_id}'
+                                                    '/perspective/{perspective_client_id}/{perspective_id}/approve_all_outer',
+                     factory='lingvodoc.models.PerspectivePublishAcl')  # ready, not tested
 
     # web-view
     config.add_route(name='edit_dictionary', pattern='/dictionary/{dictionary_client_id}/{dictionary_object_id}'

lingvodoc/models.py

@@ -359,7 +359,6 @@ def set_translation(self, request):
         search_translation_string = self.translation_string
         if not search_translation_string:
             search_translation_string = translation_string
-        print('SHEEEET', search_translation_string)
         uets = DBSession.query(UserEntitiesTranslationString).filter_by(locale_id=locale_id,
                                                                         translation_string=search_translation_string).first()
         if not translation:
@@ -373,12 +372,10 @@ def set_translation(self, request):
             self.translation_string = translation_string
             DBSession.add(uets)
             DBSession.flush()
-            print('HEY, LISTEN! WTF')
         else:
             uets.translation_string = translation_string
             self.translation_string = translation_string
             uets.translation = translation
-            print('HEY, LISTEN!')
         return
 
 

lingvodoc/scripts/approve.py

@@ -0,0 +1,92 @@
+import sqlite3
+import base64
+import requests
+import json
+import hashlib
+import logging
+
+
+def approve_batch(session, approve_url, entities):
+    status = session.patch(approve_url, json={'entities': entities})
+    return status
+
+
+def approve(session, server_url,dictionary_client_id, dictionary_object_id, client_id, object_id):
+    from time import time
+    start = time()
+    connect_url = server_url + 'dictionary/%s/%s/perspective/%s/%s/all_count' % (dictionary_client_id,
+                                                                                 dictionary_object_id,
+                                                                                 client_id, object_id)
+    status = session.get(connect_url)
+
+    count = json.loads(status.text)['count']
+    connect_url = server_url + 'dictionary/%s/%s/perspective/%s/%s/all?count=%s' % (dictionary_client_id,
+                                                                                 dictionary_object_id,
+                                                                                 client_id, object_id,
+                                                                                count)
+    approve_url = server_url + 'dictionary/%s/%s/perspective/%s/%s/approve' % (dictionary_client_id,
+                                                                                 dictionary_object_id,
+                                                                                 client_id, object_id)
+    perspective = session.get(connect_url)
+    perspective = json.loads(perspective.text)
+    entities = list()
+    for lexicalentry in perspective['lexical_entries']:
+        for entity in lexicalentry['contains']:
+            if entity['level'] in ['leveloneentity', 'groupingentity']:
+                not_publ = True
+                if entity.get('contains'):
+                    for entity2 in entity['contains']:
+                        if 'publ' in entity2['level']:
+                            if not entity2['marked_for_deletion']:
+                                not_publ = False
+                        else:
+                            if entity2['level'] == 'leveltwoentity':
+                                not_publ_2 = True
+                                if entity2.get('contains'):
+                                    for entity3 in entity2['contains']:
+                                        if 'publ' in entity3['level']:
+                                            if not entity3['marked_for_deletion']:
+                                                not_publ_2 = False
+                                                break
+                                    if not_publ_2:
+                                        entity2['type'] = entity2['level']
+                                        entities += [entity2]
+                                        if len(entities) > 100:
+                                            approve_batch(session, approve_url, entities)
+                                            entities = list()
+                if not_publ:
+                    entity['type'] = entity['level']
+                    entities += [entity]
+                    if len(entities) > 100:
+                        approve_batch(session, approve_url, entities)
+                        entities = list()
+    approve_batch(session, approve_url, entities)
+    return {'time': time() - start}
+
+
+def approve_all_outer(login, password_hash,dictionary_client_id, dictionary_object_id,
+                      perspective_client_id, perspective_object_id,  server_url="http://localhost:6543/"):
+    log = logging.getLogger(__name__)
+    log.debug("Starting convert_one")
+    log.debug("Creating session")
+    session = requests.Session()
+    session.headers.update({'Connection': 'Keep-Alive'})
+    adapter = requests.adapters.HTTPAdapter(pool_connections=3, pool_maxsize=3, max_retries=3)
+    session.mount('http://', adapter)
+    log.debug("Going to login")
+    login_data = {"login": login, "passwordhash": password_hash}
+    log.debug("Login data: " + login_data['login'] + login_data['passwordhash'])
+    cookie_set = session.post(server_url + 'cheatlogin', json=login_data)
+    log.debug("Login status:" + str(cookie_set.status_code))
+    if cookie_set.status_code != 200:
+        log.error("Cheat login for conversion was unsuccessful")
+        exit(-1)
+    try:
+        status = approve(session, server_url,
+                         dictionary_client_id, dictionary_object_id, perspective_client_id, perspective_object_id)
+    except Exception as e:
+        log.error("approving failed")
+        log.error(e.__traceback__)
+        raise
+    log.error(status)
+    return status

lingvodoc/scripts/initializedb.py

@@ -35,22 +35,11 @@ def usage(argv):
     sys.exit(1)
 
 
-def main(argv=sys.argv):
-    if len(argv) < 2:
-        usage(argv)
-    config_uri = argv[1]
-    options = parse_vars(argv[2:])
-    setup_logging(config_uri)
-    settings = get_appsettings(config_uri, options=options)
-    accounts = get_appsettings(config_uri, 'accounts')
-    engine = engine_from_config(settings, 'sqlalchemy.')
-
-    DBSession.configure(bind=engine)
-    # Base.metadata.create_all(engine)
-    with transaction.manager:
+def data_init(manager, accounts):
+    with manager:
         # Creating global administrator
         admin_account = DBSession.query(User).filter_by(login=accounts['administrator_login']).first()
-        print("Admin record not found, initializing")
+        # print("Admin record not found, initializing")
         admin_account = User(login=accounts['administrator_login'],
                              name="Администратор",
                              intl_name="System Administrator",
@@ -595,3 +584,18 @@ def main(argv=sys.argv):
                       the_page_morph, the_line_morph, notes_morph]:
             DBSession.add(field)
 
+
+def main(argv=sys.argv):
+    if len(argv) < 2:
+        usage(argv)
+    config_uri = argv[1]
+    options = parse_vars(argv[2:])
+    setup_logging(config_uri)
+    settings = get_appsettings(config_uri, options=options)
+    accounts = get_appsettings(config_uri, 'accounts')
+    engine = engine_from_config(settings, 'sqlalchemy.')
+
+    DBSession.configure(bind=engine)
+    # Base.metadata.create_all(engine)
+    # with transaction.manager:
+    data_init(transaction.manager, accounts)

lingvodoc/views.py

@@ -3809,6 +3809,36 @@ def approve_all(request):
     return {'error': str("No such perspective in the system")}
 
 
+@view_config(route_name='approve_all_outer', renderer='json', request_method='PATCH', permission='create')
+def approve_outer(request): # TODO: create test.
+    from .scripts.approve import approve_all_outer
+    client_id = request.matchdict.get('perspective_client_id')
+    object_id = request.matchdict.get('perspective_id')
+    cli_id = request.matchdict.get('dictionary_client_id')
+    obj_id = request.matchdict.get('dictionary_object_id')
+
+    # convert_one(blob.real_storage_path,
+    #             user.login,
+    #             user.password.hash,
+    #             parent_client_id,
+    #             parent_object_id)
+
+    # NOTE: doesn't work on Mac OS otherwise
+    client = DBSession.query(Client).filter_by(id=authenticated_userid(request)).first()
+    user = client.user
+    p = multiprocessing.Process(target=approve_all_outer, args=(user.login,
+                                                          user.password.hash,
+                                                          cli_id,
+                                                          obj_id,
+                                                          client_id,
+                                                          object_id))
+    log.debug("Conversion started")
+    p.start()
+    request.response.status = HTTPOk.code
+    return {"status": "Your dictionary is being approved."
+                      " Wait 5-15 minutes."}
+
+
 @view_config(route_name='approve_entity', renderer='json', request_method='PATCH', permission='create')
 def approve_entity(request):
     try:
@@ -4979,7 +5009,8 @@ def login_post(request):
     next = request.params.get('next') or request.route_url('home')
     login = request.POST.get('login', '')
     password = request.POST.get('password', '')
-    print(login)
+    # print(login)
+    log.debug(login)
     user = DBSession.query(User).filter_by(login=login).first()
     if user and user.check_password(password):
         client = Client(user_id=user.id)

requirements.txt

@@ -1,55 +1,58 @@
-bcrypt==2.0.0
-passlib==1.6.5
-Babel==2.0
+Babel==2.1.1
 Chameleon==2.22
-Mako==1.0.1
+Mako==1.0.3
 MarkupSafe==0.23
 PasteDeploy==1.5.2
 Pygments==2.0.2
 SQLAlchemy==1.0.8
 WebOb==1.5.0a1
-alembic==0.8.0
-argparse==1.3.0
-debtcollector==0.7.0
+WebTest==2.0.20
+alembic==0.8.3
+argparse==1.1
+bcrypt==2.0.0
+beautifulsoup4==4.4.1
+cffi==1.3.1
+debtcollector==1.0.0
 futures==3.0.3
-iso8601==0.1.10
-monotonic==0.3
+gunicorn==19.4.2
+iso8601==0.1.11
+monotonic==0.4
 msgpack-python==0.4.6
-netaddr==0.7.15
+netaddr==0.7.18
 netifaces==0.10.4
+nltk==3.0.5
 nose==1.3.7
-oslo.config==2.2.0
-oslo.i18n==2.3.0
-oslo.serialization==1.8.0
-oslo.utils==2.2.0
-pbr==1.5.0
-pluggy==0.3.0
+numpy==1.10.1
+oslo.config==3.0.0
+oslo.i18n==3.0.0
+oslo.serialization==2.0.0
+oslo.utils==3.0.0
+passlib==1.6.5
+pbr==1.8.1
 prettytable==0.7.2
-py==1.4.30
+psycopg2==2.6.1
+pycparser==2.14
+pympi-ling==1.40
 pyramid==1.6a2
 pyramid-chameleon==0.3
 pyramid-debugtoolbar==2.4.1
 pyramid-mako==1.0.2
 pyramid-tm==0.12
-python-editor==0.3
-python-keystoneclient==1.6.0
-python-swiftclient==2.5.0
-pytz==2015.4
+python-editor==0.4
+python-keystoneclient==1.8.1
+python-swiftclient==2.6.0
+pytz==2015.7
 repoze.lru==0.6
 requests==2.7.0
 simplejson==3.8.0
-six==1.9.0
-stevedore==1.7.0
-tox==2.1.1
+six==1.10.0
+stevedore==1.9.0
+tgt==1.3.1
 transaction==1.4.4
 translationstring==1.3
 venusian==1.0
-virtualenv==13.1.2
 waitress==0.8.9
 wrapt==1.10.5
 zope.deprecation==4.1.2
 zope.interface==4.1.2
 zope.sqlalchemy==0.7.6
-webtest==2.0.18
-psycopg2==2.6.1
-nltk==3.0.5

setup.py

@@ -19,7 +19,8 @@
     'waitress',
     'alembic',
     'passlib',
-    'py-bcrypt',
+    'bcrypt',
+
     'requests',
     'simplejson',
     'nltk',