Pharaoh: Created Admin page with upload and download

Signed-off-by: Sam Kleinman <[email protected]>

pharaoh/pharaoh/app/models.py

@@ -59,13 +59,22 @@ def get_fileIDs(source_language, target_language, curr_db=db):
                                   'target_language': target_language},
                                  {'_id': 1}).sort('priority', 1)
 
+def get_file_paths(curr_db=db):
+    '''This function  gets all of the file ids for a given pair of languages
+    :param string db: database
+    :param string source_language: source language
+    :param string target_language: target language
+    :returns: cursor of fileids
+    '''
+    return curr_db['files'].distinct('file_path')
+
 
 def get_files_for_page(page_number, num_files_per_page, fileIDs, curr_db=db):
     '''This function gets all of the stats for a list of files
     :param int page_number: current page number
+    :param int num_files_per_page: number of files per page
+    :param list fileIDS: list of file ids
     :param database db: database
-    :param string source_language: source language
-    :param string target_language: target language
     :returns: cursor of file names
     '''
     page_fileIDs = fileIDs.skip(((page_number-1)*num_files_per_page) if page_number > 0 else 0).limit(num_files_per_page)
@@ -83,6 +92,7 @@ def get_files_for_page(page_number, num_files_per_page, fileIDs, curr_db=db):
 
     return l
 
+
 def audit(action, last_editor, current_user, doc, new_target_sentence=None, curr_db=db):
     ''' This function saves an audit of the event that occurred
     :param database db': database

pharaoh/pharaoh/app/static/javascript/scripts.js

@@ -9,7 +9,97 @@ $(document).ready(function(){
     $(".target").each(check_editor);
     $('textarea').autosize();
 
-});
+    $('#upload-btn').on('click',upload);
+    $('#download-all-btn').on('click', download_all);
+    $('#download-approved-btn').on('click', download_approved);
+
+})
+
+
+function download_all(){
+    if($('#download-file-path').val() == "ALL"){
+        var url = 'download-all/'+$('#target_language_download').val();
+    }
+    else{
+        var url = 'download-all/'+$('#target_language_download').val()+'/'+$('#download-file-path').val();
+    }
+    $.ajax({
+        url: url,
+        type: 'GET',
+        success: function(data, textStatus, jqXHR){
+            if(typeof data.error === 'undefined'){
+                console.log("SUCCESS");
+                $("body").append("<iframe src='" + url + "' style='display: none;' ></iframe>");
+            }
+            else{
+                console.log('ERRORS: ' + data.error);
+            }
+        },
+        error: function(jqXHR, textStatus, errorThrown){
+            console.log('ERRORS: ' + textStatus);
+        }
+    });
+
+}
+
+function download_approved(){
+    if($('#download-file-path').val() == "ALL"){
+        var url = 'download-approved/'+$('#target_language_download').val();
+    }
+    else{
+        var url = 'download-approved/'+$('#target_language_download').val()+'/'+$('#download-file-path').val();
+    }
+    $.ajax({
+        url: url,
+        type: 'GET',
+        success: function(data, textStatus, jqXHR){
+            if(typeof data.error === 'undefined'){
+                console.log("SUCCESS");
+                $("body").append("<iframe src='" + url + "' style='display: none;' ></iframe>");
+            }
+            else{
+                console.log('ERRORS: ' + data.error);
+            }
+        },
+        error: function(jqXHR, textStatus, errorThrown){
+            console.log('ERRORS: ' + textStatus);
+        }
+    });
+
+}
+
+function upload(){
+
+    var filelist =  document.getElementById("file").files || [];
+    var data = new FormData();
+    data.append('file_name', filelist[0].webkitRelativePath);
+    data.append('file', filelist[0]);
+    data.append('username', $('#username').val());
+    data.append('status', $('#status').val());
+    data.append('source_language', $('#source_language').val());
+    data.append('target_language', $('#target_language').val());
+    $.ajax({
+        url: '/upload',
+        type: 'POST',
+        data: data,
+        cache: false,
+        dataType: 'json',
+        processData: false, // Don't process the files
+        contentType: false, // Set content type to false as jQuery will tell the server its a query string request
+        success: function(data, textStatus, jqXHR){
+            if(typeof data.error === 'undefined'){
+                console.log("SUCCESS");
+            }
+            else{
+                console.log('ERRORS: ' + data.error);
+            }
+        },
+        error: function(jqXHR, textStatus, errorThrown){
+            console.log('ERRORS: ' + textStatus);
+        }
+    });
+
+}
 
 function toggle_approved(){
     var newtext = ($(this).html() == "Show All" ? "Only Show Unapproved" : "Show All");
@@ -160,9 +250,3 @@ function unapprove_html(e){
 function language(){
     window.location.href = 'edit/'+$('#username').val()+'/'+$(this).html();
 }
-
-// Custom example logic
-function $(id) {
-return document.getElementById(id);
-}
-

pharaoh/pharaoh/app/templates/admin.html

@@ -2,13 +2,25 @@
 {% block title %}Admin{% endblock %}
 {% block content %}
     <ul id="navigation">
-    <input id="username" value="Judah"></input>
     <h1>Upload PO Files</h1>
-    <form action="" method=post enctype=multipart/form-data>
-      <p><input type=file name=file webkitdirectory directory multiple>
-         <input type=submit value=Upload>
+    <form action="upload" enctype="multipart/form-data" method="post">
+        <input type="file" name="file" id="file"><br/>
+        Username: <input type="text" id="username" name="username" value="Moses" ><br/>
+        Status: <input type="text" id="status" name="status" value="SMT" ><br/>
+        Source Language: <input type="text" id="source_language" name="source_language" value="en" ><br/>
+        Target Language: <input type="text" id="target_language" name="target_language" value="es" ><br/>
+        <input type="button" id= "upload-btn" value="Upload"><br/>
     </form>
-
+    <h1>Download PO Files</h1>
+        Target Language: <input type="text" id="target_language_download" name="target_language" value="es" ><br/>
+        <select id="download-file-path">
+            <option>ALL</option>
+            {% for file in file_list %}
+            <option>{{file}}</option>
+            {% endfor %}
+        </select><br/>
+         <input type="button" id= "download-all-btn" value="Download All Sentences"><br/>
+         <input type="button" id= "download-approved-btn" value="Download Approved Sentences"><br/>
     </ul>
 
 {% endblock %}

pharaoh/pharaoh/app/templates/base.html

@@ -2,9 +2,9 @@
 <html lang="en">
 <head>
     <title>Translation Pipeline</title>
+    <script src="http://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js" ></script>
     <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='css/style.css') }}"></link>
     <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='bootstrap/css/bootstrap.min.css') }}"></link>
-    <script src="//ajax.googleapis.com/ajax/libs/jquery/1.11.1/jquery.min.js"></script>
     <script type="text/javascript" src="{{url_for('static',filename='javascript/jquery.autosize.min.js')}}"></script>
     <script type="text/javascript" src="{{url_for('static',filename='javascript/scripts.js')}}"></script>
     {% block header %}
@@ -14,7 +14,7 @@
     <div id="content">{% block content %}{% endblock %}</div>
     <div id="footer">
         {% block footer %}
-        &copy; Copyright 2014 by MongoDB</a>.
+        &copy; Copyright 2014 by MongoDB.
         {% endblock %}
     </div>
 </body>

pharaoh/pharaoh/app/templates/language_picker.html

@@ -6,7 +6,7 @@
     <table>
         {% for language in language_list %}
             <tr>
-                <td class="target" >
+                <td class="language-cell" >
                     <button class="language btn btn-success" id="{{language}}-languageButton" >{{language}}</button>
                 </td>
             </tr>

pharaoh/pharaoh/app/views.py

@@ -23,6 +23,7 @@
 from flask_app import app, db
 import models
 from pharaoh.mongo_to_po import generate_fresh_po_text, generate_all_po_files
+from pharaoh.po_to_mongo import put_po_data_in_mongo
 
 @app.route('/')
 @app.route('/index')
@@ -135,46 +136,52 @@ def lock_error(username, language, file):
                            language=language,
                            file=file)
 
-@app.route('/download-all/<username>/<language>/<path:file>')
-def download_single_po(username, language, file):
+@app.route('/download-all/<language>/<path:file>')
+def download_single_po(language, file):
     po = generate_fresh_po_text(file, 'en', language, db, True)
     response = make_response(po)
     response.headers["Content-Disposition"] = "attachment; filename={0}.po".format(file)
     return response
 
-@app.route('/download-approved/<username>/<language>/<path:file>')
-def download_single_po_approved(username, language, file):
+@app.route('/download-approved/<language>/<path:file>')
+def download_single_po_approved(language, file):
     po = generate_fresh_po_text(file, 'en', language, db, False)
     response = make_response(po)
     response.headers["Content-Disposition"] = "attachment; filename={0}.po".format(file)
     return response
 
-@app.route('/download-all/<username>/<language>/')
-@app.route('/download-all/<username>/<language>')
-def download_all_po(username, language):
+@app.route('/download-all/<language>/')
+@app.route('/download-all/<language>')
+def download_all_po(language):
     po = generate_all_po_files( 'en', language, db, True)
     response = make_response(po)
     response.headers["Content-Disposition"] = "attachment; filename={0}.tar.gz".format(language)
     return response
 
-@app.route('/download-approved/<username>/<language>/')
-@app.route('/download-approved/<username>/<language>')
-def download_all_po_approved(username, language):
+@app.route('/download-approved/<language>/')
+@app.route('/download-approved/<language>')
+def download_all_po_approved(language):
     po = generate_all_po_files( 'en', language, db, False)
     response = make_response(po)
     response.headers["Content-Disposition"] = "attachment; filename={0}.tar.gz".format(language)
     return response
 
-@app.route('/admin/<username>')
-def admin(username):
-    if request.method == 'POST':
-        app.logger.info(request)
-        #file = request.files['file']
-        #if file and allowed_file(file.filename):
-            #filename = secure_filename(file.filename)
-            #file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
-    return render_template("admin.html",
-                           username=username)
+@app.route('/admin', methods=['GET'])
+def admin():
+    files = models.get_file_paths()
+    return render_template("admin.html", file_list=files)
+
+@app.route('/upload', methods=['POST'])
+def upload():
+    app.logger.info(request.files['file'])
+    app.logger.info(request.form)
+    put_po_data_in_mongo(request.files['file'], 
+                         request.form['username'], 
+                         request.form['status'], 
+                         request.form['source_language'], 
+                         request.form['target_language'],
+                         db)
+    return json.dumps({"code:": 200, "msg": "Unapproval Succeeded"}), 200
 
 def fix_json(json_object):
     ''' helper function to fix json from request for mongodb

pharaoh/pharaoh/config.yaml

@@ -16,7 +16,7 @@
 APPROVAL_THRESHOLD: 2
 MONGO_PORT: 28000
 MONGO_HOST: 'localhost'
-MONGO_DBNAME: 'verifier'
+MONGO_DBNAME: 'veritest'
 NUM_FILES_PER_PAGE: 20
 SESSION_LENGTH: 1
 DEBUG: False

pharaoh/pharaoh/po_to_mongo.py

@@ -15,6 +15,7 @@
 import os.path
 import logging
 import re
+import tarfile
 
 import polib
 from pymongo import MongoClient
@@ -33,26 +34,23 @@
 logger = logging.getLogger('pharaoh.po_to_mongo')
 
 
-def write_po_file_to_mongo(po_fn, userID, status, source_language, target_language, po_root, db):
+def write_po_file_to_mongo(po_fn, po_file, userID, status, source_language, target_language, db):
     '''write a po_file to mongodb
-    :param string po_fn: the file name of the current pofile
+    :param string po_fn: the file name of the current pofile as it should be put in mongodb
+    :param POFile po_file: the polib pofile instance
     :param string userID: the ID of the user that translated the po file
     :param string status: the status of the translations
     :param string source_language: The source_language of the translations
     :param string target_language: The target_language of the translations
-    :param string po_root: the root of the po_files
     :param database db: the database that you want to write to
     '''
-    po = polib.pofile(po_fn)
-    rel_fn = os.path.relpath(po_fn, po_root)
-    rel_fn = os.path.splitext(rel_fn)[0]
-    f = File({u'file_path': rel_fn,
+    f = File({u'file_path': po_fn,
               u'priority': 0,
               u'source_language': source_language,
               u'target_language': target_language}, curr_db=db)
 
     reg = re.compile('^:[a-zA-Z0-9]+:`(?!.*<.*>.*)[^`]*`$')
-    for idx, entry in enumerate(po.translated_entries()):
+    for idx, entry in enumerate(po_file.translated_entries()):
         sentence_status = status
         match = re.match(reg, entry.msgstr.encode('utf-8'))
         if match is not None and match.group() == entry.msgstr.encode('utf-8'):
@@ -72,10 +70,9 @@ def write_po_file_to_mongo(po_fn, userID, status, source_language, target_langua
         t.save()
     f.get_num_sentences()
 
-
 def put_po_files_in_mongo(path, username, status, source_language, target_language, db_host, db_port, db_name):
     '''go through directories and write the po file to mongo
-    :param path: the path to the po_files
+    :param string path: the path to the po_files
     :param string username: the username of the translator
     :param string status: the status of the translations
     :param string source_language: The source_language of the translations
@@ -99,5 +96,29 @@ def put_po_files_in_mongo(path, username, status, source_language, target_langua
         path = os.path.dirname(path)
 
     for fn in file_list:
-        write_po_file_to_mongo(fn, userID, status, source_language,
-                               target_language, path, db)
+        po = polib.pofile(fn)
+        rel_fn = os.path.relpath(fn, path)
+        rel_fn = os.path.splitext(rel_fn)[0]
+        write_po_file_to_mongo(fn, po, userID, status, source_language,
+                               target_language, db)
+
+
+def put_po_data_in_mongo(po_tar, username, status, source_language, target_language, db):
+    '''go through directories and write the po file to mongo
+    :param string po_fn: the filename for the po file
+    :param string po_data: the po_file data
+    :param string username: the username of the translator
+    :param string status: the status of the translations
+    :param string source_language: The source_language of the translations
+    :param string target_language: The target_language of the translations
+    :param database db: the mongodb database
+    '''
+    tar = tarfile.open(fileobj=po_tar)
+    for member in tar.getmembers():
+        if os.path.splitext(member.name)[1] not in ['.po', '.pot']:
+            continue
+        po_file = tar.extractfile(member)
+        po = polib.pofile(po_file.read())
+        userID = db['users'].find_one({'username': username})[u'_id']
+        write_po_file_to_mongo(os.path.splitext(member.name)[0], po, userID, status, source_language,
+                               target_language, db)