medusa.py

#!/usr/bin/env python

import os
import sys
import json
import subprocess
import time
import gzip
from flask import Flask, request, session, g, redirect, url_for, abort, \
    render_template, flash, escape, Response, send_from_directory
from werkzeug.utils import secure_filename

from utils import check_sequence, checkId, generate_hash
from utils import generate_time_hash

from store import add_job
from store import retrieve_job
from store import cumulative_jobs
from store import unique_ips
from store import unique_emails

import settings

app = Flask(__name__)

# App config from settings.py
app.config.from_object(settings)

# Production settings that override the testing ones

list_id=[]

try:
    import production
    app.config.from_object(production)
except ImportError:
    pass

# Mail log setup
try:
    import mail_log as ml
    if not app.debug:
        import logging
        mail_handler = ml.TlsSMTPHandler(ml.MAIL_HOST,
                                         ml.MAIL_FROM,
                                         ml.ADMINS, 'Medusa-webapp Failed!',
                                         credentials=(ml.MAIL_USER,
                                                      ml.MAIL_PWD))
        mail_handler.setLevel(logging.ERROR)
        app.logger.addHandler(mail_handler)
except ImportError:
    pass


@app.route('/')
def index():
    return render_template('index.html')


# DESCOMMENTARE mettere solo /run
@app.route('/medusa/run', methods=['GET', 'POST'])
def run():
    # Here handle submissions and run the analysis
    # Send emails on failures, success
    # Use redis to store user stats (hashed for privacy)
    if request.method == 'POST':
        # First things first, compute user hash
       # DESCOMMENTARE req_id = generate_time_hash(request.remote_addr)
        req_id = "2"
        # To avoid slow-downs in the running directory
        # create subdirs w/ the first 2 chars of the hash
        h2c = req_id[:2]
        try:
            os.mkdir(os.path.join(app.config['UPLOAD_FOLDER'],
                                  h2c))
        except:
            pass

        # Prepare the working directory
        # Our hash scheme ensures that it should be unique
        wdir = os.path.join(app.config['UPLOAD_FOLDER'],
                            h2c, req_id)
        wdir = os.path.abspath(wdir)
        os.mkdir(wdir)
        #MIOCOD
        if request.form['aligning_algorithm'] == 'option1':
            aligning_algorithm = ''
        else:
            aligning_algorithm = '-a' #use Minimap align
        
        random = request.form.get('graph_path_rd')
        weight = request.form.get('graph_path_wh')
        
        draft = request.files['draft']
        
        #input files checks  --SCRIVI LA FUNZIONE IN UTILS.PY
        
        #print ("qui ",draft.filename)
        #print ("qui2 ",draft.read())

        
        #print(request.files)
        
        # Save input files
        #if draft and allowed_file(draft.filename):
        if draft:
            filename = secure_filename(draft.filename)
            draftFile= os.path.join(wdir, filename)
            draft.save(draftFile)
            dname = filename
            sequence=0
            found_new_line = 0
            list_id.clear()
            if(filename.find('.gz')>-1):
                with gzip.open(draftFile,"rt") as file:
                    for line in file.readlines():
                            print(line)
                            if  found_new_line ==1:
                                if line[0]== ">":
                                    flash(u'Something went wrong with your draft genome: empty sequence found',
                                            'danger')
                                    return redirect(url_for('index'))
                            if line[0]== ">":
                                found_new_line =1
                                id = checkId(line)
                                try:
                                    if(list_id.index(id)>=0):
                                        flash(u'Something went wrong with your draft genome: duplicate id found',
                                                'danger')
                                        return redirect(url_for('index'))
                                    else:
                                        list_id.append(id)
                                except Exception as e:
                                    list_id.append(id)

                            else:
                                found_new_line =0
                                sequence = check_sequence(line)
                                if sequence == 1:
                                        flash(u'Something went wrong with your draft genome',
                                                'danger')
                                        return redirect(url_for('index'))
            else:
                try:
                    with open(draftFile,"r") as file:
                        for line in file.readlines():
                            print(line)
                            if  found_new_line ==1:
                                if line[0]== ">":
                                    flash(u'Something went wrong with your draft genome: empty sequence found',
                                            'danger')
                                    return redirect(url_for('index'))
                            if line[0]== ">":
                                found_new_line =1
                                id = checkId(line)
                                try:
                                    if(list_id.index(id)>=0):
                                        flash(u'Something went wrong with your draft genome: duplicate id found',
                                                'danger')
                                        return redirect(url_for('index'))
                                    else:
                                        list_id.append(id)
                                except Exception as e:
                                    list_id.append(id)

                            else:
                                found_new_line =0
                                sequence = check_sequence(line)
                                if sequence == 1:
                                        flash(u'Something went wrong with your draft genome',
                                                'danger')
                                        return redirect(url_for('index'))
                except Exception as e:
                    print(e)
                    flash(u'unrecognized compression type, please use GZIP for deflating your files',
                        'danger')
                    return redirect(url_for('index'))
                            
        else:
            flash(u'Something went wrong with your draft genome',
                  'danger ')
            return redirect(url_for('index'))
        
        # Save the genomes files
        genomes = set()
        try:
            for genome in request.files.getlist('genomes'):
                filename = secure_filename(genome.filename)
                genomeFile = os.path.join(wdir, filename)
                genome.save(genomeFile)
                sequence=0
                found_new_line = 0
                list_id.clear()
                if(filename.find('.gz')>-1):
                    with gzip.open(draftFile,"rt") as file:
                        for line in file.readlines():
                                print(line)
                                if  found_new_line ==1:
                                    if line[0]== ">":
                                        flash(u'Something went wrong with your target genome: empty sequence found',
                                                'danger')
                                        return redirect(url_for('index'))
                                if line[0]== ">":
                                    found_new_line =1
                                    id = checkId(line)
                                    try:
                                        if(list_id.index(id)>=0):
                                            flash(u'Something went wrong with your target genome: duplicate id found',
                                                    'danger')
                                            return redirect(url_for('index'))
                                        else:
                                            list_id.append(id)
                                    except Exception as e:
                                        list_id.append(id)

                                else:
                                    found_new_line =0
                                    sequence = check_sequence(line)
                                    if sequence == 1:
                                            flash(u'Something went wrong with your target genome: one or more sequences contain non DNA characters',
                                                    'danger')
                                            return redirect(url_for('index'))
                    genomes.add(filename)
                else:
                    try:
                        with open(genomeFile,"r") as file:
                            for line in file.readlines():
                                if  found_new_line ==1:
                                    if line[0]== ">" :
                                        flash(u'Something went wrong with your target genome: empty sequence found',
                                                'danger')
                                        return redirect(url_for('index'))
                                if line[0]== ">":
                                    found_new_line =1
                                    id = checkId(line)
                                    try:
                                        if(list_id.index(id)>=0):
                                            flash(u'Something went wrong with your target genome: duplicate id found',
                                                    'danger')
                                            return redirect(url_for('index'))
                                        else:
                                            list_id.append(id)
                                    except Exception as e:
                                        list_id.append(id)

                                else:
                                    found_new_line =0
                                    sequence = check_sequence(line)
                                    if sequence == 1:
                                            print(line)
                                            flash(u'Something went wrong with your target genome: one or more sequences contain non DNA characters',
                                                    'danger')
                                            return redirect(url_for('index'))
                        
                        genomes.add(filename)
                    except Exception as e:
                        print(e)
                        flash(u'unrecognized compression type, please use GZIP for deflating your files',
                            'danger')
                        return redirect(url_for('index'))
                
        except Exception as e:
            print(e)
            flash(u'Something went wrong with your target genomes',
                  'danger')
            return redirect(url_for('index'))
         #MIOCOD
        genomes.add(aligning_algorithm)
        if random:
            genomes.add(random)
        if weight:
            genomes.add(weight)
        # Check email, hash it
        email = request.form['email']
        # DESCOMMENTARE  if email:
        # DESCOMMENTARE    hemail = generate_hash(email)
        # DESCOMMENTARE else:
        # DESCOMMENTARE flash(u'Something went wrong with your email', 'danger')
        # DESCOMMENTARE return redirect(url_for('index'))

        # Secure my results?
        passphrase = request.form['passphrase']
        # DESCOMMENTARE if passphrase:
        # DESCOMMENTARE  hpass = generate_hash(passphrase)
        # DESCOMMENTARE else:
        # DESCOMMENTARE hpass = None
        # In case of a passphrase, don't bother the current submitter
        session['req_id'] = req_id

        # Submit the job
        # Then redirect to the waiting page
        try:
            cmd = 'python tasks.py %s %s %s %s' % (req_id,
                                                   wdir,
                                                   dname,
                                                   ' '.join(genomes))
            f = open(os.path.join(wdir, 'cmd.sh'), 'w')
            f.write(cmd + '\n')
            f.close()
            cmd = 'at -q b -M now -f %s' % os.path.join(wdir, 'cmd.sh')
            proc = subprocess.Popen(cmd,
                                    shell=(sys.platform != "win32"),
                                    stdin=subprocess.PIPE,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
            out = proc.communicate()
            return_code = proc.returncode
            if return_code != 0:
                raise Exception('%s' % str(out[1]))
        except Exception as e:
            flash(u'Could not submit your job "%s"' % e,
                  'danger')
            return redirect(url_for('index'))

        try:
            # Send details to redis
            add_job(req_id, request.remote_addr, hemail, hpass)
        except Exception as e:
            flash(u'Could not save your job details (%s)' % e, 'danger')
            return redirect(url_for('index'))

        return redirect(url_for('results',
                        req_id=req_id))

    # No POST, return to start
    flash(u'No job details given, would you like to start a new one?',
          'warning')
    return redirect(url_for('index'))


@app.route('/log/<req_id>')
def log(req_id):
    # Get details from redis
    j = retrieve_job(req_id)

    # TODO: avoid access to redirect to results
    # Check passphrase
    if 'req_id' not in session:
        # bother the user
        return redirect(url_for('access',
                        req_id=req_id))
    if 'req_id' in session and req_id != escape(session['req_id']):
        # clean the session, then bother the user
        session.pop('req_id', None)
        return redirect(url_for('access',
                        req_id=req_id))

    # Return the log, if present
    h2c = req_id[:2]
    if not os.path.exists(os.path.join(
            app.config['UPLOAD_FOLDER'],
            h2c, req_id)):
        flash('Could not retrieve the log: is your job older than one week?', 'danger')
        return render_template('index.html')
    if 'log.txt' not in os.listdir(os.path.join(
            app.config['UPLOAD_FOLDER'],
            h2c, req_id)):
        flash('Could not retrieve the log.txt file', 'danger')
        return render_template('error.html', req_id=req_id)

    path = os.path.join(app.config['UPLOAD_FOLDER'],
                        h2c, req_id, 'log.txt')
    return Response(''.join(open(path).readlines()),
                    mimetype='text/plain')


@app.route('/err/<req_id>')
def err(req_id):
    # Get details from redis
    j = retrieve_job(req_id)

    # TODO: avoid access to redirect to results
    # Check passphrase
    if 'req_id' not in session:
        # bother the user
        return redirect(url_for('access',
                        req_id=req_id))
    if 'req_id' in session and req_id != escape(session['req_id']):
        # clean the session, then bother the user
        session.pop('req_id', None)
        return redirect(url_for('access',
                        req_id=req_id))

    # Return the log, if present
    h2c = req_id[:2]
    if not os.path.exists(os.path.join(
            app.config['UPLOAD_FOLDER'],
            h2c, req_id)):
        flash('Could not retrieve the log: is your job older than one week?', 'danger')
        return render_template('index.html')
    if 'log.err' not in os.listdir(os.path.join(
            app.config['UPLOAD_FOLDER'],
            h2c, req_id)):
        flash('Could not retrieve the log.err file', 'danger')
        return render_template('error.html', req_id=req_id)

    path = os.path.join(app.config['UPLOAD_FOLDER'],
                        h2c, req_id, 'log.err')
    return Response(''.join(open(path).readlines()),
                    mimetype='text/plain')


@app.route('/scaffold/<req_id>')
def scaffold(req_id):
    # Get details from redis
    j = retrieve_job(req_id)

    # TODO: avoid access to redirect to results
    # Check passphrase
    if 'req_id' not in session:
        # bother the user
        return redirect(url_for('access',
                        req_id=req_id))
    if 'req_id' in session and req_id != escape(session['req_id']):
        # clean the session, then bother the user
        session.pop('req_id', None)
        return redirect(url_for('access',
                        req_id=req_id))

    # Return the log, if present
    h2c = req_id[:2]
    if not os.path.exists(os.path.join(
            app.config['UPLOAD_FOLDER'],
            h2c, req_id)):
        flash('Could not retrieve the scaffold: is your job older than one week?', 'danger')
        return render_template('index.html')
    if 'scaffold.fasta' not in os.listdir(os.path.join(
            app.config['UPLOAD_FOLDER'],
            h2c, req_id)):
        flash('Could not retrieve the scaffold file', 'danger')
        return render_template('error.html', req_id=req_id)

    path = os.path.join(app.config['UPLOAD_FOLDER'],
                        h2c, req_id)
    return send_from_directory(path,
                               'scaffold.fasta',
                               as_attachment=True)


@app.route('/results/<req_id>')
def results(req_id):
    # Here show the results or the wait page
    # Get the right job using the session or the hash key, a la contiguator

    # Get details from redis
    j = retrieve_job(req_id)

    # Check passphrase
    if 'req_id' not in session:
        # bother the user
        return redirect(url_for('access',
                        req_id=req_id))
    if 'req_id' in session and req_id != escape(session['req_id']):
        # clean the session, then bother the user
        session.pop('req_id', None)
        return redirect(url_for('access',
                        req_id=req_id))

    h2c = req_id[:2]
    status = j['status']
    if status == 'Job done':
        # run results logics
        try:
            result = json.load(open(os.path.join(app.config['UPLOAD_FOLDER'],
                                                 h2c, req_id, 'result.json')))
        except Exception as e:
            app.logger.error(
                'Internal server error: %s\nRequest ID: %s' % (e, req_id))
            flash(u'Internal server error: %s' % e, 'danger')
            return render_template('error.html', req_id=req_id)
        return render_template('result.html', req_id=req_id,
                               data=result)
    elif status == 'Job failed':
        error_msg = j.get('error', '')
        app.logger.error('Internal server error: %s\nRequest ID: %s' % (error_msg,
                         req_id))
        flash(u'Internal server error: %s' % error_msg,
              'danger')
        return render_template('error.html', req_id=req_id)
    else:
        # If too much time has passed, it means that the job has either failed
        # or anything like that
        cur_time = time.time()
        start_time = float(j['time'])
        delta_time = cur_time - start_time
        if (60 * 15) < delta_time < (60 * 30):
            flash(u'Your job exceeded 15 minutes, something might have gone wrong!' +
                  u'Will try 15 more minutes before giving up',
                  'danger')
        elif delta_time > (60 * 30):
            flash(u'Your job exceeded 30 minutes, something must have gone wrong!' +
                  u'If your genomes are many (and big) you might want to run Medusa locally',
                  'danger')
            return render_template('error.html', req_id=req_id)
        return render_template('waiting.html', status=status)


@app.route('/access/<req_id>', methods=['GET', 'POST'])
def access(req_id):
    # Here ask for a passphrase

    # If this is a POST request
    # Compare it and redirect accordingly

    # Get details from redis
    j = retrieve_job(req_id)

    # If no passphrase, no need to bother, just redirect
    if 'passphrase' not in j:
        session['req_id'] = req_id
        return redirect(url_for('results',
                        req_id=req_id))

    if request.method == 'POST':
        # compare passphrases, after hashing
        passphrase = request.form['passphrase']
        if passphrase:
            hpass = generate_hash(passphrase)
        else:
            flash(u'Error handling your passphrase', 'danger')
            return render_template('access.html', req_id=req_id)

        # Compare
        if hpass == j['passphrase']:
            # Correct!
            session['req_id'] = req_id
            return redirect(url_for('results',
                                    req_id=req_id))
        else:
            flash(u'Passphrase does not match', 'danger')
            session.pop('req_id', None)
            return render_template('access.html', req_id=req_id)

    # Redirect to password form
    flash('This job is protected by a passphrase', 'info')
    return render_template('access.html', req_id=req_id)


@app.route('/stats')
def stats():
    return render_template('stats.html')


@app.route('/stats/jobs')
def jobs():
    return Response(json.dumps([{'date': x[1],
                                 'jobs':x[0]} for x in cumulative_jobs()]),
                    mimetype='text/plain')


@app.route('/stats/ips')
def ips():
    return Response(json.dumps([{'date': x[1],
                                 'ips':x[0]} for x in unique_ips()]),
                    mimetype='text/plain')


@app.route('/stats/emails')
def emails():
    return Response(json.dumps([{'date': x[1],
                                 'emails':x[0]} for x in unique_emails()]),
                    mimetype='text/plain')


@app.route('/admin')
def admin():
    # Here admin section: upload a new medusa
    # Clean manually the jobs    flash('Not implemented yet', 'warning')
    return render_template('index.html')


if __name__ == '__main__':
    app.run()