Merge branch 'dev' of https://github.com/RNAcentral/rnacentral-sequence-search

Author: caduribas

.gitignore

@@ -14,6 +14,20 @@ terraform_load_balancer/load_balancer_rsa
 terraform_load_balancer/load_balancer_rsa.pub
 terraform_load_balancer/.terraform
 
+ukcloud/providers.tf
+ukcloud/terraform.tfstate.backup
+ukcloud/terraform.tfstate.d/test/*.backup
+ukcloud/sequence_search_rsa
+ukcloud/sequence_search_rsa.pub
+ukcloud/.terraform
+ukcloud/current_state
+
+ukcloud/load_balancer/providers.tf
+ukcloud/load_balancer/terraform.tfstate.backup
+ukcloud/load_balancer/load_balancer_rsa
+ukcloud/load_balancer/load_balancer_rsa.pub
+ukcloud/load_balancer/.terraform
+
 __pycache__/
 .idea/
 *.pyc
@@ -28,5 +42,9 @@ sequence_search/consumer/hmmer-3.2.1/
 sequence_search/consumer/results
 sequence_search/consumer/queries
 sequence_search/consumer/databases
+sequence_search/consumer/infernal-*
+sequence_search/consumer/rfam
+sequence_search/consumer/tests/output
+sequence_search/consumer/cmsearch_tblout_deoverlap
 sequence_search/monitor/.conf-file
 sequence_search/monitor/test*

ansible/download-sequences.yml

@@ -7,7 +7,7 @@
   tasks:
     - name: Download compressed archive from RNAcentral FTP
       get_url:
-        url: ftp://ftp.ebi.ac.uk/pub/databases/RNAcentral/current_release/sequences/.internal/sequence-database.fa.tar.gz
+        url: ftp://ftp.ebi.ac.uk/pub/databases/RNAcentral/current_release/sequences/.internal/sequences-database.fa.tar.gz
         dest: /nfs/sequence-database.fa.tar.gz
 
     - name: Uncompress the archive

ansible/main.yml

@@ -17,7 +17,7 @@
     - name: Drop the database and re-create it
       shell:
         chdir: /srv
-        cmd: ENVIRONMENT=PRODUCTION python3 -m sequence_search.db
+        cmd: ENVIRONMENT=PRODUCTION /usr/local/bin/python3.7 -m sequence_search.db
 
 
 - import_playbook: consumers.yml

ansible/postgres_restore.yml

@@ -9,5 +9,7 @@
   tasks: 
   - copy: src=producer_latest.dump dest=/tmp/producer_latest.dump mode=0777
   - shell: pg_restore -U docker -c -n public -n pg_catalog -d producer /tmp/producer_latest.dump
+  - shell: psql -U docker producer -c 'update job_chunks set consumer = NULL;'
+  - shell: psql -U docker producer -c 'update infernal_job set consumer = NULL;'
 
 - import_playbook: consumers.yml

ansible/roles/consumer/tasks/main.yml

@@ -14,30 +14,39 @@
   - name: Install yum dependencies
     yum:
       name:
+      - bzip2-devel
       - curl
       - gcc
+      - gcc-c++
       - git
       - libaio
+      - libffi-devel
+      - make
       - openssl
       - openssl-devel
+      - postgresql.x86_64
       - tar
       - unzip
       - wget
       - zlib-devel
-      - gcc-c++
-      - make
-      - python36
-      - python36-devel
-      - python36-setuptools
-      - postgresql.x86_64
     tags: [ install-deps ]
 
-  - name: Install pip3
-    shell: easy_install-3.6 pip
-    args:
-      creates: /usr/local/bin/pip
+  - name: Install build tools
+    yum:
+      name: "@Development tools"
+      state: present
     tags: [ install-deps ]
 
+  - name: Install Python 3.7
+    shell: |
+      curl -OL https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz
+      tar -xzf Python-3.7.7.tgz
+      cd Python-3.7.7
+      ./configure --enable-optimizations
+      make altinstall
+    args:
+      creates: /usr/local/bin/python3.7
+    tags: [ install-deps ]
 
 
   - name: Install nhmmer
@@ -52,6 +61,53 @@
     args:
       creates: /usr/local/bin/nhmmer
 
+  - name: Install Infernal
+    shell: |
+      curl -OL http://eddylab.org/infernal/infernal-1.1.3.tar.gz
+      tar xf infernal-1.1.3.tar.gz
+      cd infernal-1.1.3
+      ./configure --prefix /usr/local
+      make
+      make install
+
+  - name: Delete rfam directory
+    file:
+      path: /srv/sequence_search/consumer/rfam
+      state: absent
+    ignore_errors: yes
+    tags: [ rfam ]
+
+  - name: Create rfam directory
+    file:
+      path: /srv/sequence_search/consumer/rfam
+      state: directory
+      owner: centos
+      group: centos
+      mode: 0755
+    tags: [ rfam ]
+
+  - name: Install Rfam
+    shell:
+      chdir: /srv/sequence_search/consumer/rfam
+      cmd: curl -OL ftp://ftp.ebi.ac.uk/pub/databases/Rfam/CURRENT/Rfam.cm.gz && gunzip Rfam.cm.gz
+    tags: [ rfam ]
+
+  - name: Create the cm binaries
+    shell: /usr/local/bin/cmpress /srv/sequence_search/consumer/rfam/Rfam.cm
+    tags: [ rfam ]
+
+  - name: Delete cmsearch_tblout_deoverlap directory
+    file:
+      path: /srv/sequence_search/consumer/cmsearch_tblout_deoverlap
+      state: absent
+    ignore_errors: yes
+    tags: [ deoverlap ]
+
+  - name: Install cmsearch_tblout_deoverlap
+    shell:
+      chdir: /srv/sequence_search/consumer
+      cmd: git clone https://github.com/nawrockie/cmsearch_tblout_deoverlap.git
+    tags: [ deoverlap ]
 
 
   - name: Rsync aiohttp code
@@ -62,11 +118,14 @@
         - "--exclude=/sequence_search/consumer/databases"
         - "--exclude=/sequence_search/consumer/queries/*.fasta"
         - "--exclude=/sequence_search/consumer/results/*.fasta"
+        - "--exclude=/sequence_search/consumer/infernal-queries"
+        - "--exclude=/sequence_search/consumer/infernal-results"
+        - "--exclude=/sequence_search/consumer/cmsearch_tblout_deoverlap"
+        - "--exclude=/sequence_search/consumer/rfam"
         - "--exclude=/sequence_search/producer/static/node_modules"
     tags: [ rsync, quick ]
 
 
-
   - name: Delete logs directory
     file:
       path: /var/log/gunicorn
@@ -84,7 +143,6 @@
     tags: [ logs ]
 
 
-
   - name: Delete queries directory
     file:
       path: /srv/sequence_search/consumer/queries
@@ -101,7 +159,6 @@
     tags: [ queries, quick ]
 
 
-
   - name: Delete results directory
     file:
       path: /srv/sequence_search/consumer/results
@@ -118,16 +175,37 @@
     tags: [ results, quick ]
 
 
-# Ansible chown is slow, cause it checks ownership before assigning it - so just run chown imperatively
-# https://groups.google.com/forum/#!topic/ansible-project/2O5u8GIZ5Rw
+  - name: Delete infernal queries directory
+    file:
+      path: /srv/sequence_search/consumer/infernal-queries
+      state: absent
+    tags: [ infernal-queries, quick ]
+
+  - name: Create infernal queries directory
+    file:
+      path: /srv/sequence_search/consumer/infernal-queries
+      state: directory
+      owner: centos
+      group: centos
+      mode: 0755
+    tags: [ infernal-queries, quick ]
+
+
+  - name: Delete infernal results directory
+    file:
+      path: /srv/sequence_search/consumer/infernal-results
+      state: absent
+    tags: [ infernal-results, quick ]
+
+  - name: Create infernal results directory
+    file:
+      path: /srv/sequence_search/consumer/infernal-results
+      state: directory
+      owner: centos
+      group: centos
+      mode: 0755
+    tags: [ infernal-results, quick ]
 
-# Alternatively:
-#    file:
-#      path: /srv/sequence_search
-#      owner: centos
-#      group: centos
-#      mode: 0755
-#      recurse: yes
 
   - name: Chown aiohttp code
     command: "chown -R centos:centos /srv/sequence_search"
@@ -140,9 +218,10 @@
 
 
   - name: Install pip3 requirements
-    shell: /usr/local/bin/pip install -r requirements.txt
+    shell: /usr/local/bin/pip3.7 install -r requirements.txt
     args:
       chdir: /srv/sequence_search
+    tags: [ pip-requirements ]
 
 
   - name: Create a symbolic link to sequence databases
@@ -189,5 +268,5 @@
     shell:
       chdir: /srv
 #      cmd: /usr/local/bin/gunicorn consumer.main:app --daemon --bind 0.0.0.0:8000 --worker-class aiohttp.worker.GunicornWebWorker --access-logfile /var/log/gunicorn/access_log --error-logfile /var/log/gunicorn/error_log --env ENVIRONMENT=PRODUCTION --log-level=DEBUG
-      cmd: ENVIRONMENT=PRODUCTION nohup python3 -m sequence_search.consumer &> /var/log/gunicorn/access_log &
+      cmd: ENVIRONMENT=PRODUCTION nohup /usr/local/bin/python3.7 -m sequence_search.consumer &> /var/log/gunicorn/access_log &
     tags: [ restart, quick ]