Bug 1527868 - Vagrant/Travis: Remove Elasticsearch server

To ensure the CI and local environments match stage/production (where
Elasticsearch is not configured) and don't run differing code-paths due
to the various `ELASTICSEARCH_URL` conditionals used in auto-classify.

At such time when work on the fuzzy auto-classify matcher continues,
this can be reverted.

Author: edmorley

Vagrantfile

@@ -13,8 +13,6 @@ Vagrant.configure("2") do |config|
   config.vm.network "forwarded_port", guest: 8000, host: 8000, host_ip: "127.0.0.1"
   # MySQL
   config.vm.network "forwarded_port", guest: 3306, host: 3308, host_ip: "127.0.0.1"
-  # Elasticsearch
-  config.vm.network "forwarded_port", guest: 9200, host: 9201, host_ip: "127.0.0.1"
 
   if !Vagrant::Util::Platform.windows?
     # On platforms where NFS is used (ie all but Windows), we still have to use

bin/travis-setup.sh

@@ -5,7 +5,6 @@ set -euo pipefail
 # This script must be sourced, so that the environment variables are set in the calling shell.
 export BROKER_URL='amqp://guest:guest@localhost:5672//'
 export DATABASE_URL='mysql://root@localhost/test_treeherder'
-export ELASTICSEARCH_URL='http://127.0.0.1:9200'
 export REDIS_URL='redis://localhost:6379'
 export TREEHERDER_DJANGO_SECRET_KEY='secretkey-of-at-50-characters-to-pass-check-deploy'
 # Suppress warnings shown during pytest startup when using `python2 -3` mode.
@@ -18,16 +17,6 @@ setup_services() {
     echo '-----> Installing RabbitMQ'
     sudo apt-get install --no-install-recommends rabbitmq-server
 
-    ELASTICSEARCH_VERSION="6.6.0"
-    if [[ "$(dpkg-query --show --showformat='${Version}' elasticsearch 2>&1)" != "$ELASTICSEARCH_VERSION" ]]; then
-        echo '-----> Installing Elasticsearch'
-        curl -sSfo /tmp/elasticsearch.deb "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ELASTICSEARCH_VERSION}.deb"
-        sudo dpkg -i --force-confnew /tmp/elasticsearch.deb
-        sudo systemctl restart elasticsearch
-    else
-        sudo systemctl start elasticsearch
-    fi
-
     echo '-----> Configuring MySQL'
     # Using tmpfs for the MySQL data directory reduces pytest runtime by 30%.
     sudo mkdir /mnt/ramdisk
@@ -39,9 +28,6 @@ setup_services() {
 
     echo '-----> Starting redis-server'
     sudo systemctl start redis-server
-
-    echo '-----> Waiting for Elasticsearch to be ready'
-    while ! curl "${ELASTICSEARCH_URL}" &> /dev/null; do sleep 1; done
 }
 
 setup_python_env() {

docs/backend_tasks.md

@@ -68,7 +68,7 @@ hidden by default. For TaskCluster, edit the task definition to include the
 
 ## Connecting to Services Running inside Vagrant
 
-Treeherder uses various services to function, eg MySQL, Elasticsearch, etc.
+Treeherder uses various services to function, eg MySQL, etc.
 At times it can be useful to connect to them from outside the Vagrant VM.
 
 The Vagrantfile defines how internal ports are mapped to the host OS' ports.
@@ -91,7 +91,7 @@ With MySQL exposed at port 3308 you can connect to it from your host OS with the
 - user: `root`
 - password: leave blank
 
-Other services running inside the VM, such as Elasticsearch, can be accessed in the same way.
+Other services running inside the VM, can be accessed in the same way.
 
 [client git log]: https://github.com/mozilla/treeherder/commits/master/treeherder/client
 [client.py]: https://github.com/mozilla/treeherder/blob/master/treeherder/client/thclient/client.py

treeherder/autoclassify/matchers.py

@@ -62,6 +62,7 @@ def elasticsearch_matcher(text_log_error):
     Uses a filtered search checking test, status, expected, and the message
     as a phrase query with non-alphabet tokens removed.
     """
+    # Note: Elasticsearch is currently disabled in all environments (see bug 1527868).
     if not settings.ELASTICSEARCH_URL:
         return []
 

treeherder/config/settings.py

@@ -352,7 +352,8 @@
     INSTALLED_APPS.append('debug_toolbar')
 
 # Elasticsearch
-ELASTICSEARCH_URL = env.str('ELASTICSEARCH_URL', default='')
+# Note: Elasticsearch is currently disabled in all environments (see bug 1527868).
+ELASTICSEARCH_URL = env.str('ELASTICSEARCH_URL', default=None)
 
 # Rest Framework
 REST_FRAMEWORK = {

treeherder/config/utils.py

@@ -2,7 +2,7 @@
 
 
 def connection_should_use_tls(url):
-    # Services such as RabbitMQ/Elasticsearch running on Travis do not yet have TLS
+    # Services such as RabbitMQ/MySQL running on Travis do not yet have TLS
     # certificates set up. We could try using TLS locally using self-signed certs,
     # but until Travis has support it's not overly useful.
     return furl(url).host != 'localhost'

vagrant/env.sh

@@ -4,7 +4,6 @@ export PATH="${HOME}/firefox:${HOME}/python/bin:${PATH}"
 
 export BROKER_URL='amqp://guest:guest@localhost//'
 export DATABASE_URL='mysql://root@localhost/treeherder'
-export ELASTICSEARCH_URL='http://localhost:9200'
 export REDIS_URL='redis://localhost:6379'
 
 export TREEHERDER_DEBUG='True'

vagrant/setup.sh

@@ -15,7 +15,6 @@ PYTHON_DIR="$HOME/python"
 
 cd "$SRC_DIR"
 
-ELASTICSEARCH_VERSION="6.6.0"
 GECKODRIVER_VERSION="0.24.0"
 PYTHON_VERSION="$(sed 's/python-//' runtime.txt)"
 PIP_VERSION="19.0.2"
@@ -56,7 +55,6 @@ sudo -E apt-get -yqq update
 sudo -E apt-get -yqq -o 'Dpkg::Options::=--force-confnew' dist-upgrade
 # libdbus-glib-1-2, libgtk-3.0 and libxt6 are required by Firefox
 # gcc and libmysqlclient-dev are required by mysqlclient
-# openjdk-8-jre-headless is required by Elasticsearch
 sudo -E apt-get -yqq install --no-install-recommends \
     gcc \
     libdbus-glib-1-2 \
@@ -65,21 +63,10 @@ sudo -E apt-get -yqq install --no-install-recommends \
     libxt6 \
     mysql-server-5.7 \
     nodejs \
-    openjdk-8-jre-headless \
     rabbitmq-server \
     redis-server \
     yarn
 
-if [[ "$(dpkg-query --show --showformat='${Version}' elasticsearch 2>&1)" != "$ELASTICSEARCH_VERSION" ]]; then
-    echo '-----> Installing Elasticsearch'
-    curl -sSfo /tmp/elasticsearch.deb "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ELASTICSEARCH_VERSION.deb"
-    sudo dpkg -i --force-confnew /tmp/elasticsearch.deb
-    # Override the new ES 5.x default minimum heap size of 2GB.
-    sudo sed -i 's/.*ES_JAVA_OPTS=.*/ES_JAVA_OPTS="-Xms256m -Xmx1g"/' /etc/default/elasticsearch
-    sudo systemctl enable elasticsearch.service 2>&1
-    sudo systemctl restart elasticsearch.service
-fi
-
 if ! cmp -s vagrant/mysql.cnf /etc/mysql/conf.d/treeherder.cnf; then
     echo '-----> Configuring MySQL'
     sudo cp vagrant/mysql.cnf /etc/mysql/conf.d/treeherder.cnf
@@ -131,9 +118,6 @@ sudo mysql -e 'ALTER USER root@localhost IDENTIFIED WITH mysql_native_password B
 mysql -u root -e 'GRANT ALL PRIVILEGES ON *.* to root@"%"'
 mysql -u root -e 'CREATE DATABASE IF NOT EXISTS treeherder'
 
-echo '-----> Waiting for Elasticsearch to be ready'
-while ! curl "$ELASTICSEARCH_URL" &> /dev/null; do sleep 1; done
-
 echo '-----> Running Django migrations and loading reference data'
 ./manage.py migrate --noinput
 ./manage.py load_initial_data