Revert "Revert back to b83cb96, remove all (Inline::)Python code"

This reverts commit 75de461.

Author: pypt

.gitignore

@@ -12,6 +12,9 @@ cache/translate
 # all caches
 data/cache/
 
+# Media Cloud virtualenv
+mc-venv/
+
 # Python bytecode files
 *.py[cod]
 
@@ -26,6 +29,9 @@ Session.vim
 # OS X
 .DS_Store
 
+# Inline::* bytecode (there might be multiple directories that start with this prefix)
+_Inline*
+
 # Devel::NYTProf output
 nytprof*
 
@@ -48,7 +54,3 @@ script/vagrant/temp-vagrant-mediacloud
 
 # atom sync config
 .remote-sync.json
-
-# Inline:: binaries
-_Inline*
-

.travis.yml

@@ -50,19 +50,14 @@ env:
 before_install:
     # Install APT packages (not PostgreSQL though)
     - ./install_scripts/install_mediacloud_system_package_dependencies.sh
+    # Install Python dependencies
+    - travis_retry ./install_scripts/install_python_dependencies.sh
     # Set up kernel and PostgreSQL parameters
     - ./install_scripts/set_kernel_parameters.sh
     - ./install_scripts/set_postgresql_parameters.sh
-    # Set up local Media Cloud environment for Python
-    - mkdir -p $HOME/mc-python
-    - virtualenv $HOME/mc-python
-    - source $HOME/mc-python/bin/activate
-    # Install Python dependencies
-    - travis_retry pip install --upgrade pip setuptools
-    - travis_retry pip install --upgrade -r python_scripts/requirements.txt
     # Install Media Cloud testing variant of the Python Client
     - travis_retry git clone https://github.com/berkmancenter/mediacloud-api-testing-client.git MediaCloud-API-Client/
-    - travis_retry pip install --editable MediaCloud-API-Client/
+    - travis_retry sudo pip2 install --editable MediaCloud-API-Client/
     # Create + switch to the Media Cloud's Perlbrew library
     - perlbrew lib create mediacloud
     - perlbrew switch @mediacloud
@@ -80,6 +75,6 @@ before_script:
     - ./script/run_with_carton.sh ./script/mediawords_create_db.pl
 script:
     # Start Readability service for testing "PythonReadability" extractor method on t/test_extractor.t
-    - "python python_scripts/extractor_python_readability_server.py &"
+    - "python2 python_scripts/extractor_python_readability_server.py &"
     # Run Media Cloud's test suite, report test coverage to https://coveralls.io/r/berkmancenter/mediacloud
     - ./script/run_test_suite_for_devel_cover.sh coveralls --destroy-solr

api_test/api_media.t

@@ -108,7 +108,7 @@ sub test_media
             }
         ];
 
-        cmp_deeply( $actual_response, $expected_response, "response format mismatch for $url" );
+        cmp_deeply( $actual_response, $expected_response, "response format mismatch for $url; actual response: " . Dumper( $actual_response ) );
 
         foreach my $medium ( @{ $expected_response } )
         {
@@ -135,7 +135,7 @@ sub test_media
 
             my $feed_actual_response = decode_json( $response->decoded_content() );
 
-            cmp_deeply( $feed_actual_response, $expected_feed, 'response format mismatch for feed' );
+            cmp_deeply( $feed_actual_response, $expected_feed, 'response format mismatch for feed; actual response: ' . Dumper( $feed_actual_response ) );
         }
     }
 
@@ -178,7 +178,7 @@ sub test_tags
             }
         ];
 
-        cmp_deeply( $actual_response, $expected_response, "response format mismatch for $url" );
+        cmp_deeply( $actual_response, $expected_response, "response format mismatch for $url; actual response: " . Dumper( $actual_response ) );
     }
 }
 

api_test/run_api_test.sh

@@ -41,7 +41,7 @@ if ps aux | grep java | grep 8983 > /dev/null; then
 fi
 
 echo "Starting Solr..."
-python ./solr/run_solr_standalone.py &
+python3 ./solr/run_solr_standalone.py &
 solr_pid=$!
 
 SOLR_IS_UP=0
@@ -119,7 +119,7 @@ if MEDIACLOUD_IGNORE_DB_SCHEMA_VERSION=1 MEDIAWORDS_FORCE_USING_TEST_DATABASE=1
             done
 
         cd MediaCloud-API-Client
-        if python test.py; then
+        if python2 test.py; then
             echo "Python API test succeeded"
         else
             TEST_RETURN_STATUS=$?

cpanfile

@@ -108,6 +108,7 @@ requires 'Module::Install';
 requires 'Module::Pluggable::Object';
 requires 'Moo', '2.000001';
 requires 'Moose';
+requires 'Mouse', '2.4.5';
 requires 'namespace::autoclean';
 requires 'Net::Amazon::S3';
 requires 'Net::Calais';

cpanfile.snapshot

@@ -6268,10 +6268,10 @@ DISTRIBUTIONS
       MooseX::Types 0.04
       Path::Class 0.16
       Test::More 0.88
-  Mouse-2.4.1
-    pathname: G/GF/GFUJI/Mouse-2.4.1.tar.gz
+  Mouse-v2.4.5
+    pathname: S/SY/SYOHEX/Mouse-v2.4.5.tar.gz
     provides:
-      Mouse v2.4.1
+      Mouse v2.4.5
       Mouse::Exporter undef
       Mouse::Meta::Attribute undef
       Mouse::Meta::Class undef
@@ -6289,19 +6289,19 @@ DISTRIBUTIONS
       Mouse::Meta::TypeConstraint undef
       Mouse::Object undef
       Mouse::PurePerl undef
-      Mouse::Role v2.4.1
-      Mouse::Spec v2.4.1
-      Mouse::Tiny v2.2.0
+      Mouse::Role v2.4.5
+      Mouse::Spec v2.4.5
+      Mouse::Tiny v2.4.5
       Mouse::TypeRegistry undef
-      Mouse::Util v2.4.1
+      Mouse::Util v2.4.5
       Mouse::Util::MetaRole undef
       Mouse::Util::TypeConstraints undef
       Squirrel undef
       Squirrel::Role undef
       Test::Mouse undef
       ouse undef
     requirements:
-      Devel::PPPort 3.19
+      Devel::PPPort 3.22
       ExtUtils::ParseXS 3.22
       Module::Build 0.4005
       Module::Build::XSUtil 0

doc/temporarycrawler.markdown

@@ -62,7 +62,7 @@ Following are instructions for setting up and running a temporary crawler (backu
 
 6. Import media and feeds from the production server by running:
 
-        python python_scripts/media_import.py --api-key API-KEY
+        python2 python_scripts/media_import.py --api-key API-KEY
 
     where `API-KEY` is an API key for mediacloud.org.
 
@@ -102,7 +102,7 @@ When you're ready to export:
 
 4. Export `feed_downloads` to the production system by running:
 
-        python python_scripts/export_feed_downloads_through_api.py \
+        python2 python_scripts/export_feed_downloads_through_api.py \
             --source-api-key SOURCE_API_KEY \
             --dest-api-key DEST_API_KEY \
             --source-media-cloud-api_url SOURCE_MEDIA_CLOUD_API_URL \
@@ -117,7 +117,7 @@ When you're ready to export:
 
     E.g.:
 
-        python python_scripts/export_feed_downloads_through_api.py \
+        python2 python_scripts/export_feed_downloads_through_api.py \
             --source-api-key 'e07cf98dd0d457351354ee520635c226acd238ecf15ec9e853346e185343bf7b' \
             --dest-api-key '1161251f5de4f381a198eea4dc20350fd992f5eef7cb2fdc284c245ff3d4f3ca' \
             --source-media-cloud-api_url 'http://localhost:3000/' \

install.sh

@@ -19,18 +19,11 @@ if [ ! -f mediawords.yml ]; then
 fi
 
 ./install_scripts/install_mediacloud_package_dependencies.sh
+./install_scripts/install_python_dependencies.sh
 ./install_scripts/set_kernel_parameters.sh
 ./install_scripts/set_postgresql_parameters.sh
 ./install_scripts/install_mc_perlbrew_and_modules.sh
 
-# Install Python dependencies
-sudo pip install --upgrade pip # make sure pip is latest version to avoid errors
-sudo pip install --upgrade -r python_scripts/requirements.txt || {
-    # Sometimes fails with some sort of Setuptools error
-    echo "'pip install' failed the first time, retrying..."
-    sudo pip install --upgrade -r python_scripts/requirements.txt
-}
-
 echo "install complete"
 echo "running compile test"
 

install_scripts/install_mediacloud_system_package_dependencies.sh

@@ -4,7 +4,7 @@ set -u
 set -o errexit
 
 
-VAGRANT_URL_DEBIAN="https://releases.hashicorp.com/vagrant/1.8.5/vagrant_1.8.5_x86_64.deb"
+VAGRANT_URL_DEBIAN="https://releases.hashicorp.com/vagrant/1.8.6/vagrant_1.8.6_x86_64.deb"
 ERLANG_APT_GPG_KEY_URL="http://packages.erlang-solutions.com/ubuntu/erlang_solutions.asc"
 ERLANG_APT_REPOSITORY_URL="http://packages.erlang-solutions.com/ubuntu"
 RABBITMQ_PACKAGECLOUD_SCRIPT="https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.deb.sh"
@@ -59,7 +59,7 @@ function verlt() {
 }
 
 
-echo "installing media cloud system dependencies"
+echo "Installing Media Cloud system dependencies..."
 echo
 
 if [ `uname` == 'Darwin' ]; then
@@ -87,6 +87,7 @@ EOF
         exit 1
     fi
 
+    echo "Installing Media Cloud dependencies with Homebrew..."
     brew install \
         coreutils \
         cpanminus \
@@ -96,12 +97,13 @@ EOF
         homebrew/dupes/tidy \
         hunspell \
         libyaml \
-        libyaml \
         netcat \
         openssl \
+        python3 \
         rabbitmq \
         #
 
+    echo "Installing Media Cloud dependencies with cpanm..."
     sudo cpanm \
         DBD::Pg \
         Graph \
@@ -138,6 +140,8 @@ else
 
     # assume Ubuntu
     source /etc/lsb-release
+
+    echo "Installing curl..."
     sudo apt-get -y install curl
 
     # Apt's versions of Supervisor, Vagrant, RabbitMQ are too old
@@ -154,7 +158,11 @@ else
 
     if verlt "$DISTRIB_RELEASE" "14.04"; then
         # Ubuntu < 14.04 APT's version of Erlang is too old (needed by RabbitMQ)
+        echo "Removing system package Erlang on Ubuntu 12.04 because it's too old..."
         sudo apt-get -y remove erlang*
+
+        # Install and hold specific version of Erlang
+        echo "Installing Erlang from Erlang Solutions..."
         curl "$ERLANG_APT_GPG_KEY_URL" | sudo apt-key add -
         echo "deb $ERLANG_APT_REPOSITORY_URL precise contrib" | \
             sudo tee -a /etc/apt/sources.list.d/erlang-solutions.list
@@ -169,6 +177,7 @@ else
 
     # Ubuntu (all versions) APT's version of RabbitMQ is too old
     # (we need 3.6.0+ to support priorities and lazy queues)
+    echo "Adding RabbitMQ GPG key for Apt..."
     curl -s "$RABBITMQ_PACKAGECLOUD_SCRIPT" | sudo bash
 
     if verlt "$DISTRIB_RELEASE" "14.04"; then
@@ -184,12 +193,23 @@ else
 
     if verlt "$DISTRIB_RELEASE" "16.04"; then
         # Solr 6+ requires Java 8 which is unavailable before 16.04
+        echo "Adding Java 8 PPA repository to Ubuntu 12.04..."
         sudo apt-get -y install python-software-properties
         sudo add-apt-repository -y ppa:openjdk-r/ppa
         sudo apt-get update
     fi
 
+    # Python version to install
+    if verlt "$DISTRIB_RELEASE" "16.04"; then
+        # We require at least Python 3.5 (12.04 only has 3.2 which doesn't work with newest Pip)
+        echo "Adding Python 3.5 PPA repository to Ubuntu 12.04..."
+        sudo apt-get -y install python-software-properties
+        sudo add-apt-repository -y ppa:fkrull/deadsnakes
+        sudo apt-get update
+    fi
+
     # Install the rest of the packages
+    echo "Installing Media Cloud dependencies with APT..."
     sudo apt-get --assume-yes install \
         build-essential \
         cpanminus \
@@ -231,17 +251,21 @@ else
         python-pip \
         python2.7 \
         python2.7-dev \
+        python3.5 \
+        python3.5-dev \
         unzip \
         #
 
     # Choose to use OpenJDK 8 by default
     PATH="$PATH:/usr/sbin"
+    echo "Selecting Java 8..."
     sudo update-java-alternatives -s `update-java-alternatives --list | grep java-1.8 | awk '{ print $3 }'`
 
     # Install / upgrade Setuptools before installing Python dependencies
     wget https://bootstrap.pypa.io/ez_setup.py -O - | sudo python2.7 -
 
     # Disable system-wide RabbitMQ server (we will start and use our very own instance)
+    echo "Stopping and disabling system's RabbitMQ instance..."
     sudo update-rc.d rabbitmq-server disable
     sudo service rabbitmq-server stop
 
@@ -284,12 +308,3 @@ else
     fi
 
 fi
-
-# Install (upgrade) Supervisor
-# (change dir, otherwise the installer might think we're trying to install
-# from the supervisor/ directory)
-if [ `uname` == 'Darwin' ]; then
-    ( cd /tmp; pip install --upgrade supervisor )
-else
-    ( cd /tmp; sudo pip install --upgrade supervisor )
-fi

install_scripts/install_modules_with_carton.sh

@@ -35,12 +35,17 @@ if [ `uname` == 'Darwin' ]; then
         Net::AMQP::RabbitMQ~2.100001
 fi
 
+# Inline::Python needs to use virtualenv's Python 3 instead of the default Python2
+set +u; source mc-venv/bin/activate; set -u
+INLINE_PYTHON_EXECUTABLE=`which python3`
+
 # Install dependency modules; run the command twice because the first
 # attempt might fail
 ATTEMPT=1
 CARTON_INSTALL_SUCCEEDED=0
 until [ $ATTEMPT -ge 3 ]; do
     OPENSSL_PREFIX=$OPENSSL_PREFIX \
+    INLINE_PYTHON_EXECUTABLE=$INLINE_PYTHON_EXECUTABLE \
     ./script/run_carton.sh install && {
         echo "Successfully installed Carton modules"
         CARTON_INSTALL_SUCCEEDED=1

install_scripts/install_python_dependencies.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -e
+# no "set -u" because installation scripts address various undefined variables
+
+# Version comparison functions
+function verlte() {
+    [  "$1" = "`echo -e "$1\n$2" | sort -V | head -n1`" ]
+}
+
+function verlt() {
+    [ "$1" = "$2" ] && return 1 || verlte "$1" "$2"
+}
+
+# Install / upgrade Setuptools (easy_install) to both Python versions before installing dependencies
+wget https://bootstrap.pypa.io/ez_setup.py -O - | sudo python2.7 -
+wget https://bootstrap.pypa.io/ez_setup.py -O - | sudo python3.5 -
+
+# Install / upgrade Pip to both Python versions before installing dependencies
+sudo easy_install-2.7 pip
+sudo easy_install-3.5 pip
+
+# Install (upgrade) Supervisor:
+# * change dir, otherwise the installer might think we're trying to install from the supervisor/ directory
+# * also, Supervisor only supports Python 2.7 at the moment
+( cd /tmp; sudo pip2.7 install --upgrade supervisor )
+
+# Install (upgrade Virtualenv)
+sudo pip2.7 install --upgrade virtualenv
+sudo pip3.5 install --upgrade virtualenv
+
+# Install Python 2 dependencies
+sudo pip2.7 install --upgrade -r python_scripts/requirements.txt || {
+    # Sometimes fails with some sort of Setuptools error
+    echo "'pip2.7 install' failed the first time, retrying..."
+    sudo pip2.7 install --upgrade -r python_scripts/requirements.txt
+}
+
+# Create virtualenv and activate it
+virtualenv --python=python3.5 mc-venv
+source mc-venv/bin/activate
+
+# Install Python 3 dependencies (no sudo because we're in virtualenv)
+pip3.5 install --upgrade -r mediacloud/requirements.txt || {
+    # Sometimes fails with some sort of Setuptools error
+    echo "'pip3.5 install' failed the first time, retrying..."
+    pip3.5 install --upgrade -r mediacloud/requirements.txt
+}