PIG-4526: Make setting up the build environment easier (nielsbasjes v…

…ia rohini)

git-svn-id: https://svn.apache.org/repos/asf/pig/trunk@1741275 13f79535-47bb-0310-9956-ffa450edef68

BUILDING.md

@@ -0,0 +1,75 @@
+# Building Apache Pig
+
+## Requirements:
+
+* Unix System
+* JDK 1.7+
+* Ant 1.8.1+
+* Findbugs 3.x+
+* Forrest 0.9 (for building the documentation)
+* Internet connection for first build (to fetch all dependencies)
+
+**Note**: Further down this document you can read about the _ready to run build environment_.
+
+## Building Pig
+
+To compile with Hadoop 1.x 
+
+    ant clean jar piggybank 
+
+To compile with Hadoop 2.x 
+
+    ant clean jar piggybank -Dhadoopversion=23
+
+Building and running the tests needed before submitting a patch.
+For more details https://cwiki.apache.org/confluence/display/PIG/HowToContribute
+
+    ANT_OPTS='-Djavac.args="-Xlint -Xmaxwarns 1000" -Dhadoopversion=23'
+    ant ${ANT_OPTS} clean piggybank jar compile-test test-commit
+    cd contrib/piggybank/java && ant ${ANT_OPTS} test
+
+Generate documentation
+
+    ant docs
+
+# Ready to run build environment
+The easiest way to get an environment with all the appropriate tools is by means
+of the provided Docker config.
+This requires a recent version of docker ( 1.4.1 and higher are known to work ).
+
+## How it works
+By using the mounted volumes feature of Docker this image will wrap itself around the directory from which it is started.
+So the files within the docker environment are actually the same as outsite.
+
+A very valid way of working is by having your favourite IDE that has the project 
+open and a commandline into the docker that has the exact right tools to do the full build.
+
+## Using it on Linux:
+Install Docker and run this command:
+
+    $ ./start-build-env.sh
+
+## Using it on Mac:
+First make sure Homebrew has been installed ( http://brew.sh/ )
+
+    $ brew install docker boot2docker
+    $ boot2docker init -m 4096
+    $ boot2docker start
+    $ $(boot2docker shellinit)
+    $ ./start-build-env.sh
+
+The prompt which is then presented is located at a mounted version of the source tree
+and all required tools for testing and building have been installed and configured.
+
+Note that from within this docker environment you ONLY have access to the source
+tree from where you started. 
+
+## Known issues:
+On Mac with Boot2Docker the performance on the mounted directory is currently extremely slow.
+This is a known problem related to boot2docker on the Mac.
+https://github.com/boot2docker/boot2docker/issues/593
+    This issue has been resolved as a duplicate, and they point to a new feature for utilizing NFS mounts as the proposed solution:
+
+https://github.com/boot2docker/boot2docker/issues/64
+    An alternative solution to this problem is when you install Linux native inside a virtual machine and run your IDE and Docker etc in side that VM.
+

CHANGES.txt

@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
 
 IMPROVEMENTS
 
+PIG-4526: Make setting up the build environment easier (nielsbasjes via rohini)
+
 PIG-4641: Print the instance of Object without using toString() (sandyridgeracer via rohini)
 
 PIG-4455: Should use DependencyOrderWalker instead of DepthFirstWalker in MRPrinter (zjffdu via rohini)

dev-support/docker/Dockerfile

@@ -0,0 +1,94 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Dockerfile for installing the necessary dependencies for building Apache Pig.
+# See BUILDING.md.
+
+FROM ubuntu:trusty
+
+# Define working directory.
+WORKDIR /root
+
+RUN apt-get update
+
+# Install dependencies from packages
+RUN sed -i 's/# \(.*multiverse$\)/\1/g' /etc/apt/sources.list && \
+    apt-get install -y build-essential && \
+    apt-get install -y software-properties-common && \
+    apt-get install --no-install-recommends -y \
+            git subversion \
+            byobu htop man unzip vim \
+            cabal-install \
+            curl wget \
+            openjdk-7-jdk \
+            ant ant-contrib ant-optional make maven \
+            cmake gcc g++ protobuf-compiler \
+            build-essential libtool \
+            zlib1g-dev pkg-config libssl-dev \
+            snappy libsnappy-dev \
+            bzip2 libbz2-dev \
+            libjansson-dev \
+            fuse libfuse-dev \
+            libcurl4-openssl-dev \
+            python python2.7 && \
+    rm -rf /var/lib/apt/lists/*
+
+# Define commonly used JAVA_HOME variable
+ENV JAVA_HOME /usr/lib/jvm/java-7-openjdk-amd64
+
+# Fixing the Apache commons / Maven dependency problem under Ubuntu:
+# See http://wiki.apache.org/commons/VfsProblems
+RUN cd /usr/share/maven/lib && ln -s ../../java/commons-lang.jar .
+
+# Avoid out of memory errors in builds
+ENV MAVEN_OPTS -Xms256m -Xmx512m
+
+# Install findbugs
+RUN mkdir -p /opt/findbugs && \
+    wget http://sourceforge.net/projects/findbugs/files/findbugs/3.0.1/findbugs-noUpdateChecks-3.0.1.tar.gz/download \
+         -O /opt/findbugs.tar.gz && \
+    tar xzf /opt/findbugs.tar.gz --strip-components 1 -C /opt/findbugs
+ENV FINDBUGS_HOME /opt/findbugs
+
+# Install Forrest in /usr/local/apache-forrest
+# Screenscrape the download page for a local mirror URL
+RUN cd /usr/local/ && \
+    curl https://forrest.apache.org/mirrors.cgi | \
+    fgrep href | fgrep apache-forrest-0.9 | \
+    sed 's@^.*"\(http[^"]*apache-forrest-[^"]*.tar.gz\)".*@\1@' | \
+    xargs -n1 -r wget
+
+# Unpack Apache Forrest
+RUN cd /usr/local/ && \
+    tar xzf apache-forrest-0.9-sources.tar.gz && \
+    tar xzf apache-forrest-0.9-dependencies.tar.gz && \
+    mv apache-forrest-0.9 apache-forrest
+RUN cd /usr/local/apache-forrest/main && ./build.sh
+
+# The solution for https://issues.apache.org/jira/browse/PIG-3906
+RUN mkdir -p /usr/local/apache-forrest/plugins       && chmod a+rwX -R /usr/local/apache-forrest/plugins
+RUN mkdir -p /usr/local/apache-forrest/build/plugins && chmod a+rwX -R /usr/local/apache-forrest/build/plugins
+
+# Configure where forrest can be found
+RUN echo 'forrest.home=/usr/local/apache-forrest' > build.properties
+ENV FORREST_HOME /usr/local/apache-forrest
+
+# Add a welcome message and environment checks.
+ADD build_env_checks.sh /root/build_env_checks.sh
+RUN chmod 755 /root/build_env_checks.sh
+ADD configure-for-user.sh /root/configure-for-user.sh
+RUN chmod 755 /root/configure-for-user.sh
+RUN echo '~/build_env_checks.sh' >> /root/.bashrc

dev-support/docker/build_env_checks.sh

@@ -0,0 +1,120 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# -------------------------------------------------------
+function showWelcome {
+
+# http://patorjk.com/software/taag/#p=display&f=Doom&t=Pig%20Builder
+cat <<Welcome-message
+
+______ _        ______       _ _     _
+| ___ (_)       | ___ \\     (_) |   | |
+| |_/ /_  __ _  | |_/ /_   _ _| | __| | ___ _ __
+|  __/| |/ _\` | | ___ \\ | | | | |/ _\` |/ _ \\ '__|
+| |   | | (_| | | |_/ / |_| | | | (_| |  __/ |_
+\\_|   |_|\\__, | \\____/ \\__,_|_|_|\\__,_|\\___|_|
+          __/ |
+         |___/
+
+This is the standard Apache Pig Developer build environment.
+This has all the right tools installed required to build
+Pig from source.
+
+Welcome-message
+}
+
+# -------------------------------------------------------
+
+function showAbort {
+  cat <<Abort-message
+
+  ___  _                _   _
+ / _ \\| |              | | (_)
+/ /_\\ \\ |__   ___  _ __| |_ _ _ __   __ _
+|  _  | '_ \\ / _ \\| '__| __| | '_ \\ / _\` |
+| | | | |_) | (_) | |  | |_| | | | | (_| |
+\\_| |_/_.__/ \\___/|_|   \\__|_|_| |_|\\__, |
+                                     __/ |
+                                    |___/
+
+Abort-message
+}
+
+# -------------------------------------------------------
+
+function failIfUserIsRoot {
+    if [ "$(id -u)" -eq "0" ]; # If you are root then something went wrong.
+    then
+        cat <<End-of-message
+
+Apparently you are inside this docker container as the user root.
+Putting it simply:
+
+   This should not occur.
+
+Known possible causes of this are:
+1) Running this script as the root user ( Just don't )
+2) Running an old docker version ( upgrade to 1.4.1 or higher )
+
+End-of-message
+
+    showAbort
+
+    logout
+
+    fi
+}
+
+# -------------------------------------------------------
+
+# Configurable low water mark in GiB
+MINIMAL_MEMORY_GiB=2
+
+function warnIfLowMemory {
+    MINIMAL_MEMORY=$((MINIMAL_MEMORY_GiB*1024*1024)) # Convert to KiB
+    INSTALLED_MEMORY=$(fgrep MemTotal /proc/meminfo | awk '{print $2}')
+    if [ $((INSTALLED_MEMORY)) -le $((MINIMAL_MEMORY)) ];
+    then
+        cat <<End-of-message
+
+ _                    ___  ___
+| |                   |  \\/  |
+| |     _____      __ | .  . | ___ _ __ ___   ___  _ __ _   _
+| |    / _ \\ \\ /\\ / / | |\\/| |/ _ \\ '_ \` _ \\ / _ \\| '__| | | |
+| |___| (_) \\ V  V /  | |  | |  __/ | | | | | (_) | |  | |_| |
+\\_____/\\___/ \\_/\\_/   \\_|  |_/\\___|_| |_| |_|\\___/|_|   \\__, |
+                                                         __/ |
+                                                        |___/
+
+Your system is running on very little memory.
+This means it may work but it wil most likely be slower than needed.
+
+If you are running this via boot2docker you can simply increase
+the available memory to atleast ${MINIMAL_MEMORY_GiB} GiB (you have $((INSTALLED_MEMORY/(1024*1024))) GiB )
+
+End-of-message
+    fi
+}
+
+# -------------------------------------------------------
+
+showWelcome
+warnIfLowMemory
+failIfUserIsRoot
+
+# -------------------------------------------------------

dev-support/docker/configure-for-user.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is used to tweak the environment at the moment we know 
+# the real username of the person using this.
+# By making this script a part of the image we can extend and update 
+# it to fit future needs more easily.
+
+# Native Linux (direct or via sudo)
+USER_NAME=$1
+USER_ID=$2
+GROUP_ID=$3
+
+groupadd --non-unique -g ${GROUP_ID} ${USER_NAME}
+useradd -g ${GROUP_ID} -u ${USER_ID} -k /root -m ${USER_NAME}
+echo "export HOME=/home/${USER_NAME}" >> ~/.bashrc
+echo "export USER=${USER_NAME}" >> ~/.bashrc
+
+VBOXSF_GROUP_LINE=$4
+if [ -n ${VBOXSF_GROUP_LINE} ];
+then
+    echo ${VBOXSF_GROUP_LINE} >> /etc/group
+    usermod -aG vboxsf ${USER_NAME}
+fi
+
+echo "${USER_NAME}    ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers.d/${USER_NAME}