after running map-red successfully.

.gitignore

@@ -0,0 +1,2 @@
+*.swp
+local/

Dockerfile

@@ -0,0 +1,53 @@
+from ubuntu
+LABEL maintainer="[email protected]"
+
+RUN  apt-get update \
+  && apt-get install -y wget \
+  && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /home/
+RUN wget https://archive.apache.org/dist/hadoop/core/hadoop-2.7.1/hadoop-2.7.1.tar.gz
+
+RUN apt-get update && \
+    apt-get install -y software-properties-common python-software-properties debconf-utils && \
+    add-apt-repository ppa:webupd8team/java && \
+    apt-get update && \
+    echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | sudo debconf-set-selections && \
+    apt-get install -y oracle-java8-installer && \
+    java -version
+
+RUN apt-get update && \
+    apt-get install --no-install-recommends  -y ssh rsync &&  \
+    rm -rf /var/lib/apt/lists/*
+
+
+RUN cp ./hadoop-2.7.1.tar.gz /usr/local/hadoop.tar.gz && \
+    cd /usr/local && \
+    tar xzf hadoop.tar.gz && \
+    mv hadoop-2.7.1 hadoop
+
+
+# set up environment variables and conffig files
+ARG RECONFIG=1
+ENV HADOOP_HOME /usr/local/hadoop
+ENV JAVA_HOME /usr/lib/jvm/java-8-oracle
+ENV HADOOP_CLASSPATH=/usr/lib/jvm/java-8-oracle/lib/tools.jar
+RUN echo "# Some convenient aliases and functions for running Hadoop-related commands\nunalias fs &> /dev/null\nalias fs=\"hadoop fs\"\nunalias hls &> /dev/null\nalias hls=\"fs -ls\"" >> ~/.bashrc
+ENV PATH="${PATH}:$HADOOP_HOME/bin"
+COPY start.sh yarn-site.xml hadoop-env.sh hdfs-site.xml mapred-site.xml core-site.xml $HADOOP_HOME/etc/hadoop/
+ADD WordCount.java mahdiz.big ./
+
+RUN rm -f /etc/ssh/ssh_host_dsa_key && ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key && \
+    rm -f /etc/ssh/ssh_host_rsa_key && ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key && \
+    rm -f /root/.ssh/id_rsa | ssh-keygen -q -N "" -t rsa -f /root/.ssh/id_rsa && \
+    cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys 
+
+RUN mkdir -p /app/hadoop/tmp && \
+    mkdir -p /root/.ssh/ && \
+    service ssh start && \
+    ssh-keyscan localhost  >> ~/.ssh/known_hosts && \
+    ssh-keyscan 0.0.0.0  >> ~/.ssh/known_hosts && \
+    hdfs namenode -format && \
+    chmod +x $HADOOP_HOME/etc/hadoop/start.sh
+
+ENTRYPOINT $HADOOP_HOME/etc/hadoop/start.sh && /bin/bash

WordCount.java

@@ -0,0 +1,61 @@
+import java.io.IOException;
+import java.util.StringTokenizer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+
+public class WordCount {
+
+  public static class TokenizerMapper
+       extends Mapper<Object, Text, Text, IntWritable>{
+
+    private final static IntWritable one = new IntWritable(1);
+    private Text word = new Text();
+
+    public void map(Object key, Text value, Context context
+                    ) throws IOException, InterruptedException {
+      StringTokenizer itr = new StringTokenizer(value.toString());
+      while (itr.hasMoreTokens()) {
+        word.set(itr.nextToken());
+        context.write(word, one);
+      }
+    }
+  }
+
+  public static class IntSumReducer
+       extends Reducer<Text,IntWritable,Text,IntWritable> {
+    private IntWritable result = new IntWritable();
+
+    public void reduce(Text key, Iterable<IntWritable> values,
+                       Context context
+                       ) throws IOException, InterruptedException {
+      int sum = 0;
+      for (IntWritable val : values) {
+        sum += val.get();
+      }
+      result.set(sum);
+      context.write(key, result);
+    }
+  }
+
+  public static void main(String[] args) throws Exception {
+    Configuration conf = new Configuration();
+    Job job = Job.getInstance(conf, "word count");
+    job.setJarByClass(WordCount.class);
+    job.setMapperClass(TokenizerMapper.class);
+    job.setCombinerClass(IntSumReducer.class);
+    job.setReducerClass(IntSumReducer.class);
+    job.setOutputKeyClass(Text.class);
+    job.setOutputValueClass(IntWritable.class);
+    FileInputFormat.addInputPath(job, new Path(args[0]));
+    FileOutputFormat.setOutputPath(job, new Path(args[1]));
+    System.exit(job.waitForCompletion(true) ? 0 : 1);
+  }
+}

core-site.multi-node.xml

@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?> 
+
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> 
+
+<configuration>
+
+    <property>
+        <name>hadoop.tmp.dir</name>
+        <value>/app/hadoop/tmp</value>
+        <description>A base for other temporary directories.</description>
+    </property> 
+
+    <property>
+        <name>fs.defaultFS</name>
+        <value>hdfs://localhost:54310</value> 
+        <description>
+            The name of the default file system. A URI whose scheme and authority
+            determine the FileSystem implementation. The uri's scheme determines
+            the config property (fs.SCHEME.impl) naming the FileSystem implementation
+            class. The uri's authority is used to determine the host, port, etc. for a filesystem.
+        </description> 
+    </property> 
+
+    <property>
+        <name>dfs.http.address</name>
+        <value>50077</value>
+    </property> 
+
+    <property>
+        <name>dfs.datanode.http.address</name>
+        <value>50075</value>
+    </property> 
+
+    <property>
+        <name>dfs.secondary.http.address</name>
+        <value>50090</value>
+    </property> 
+
+    <property>
+        <name>dfs.backup.http.address</name>
+        <value>50105</value>
+    </property>
+
+</configuration> 

core-site.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+	<property>
+		<name>hadoop.tmp.dir</name>
+		<value>/app/hadoop/tmp</value>
+		<description>A base for other temporary directories.</description>
+	</property>
+
+	<property>
+		<name>fs.defaultFS</name>
+		<value>hdfs://localhost:54310</value>
+		<description>
+			The name of the default file system. A URI whose scheme and
+			authority determine the FileSystem implementation. The uri's scheme
+			determines the config property (fs.SCHEME.impl) naming the FileSystem
+			implementation class. The uri's authority is used to determine the host,
+			port, etc. for a filesystem.
+		</description>
+	</property>
+
+</configuration>
+
+

core-site.xml.multy

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?> 
+<?xml-stylesheet type="text/xsl" href="configuration.xs1"7> 
+<configuration> <property> <name>hadoop.tmp.dir</name> <value>/app/hadoop/tmp</value> <description>A base for other temporary directories.</description> </property> 
+<property> <name>fs.defaultFS</name> <value>hdfs://localhost:54310</value> 

hadoop-env.sh

@@ -0,0 +1,99 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Set Hadoop-specific environment variables here.
+
+# The only required environment variable is JAVA_HOME.  All others are
+# optional.  When running a distributed configuration it is best to
+# set JAVA_HOME in this file, so that it is correctly defined on
+# remote nodes.
+
+# The java implementation to use.
+# export JAVA_HOME=${JAVA_HOME}
+export JAVA_HOME=/usr/lib/jvm/java-8-oracle
+
+# The jsvc implementation to use. Jsvc is required to run secure datanodes
+# that bind to privileged ports to provide authentication of data transfer
+# protocol.  Jsvc is not required if SASL is configured for authentication of
+# data transfer protocol using non-privileged ports.
+#export JSVC_HOME=${JSVC_HOME}
+
+export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/etc/hadoop"}
+
+# Extra Java CLASSPATH elements.  Automatically insert capacity-scheduler.
+for f in $HADOOP_HOME/contrib/capacity-scheduler/*.jar; do
+  if [ "$HADOOP_CLASSPATH" ]; then
+    export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$f
+  else
+    export HADOOP_CLASSPATH=$f
+  fi
+done
+
+# The maximum amount of heap to use, in MB. Default is 1000.
+#export HADOOP_HEAPSIZE=
+#export HADOOP_NAMENODE_INIT_HEAPSIZE=""
+
+# Extra Java runtime options.  Empty by default.
+export HADOOP_OPTS="$HADOOP_OPTS -Djava.net.preferIPv4Stack=true"
+
+# Command specific options appended to HADOOP_OPTS when specified
+export HADOOP_NAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_NAMENODE_OPTS"
+export HADOOP_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS $HADOOP_DATANODE_OPTS"
+
+export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_SECONDARYNAMENODE_OPTS"
+
+export HADOOP_NFS3_OPTS="$HADOOP_NFS3_OPTS"
+export HADOOP_PORTMAP_OPTS="-Xmx512m $HADOOP_PORTMAP_OPTS"
+
+# The following applies to multiple commands (fs, dfs, fsck, distcp etc)
+export HADOOP_CLIENT_OPTS="-Xmx512m $HADOOP_CLIENT_OPTS"
+#HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData $HADOOP_JAVA_PLATFORM_OPTS"
+
+# On secure datanodes, user to run the datanode as after dropping privileges.
+# This **MUST** be uncommented to enable secure HDFS if using privileged ports
+# to provide authentication of data transfer protocol.  This **MUST NOT** be
+# defined if SASL is configured for authentication of data transfer protocol
+# using non-privileged ports.
+export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER}
+
+# Where log files are stored.  $HADOOP_HOME/logs by default.
+#export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}/$USER
+
+# Where log files are stored in the secure data environment.
+export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
+
+###
+# HDFS Mover specific parameters
+###
+# Specify the JVM options to be used when starting the HDFS Mover.
+# These options will be appended to the options specified as HADOOP_OPTS
+# and therefore may override any similar flags set in HADOOP_OPTS
+#
+# export HADOOP_MOVER_OPTS=""
+
+###
+# Advanced Users Only!
+###
+
+# The directory where pid files are stored. /tmp by default.
+# NOTE: this should be set to a directory that can only be written to by
+#       the user that will run the hadoop daemons.  Otherwise there is the
+#       potential for a symlink attack.
+export HADOOP_PID_DIR=${HADOOP_PID_DIR}
+export HADOOP_SECURE_DN_PID_DIR=${HADOOP_PID_DIR}
+
+# A string representing this instance of hadoop. $USER by default.
+export HADOOP_IDENT_STRING=$USER

hdfs-site.multi-node.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+	<property> 
+		<name>dfs.replication</name>
+		<value>2</value>
+		<description>
+			Default block replication. The actual number of replications can be
+			specified when the file is created. The default is used if replication is
+			not specified in create time.
+		</description>
+	</property>
+
+	<property>
+		<name>dfs.namenode.name.dir</name>
+		<value>/home/hduser/Hadoop/hdfs/namenode</value>
+	</property>
+
+	<property>
+		<name>dfs.datanode.name.dir</name>
+		<value>/home/hduser/Hadoop/hdfs/datanode</value>
+	</property>
+
+</configuration>
+
+
+
+

hdfs-site.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+	<property> 
+		<name>dfs.replication</name>
+		<value>1</value>
+		<description>
+			Default block replication. The actual number of replications can be
+			specified when the file is created. The default is used if replication is
+			not specified in create time.
+		</description>
+	</property>
+
+	<property>
+		<name>dfs.namenode.name.dir</name>
+		<value>/home/hduser/Hadoop/hdfs/namenode</value>
+	</property>
+
+	<property>
+		<name>dfs.datanode.name.dir</name>
+		<value>/home/hduser/Hadoop/hdfs/datanode</value>
+	</property>
+
+</configuration>
+
+
+
+

mapred-site.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+	<property>
+		<name>mapreduce.framework.name</name>
+		<value>yarn</value>
+		<description>Execution framework set to Hadoop YARN</description>
+	</property>
+
+	<property>
+		<name>mapreduce.jobhistory.address</name>
+		<value>127.0.0.1:10020</value>
+	</property>
+
+	<property>
+		<name>mapreduce.jobhistory.webapp.address</name>
+		<value>127.0.0.1:19888</value>
+	</property>
+
+</configuration>
+
+