PIG-4948: Pig on Tez AM use too much memory on a small cluster

git-svn-id: https://svn.apache.org/repos/asf/pig/trunk@1753329 13f79535-47bb-0310-9956-ffa450edef68
lujiefsi · Jul 19, 2016 · 1ad1ecd · 1ad1ecd
1 parent 8771726
commit 1ad1ecd
Show file tree

Hide file tree

Showing 5 changed files with 16 additions and 1 deletion.
diff --git a/CHANGES.txt b/CHANGES.txt
@@ -60,6 +60,8 @@ OPTIMIZATIONS
 
 BUG FIXES
 
+PIG-4948: Pig on Tez AM use too much memory on a small cluster (daijy)
+
 PIG-4949: Fix registering jar in S3 which was broken by PIG-4417 in Pig 0.16 (yangyishan0901m via daijy)
 
 PIG-4950: Fix minor issues with running scripts in non-local FileSystems (petersla via daijy)

diff --git a/conf/pig.properties b/conf/pig.properties
@@ -678,3 +678,6 @@ pig.ats.enabled=true
 
 pig.sort.readonce.loadfuncs=org.apache.pig.backend.hadoop.hbase.HBaseStorage,org.apache.pig.backend.hadoop.accumulo.AccumuloStorage
 
+# If set, Pig will override tez.am.launch.cmd-opts and tez.am.resource.memory.mb to optimal
+# even they are set to a different value. Default value is true.
+#pig.tez.configure.am.memory=false
diff --git a/src/org/apache/pig/PigConfiguration.java b/src/org/apache/pig/PigConfiguration.java
@@ -434,6 +434,11 @@ private PigConfiguration() {}
      */
     public static final String ENABLE_ATS = "pig.ats.enabled";
 
+    /**
+     * Enable ATS for Pig
+     */
+    public static final String PIG_TEZ_CONFIGURE_AM_MEMORY = "pig.tez.configure.am.memory";
+
     // Deprecated settings of Pig 0.13
 
     /**

diff --git a/src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java b/src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java
@@ -31,6 +31,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.pig.PigConfiguration;
 import org.apache.pig.backend.hadoop.executionengine.tez.TezJob.TezJobConfig;
 import org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper;
 import org.apache.pig.impl.PigContext;
@@ -99,7 +100,9 @@ private static SessionInfo createSession(TezConfiguration amConf,
         MRToTezHelper.translateMRSettingsForTezAM(amConf);
         TezScriptState ss = TezScriptState.get();
         ss.addDAGSettingsToConf(amConf);
-        adjustAMConfig(amConf, tezJobConf);
+        if (amConf.getBoolean(PigConfiguration.PIG_TEZ_CONFIGURE_AM_MEMORY, true)) {
+            adjustAMConfig(amConf, tezJobConf);
+        }
         String jobName = amConf.get(PigContext.JOB_NAME, "pig");
         TezClient tezClient = TezClient.create(jobName, amConf, true, requestedAMResources, creds);
         try {

diff --git a/src/pig-default.properties b/src/pig-default.properties
@@ -64,3 +64,5 @@ pig.tez.opt.union.unsupported.storefuncs=org.apache.hcatalog.pig.HCatStorer,org.
 pig.sort.readonce.loadfuncs=org.apache.pig.backend.hadoop.hbase.HBaseStorage,org.apache.pig.backend.hadoop.accumulo.AccumuloStorage
 
 pig.ats.enabled=true
+
+pig.tez.configure.am.memory=true
Original file line number	Diff line number	Diff line change
Expand Up		@@ -64,3 +64,5 @@ pig.tez.opt.union.unsupported.storefuncs=org.apache.hcatalog.pig.HCatStorer,org.
		pig.sort.readonce.loadfuncs=org.apache.pig.backend.hadoop.hbase.HBaseStorage,org.apache.pig.backend.hadoop.accumulo.AccumuloStorage

		pig.ats.enabled=true

		pig.tez.configure.am.memory=true