pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1753330 - in /pig/branches/branch-0.16: CHANGES.txt conf/pig.properties src/org/apache/pig/PigConfiguration.java src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java src/pig-default.properties
Date Tue, 19 Jul 2016 01:29:07 GMT
Author: daijy
Date: Tue Jul 19 01:29:07 2016
New Revision: 1753330

URL: http://svn.apache.org/viewvc?rev=1753330&view=rev
Log:
PIG-4948: Pig on Tez AM use too much memory on a small cluster

Modified:
    pig/branches/branch-0.16/CHANGES.txt
    pig/branches/branch-0.16/conf/pig.properties
    pig/branches/branch-0.16/src/org/apache/pig/PigConfiguration.java
    pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java
    pig/branches/branch-0.16/src/pig-default.properties

Modified: pig/branches/branch-0.16/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/CHANGES.txt?rev=1753330&r1=1753329&r2=1753330&view=diff
==============================================================================
--- pig/branches/branch-0.16/CHANGES.txt (original)
+++ pig/branches/branch-0.16/CHANGES.txt Tue Jul 19 01:29:07 2016
@@ -30,6 +30,8 @@ OPTIMIZATIONS
 
 BUG FIXES
 
+PIG-4948: Pig on Tez AM use too much memory on a small cluster (daijy)
+
 PIG-4942: Fix mvn-install ant target (petersla via daijy)
 
 PIG-4949: Fix registering jar in S3 which was broken by PIG-4417 in Pig 0.16 (yangyishan0901m
via daijy)

Modified: pig/branches/branch-0.16/conf/pig.properties
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/conf/pig.properties?rev=1753330&r1=1753329&r2=1753330&view=diff
==============================================================================
--- pig/branches/branch-0.16/conf/pig.properties (original)
+++ pig/branches/branch-0.16/conf/pig.properties Tue Jul 19 01:29:07 2016
@@ -678,3 +678,6 @@ pig.ats.enabled=true
      
 pig.sort.readonce.loadfuncs=org.apache.pig.backend.hadoop.hbase.HBaseStorage,org.apache.pig.backend.hadoop.accumulo.AccumuloStorage
 
+# If set, Pig will override tez.am.launch.cmd-opts and tez.am.resource.memory.mb to optimal
+# even they are set to a different value. Default value is true.
+#pig.tez.configure.am.memory=false

Modified: pig/branches/branch-0.16/src/org/apache/pig/PigConfiguration.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/src/org/apache/pig/PigConfiguration.java?rev=1753330&r1=1753329&r2=1753330&view=diff
==============================================================================
--- pig/branches/branch-0.16/src/org/apache/pig/PigConfiguration.java (original)
+++ pig/branches/branch-0.16/src/org/apache/pig/PigConfiguration.java Tue Jul 19 01:29:07
2016
@@ -428,6 +428,11 @@ public class PigConfiguration {
      */
     public static final String ENABLE_ATS = "pig.ats.enabled";
 
+    /**
+     * Enable ATS for Pig
+     */
+    public static final String PIG_TEZ_CONFIGURE_AM_MEMORY = "pig.tez.configure.am.memory";
+
     // Deprecated settings of Pig 0.13
 
     /**

Modified: pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java?rev=1753330&r1=1753329&r2=1753330&view=diff
==============================================================================
--- pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java
(original)
+++ pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java
Tue Jul 19 01:29:07 2016
@@ -29,6 +29,7 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.pig.PigConfiguration;
 import org.apache.pig.backend.hadoop.executionengine.tez.TezJob.TezJobConfig;
 import org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper;
 import org.apache.pig.impl.PigContext;
@@ -89,7 +90,9 @@ public class TezSessionManager {
         TezConfiguration amConf = MRToTezHelper.getDAGAMConfFromMRConf(conf);
         TezScriptState ss = TezScriptState.get();
         ss.addDAGSettingsToConf(amConf);
-        adjustAMConfig(amConf, tezJobConf);
+        if (amConf.getBoolean(PigConfiguration.PIG_TEZ_CONFIGURE_AM_MEMORY, true)) {
+            adjustAMConfig(amConf, tezJobConf);
+        }
         String jobName = conf.get(PigContext.JOB_NAME, "pig");
         TezClient tezClient = TezClient.create(jobName, amConf, true, requestedAMResources,
creds);
         try {

Modified: pig/branches/branch-0.16/src/pig-default.properties
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/src/pig-default.properties?rev=1753330&r1=1753329&r2=1753330&view=diff
==============================================================================
--- pig/branches/branch-0.16/src/pig-default.properties (original)
+++ pig/branches/branch-0.16/src/pig-default.properties Tue Jul 19 01:29:07 2016
@@ -64,3 +64,5 @@ pig.tez.opt.union.unsupported.storefuncs
 pig.sort.readonce.loadfuncs=org.apache.pig.backend.hadoop.hbase.HBaseStorage,org.apache.pig.backend.hadoop.accumulo.AccumuloStorage
 
 pig.ats.enabled=true
+
+pig.tez.configure.am.memory=true



Mime
View raw message