pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From roh...@apache.org
Subject svn commit: r1575499 - in /pig/branches/tez: ./ ivy/ shims/test/hadoop23/org/apache/pig/test/ src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/ src/org/apache/pig/backend/...
Date Sat, 08 Mar 2014 08:41:03 GMT
Author: rohini
Date: Sat Mar  8 08:41:03 2014
New Revision: 1575499

URL: http://svn.apache.org/r1575499
Log:
PIG-3797: Fix some memory leaks affecting container reuse (rohini)

Modified:
    pig/branches/tez/ivy.xml
    pig/branches/tez/ivy/libraries.properties
    pig/branches/tez/shims/test/hadoop23/org/apache/pig/test/TezMiniCluster.java
    pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java
    pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java
    pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java
    pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/PigProcessor.java
    pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
    pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
    pig/branches/tez/test/tez-tests

Modified: pig/branches/tez/ivy.xml
URL: http://svn.apache.org/viewvc/pig/branches/tez/ivy.xml?rev=1575499&r1=1575498&r2=1575499&view=diff
==============================================================================
--- pig/branches/tez/ivy.xml (original)
+++ pig/branches/tez/ivy.xml Sat Mar  8 08:41:03 2014
@@ -415,8 +415,6 @@
        conf="hadoop23->master"/>
     <dependency org="org.apache.tez" name="tez-dag" rev="${tez.version}"
        conf="hadoop23->master"/>
-    <dependency org="org.apache.tez" name="tez-dist" rev="${tez.version}"
-       conf="hadoop23->master"/>
     <dependency org="org.apache.tez" name="tez-runtime-internals" rev="${tez.version}"
        conf="hadoop23->master"/>
     <dependency org="org.apache.tez" name="tez-runtime-library" rev="${tez.version}"

Modified: pig/branches/tez/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/pig/branches/tez/ivy/libraries.properties?rev=1575499&r1=1575498&r2=1575499&view=diff
==============================================================================
--- pig/branches/tez/ivy/libraries.properties (original)
+++ pig/branches/tez/ivy/libraries.properties Sat Mar  8 08:41:03 2014
@@ -91,5 +91,5 @@ mockito.version=1.8.4
 jansi.version=1.9
 asm.version=3.3.1
 snappy.version=1.0.5-M3
-tez.version=0.3.0-incubating-SNAPSHOT
+tez.version=0.3.0-incubating
 parquet-pig-bundle.version=1.2.3

Modified: pig/branches/tez/shims/test/hadoop23/org/apache/pig/test/TezMiniCluster.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/shims/test/hadoop23/org/apache/pig/test/TezMiniCluster.java?rev=1575499&r1=1575498&r2=1575499&view=diff
==============================================================================
--- pig/branches/tez/shims/test/hadoop23/org/apache/pig/test/TezMiniCluster.java (original)
+++ pig/branches/tez/shims/test/hadoop23/org/apache/pig/test/TezMiniCluster.java Sat Mar 
8 08:41:03 2014
@@ -27,7 +27,6 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.pig.ExecType;
@@ -90,7 +89,7 @@ public class TezMiniCluster extends Mini
             m_mr_conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
                     System.getProperty("java.class.path"));
             // TODO PIG-3659 - Remove this once memory management is fixed
-            m_mr_conf.set(MRJobConfig.MAP_JAVA_OPTS, "-Xmx384M");// -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8005
-Xnoagent -Djava.compiler=NONE");
+            // m_mr_conf.set(MRJobConfig.MAP_JAVA_OPTS, "-Xmx384M");// -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8005
-Xnoagent -Djava.compiler=NONE");
             m_mr_conf.writeXml(new FileOutputStream(MAPRED_CONF_FILE));
             m_fileSys.copyFromLocalFile(
                     new Path(MAPRED_CONF_FILE.getAbsoluteFile().toString()),

Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java?rev=1575499&r1=1575498&r2=1575499&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java
(original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java
Sat Mar  8 08:41:03 2014
@@ -25,7 +25,6 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.log4j.PropertyConfigurator;
 import org.apache.pig.PigException;
@@ -46,8 +45,6 @@ import org.apache.pig.tools.pigstats.Pig
 
 public class PigCombiner {
 
-    public static JobContext sJobContext = null;
-
     public static class Combine
             extends Reducer<PigNullableWritable, NullableTuple, PigNullableWritable, Writable>
{
 
@@ -84,7 +81,6 @@ public class PigCombiner {
         @Override
         protected void setup(Context context) throws IOException, InterruptedException {
             super.setup(context);
-            sJobContext = context;
             Configuration jConf = context.getConfiguration();
             try {
                 PigContext.setPackageImportList((ArrayList<String>)ObjectSerializer.deserialize(jConf.get("udf.import.list")));

Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java?rev=1575499&r1=1575498&r2=1575499&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java
(original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java
Sat Mar  8 08:41:03 2014
@@ -98,7 +98,7 @@ public class PigGenericMapReduce {
     @Deprecated
     public static Configuration sJobConf = null;
 
-    public static final ThreadLocal<Configuration> sJobConfInternal = new ThreadLocal<Configuration>();
+    public static ThreadLocal<Configuration> sJobConfInternal = new ThreadLocal<Configuration>();
 
     public static class Map extends PigMapBase {
 

Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java?rev=1575499&r1=1575498&r2=1575499&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java
(original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java
Sat Mar  8 08:41:03 2014
@@ -100,7 +100,7 @@ public abstract class PhysicalOperator e
     // Will be used by operators to report status or transmit heartbeat
     // Should be set by the backends to appropriate implementations that
     // wrap their own version of a reporter.
-    private static ThreadLocal<PigProgressable> reporter = new ThreadLocal<PigProgressable>();
+    public static ThreadLocal<PigProgressable> reporter = new ThreadLocal<PigProgressable>();
 
     // Will be used by operators to aggregate warning messages
     // Should be set by the backends to appropriate implementations that

Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/PigProcessor.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/PigProcessor.java?rev=1575499&r1=1575498&r2=1575499&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/PigProcessor.java
(original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/PigProcessor.java
Sat Mar  8 08:41:03 2014
@@ -34,6 +34,7 @@ import org.apache.pig.backend.hadoop.exe
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PigProgressable;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserFunc;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
@@ -74,10 +75,16 @@ public class PigProcessor implements Log
     @Override
     public void initialize(TezProcessorContext processorContext)
             throws Exception {
-        // Reset any static variables to avoid conflic in container-reuse.
+        // Reset any static variables to avoid conflict in container-reuse.
         sampleVertex = null;
         sampleMap = null;
 
+        // Reset static variables cleared for avoiding OOM.
+        // TODO: Figure out a cleaner way to do this. ThreadLocals actually can be avoided
all together
+        // for mapreduce/tez mode and just used for Local mode.
+        PhysicalOperator.reporter = new ThreadLocal<PigProgressable>();
+        PigMapReduce.sJobConfInternal = new ThreadLocal<Configuration>();
+
         byte[] payload = processorContext.getUserPayload();
         conf = TezUtils.createConfFromUserPayload(payload);
         PigContext pc = (PigContext) ObjectSerializer.deserialize(conf.get("pig.pigContext"));
@@ -103,8 +110,16 @@ public class PigProcessor implements Log
 
     @Override
     public void close() throws Exception {
-        // TODO Auto-generated method stub
-
+        // Avoid memory leak. ThreadLocals especially leak a lot of memory.
+        PhysicalOperator.reporter = new ThreadLocal<PigProgressable>();
+        PigMapReduce.sJobConfInternal = new ThreadLocal<Configuration>();
+        PigMapReduce.sJobContext = null;
+        execPlan = null;
+        fileOutputs = null;
+        leaf = null;
+        conf = null;
+        sampleMap = null;
+        sampleVertex = null;
     }
 
     @SuppressWarnings("rawtypes")

Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java?rev=1575499&r1=1575498&r2=1575499&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
(original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
Sat Mar  8 08:41:03 2014
@@ -257,6 +257,9 @@ public class TezDagBuilder extends TezOp
                     edge.partitionerClass.getName());
         }
 
+        conf.set("udf.import.list",
+                ObjectSerializer.serialize(PigContext.getPackageImportList()));
+
         MRToTezHelper.convertMRToTezRuntimeConf(conf, globalConf);
 
         in.setUserPayload(TezUtils.createUserPayloadFromConf(conf));

Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java?rev=1575499&r1=1575498&r2=1575499&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
(original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
Sat Mar  8 08:41:03 2014
@@ -59,6 +59,7 @@ public class TezLauncher extends Launche
 
         TezResourceManager tezResourceManager = new TezResourceManager(stagingDir, pc, conf);
 
+        log.info("Tez staging directory is " + stagingDir.toString());
         conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir.toString());
 
         List<TezOperPlan> processedPlans = new ArrayList<TezOperPlan>();

Modified: pig/branches/tez/test/tez-tests
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/tez-tests?rev=1575499&r1=1575498&r2=1575499&view=diff
==============================================================================
--- pig/branches/tez/test/tez-tests (original)
+++ pig/branches/tez/test/tez-tests Sat Mar  8 08:41:03 2014
@@ -18,5 +18,4 @@
 **/TestBZip.java
 **/TestCompressedFiles.java
 **/TestCharArrayToNumeric.java
-## TODO: Runs fine individually. Hangs with file.out.index not found when run together. Likely
Tez Bug
-##**/TestSecondarySortTez.java
+**/TestSecondarySortTez.java



Mime
View raw message