pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From roh...@apache.org
Subject svn commit: r1714257 - in /pig/trunk: ./ src/org/apache/pig/ src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/ src/org/apache/pig/backend/hadoop/executionengine/tez/runtim...
Date Fri, 13 Nov 2015 19:44:16 GMT
Author: rohini
Date: Fri Nov 13 19:44:16 2015
New Revision: 1714257

URL: http://svn.apache.org/viewvc?rev=1714257&view=rev
Log:
PIG-4733: Avoid NullPointerException in JVMReuseImpl for builtin classes (rohini)

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/JVMReuseImpl.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java
    pig/trunk/src/org/apache/pig/builtin/Distinct.java
    pig/trunk/src/org/apache/pig/impl/PigContext.java
    pig/trunk/src/org/apache/pig/impl/util/SpillableMemoryManager.java
    pig/trunk/src/org/apache/pig/impl/util/UDFContext.java
    pig/trunk/src/org/apache/pig/tools/pigstats/PigStatusReporter.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1714257&r1=1714256&r2=1714257&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Nov 13 19:44:16 2015
@@ -69,6 +69,8 @@ PIG-4639: Add better parser for Apache H
 
 BUG FIXES
 
+PIG-4733: Avoid NullPointerException in JVMReuseImpl for builtin classes (rohini)
+
 PIG-4722: [Pig on Tez] NPE while running Combiner (rohini)
 
 PIG-4730: [Pig on Tez] Total parallelism estimation does not account load parallelism (rohini)

Modified: pig/trunk/src/org/apache/pig/JVMReuseImpl.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/JVMReuseImpl.java?rev=1714257&r1=1714256&r2=1714257&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/JVMReuseImpl.java (original)
+++ pig/trunk/src/org/apache/pig/JVMReuseImpl.java Fri Nov 13 19:44:16 2015
@@ -22,12 +22,34 @@ import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigCombiner;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapReduce;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
+import org.apache.pig.classification.InterfaceAudience;
+import org.apache.pig.classification.InterfaceStability;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.util.SpillableMemoryManager;
+import org.apache.pig.impl.util.UDFContext;
+import org.apache.pig.tools.pigstats.PigStatusReporter;
 
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
 public class JVMReuseImpl {
 
     private static Log LOG = LogFactory.getLog(JVMReuseImpl.class);
 
     public void cleanupStaticData() {
+
+        // Calling Pig builtin ones directly without reflection for optimization
+        // and to reduce probability of NPE in PIG-4418
+        SpillableMemoryManager.staticDataCleanup();
+        PhysicalOperator.staticDataCleanup();
+        PigContext.staticDataCleanup();
+        UDFContext.staticDataCleanup();
+        PigGenericMapReduce.staticDataCleanup();
+        PigStatusReporter.staticDataCleanup();
+        PigCombiner.Combine.staticDataCleanup();
+
         String className = null;
         String msg = null;
         List<Method> staticCleanupMethods = JVMReuseManager.getInstance()
@@ -45,15 +67,16 @@ public class JVMReuseImpl {
                 m.invoke(null);
                 msg = null;
             } catch (Exception e) {
-                LOG.error("Exception while calling static methods:" + getMethodNames() +
". " + msg, e);
+                LOG.error("Exception while calling static methods:"
+                        + getMethodNames(staticCleanupMethods) + ". " + msg, e);
                 throw new RuntimeException("Error while " + msg, e);
             }
         }
     }
 
-    private String getMethodNames() {
+    private String getMethodNames(List<Method> staticCleanupMethods) {
         StringBuilder sb = new StringBuilder();
-        for (Method m : JVMReuseManager.getInstance().getStaticDataCleanupMethods()) {
+        for (Method m : staticCleanupMethods) {
             if (m == null) {
                 sb.append("null,");
             } else {

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java?rev=1714257&r1=1714256&r2=1714257&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java
(original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java
Fri Nov 13 19:44:16 2015
@@ -27,9 +27,7 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.log4j.PropertyConfigurator;
-import org.apache.pig.JVMReuseManager;
 import org.apache.pig.PigException;
-import org.apache.pig.StaticDataCleanup;
 import org.apache.pig.backend.executionengine.ExecException;
 import org.apache.pig.backend.hadoop.HDataType;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
@@ -76,11 +74,7 @@ public class PigCombiner {
         PigContext pigContext = null;
         private volatile boolean initialized = false;
 
-        static {
-            JVMReuseManager.getInstance().registerForStaticDataCleanup(Combine.class);
-        }
-
-        @StaticDataCleanup
+        //@StaticDataCleanup
         public static void staticDataCleanup() {
             firstTime = true;
         }

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java?rev=1714257&r1=1714256&r2=1714257&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java
(original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java
Fri Nov 13 19:44:16 2015
@@ -30,10 +30,8 @@ import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.mapred.jobcontrol.Job;
 import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.pig.JVMReuseManager;
 import org.apache.pig.PigConstants;
 import org.apache.pig.PigException;
-import org.apache.pig.StaticDataCleanup;
 import org.apache.pig.backend.executionengine.ExecException;
 import org.apache.pig.backend.hadoop.HDataType;
 import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
@@ -103,11 +101,7 @@ public class PigGenericMapReduce {
 
     public static ThreadLocal<Configuration> sJobConfInternal = new ThreadLocal<Configuration>();
 
-    static {
-        JVMReuseManager.getInstance().registerForStaticDataCleanup(PigGenericMapReduce.class);
-    }
-
-    @StaticDataCleanup
+    //@StaticDataCleanup
     public static void staticDataCleanup() {
         sJobContext = null;
         sJobConf = null;

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java?rev=1714257&r1=1714256&r2=1714257&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java
(original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java
Fri Nov 13 19:44:16 2015
@@ -24,8 +24,6 @@ import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.pig.JVMReuseManager;
-import org.apache.pig.StaticDataCleanup;
 import org.apache.pig.backend.executionengine.ExecException;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
@@ -122,10 +120,6 @@ public abstract class PhysicalOperator e
 
     private List<OriginalLocation> originalLocations =  new ArrayList<OriginalLocation>();
 
-    static {
-        JVMReuseManager.getInstance().registerForStaticDataCleanup(PhysicalOperator.class);
-    }
-
     public PhysicalOperator(OperatorKey k) {
         this(k, -1, null);
     }
@@ -458,7 +452,7 @@ public abstract class PhysicalOperator e
         PhysicalOperator.reporter.set(reporter);
     }
 
-    @StaticDataCleanup
+    //@StaticDataCleanup
     public static void staticDataCleanup() {
         reporter = new ThreadLocal<PigProgressable>();
     }

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java?rev=1714257&r1=1714256&r2=1714257&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java
(original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java
Fri Nov 13 19:44:16 2015
@@ -162,7 +162,6 @@ public class PigProcessor extends Abstra
 
     @Override
     public void close() throws Exception {
-
         execPlan = null;
         fileOutputs = null;
         leaf = null;

Modified: pig/trunk/src/org/apache/pig/builtin/Distinct.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/builtin/Distinct.java?rev=1714257&r1=1714256&r2=1714257&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/builtin/Distinct.java (original)
+++ pig/trunk/src/org/apache/pig/builtin/Distinct.java Fri Nov 13 19:44:16 2015
@@ -22,9 +22,7 @@ import java.io.IOException;
 
 import org.apache.pig.Algebraic;
 import org.apache.pig.EvalFunc;
-import org.apache.pig.JVMReuseManager;
 import org.apache.pig.PigConfiguration;
-import org.apache.pig.StaticDataCleanup;
 import org.apache.pig.backend.executionengine.ExecException;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce;
 import org.apache.pig.data.BagFactory;
@@ -45,16 +43,6 @@ public class Distinct  extends EvalFunc<
     private static boolean initialized = false;
     private static boolean useDefaultBag = false;
 
-    static {
-        JVMReuseManager.getInstance().registerForStaticDataCleanup(Distinct.class);
-    }
-
-    @StaticDataCleanup
-    public static void staticDataCleanup() {
-        initialized = false;
-        useDefaultBag = false;
-    }
-
     @Override
     public DataBag exec(Tuple input) throws IOException {
         return getDistinct(input);

Modified: pig/trunk/src/org/apache/pig/impl/PigContext.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/PigContext.java?rev=1714257&r1=1714256&r2=1714257&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/PigContext.java (original)
+++ pig/trunk/src/org/apache/pig/impl/PigContext.java Fri Nov 13 19:44:16 2015
@@ -52,9 +52,7 @@ import org.apache.log4j.Level;
 import org.apache.pig.ExecType;
 import org.apache.pig.ExecTypeProvider;
 import org.apache.pig.FuncSpec;
-import org.apache.pig.JVMReuseManager;
 import org.apache.pig.PigException;
-import org.apache.pig.StaticDataCleanup;
 import org.apache.pig.backend.datastorage.DataStorage;
 import org.apache.pig.backend.datastorage.DataStorageException;
 import org.apache.pig.backend.datastorage.ElementDescriptor;
@@ -171,11 +169,7 @@ public class PigContext implements Seria
     // List of paths skipped for automatic shipping
     List<String> skippedShipPaths = new ArrayList<String>();
 
-    static {
-        JVMReuseManager.getInstance().registerForStaticDataCleanup(PigContext.class);
-    }
-
-    @StaticDataCleanup
+    //@StaticDataCleanup
     public static void staticDataCleanup() {
         packageImportList.set(null);
     }

Modified: pig/trunk/src/org/apache/pig/impl/util/SpillableMemoryManager.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/util/SpillableMemoryManager.java?rev=1714257&r1=1714256&r2=1714257&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/util/SpillableMemoryManager.java (original)
+++ pig/trunk/src/org/apache/pig/impl/util/SpillableMemoryManager.java Fri Nov 13 19:44:16
2015
@@ -36,8 +36,6 @@ import javax.management.openmbean.Compos
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.pig.JVMReuseManager;
-import org.apache.pig.StaticDataCleanup;
 
 /**
  * This class Tracks the tenured pool and a list of Spillable objects. When memory gets low,
this
@@ -93,12 +91,8 @@ public class SpillableMemoryManager impl
 
     private static final SpillableMemoryManager manager = new SpillableMemoryManager();
 
-    static {
-        JVMReuseManager.getInstance().registerForStaticDataCleanup(SpillableMemoryManager.class);
-    }
-
-    @StaticDataCleanup
-    public static void cleanupStaticData() {
+    //@StaticDataCleanup
+    public static void staticDataCleanup() {
         manager.spillables.clear();
         manager.accumulatedFreeSize = 0L;
     }

Modified: pig/trunk/src/org/apache/pig/impl/util/UDFContext.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/util/UDFContext.java?rev=1714257&r1=1714256&r2=1714257&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/util/UDFContext.java (original)
+++ pig/trunk/src/org/apache/pig/impl/util/UDFContext.java Fri Nov 13 19:44:16 2015
@@ -26,8 +26,6 @@ import java.util.Map.Entry;
 import java.util.Properties;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.pig.JVMReuseManager;
-import org.apache.pig.StaticDataCleanup;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRConfiguration;
 
 public class UDFContext {
@@ -46,10 +44,6 @@ public class UDFContext {
         }
     };
 
-    static {
-        JVMReuseManager.getInstance().registerForStaticDataCleanup(UDFContext.class);
-    }
-
     private UDFContext() {
         udfConfs = new HashMap<UDFContextKey, Properties>();
     }
@@ -71,8 +65,8 @@ public class UDFContext {
     /*
      *  internal pig use only - should NOT be called from user code
      */
-    @StaticDataCleanup
-    public static void cleanupStaticData() {
+    //@StaticDataCleanup
+    public static void staticDataCleanup() {
         tss = new ThreadLocal<UDFContext>() {
             @Override
             public UDFContext initialValue() {

Modified: pig/trunk/src/org/apache/pig/tools/pigstats/PigStatusReporter.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/PigStatusReporter.java?rev=1714257&r1=1714256&r2=1714257&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/PigStatusReporter.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/PigStatusReporter.java Fri Nov 13 19:44:16
2015
@@ -21,8 +21,6 @@ package org.apache.pig.tools.pigstats;
 import org.apache.hadoop.mapreduce.Counter;
 import org.apache.hadoop.mapreduce.StatusReporter;
 import org.apache.hadoop.util.Progressable;
-import org.apache.pig.JVMReuseManager;
-import org.apache.pig.StaticDataCleanup;
 import org.apache.pig.backend.hadoop.executionengine.TaskContext;
 import org.apache.pig.classification.InterfaceAudience;
 import org.apache.pig.classification.InterfaceStability;
@@ -35,11 +33,7 @@ public class PigStatusReporter extends S
 
     private TaskContext<?> context = null;
 
-    static {
-        JVMReuseManager.getInstance().registerForStaticDataCleanup(PigStatusReporter.class);
-    }
-
-    @StaticDataCleanup
+    //@StaticDataCleanup
     public static void staticDataCleanup() {
         reporter = null;
     }



Mime
View raw message