pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From knogu...@apache.org
Subject svn commit: r1734296 - in /pig/trunk: CHANGES.txt src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java test/org/apache/pig/test/TestBuiltin.java test/org/apache/pig/test/Util.java
Date Wed, 09 Mar 2016 18:30:03 GMT
Author: knoguchi
Date: Wed Mar  9 18:30:02 2016
New Revision: 1734296

URL: http://svn.apache.org/viewvc?rev=1734296&view=rev
Log:
PIG-4833 TestBuiltin.testURIWithCurlyBrace in TEZ failing after PIG-4819 (knoguchi)

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java
    pig/trunk/test/org/apache/pig/test/TestBuiltin.java
    pig/trunk/test/org/apache/pig/test/Util.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1734296&r1=1734295&r2=1734296&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Mar  9 18:30:02 2016
@@ -97,6 +97,8 @@ PIG-4639: Add better parser for Apache H
 
 BUG FIXES
 
+PIG-4833 TestBuiltin.testURIWithCurlyBrace in TEZ failing after PIG-4819 (knoguchi)
+
 PIG-4819: RANDOM() udf can lead to missing or redundant records (knoguchi)
 
 PIG-4816: Read a null scalar causing a Tez failure (daijy)

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java?rev=1734296&r1=1734295&r2=1734296&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java
(original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java
Wed Mar  9 18:30:02 2016
@@ -55,6 +55,14 @@ public class TezResourceManager {
         return instance;
     }
 
+
+    /**
+     * This method is only used by test code to reset state.
+     */
+    public static void dropInstance() {
+        instance = null;
+    }
+
     public void init(PigContext pigContext, Configuration conf) throws IOException {
         if (!inited) {
             this.resourcesDir = FileLocalizer.getTemporaryResourcePath(pigContext);

Modified: pig/trunk/test/org/apache/pig/test/TestBuiltin.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestBuiltin.java?rev=1734296&r1=1734295&r2=1734296&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestBuiltin.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestBuiltin.java Wed Mar  9 18:30:02 2016
@@ -3209,7 +3209,9 @@ public class TestBuiltin {
         String inputFileName = "testUniqueID.txt";
         Util.createInputFile(cluster, inputFileName, new String[]
             {"1\n2\n3\n4\n5\n1\n2\n3\n4\n5\n"});
-        PigServer pigServer = new PigServer(cluster.getExecType(), cluster.getProperties());
+        Properties copyproperties = new Properties();
+        copyproperties.putAll(cluster.getProperties());
+        PigServer pigServer = new PigServer(cluster.getExecType(), copyproperties);
         pigServer.getPigContext().getProperties().setProperty("mapred.max.split.size", "10");
         pigServer.getPigContext().getProperties().setProperty("pig.noSplitCombination", "true");
         pigServer.registerQuery("A = load '" + inputFileName + "' as (name);");
@@ -3225,6 +3227,7 @@ public class TestBuiltin {
         assertEquals(iter.next().get(1),"1-2");
         assertEquals(iter.next().get(1),"1-3");
         assertEquals(iter.next().get(1),"1-4");
+        Util.deleteFile(cluster, inputFileName);
     }
 
     @Test
@@ -3233,7 +3236,10 @@ public class TestBuiltin {
         String inputFileName = "testRANDOM.txt";
         Util.createInputFile(cluster, inputFileName, new String[]
             {"1\n2\n3\n4\n5\n1\n2\n3\n4\n5\n"});
-        PigServer pigServer = new PigServer(cluster.getExecType(), cluster.getProperties());
+
+        Properties copyproperties = new Properties();
+        copyproperties.putAll(cluster.getProperties());
+        PigServer pigServer = new PigServer(cluster.getExecType(), copyproperties);
         // running with two mappers
         pigServer.getPigContext().getProperties().setProperty("mapred.max.split.size", "10");
         pigServer.getPigContext().getProperties().setProperty("pig.noSplitCombination", "true");
@@ -3260,6 +3266,7 @@ public class TestBuiltin {
         for( int i = 0; i < 5; i++ ){
             assertNotEquals(mapper1[i], mapper2[i], 0.0001);
         }
+        Util.deleteFile(cluster, inputFileName);
     }
 
 

Modified: pig/trunk/test/org/apache/pig/test/Util.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/Util.java?rev=1734296&r1=1734295&r2=1734296&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/Util.java (original)
+++ pig/trunk/test/org/apache/pig/test/Util.java Wed Mar  9 18:30:02 2016
@@ -75,6 +75,7 @@ import org.apache.pig.backend.hadoop.exe
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.TezResourceManager;
 import org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil;
 import org.apache.pig.builtin.Utf8StorageConverter;
 import org.apache.pig.data.BagFactory;
@@ -1340,6 +1341,11 @@ public class Util {
      */
     public static void resetStateForExecModeSwitch() {
         FileLocalizer.setInitialized(false);
+
+        // For tez testing, we want to avoid TezResourceManager/LocalResource reuse
+        // (when switching between local and mapreduce/tez)
+        TezResourceManager.dropInstance();
+
         // TODO: once we have Tez local mode, we can get rid of this. For now,
         // if we run this test suite in Tez mode and there are some tests
         // in LOCAL mode, we need to set ScriptState to



Mime
View raw message