pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From roh...@apache.org
Subject svn commit: r1797104 - in /pig/branches/branch-0.17: CHANGES.txt test/org/apache/pig/test/MiniCluster.java test/org/apache/pig/test/TestGrunt.java test/org/apache/pig/test/TezMiniCluster.java test/org/apache/pig/test/YarnMiniCluster.java
Date Wed, 31 May 2017 20:28:52 GMT
Author: rohini
Date: Wed May 31 20:28:52 2017
New Revision: 1797104

URL: http://svn.apache.org/viewvc?rev=1797104&view=rev
Log:
PIG-5245: TestGrunt.testStopOnFailure is flaky (rohini)

Modified:
    pig/branches/branch-0.17/CHANGES.txt
    pig/branches/branch-0.17/test/org/apache/pig/test/MiniCluster.java
    pig/branches/branch-0.17/test/org/apache/pig/test/TestGrunt.java
    pig/branches/branch-0.17/test/org/apache/pig/test/TezMiniCluster.java
    pig/branches/branch-0.17/test/org/apache/pig/test/YarnMiniCluster.java

Modified: pig/branches/branch-0.17/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.17/CHANGES.txt?rev=1797104&r1=1797103&r2=1797104&view=diff
==============================================================================
--- pig/branches/branch-0.17/CHANGES.txt (original)
+++ pig/branches/branch-0.17/CHANGES.txt Wed May 31 20:28:52 2017
@@ -111,6 +111,8 @@ OPTIMIZATIONS
  
 BUG FIXES
 
+PIG-5245: TestGrunt.testStopOnFailure is flaky (rohini)
+
 PIG-5244: Several unit tests are failing in Tez mode after merging spark branch (nkollar
via szita)
 
 PIG-5207: BugFix e2e tests fail on spark (szita)

Modified: pig/branches/branch-0.17/test/org/apache/pig/test/MiniCluster.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.17/test/org/apache/pig/test/MiniCluster.java?rev=1797104&r1=1797103&r2=1797104&view=diff
==============================================================================
--- pig/branches/branch-0.17/test/org/apache/pig/test/MiniCluster.java (original)
+++ pig/branches/branch-0.17/test/org/apache/pig/test/MiniCluster.java Wed May 31 20:28:52
2017
@@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.filecache.DistributedCache;
 import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster;
 import org.apache.pig.ExecType;
@@ -79,6 +80,7 @@ public class MiniCluster extends MiniGen
             Configuration config = new Configuration();
             config.set("yarn.scheduler.capacity.root.queues", "default");
             config.set("yarn.scheduler.capacity.root.default.capacity", "100");
+            config.set("yarn.scheduler.capacity.maximum-am-resource-percent", "0.1");
             m_dfs = new MiniDFSCluster(config, dataNodes, true, null);
             m_fileSys = m_dfs.getFileSystem();
             m_dfs_conf = m_dfs.getConfiguration(0);
@@ -97,8 +99,12 @@ public class MiniCluster extends MiniGen
             m_conf.set(FileSystem.FS_DEFAULT_NAME_KEY, m_dfs_conf.get(FileSystem.FS_DEFAULT_NAME_KEY));
             m_conf.unset(MRConfiguration.JOB_CACHE_FILES);
 
-            m_conf.setInt(MRConfiguration.IO_SORT_MB, 200);
-            m_conf.set(MRConfiguration.CHILD_JAVA_OPTS, "-Xmx512m");
+            m_conf.setInt(MRConfiguration.IO_SORT_MB, 50);
+            m_conf.set(MRConfiguration.CHILD_JAVA_OPTS, "-Xmx384m");
+            m_conf.setInt(MRJobConfig.MAP_MEMORY_MB, 512);
+            m_conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 512);
+            m_conf.set(MRJobConfig.MR_AM_COMMAND_OPTS, "-Xmx384m");
+            m_conf.setInt(MRJobConfig.MR_AM_VMEM_MB, 512);
 
             m_conf.setInt(MRConfiguration.SUMIT_REPLICATION, 2);
             m_conf.setInt(MRConfiguration.MAP_MAX_ATTEMPTS, 2);

Modified: pig/branches/branch-0.17/test/org/apache/pig/test/TestGrunt.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.17/test/org/apache/pig/test/TestGrunt.java?rev=1797104&r1=1797103&r2=1797104&view=diff
==============================================================================
--- pig/branches/branch-0.17/test/org/apache/pig/test/TestGrunt.java (original)
+++ pig/branches/branch-0.17/test/org/apache/pig/test/TestGrunt.java Wed May 31 20:28:52 2017
@@ -981,6 +981,7 @@ public class TestGrunt {
 
     @Test
     public void testStopOnFailure() throws Throwable {
+        Assume.assumeFalse("Skip this test for Spark", Util.isSparkExecType(cluster.getExecType()));
         PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
         PigContext context = server.getPigContext();
         context.getProperties().setProperty("stop.on.failure", ""+true);

Modified: pig/branches/branch-0.17/test/org/apache/pig/test/TezMiniCluster.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.17/test/org/apache/pig/test/TezMiniCluster.java?rev=1797104&r1=1797103&r2=1797104&view=diff
==============================================================================
--- pig/branches/branch-0.17/test/org/apache/pig/test/TezMiniCluster.java (original)
+++ pig/branches/branch-0.17/test/org/apache/pig/test/TezMiniCluster.java Wed May 31 20:28:52
2017
@@ -59,8 +59,6 @@ public class TezMiniCluster extends Yarn
         try {
             // Write tez-site.xml
             Configuration tez_conf = new Configuration(false);
-            // TODO PIG-3659 - Remove this once memory management is fixed
-            tez_conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, "20");
             tez_conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, "false");
             tez_conf.set("tez.lib.uris", "hdfs:///tez,hdfs:///tez/lib");
             // Set to a lower value so that tests don't get stuck for long because of 1 AM
running at a time

Modified: pig/branches/branch-0.17/test/org/apache/pig/test/YarnMiniCluster.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.17/test/org/apache/pig/test/YarnMiniCluster.java?rev=1797104&r1=1797103&r2=1797104&view=diff
==============================================================================
--- pig/branches/branch-0.17/test/org/apache/pig/test/YarnMiniCluster.java (original)
+++ pig/branches/branch-0.17/test/org/apache/pig/test/YarnMiniCluster.java Wed May 31 20:28:52
2017
@@ -75,20 +75,33 @@ public abstract class YarnMiniCluster ex
             }
             hdfs_site.writeXml(new FileOutputStream(HDFS_CONF_FILE));
 
+            m_dfs_conf.set("yarn.scheduler.capacity.root.queues", "default");
+            m_dfs_conf.set("yarn.scheduler.capacity.root.default.capacity", "100");
+            m_dfs_conf.set("yarn.scheduler.capacity.maximum-am-resource-percent", "0.1");
             // Build mini YARN cluster
             m_mr = new MiniMRYarnCluster("PigMiniCluster", 2);
             m_mr.init(m_dfs_conf);
             m_mr.start();
             m_mr_conf = m_mr.getConfig();
+
             File libDir = new File(System.getProperty("ivy.lib.dir", "build/ivy/lib/Pig"));
+            File sparkLibDir = new File(System.getProperty("ivy.lib.dir.spark", "build/ivy/lib/Pig/spark"));
             File classesDir = new File(System.getProperty("build.classes", "build/classes"));
             File testClassesDir = new File(System.getProperty("test.build.classes", "test/build/classes"));
+
             String classpath = libDir.getAbsolutePath() + "/*"
-                + File.pathSeparator + classesDir.getAbsolutePath()
-                + File.pathSeparator + testClassesDir.getAbsolutePath();
+                    + (Util.isSparkExecType(getExecType()) ? File.pathSeparator + sparkLibDir.getAbsolutePath()
+ "/*" : "")
+                    + File.pathSeparator + classesDir.getAbsolutePath()
+                    + File.pathSeparator + testClassesDir.getAbsolutePath();
+
             m_mr_conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, classpath);
-            m_mr_conf.set(MRJobConfig.MAP_JAVA_OPTS, "-Xmx512m");
-            m_mr_conf.set(MRJobConfig.REDUCE_JAVA_OPTS, "-Xmx512m");
+            m_mr_conf.setInt(MRJobConfig.IO_SORT_MB, 50);
+            m_mr_conf.set(MRJobConfig.MAP_JAVA_OPTS, "-Xmx384m");
+            m_mr_conf.set(MRJobConfig.REDUCE_JAVA_OPTS, "-Xmx384m");
+            m_mr_conf.setInt(MRJobConfig.MAP_MEMORY_MB, 512);
+            m_mr_conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 512);
+            m_mr_conf.set(MRJobConfig.MR_AM_COMMAND_OPTS, "-Xmx384m");
+            m_mr_conf.setInt(MRJobConfig.MR_AM_VMEM_MB, 512);
 
             Configuration mapred_site = new Configuration(false);
             Configuration yarn_site = new Configuration(false);
@@ -137,4 +150,4 @@ public abstract class YarnMiniCluster ex
             m_mr = null;
         }
     }
-}
\ No newline at end of file
+}



Mime
View raw message