tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rbalamo...@apache.org
Subject git commit: TEZ-1368. TestSecureShuffle failing
Date Wed, 06 Aug 2014 04:16:21 GMT
Repository: tez
Updated Branches:
  refs/heads/master 675b23724 -> 45420856e


TEZ-1368. TestSecureShuffle failing


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/45420856
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/45420856
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/45420856

Branch: refs/heads/master
Commit: 45420856ee967dc777338020dcdfbbc6dd7b20b0
Parents: 675b237
Author: Rajesh Balamohan <rbalamohan@apache.org>
Authored: Wed Aug 6 09:46:01 2014 +0530
Committer: Rajesh Balamohan <rbalamohan@apache.org>
Committed: Wed Aug 6 09:46:01 2014 +0530

----------------------------------------------------------------------
 .../mapreduce/examples/OrderedWordCount.java    |  9 +-
 .../org/apache/tez/test/TestSecureShuffle.java  | 92 ++++++++++++++------
 2 files changed, 71 insertions(+), 30 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/45420856/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
index 3513c0e..5c20e5b 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
@@ -266,14 +266,16 @@ public class OrderedWordCount extends Configured implements Tool {
 
     OrderedPartitionedKVEdgeConfigurer edgeConf1 = OrderedPartitionedKVEdgeConfigurer
         .newBuilder(Text.class.getName(), IntWritable.class.getName(),
-            HashPartitioner.class.getName()).configureInput().useLegacyInput().done().build();
+            HashPartitioner.class.getName()).setFromConfiguration(conf)
+	    .configureInput().useLegacyInput().done().build();
     dag.addEdge(
         new Edge(dag.getVertex("initialmap"), dag.getVertex("intermediate_reducer"),
             edgeConf1.createDefaultEdgeProperty()));
 
     OrderedPartitionedKVEdgeConfigurer edgeConf2 = OrderedPartitionedKVEdgeConfigurer
         .newBuilder(IntWritable.class.getName(), Text.class.getName(),
-            HashPartitioner.class.getName()).configureInput().useLegacyInput().done().build();
+            HashPartitioner.class.getName()).setFromConfiguration(conf)
+            .configureInput().useLegacyInput().done().build();
     dag.addEdge(
         new Edge(dag.getVertex("intermediate_reducer"), dag.getVertex("finalreduce"),
             edgeConf2.createDefaultEdgeProperty()));
@@ -511,9 +513,8 @@ public class OrderedWordCount extends Configured implements Tool {
     if (!useTezSession) {
       ExampleDriver.printDAGStatus(dagClient, vNames);
       LOG.info("Application completed. " + "FinalState=" + dagStatus.getState());
-      return dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1;
     }
-    return 0;
+    return dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1;
   }
 
   private static void waitForTezSessionReady(TezClient tezSession)

http://git-wip-us.apache.org/repos/asf/tez/blob/45420856/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java b/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java
index 9d416fb..9df1580 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java
@@ -24,6 +24,8 @@ import java.io.BufferedWriter;
 import java.io.File;
 import java.io.IOException;
 import java.io.OutputStreamWriter;
+import java.util.ArrayList;
+import java.util.Collection;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -34,14 +36,16 @@ import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
 import org.apache.tez.mapreduce.examples.OrderedWordCount;
 import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
 /**
  * Test to verify secure-shuffle (SSL mode) in Tez
  */
+@RunWith(Parameterized.class)
 public class TestSecureShuffle {
 
   private static MiniDFSCluster miniDFSCluster;
@@ -55,22 +59,55 @@ public class TestSecureShuffle {
       + TestSecureShuffle.class.getName() + "-tmpDir";
   private static File keysStoresDir = new File(TEST_ROOT_DIR, "keystores");
 
-  @BeforeClass
-  public static void setup() throws Exception {
-    // Enable SSL debugging
-    System.setProperty("javax.net.debug", "all");
+  private boolean enableSSLInCluster; //To set ssl config in cluster
+  private boolean enableSSLInTez; //To set ssl config in Tez
+  private int expectedResult;
+
+  public TestSecureShuffle(boolean sslInCluster, boolean sslInTez, int expectedResult) {
+    this.enableSSLInCluster = sslInCluster;
+    this.enableSSLInTez = sslInTez;
+    this.expectedResult = expectedResult;
+  }
+
+  @Parameterized.Parameters(name = "test[sslInCluster:{0}, sslInTez:{1}, expectedResult:{2}]")
+  public static Collection<Object[]> getParameters() {
+    Collection<Object[]> parameters = new ArrayList<Object[]>();
+    //enable ssl in cluster and in tez.
+    parameters.add(new Object[] { true, true, 0 });
+    //disable ssl in cluster and in tez.
+    parameters.add(new Object[] { false, false, 0 });
+
+    //Negative test cases
+
+    //enable ssl in cluster, but disable in tez side.
+    parameters.add(new Object[] { true, false, 1 });
+    //disable ssl config in cluster, but enable from tez side.
+    parameters.add(new Object[] { false, true, 1 });
+
+    return parameters;
+  }
+
+
+  @Before
+  public void setup() throws Exception {
     conf = new Configuration();
-    setupKeyStores();
     conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
+    if (enableSSLInCluster) {
+      // Enable SSL debugging
+      System.setProperty("javax.net.debug", "all");
+      setupKeyStores();
+    }
 
     miniDFSCluster =
         new MiniDFSCluster.Builder(conf).numDataNodes(3).format(true).build();
     fs = miniDFSCluster.getFileSystem();
     conf.set("fs.defaultFS", fs.getUri().toString());
 
-    // 15 seconds should be good enough in local machine
-    conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_CONNECT_TIMEOUT, 15 * 1000);
-    conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT, 15 * 1000);
+    // 5 seconds should be good enough in local machine
+    conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_CONNECT_TIMEOUT, 5 * 1000);
+    conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT, 5 * 1000);
+    //set to low value so that it can detect failures quickly
+    conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES_LIMIT, 2);
 
     miniTezCluster =
         new MiniTezCluster(TestSecureShuffle.class.getName(), 3, 3, 1);
@@ -80,37 +117,40 @@ public class TestSecureShuffle {
     createSampleFile(inputLoc);
   }
 
+  @After
+  public void tearDown() throws IOException {
+    fs.delete(inputLoc, true);
+    fs.delete(outputLoc, true);
+    if (miniTezCluster != null) {
+      miniTezCluster.stop();
+    }
+    if (miniDFSCluster != null) {
+      miniDFSCluster.shutdown();
+    }
+  }
+
   /**
-   * Verify whether shuffle works on SSL mode.
+   * Verify whether shuffle works in mini cluster
    *
    * @throws Exception
    */
   @Test(timeout = 60000)
-  @Ignore // TODO TEZ-1368
   public void testSecureShuffle() throws Exception {
-    miniTezCluster.getConfig().setBoolean(
-      TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_SSL, true);
+    if (enableSSLInTez) {
+      miniTezCluster.getConfig().setBoolean(
+          TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_SSL, true);
+    }
 
     OrderedWordCount wordCount = new OrderedWordCount();
     wordCount.setConf(new Configuration(miniTezCluster.getConfig()));
 
     String[] args = new String[] { inputLoc.toString(), outputLoc.toString() };
-    assertEquals(0, wordCount.run(args));
+    assertEquals(this.expectedResult, wordCount.run(args));
 
     // cleanup output
     fs.delete(outputLoc, true);
   }
 
-  @AfterClass
-  public static void tearDown() {
-    if (miniTezCluster != null) {
-      miniTezCluster.stop();
-    }
-    if (miniDFSCluster != null) {
-      miniDFSCluster.shutdown();
-    }
-  }
-
   /**
    * Create sample file for wordcount program
    *


Mime
View raw message