crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chao...@apache.org
Subject git commit: CRUNCH-249: Fix HFileTargetIT failure under hadoop2
Date Mon, 12 Aug 2013 03:39:21 GMT
Updated Branches:
  refs/heads/master fd957a920 -> 3f9841136


CRUNCH-249: Fix HFileTargetIT failure under hadoop2


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/3f984113
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/3f984113
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/3f984113

Branch: refs/heads/master
Commit: 3f98411364cec32a0a8c6681dfaabd43caa4dd60
Parents: fd957a9
Author: Chao Shi <chaoshi@apache.org>
Authored: Mon Aug 12 11:35:23 2013 +0800
Committer: Chao Shi <chaoshi@apache.org>
Committed: Mon Aug 12 11:35:23 2013 +0800

----------------------------------------------------------------------
 .../apache/crunch/io/hbase/HFileTargetIT.java   | 36 ++++++++++++++++++++
 pom.xml                                         |  4 +--
 2 files changed, 38 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/3f984113/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java b/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java
index 86d965b..8c1b3f4 100644
--- a/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java
+++ b/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java
@@ -53,6 +53,8 @@ import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.StoreFileScanner;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MiniMRCluster;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -63,11 +65,13 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.Serializable;
+import java.lang.reflect.Field;
 import java.util.List;
 import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 public class HFileTargetIT implements Serializable {
 
@@ -96,6 +100,38 @@ public class HFileTargetIT implements Serializable {
       splits[i] = new byte[] { b };
     }
     admin.createTable(htable, splits);
+
+    // Set classpath for yarn, otherwise it won't be able to find MRAppMaster
+    // (see CRUNCH-249 and HBASE-8528).
+    HBASE_TEST_UTILITY.getConfiguration().setBoolean("yarn.is.minicluster", true);
+    dirtyFixForJobHistoryServerAddress();
+  }
+
+  /**
+   * We need to set the address of JobHistory server, as it randomly picks a unused port
+   * to listen. Unfortunately, HBaseTestingUtility neither does that nor provides a way
+   * for us to know the picked address. We have to access it using reflection.
+   *
+   * This is necessary when testing with MRv2, but does no harm to MRv1.
+   */
+  private static void dirtyFixForJobHistoryServerAddress() {
+    try {
+      // Retrieve HBASE_TEST_UTILITY.mrCluster via reflection, as it is private.
+      Field mrClusterField = HBaseTestingUtility.class.getDeclaredField("mrCluster");
+      mrClusterField.setAccessible(true);
+      MiniMRCluster mrCluster = (MiniMRCluster) mrClusterField.get(HBASE_TEST_UTILITY);
+      JobConf jobConf = mrCluster.createJobConf();
+      Configuration conf = HBASE_TEST_UTILITY.getConfiguration();
+      String proprety = "mapreduce.jobhistory.address";
+      String value = jobConf.get(proprety);
+      if (value != null) { // maybe null if we're running MRv1
+        conf.set(proprety, value);
+      }
+    } catch (IllegalAccessException e) {
+      throw new AssertionError(e);
+    } catch (NoSuchFieldException e) {
+      throw new AssertionError(e);
+    }
   }
 
   @AfterClass

http://git-wip-us.apache.org/repos/asf/crunch/blob/3f984113/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 369041f..e972ce2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -433,10 +433,10 @@ under the License.
       </activation>
       <properties>
         <hadoop.version>2.0.4-alpha</hadoop.version>
-        <!-- NOTE: You must build HBase 0.94.1 from source using:
+        <!-- NOTE: You must build HBase 0.94.3 from source using:
              mvn clean install -Dhadoop.profile=2.0
              for this to work. -->
-        <hbase.version>0.94.1</hbase.version>
+        <hbase.version>0.94.3</hbase.version>
         <commons-lang.version>2.5</commons-lang.version>
         <slf4j.version>1.6.1</slf4j.version>
       </properties>


Mime
View raw message