incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: Fixing the hive bulk loader test.
Date Tue, 02 Jun 2015 12:32:55 GMT
Repository: incubator-blur
Updated Branches:
  refs/heads/master 3958192bc -> 9243fe09f


Fixing the hive bulk loader test.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/9243fe09
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/9243fe09
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/9243fe09

Branch: refs/heads/master
Commit: 9243fe09f5cc015bdc75466816cc8fcc86b0403e
Parents: 3958192
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Tue Jun 2 08:32:48 2015 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Tue Jun 2 08:32:48 2015 -0400

----------------------------------------------------------------------
 .../test/java/org/apache/blur/MiniCluster.java  |  8 +++---
 .../hive/BlurHiveMRLoaderOutputCommitter.java   |  6 +++--
 .../org/apache/blur/hive/BlurSerDeTest.java     | 26 ++++++++++++++++++++
 .../lib/update/BulkTableUpdateCommand.java      |  3 +++
 4 files changed, 37 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9243fe09/blur-core/src/test/java/org/apache/blur/MiniCluster.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/MiniCluster.java b/blur-core/src/test/java/org/apache/blur/MiniCluster.java
index 8db632f..2cc805a 100644
--- a/blur-core/src/test/java/org/apache/blur/MiniCluster.java
+++ b/blur-core/src/test/java/org/apache/blur/MiniCluster.java
@@ -219,7 +219,7 @@ public class MiniCluster {
     ScriptEngine engine = manager.getEngineByName("js");
 
     if (useYarn()) {
-      int nodeManagers = 1;
+      int nodeManagers = 2;
       Class<?> c = getClass();
       engine.put("c", c);
       engine.put("nodeManagers", nodeManagers);
@@ -239,7 +239,7 @@ public class MiniCluster {
       mrMiniCluster = bindings.get("mrMiniCluster");
       return (Configuration) bindings.get("configuration");
     } else {
-      int numTaskTrackers = 1;
+      int numTaskTrackers = 2;
       int numDir = 1;
       engine.put("fileSystemUri", fileSystemUri);
       engine.put("numTaskTrackers", numTaskTrackers);
@@ -258,7 +258,7 @@ public class MiniCluster {
     }
   }
 
-  private boolean useYarn() {
+  public boolean useYarn() {
     String version = VersionInfo.getVersion();
     if (version.startsWith("0.20.") || version.startsWith("1.")) {
       return false;
@@ -270,7 +270,7 @@ public class MiniCluster {
     return true;
   }
 
-  private boolean isMr1Hadoop2() {
+  public boolean isMr1Hadoop2() {
     try {
       Enumeration<URL> e = ClassLoader.getSystemClassLoader().getResources(
           "META-INF/maven/org.apache.hadoop/hadoop-client/pom.properties");

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9243fe09/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveMRLoaderOutputCommitter.java
----------------------------------------------------------------------
diff --git a/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveMRLoaderOutputCommitter.java
b/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveMRLoaderOutputCommitter.java
index 936c822..94a6415 100644
--- a/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveMRLoaderOutputCommitter.java
+++ b/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveMRLoaderOutputCommitter.java
@@ -34,8 +34,9 @@ import org.apache.hadoop.mapred.TaskAttemptContext;
 
 public class BlurHiveMRLoaderOutputCommitter extends OutputCommitter {
 
-  private static final String YARN_SITE_XML = "yarn-site.xml";
-  private static final String HDFS_SITE_XML = "hdfs-site.xml";
+  public static final String YARN_SITE_XML = "yarn-site.xml";
+  public static final String MAPRED_SITE_XML = "mapred-site.xml";
+  public static final String HDFS_SITE_XML = "hdfs-site.xml";
 
   private static final Log LOG = LogFactory.getLog(BlurHiveMRLoaderOutputCommitter.class);
 
@@ -103,6 +104,7 @@ public class BlurHiveMRLoaderOutputCommitter extends OutputCommitter {
       Configuration config = new Configuration(false);
       config.addResource(HDFS_SITE_XML);
       config.addResource(YARN_SITE_XML);
+      config.addResource(MAPRED_SITE_XML);
 
       bulkTableUpdateCommand.addExtraConfig(config);
       if (bulkTableUpdateCommand.run(BlurClient.getClient(connectionStr)) != 0) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9243fe09/blur-hive/src/test/java/org/apache/blur/hive/BlurSerDeTest.java
----------------------------------------------------------------------
diff --git a/blur-hive/src/test/java/org/apache/blur/hive/BlurSerDeTest.java b/blur-hive/src/test/java/org/apache/blur/hive/BlurSerDeTest.java
index 96609f2..53209a1 100644
--- a/blur-hive/src/test/java/org/apache/blur/hive/BlurSerDeTest.java
+++ b/blur-hive/src/test/java/org/apache/blur/hive/BlurSerDeTest.java
@@ -19,6 +19,8 @@ package org.apache.blur.hive;
 import static org.junit.Assert.assertEquals;
 
 import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.lang.reflect.Field;
@@ -69,6 +71,8 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import com.google.common.base.Splitter;
+
 public class BlurSerDeTest {
 
   public static final File WAREHOUSE = new File("./target/tmp/warehouse");
@@ -292,6 +296,7 @@ public class BlurSerDeTest {
       SQLException {
 
     Configuration configuration = miniCluster.getMRConfiguration();
+    writeSiteFiles(configuration);
     HiveConf hiveConf = new HiveConf(configuration, getClass());
     hiveConf.set("hive.server2.thrift.port", "0");
     HiveServer2 hiveServer2 = new HiveServer2();
@@ -331,6 +336,27 @@ public class BlurSerDeTest {
     return totalRecords;
   }
 
+  private void writeSiteFiles(Configuration configuration) throws FileNotFoundException,
IOException {
+    String name = BlurHiveMRLoaderOutputCommitter.MAPRED_SITE_XML;
+    if (miniCluster.useYarn()) {
+      name = BlurHiveMRLoaderOutputCommitter.YARN_SITE_XML;
+    }
+    String classPath = System.getProperty("java.class.path");
+    for (String path : Splitter.on(":").split(classPath)) {
+      File file = new File(path);
+      if (file.getName().equals("test-classes")) {
+        writeFile(new File(file, name), configuration);
+        return;
+      }
+    }
+  }
+
+  private void writeFile(File file, Configuration configuration) throws FileNotFoundException,
IOException {
+    FileOutputStream outputStream = new FileOutputStream(file);
+    configuration.writeXml(outputStream);
+    outputStream.close();
+  }
+
   private void generateData(File file, int totalRecords) throws IOException {
     SimpleDateFormat simpleDateFormat = new SimpleDateFormat(YYYY_MM_DD);
     file.mkdirs();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9243fe09/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/update/BulkTableUpdateCommand.java
----------------------------------------------------------------------
diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/update/BulkTableUpdateCommand.java
b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/update/BulkTableUpdateCommand.java
index 98b5967..dc8f9b7 100644
--- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/update/BulkTableUpdateCommand.java
+++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/lib/update/BulkTableUpdateCommand.java
@@ -43,6 +43,7 @@ public class BulkTableUpdateCommand extends ClusterExecuteCommandSingleTable<Int
 
   private static final String YARN_SITE_XML = "yarn-site.xml";
   private static final String HDFS_SITE_XML = "hdfs-site.xml";
+  private static final String MAPRED_SITE_XML = "mapred-site.xml";
   private static final String IMPORT = "import";
   private static final String BULK_UPDATE = "bulk-update";
 
@@ -78,6 +79,7 @@ public class BulkTableUpdateCommand extends ClusterExecuteCommandSingleTable<Int
     Configuration configuration = new Configuration();
     configuration.addResource(HDFS_SITE_XML);
     configuration.addResource(YARN_SITE_XML);
+    configuration.addResource(MAPRED_SITE_XML);
     for (String s : extraConfigs) {
       if (s != null) {
         InputStream inputStream = IOUtils.toInputStream(s);
@@ -90,6 +92,7 @@ public class BulkTableUpdateCommand extends ClusterExecuteCommandSingleTable<Int
       run = ToolRunner.run(configuration, new Driver(), new String[] { table, mrIncWorkingPath.toString(),
           outputPathStr, blurZkConnection, Integer.toString(reducerMultipler) });
     } catch (Exception e) {
+      e.printStackTrace();
       throw new IOException(e);
     }
 


Mime
View raw message