incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: Attempting to make unit tests more consistent by adding an option to make hive output format block waiting for commit.
Date Mon, 09 Feb 2015 13:02:17 GMT
Repository: incubator-blur
Updated Branches:
  refs/heads/master b930ea235 -> 774d143c6


Attempting to make unit tests more consistent by adding an option to make hive output format
block waiting for commit.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/774d143c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/774d143c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/774d143c

Branch: refs/heads/master
Commit: 774d143c62ef8131d0d882484a97ce828836c8d2
Parents: b930ea2
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Mon Feb 9 08:02:09 2015 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Mon Feb 9 08:02:09 2015 -0500

----------------------------------------------------------------------
 .../java/org/apache/blur/hive/BlurHiveOutputCommitter.java   | 8 ++++++--
 blur-hive/src/main/java/org/apache/blur/hive/BlurSerDe.java  | 4 +++-
 .../src/test/java/org/apache/blur/hive/BlurSerDeTest.java    | 4 +++-
 3 files changed, 12 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/774d143c/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveOutputCommitter.java
----------------------------------------------------------------------
diff --git a/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveOutputCommitter.java b/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveOutputCommitter.java
index df182b3..bfeaaec 100644
--- a/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveOutputCommitter.java
+++ b/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveOutputCommitter.java
@@ -16,6 +16,9 @@
  */
 package org.apache.blur.hive;
 
+import static org.apache.blur.hive.BlurSerDe.BLUR_BLOCKING_APPLY;
+import static org.apache.blur.hive.BlurSerDe.BLUR_CONTROLLER_CONNECTION_STR;
+
 import java.io.IOException;
 
 import org.apache.blur.thirdparty.thrift_0_9_0.TException;
@@ -68,11 +71,12 @@ public class BlurHiveOutputCommitter extends OutputCommitter {
 
   private void finishBulkJob(JobContext context, boolean apply) throws IOException {
     Configuration configuration = context.getConfiguration();
-    String connectionStr = configuration.get(BlurSerDe.BLUR_CONTROLLER_CONNECTION_STR);
+    String connectionStr = configuration.get(BLUR_CONTROLLER_CONNECTION_STR);
+    boolean blocking = configuration.getBoolean(BLUR_BLOCKING_APPLY, false);
     Iface client = BlurClient.getClient(connectionStr);
     String bulkId = BlurHiveOutputFormat.getBulkId(configuration);
     try {
-      client.bulkMutateFinish(bulkId, apply, false);
+      client.bulkMutateFinish(bulkId, apply, blocking);
     } catch (BlurException e) {
       throw new IOException(e);
     } catch (TException e) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/774d143c/blur-hive/src/main/java/org/apache/blur/hive/BlurSerDe.java
----------------------------------------------------------------------
diff --git a/blur-hive/src/main/java/org/apache/blur/hive/BlurSerDe.java b/blur-hive/src/main/java/org/apache/blur/hive/BlurSerDe.java
index 055c6f4..150ad20 100644
--- a/blur-hive/src/main/java/org/apache/blur/hive/BlurSerDe.java
+++ b/blur-hive/src/main/java/org/apache/blur/hive/BlurSerDe.java
@@ -43,10 +43,12 @@ import org.apache.hadoop.io.Writable;
 
 public class BlurSerDe extends AbstractSerDe {
 
-  public static final String BLUR_CONTROLLER_CONNECTION_STR = "BLUR_CONTROLLER_CONNECTION_STR";
+  public static final String BLUR_BLOCKING_APPLY = "blur.blocking.apply";
+  public static final String BLUR_CONTROLLER_CONNECTION_STR = "blur.controller.connection.str";
   public static final String FAMILY = "blur.family";
   public static final String TABLE = "blur.table";
   public static final String ZK = BlurConstants.BLUR_ZOOKEEPER_CONNECTION;
+  
   private String _family;
   private Map<String, ColumnDefinition> _schema;
   private ObjectInspector _objectInspector;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/774d143c/blur-hive/src/test/java/org/apache/blur/hive/BlurSerDeTest.java
----------------------------------------------------------------------
diff --git a/blur-hive/src/test/java/org/apache/blur/hive/BlurSerDeTest.java b/blur-hive/src/test/java/org/apache/blur/hive/BlurSerDeTest.java
index b504ffe..8f37db3 100644
--- a/blur-hive/src/test/java/org/apache/blur/hive/BlurSerDeTest.java
+++ b/blur-hive/src/test/java/org/apache/blur/hive/BlurSerDeTest.java
@@ -256,7 +256,8 @@ public class BlurSerDeTest {
   }
 
   @Test
-  public void test2() throws SQLException, ClassNotFoundException, IOException, BlurException,
TException, InterruptedException {
+  public void test2() throws SQLException, ClassNotFoundException, IOException, BlurException,
TException,
+      InterruptedException {
     miniCluster.startMrMiniCluster();
     Configuration configuration = miniCluster.getMRConfiguration();
     HiveConf hiveConf = new HiveConf(configuration, getClass());
@@ -290,6 +291,7 @@ public class BlurSerDeTest {
     generateData(tableDir, totalRecords);
 
     run(connection, "select * from loadtable");
+    run(connection, "set " + BlurSerDe.BLUR_BLOCKING_APPLY + " = true");
     run(connection, "insert into table testtable select * from loadtable");
     miniCluster.stopMrMiniCluster();
     connection.close();


Mime
View raw message