incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [1/2] git commit: Adding some more updates to the hive integration. Also moved some basic shard id logic out to it's own util class so that all of Lucene doesn't need to be dragged into the mr job.
Date Fri, 19 Dec 2014 15:33:38 GMT
Repository: incubator-blur
Updated Branches:
  refs/heads/master a0c27fc48 -> b72ea64d5


Adding some more updates to the hive integration.  Also moved some basic shard id logic out
to it's own util class so that all of Lucene doesn't need to be dragged into the mr job.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/f31bd309
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/f31bd309
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/f31bd309

Branch: refs/heads/master
Commit: f31bd309e8aacf0d61c60284b0cbf6b15a840db9
Parents: f81e010
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Fri Dec 19 10:33:35 2014 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Fri Dec 19 10:33:35 2014 -0500

----------------------------------------------------------------------
 .../manager/indexserver/LocalIndexServer.java   |   4 +-
 .../blur/manager/writer/IndexImporter.java      |   4 +-
 .../org/apache/blur/server/ShardContext.java    |   4 +-
 .../org/apache/blur/server/TableContext.java    |   7 +-
 .../org/apache/blur/thrift/MutationHelper.java  |   4 +-
 .../java/org/apache/blur/thrift/TableAdmin.java |   4 +-
 .../java/org/apache/blur/utils/BlurUtil.java    |  85 +--------------
 .../org/apache/blur/utils/RowDocumentUtil.java  |   2 +-
 .../java/org/apache/blur/utils/ShardUtil.java   | 103 +++++++++++++++++++
 .../blur/utils/TableShardCountCollapser.java    |   2 +-
 .../blur/command/ShardCommandManagerTest.java   |   4 +-
 .../apache/blur/manager/IndexManagerTest.java   |   4 +-
 .../indexserver/MasterBasedLevelerTest.java     |   4 +-
 .../results/MultipleBlurResultIterableTest.java |   5 +-
 .../org/apache/blur/utils/BlurUtilsTest.java    |   2 +-
 .../utils/TableShardCountCollapserTest.java     |   4 +-
 contrib/blur-hive/pom.xml                       |  46 +++++++++
 contrib/blur-hive/src/assemble/bin.xml          |  16 +++
 .../apache/blur/hive/BlurHiveOutputFormat.java  |   4 +-
 contrib/blur-hive/src/test/java/test.hive       |  37 +------
 20 files changed, 198 insertions(+), 147 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f31bd309/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
b/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
index 7e1d57f..8cce939 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/indexserver/LocalIndexServer.java
@@ -45,7 +45,7 @@ import org.apache.blur.server.TableContext;
 import org.apache.blur.thrift.generated.ShardState;
 import org.apache.blur.thrift.generated.TableDescriptor;
 import org.apache.blur.utils.BlurConstants;
-import org.apache.blur.utils.BlurUtil;
+import org.apache.blur.utils.ShardUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.lucene.analysis.core.KeywordAnalyzer;
 import org.apache.lucene.index.CorruptIndexException;
@@ -147,7 +147,7 @@ public class LocalIndexServer extends AbstractIndexServer {
       int shardCount = _tableContext.getDescriptor().getShardCount();
       for (int i = 0; i < shardCount; i++) {
         Directory directory;
-        String shardName = BlurUtil.getShardName(BlurConstants.SHARD_PREFIX, i);
+        String shardName = ShardUtil.getShardName(BlurConstants.SHARD_PREFIX, i);
         if (_ramDir) {
           directory = new RAMDirectory();
         } else {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f31bd309/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java b/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java
index c2a7398..92f623f 100644
--- a/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java
+++ b/blur-core/src/main/java/org/apache/blur/manager/writer/IndexImporter.java
@@ -39,7 +39,7 @@ import org.apache.blur.server.ShardContext;
 import org.apache.blur.server.TableContext;
 import org.apache.blur.store.hdfs.HdfsDirectory;
 import org.apache.blur.utils.BlurConstants;
-import org.apache.blur.utils.BlurUtil;
+import org.apache.blur.utils.ShardUtil;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -229,7 +229,7 @@ public class IndexImporter extends TimerTask implements Closeable {
       BlurPartitioner blurPartitioner = new BlurPartitioner();
       Text key = new Text();
       int numberOfShards = _shardContext.getTableContext().getDescriptor().getShardCount();
-      int shardId = BlurUtil.getShardIndex(shard);
+      int shardId = ShardUtil.getShardIndex(shard);
       for (AtomicReaderContext context : leaves) {
         AtomicReader atomicReader = context.reader();
         Fields fields = atomicReader.fields();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f31bd309/blur-core/src/main/java/org/apache/blur/server/ShardContext.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/server/ShardContext.java b/blur-core/src/main/java/org/apache/blur/server/ShardContext.java
index b8925f1..50448f9 100644
--- a/blur-core/src/main/java/org/apache/blur/server/ShardContext.java
+++ b/blur-core/src/main/java/org/apache/blur/server/ShardContext.java
@@ -19,7 +19,7 @@ package org.apache.blur.server;
 import java.io.IOException;
 
 import org.apache.blur.store.hdfs.HdfsDirectory;
-import org.apache.blur.utils.BlurUtil;
+import org.apache.blur.utils.ShardUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.lucene.store.Directory;
 
@@ -67,7 +67,7 @@ public class ShardContext {
   }
 
   public static ShardContext create(TableContext tableContext, String shard) throws IOException
{
-    BlurUtil.validateShardName(shard);
+    ShardUtil.validateShardName(shard);
     ShardContext shardContext = new ShardContext();
     shardContext.tableContext = tableContext;
     shardContext.hdfsDirPath = new Path(tableContext.getTablePath(), shard);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f31bd309/blur-core/src/main/java/org/apache/blur/server/TableContext.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/server/TableContext.java b/blur-core/src/main/java/org/apache/blur/server/TableContext.java
index 8a2b503..54186e8 100644
--- a/blur-core/src/main/java/org/apache/blur/server/TableContext.java
+++ b/blur-core/src/main/java/org/apache/blur/server/TableContext.java
@@ -30,9 +30,9 @@ import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Timer;
 import java.util.Map.Entry;
 import java.util.Set;
+import java.util.Timer;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeUnit;
@@ -53,12 +53,11 @@ import org.apache.blur.manager.writer.BlurIndexSimpleWriter;
 //import org.apache.blur.manager.writer.BlurNRTIndex;
 import org.apache.blur.manager.writer.SharedMergeScheduler;
 import org.apache.blur.store.hdfs.HdfsDirectory;
-import org.apache.blur.thrift.BlurClient;
 import org.apache.blur.thrift.generated.Blur.Iface;
 import org.apache.blur.thrift.generated.ScoreType;
 import org.apache.blur.thrift.generated.TableDescriptor;
 import org.apache.blur.utils.BlurConstants;
-import org.apache.blur.utils.BlurUtil;
+import org.apache.blur.utils.ShardUtil;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -413,7 +412,7 @@ public class TableContext implements Cloneable {
     for (FileStatus fileStatus : listStatus) {
       Path shardPath = fileStatus.getPath();
       String shardId = shardPath.getName();
-      int shardIndex = BlurUtil.getShardIndex(shardId);
+      int shardIndex = ShardUtil.getShardIndex(shardId);
       if (shardIndex >= shardCount) {
         throw new IOException("Too many shards [" + shardIndex + "].");
       }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f31bd309/blur-core/src/main/java/org/apache/blur/thrift/MutationHelper.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/MutationHelper.java b/blur-core/src/main/java/org/apache/blur/thrift/MutationHelper.java
index 3ca394e..2d64f74 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/MutationHelper.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/MutationHelper.java
@@ -24,13 +24,13 @@ import org.apache.blur.thrift.generated.RecordMutation;
 import org.apache.blur.thrift.generated.Row;
 import org.apache.blur.thrift.generated.RowMutation;
 import org.apache.blur.utils.BlurConstants;
-import org.apache.blur.utils.BlurUtil;
+import org.apache.blur.utils.ShardUtil;
 
 public class MutationHelper {
 
   public static String getShardName(String table, String rowId, int numberOfShards, BlurPartitioner
blurPartitioner) {
     int partition = blurPartitioner.getShard(rowId, numberOfShards);
-    return BlurUtil.getShardName(BlurConstants.SHARD_PREFIX, partition);
+    return ShardUtil.getShardName(BlurConstants.SHARD_PREFIX, partition);
   }
 
   public static void validateMutation(RowMutation mutation) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f31bd309/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
index 7583160..67271da 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
@@ -54,8 +54,8 @@ import org.apache.blur.thrift.generated.ShardState;
 import org.apache.blur.thrift.generated.TableDescriptor;
 import org.apache.blur.trace.Trace;
 import org.apache.blur.trace.TraceStorage;
-import org.apache.blur.utils.BlurUtil;
 import org.apache.blur.utils.MemoryReporter;
+import org.apache.blur.utils.ShardUtil;
 import org.apache.log4j.LogManager;
 import org.apache.log4j.Logger;
 import org.apache.log4j.xml.DOMConfigurator;
@@ -117,7 +117,7 @@ public abstract class TableAdmin implements Iface {
   public final void createTable(TableDescriptor tableDescriptor) throws BlurException, TException
{
     try {
       TableContext.clear(tableDescriptor.getName());
-      BlurUtil.validateTableName(tableDescriptor.getName());
+      ShardUtil.validateTableName(tableDescriptor.getName());
       assignClusterIfNull(tableDescriptor);
       _clusterStatus.createTable(tableDescriptor);
     } catch (Exception e) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f31bd309/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
index 315b841..44778f6 100644
--- a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
+++ b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
@@ -88,7 +88,6 @@ import org.apache.blur.thrift.generated.BlurException;
 import org.apache.blur.thrift.generated.BlurQuery;
 import org.apache.blur.thrift.generated.BlurResult;
 import org.apache.blur.thrift.generated.BlurResults;
-import org.apache.blur.thrift.generated.Column;
 import org.apache.blur.thrift.generated.FetchResult;
 import org.apache.blur.thrift.generated.Record;
 import org.apache.blur.thrift.generated.RecordMutation;
@@ -650,22 +649,6 @@ public class BlurUtil {
     }
   }
 
-  public static String getShardName(int id) {
-    return getShardName(BlurConstants.SHARD_PREFIX, id);
-  }
-
-  public static String getShardName(String prefix, int id) {
-    return prefix + buffer(id, 8);
-  }
-
-  private static String buffer(int value, int length) {
-    String str = Integer.toString(value);
-    while (str.length() < length) {
-      str = "0" + str;
-    }
-    return str;
-  }
-
   public static String humanizeTime(long time, TimeUnit unit) {
     long seconds = unit.toSeconds(time);
     long hours = getHours(seconds);
@@ -720,7 +703,7 @@ public class BlurUtil {
       validateShardCount(shardCount, fileSystem, tablePath);
     }
     for (int i = 0; i < shardCount; i++) {
-      String shardName = BlurUtil.getShardName(SHARD_PREFIX, i);
+      String shardName = ShardUtil.getShardName(SHARD_PREFIX, i);
       Path shardPath = new Path(tablePath, shardName);
       createPath(fileSystem, shardPath);
     }
@@ -729,7 +712,7 @@ public class BlurUtil {
   public static void validateShardCount(int shardCount, FileSystem fileSystem, Path tablePath)
throws IOException {
     // Check that all the directories that should be are in fact there.
     for (int i = 0; i < shardCount; i++) {
-      Path path = new Path(tablePath, BlurUtil.getShardName(BlurConstants.SHARD_PREFIX, i));
+      Path path = new Path(tablePath, ShardUtil.getShardName(BlurConstants.SHARD_PREFIX,
i));
       if (!fileSystem.exists(path)) {
         LOG.error("Path [{0}] for shard [{1}] does not exist.", path, i);
         throw new RuntimeException("Path [" + path + "] for shard [" + i + "] does not exist.");
@@ -1121,70 +1104,6 @@ public class BlurUtil {
     return SlowCompositeReaderWrapper.wrap(reader);
   }
 
-  public static int getShardIndex(String shard) {
-    int index = shard.indexOf('-');
-    return Integer.parseInt(shard.substring(index + 1));
-  }
-
-  public static void validateRowIdAndRecord(String rowId, Record record) {
-    if (!validate(record.family)) {
-      throw new IllegalArgumentException("Invalid column family name [ " + record.family
-          + " ]. It should contain only this pattern [A-Za-z0-9_-]");
-    }
-
-    for (Column column : record.getColumns()) {
-      if (!validate(column.name)) {
-        throw new IllegalArgumentException("Invalid column name [ " + column.name
-            + " ]. It should contain only this pattern [A-Za-z0-9_-]");
-      }
-    }
-  }
-
-  public static boolean validate(String s) {
-    int length = s.length();
-    for (int i = 0; i < length; i++) {
-      char c = s.charAt(i);
-      if (!validate(c)) {
-        return false;
-      }
-    }
-    return true;
-  }
-
-  private static boolean validate(char c) {
-    if (c >= 'a' && c <= 'z') {
-      return true;
-    }
-    if (c >= 'A' && c <= 'Z') {
-      return true;
-    }
-    if (c >= '0' && c <= '9') {
-      return true;
-    }
-    switch (c) {
-    case '_':
-      return true;
-    case '-':
-      return true;
-    default:
-      return false;
-    }
-  }
-
-  public static void validateTableName(String tableName) {
-    if (!validate(tableName)) {
-      throw new IllegalArgumentException("Invalid table name [ " + tableName
-          + " ]. It should contain only this pattern [A-Za-z0-9_-]");
-    }
-  }
-
-  public static void validateShardName(String shardName) {
-    if (!validate(shardName)) {
-      throw new IllegalArgumentException("Invalid shard name [ " + shardName
-          + " ]. It should contain only this pattern [A-Za-z0-9_-]");
-    }
-  }
-
   public static String getPid() {
     return ManagementFactory.getRuntimeMXBean().getName();
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f31bd309/blur-core/src/main/java/org/apache/blur/utils/RowDocumentUtil.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/RowDocumentUtil.java b/blur-core/src/main/java/org/apache/blur/utils/RowDocumentUtil.java
index 085147c..a8f134a 100644
--- a/blur-core/src/main/java/org/apache/blur/utils/RowDocumentUtil.java
+++ b/blur-core/src/main/java/org/apache/blur/utils/RowDocumentUtil.java
@@ -125,7 +125,7 @@ public class RowDocumentUtil {
   }
 
   public static List<Field> getDoc(FieldManager fieldManager, final String rowId, Record
record) throws IOException {
-    BlurUtil.validateRowIdAndRecord(rowId, record);
+    ShardUtil.validateRowIdAndRecord(rowId, record);
     List<Field> fields = fieldManager.getFields(rowId, record);
     return fields;
   }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f31bd309/blur-core/src/main/java/org/apache/blur/utils/ShardUtil.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/ShardUtil.java b/blur-core/src/main/java/org/apache/blur/utils/ShardUtil.java
new file mode 100644
index 0000000..0656bc4
--- /dev/null
+++ b/blur-core/src/main/java/org/apache/blur/utils/ShardUtil.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.utils;
+
+import org.apache.blur.thrift.generated.Column;
+import org.apache.blur.thrift.generated.Record;
+
+public class ShardUtil {
+  
+  public static String getShardName(int id) {
+    return getShardName(BlurConstants.SHARD_PREFIX, id);
+  }
+
+  public static String getShardName(String prefix, int id) {
+    return prefix + buffer(id, 8);
+  }
+
+  private static String buffer(int value, int length) {
+    String str = Integer.toString(value);
+    while (str.length() < length) {
+      str = "0" + str;
+    }
+    return str;
+  }
+
+  public static int getShardIndex(String shard) {
+    int index = shard.indexOf('-');
+    return Integer.parseInt(shard.substring(index + 1));
+  }
+
+  public static void validateRowIdAndRecord(String rowId, Record record) {
+    if (!validate(record.family)) {
+      throw new IllegalArgumentException("Invalid column family name [ " + record.family
+          + " ]. It should contain only this pattern [A-Za-z0-9_-]");
+    }
+
+    for (Column column : record.getColumns()) {
+      if (!validate(column.name)) {
+        throw new IllegalArgumentException("Invalid column name [ " + column.name
+            + " ]. It should contain only this pattern [A-Za-z0-9_-]");
+      }
+    }
+  }
+
+  public static boolean validate(String s) {
+    int length = s.length();
+    for (int i = 0; i < length; i++) {
+      char c = s.charAt(i);
+      if (!validate(c)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  private static boolean validate(char c) {
+    if (c >= 'a' && c <= 'z') {
+      return true;
+    }
+    if (c >= 'A' && c <= 'Z') {
+      return true;
+    }
+    if (c >= '0' && c <= '9') {
+      return true;
+    }
+    switch (c) {
+    case '_':
+      return true;
+    case '-':
+      return true;
+    default:
+      return false;
+    }
+  }
+
+  public static void validateTableName(String tableName) {
+    if (!validate(tableName)) {
+      throw new IllegalArgumentException("Invalid table name [ " + tableName
+          + " ]. It should contain only this pattern [A-Za-z0-9_-]");
+    }
+  }
+
+  public static void validateShardName(String shardName) {
+    if (!validate(shardName)) {
+      throw new IllegalArgumentException("Invalid shard name [ " + shardName
+          + " ]. It should contain only this pattern [A-Za-z0-9_-]");
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f31bd309/blur-core/src/main/java/org/apache/blur/utils/TableShardCountCollapser.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/TableShardCountCollapser.java b/blur-core/src/main/java/org/apache/blur/utils/TableShardCountCollapser.java
index adbb8f8..cc25ff6 100644
--- a/blur-core/src/main/java/org/apache/blur/utils/TableShardCountCollapser.java
+++ b/blur-core/src/main/java/org/apache/blur/utils/TableShardCountCollapser.java
@@ -114,7 +114,7 @@ public class TableShardCountCollapser extends Configured implements Tool
{
   private void validateShards(SortedSet<String> shards) {
     int count = shards.size();
     for (int i = 0; i < count; i++) {
-      if (!shards.contains(BlurUtil.getShardName(i))) {
+      if (!shards.contains(ShardUtil.getShardName(i))) {
         throw new RuntimeException("Invalid table");
       }
     }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f31bd309/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java
b/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java
index 673838b..99c8ac9 100644
--- a/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java
+++ b/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java
@@ -46,7 +46,7 @@ import org.apache.blur.thrift.generated.BlurException;
 import org.apache.blur.thrift.generated.RowMutation;
 import org.apache.blur.thrift.generated.ShardState;
 import org.apache.blur.thrift.generated.TableDescriptor;
-import org.apache.blur.utils.BlurUtil;
+import org.apache.blur.utils.ShardUtil;
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.lucene.analysis.core.KeywordAnalyzer;
@@ -317,7 +317,7 @@ public class ShardCommandManagerTest {
       public Map<String, BlurIndex> getIndexes(String table) throws IOException {
         Map<String, BlurIndex> indexes = new HashMap<String, BlurIndex>();
         for (int i = 0; i < 3; i++) {
-          String shardName = BlurUtil.getShardName(i);
+          String shardName = ShardUtil.getShardName(i);
           indexes.put(shardName, getNullBlurIndex(shardName));
         }
         return indexes;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f31bd309/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java b/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
index e91d16f..78fe48e 100644
--- a/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
+++ b/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
@@ -76,7 +76,7 @@ import org.apache.blur.trace.TraceCollector;
 import org.apache.blur.trace.TraceStorage;
 import org.apache.blur.utils.BlurConstants;
 import org.apache.blur.utils.BlurIterator;
-import org.apache.blur.utils.BlurUtil;
+import org.apache.blur.utils.ShardUtil;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Filter;
 import org.apache.lucene.search.QueryWrapperFilter;
@@ -90,7 +90,7 @@ public class IndexManagerTest {
 
   private static final File TMPDIR = new File("./target/tmp");
 
-  private static final String SHARD_NAME = BlurUtil.getShardName(BlurConstants.SHARD_PREFIX,
0);
+  private static final String SHARD_NAME = ShardUtil.getShardName(BlurConstants.SHARD_PREFIX,
0);
   private static final String TABLE = "table";
   private static final String FAMILY = "test-family";
   private static final String FAMILY2 = "test-family2";

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f31bd309/blur-core/src/test/java/org/apache/blur/manager/indexserver/MasterBasedLevelerTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/indexserver/MasterBasedLevelerTest.java
b/blur-core/src/test/java/org/apache/blur/manager/indexserver/MasterBasedLevelerTest.java
index 712a218..86bfb80 100644
--- a/blur-core/src/test/java/org/apache/blur/manager/indexserver/MasterBasedLevelerTest.java
+++ b/blur-core/src/test/java/org/apache/blur/manager/indexserver/MasterBasedLevelerTest.java
@@ -29,7 +29,7 @@ import java.util.Random;
 import java.util.Set;
 import java.util.TreeMap;
 
-import org.apache.blur.utils.BlurUtil;
+import org.apache.blur.utils.ShardUtil;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -258,7 +258,7 @@ public class MasterBasedLevelerTest {
 
   private void populateCurrentLayout(Random random, Map<String, String> newLayoutMap,
int shards, List<String> servers) {
     for (int i = 0; i < shards; i++) {
-      String shardName = BlurUtil.getShardName(i);
+      String shardName = ShardUtil.getShardName(i);
       int server = random.nextInt(servers.size());
       newLayoutMap.put(shardName, servers.get(server));
     }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f31bd309/blur-core/src/test/java/org/apache/blur/manager/results/MultipleBlurResultIterableTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/results/MultipleBlurResultIterableTest.java
b/blur-core/src/test/java/org/apache/blur/manager/results/MultipleBlurResultIterableTest.java
index 262552c..8008574 100644
--- a/blur-core/src/test/java/org/apache/blur/manager/results/MultipleBlurResultIterableTest.java
+++ b/blur-core/src/test/java/org/apache/blur/manager/results/MultipleBlurResultIterableTest.java
@@ -31,6 +31,7 @@ import org.apache.blur.thrift.generated.BlurResult;
 import org.apache.blur.thrift.generated.SortFieldResult;
 import org.apache.blur.utils.BlurIterator;
 import org.apache.blur.utils.BlurUtil;
+import org.apache.blur.utils.ShardUtil;
 import org.junit.Test;
 
 public class MultipleBlurResultIterableTest {
@@ -103,7 +104,7 @@ public class MultipleBlurResultIterableTest {
       Entry<Double, List<SortFieldResult>>... entries) {
     List<BlurResult> results = new ArrayList<BlurResult>();
     for (Entry<Double, List<SortFieldResult>> entry : entries) {
-      String shardName = BlurUtil.getShardName(shard);
+      String shardName = ShardUtil.getShardName(shard);
       int docId = random.nextInt(Integer.MAX_VALUE);
       Double score = entry.getKey();
       results.add(new BlurResult(shardName + "/" + docId, score, null, entry.getValue()));
@@ -138,7 +139,7 @@ public class MultipleBlurResultIterableTest {
   private BlurResultIterable newBlurResultIterableNoSort(int shard, Random random, double...
ds) {
     List<BlurResult> results = new ArrayList<BlurResult>();
     for (double d : ds) {
-      String shardName = BlurUtil.getShardName(shard);
+      String shardName = ShardUtil.getShardName(shard);
       int docId = random.nextInt(Integer.MAX_VALUE);
       results.add(new BlurResult(shardName + "/" + docId, d, null, null));
     }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f31bd309/blur-core/src/test/java/org/apache/blur/utils/BlurUtilsTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/utils/BlurUtilsTest.java b/blur-core/src/test/java/org/apache/blur/utils/BlurUtilsTest.java
index f91633d..dce02e8 100644
--- a/blur-core/src/test/java/org/apache/blur/utils/BlurUtilsTest.java
+++ b/blur-core/src/test/java/org/apache/blur/utils/BlurUtilsTest.java
@@ -241,7 +241,7 @@ public class BlurUtilsTest {
 
   private void createShardDirs(int shardCount, FileSystem fileSystem, Path path) throws IOException
{
     for (int i = 0; i < shardCount; i++) {
-      fileSystem.mkdirs(new Path(path, BlurUtil.getShardName(BlurConstants.SHARD_PREFIX,
i)));
+      fileSystem.mkdirs(new Path(path, ShardUtil.getShardName(BlurConstants.SHARD_PREFIX,
i)));
     }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f31bd309/blur-core/src/test/java/org/apache/blur/utils/TableShardCountCollapserTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/utils/TableShardCountCollapserTest.java
b/blur-core/src/test/java/org/apache/blur/utils/TableShardCountCollapserTest.java
index a2f3be1..782c612 100644
--- a/blur-core/src/test/java/org/apache/blur/utils/TableShardCountCollapserTest.java
+++ b/blur-core/src/test/java/org/apache/blur/utils/TableShardCountCollapserTest.java
@@ -59,7 +59,7 @@ public class TableShardCountCollapserTest {
 
   private void createShards(int shardCount) throws IOException {
     for (int i = 0; i < shardCount; i++) {
-      String shardName = BlurUtil.getShardName(i);
+      String shardName = ShardUtil.getShardName(i);
       createShard(configuration, i, new Path(path, shardName), shardCount);
     }
   }
@@ -78,7 +78,7 @@ public class TableShardCountCollapserTest {
   private void assertData(int totalShardCount) throws IOException {
     Partitioner<IntWritable, IntWritable> partitioner = new HashPartitioner<IntWritable,
IntWritable>();
     for (int i = 0; i < totalShardCount; i++) {
-      HdfsDirectory directory = new HdfsDirectory(configuration, new Path(path, BlurUtil.getShardName(i)));
+      HdfsDirectory directory = new HdfsDirectory(configuration, new Path(path, ShardUtil.getShardName(i)));
       DirectoryReader reader = DirectoryReader.open(directory);
       int numDocs = reader.numDocs();
       for (int d = 0; d < numDocs; d++) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f31bd309/contrib/blur-hive/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/blur-hive/pom.xml b/contrib/blur-hive/pom.xml
index 3261e9a..945da1f 100644
--- a/contrib/blur-hive/pom.xml
+++ b/contrib/blur-hive/pom.xml
@@ -27,9 +27,22 @@
 
 	<dependencies>
 		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-api</artifactId>
+			<version>${slf4j.version}</version>
+			<scope>provided</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-log4j12</artifactId>
+			<version>${slf4j.version}</version>
+			<scope>provided</scope>
+		</dependency>
+		<dependency>
 			<groupId>org.apache.hive</groupId>
 			<artifactId>hive-exec</artifactId>
 			<version>0.13.1</version>
+			<scope>provided</scope>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.hive</groupId>
@@ -41,6 +54,18 @@
 			<groupId>org.apache.blur</groupId>
 			<artifactId>blur-thrift</artifactId>
 			<version>${project.version}</version>
+			<exclusions>
+				<exclusion>
+					<groupId>org.apache.httpcomponents</groupId>
+					<artifactId>httpclient</artifactId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+		<dependency>
+			<groupId>log4j</groupId>
+			<artifactId>log4j</artifactId>
+			<version>${log4j.version}</version>
+			<scope>provided</scope>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.blur</groupId>
@@ -101,11 +126,30 @@
 					<groupId>org.apache.hadoop</groupId>
 					<artifactId>hadoop-core</artifactId>
 					<version>${hadoop.version}</version>
+					<scope>provided</scope>
 				</dependency>
 				<dependency>
 					<groupId>org.apache.blur</groupId>
 					<artifactId>blur-mapred-hadoop1</artifactId>
 					<version>${project.version}</version>
+					<exclusions>
+						<exclusion>
+							<groupId>org.apache.blur</groupId>
+							<artifactId>blur-store</artifactId>
+						</exclusion>
+						<exclusion>
+							<groupId>org.apache.blur</groupId>
+							<artifactId>blur-query</artifactId>
+						</exclusion>
+						<exclusion>
+							<groupId>org.apache.blur</groupId>
+							<artifactId>blur-gui</artifactId>
+						</exclusion>
+						<exclusion>
+							<groupId>org.eclipse.jetty</groupId>
+							<artifactId>jetty-webapp</artifactId>
+						</exclusion>
+					</exclusions>
 				</dependency>
 				<dependency>
 					<groupId>org.apache.hadoop</groupId>
@@ -131,6 +175,7 @@
 					<groupId>org.apache.hadoop</groupId>
 					<artifactId>hadoop-client</artifactId>
 					<version>${hadoop.version}</version>
+					<scope>provided</scope>
 				</dependency>
 				<dependency>
 					<groupId>org.apache.hadoop</groupId>
@@ -155,6 +200,7 @@
 					<groupId>org.apache.hadoop</groupId>
 					<artifactId>hadoop-client</artifactId>
 					<version>${hadoop.version}</version>
+					<scope>provided</scope>
 				</dependency>
 				<dependency>
 					<groupId>org.apache.hadoop</groupId>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f31bd309/contrib/blur-hive/src/assemble/bin.xml
----------------------------------------------------------------------
diff --git a/contrib/blur-hive/src/assemble/bin.xml b/contrib/blur-hive/src/assemble/bin.xml
new file mode 100644
index 0000000..1b65be3
--- /dev/null
+++ b/contrib/blur-hive/src/assemble/bin.xml
@@ -0,0 +1,16 @@
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor 
+	license agreements. See the NOTICE file distributed with this work for additional 
+	information regarding copyright ownership. The ASF licenses this file to 
+	you under the Apache License, Version 2.0 (the "License"); you may not use 
+	this file except in compliance with the License. You may obtain a copy of 
+	the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required 
+	by applicable law or agreed to in writing, software distributed under the 
+	License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS 
+	OF ANY KIND, either express or implied. See the License for the specific 
+	language governing permissions and limitations under the License. -->
+<assembly
+	xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2
http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+
+</assembly>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f31bd309/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveOutputFormat.java
----------------------------------------------------------------------
diff --git a/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveOutputFormat.java
b/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveOutputFormat.java
index 8561aa5..b2f9267 100644
--- a/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveOutputFormat.java
+++ b/contrib/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveOutputFormat.java
@@ -38,7 +38,7 @@ import org.apache.blur.thrift.generated.RecordMutationType;
 import org.apache.blur.thrift.generated.RowMutation;
 import org.apache.blur.thrift.generated.RowMutationType;
 import org.apache.blur.thrift.generated.TableDescriptor;
-import org.apache.blur.utils.BlurUtil;
+import org.apache.blur.utils.ShardUtil;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -110,7 +110,7 @@ public class BlurHiveOutputFormat implements HiveOutputFormat<Text,
BlurRecord>
 
       private Iface getClient(String rowId) throws BlurException, TException {
         int shard = _blurPartitioner.getShard(rowId, numberOfShardsInTable);
-        String shardId = BlurUtil.getShardName(shard);
+        String shardId = ShardUtil.getShardName(shard);
         return getClientFromShardId(table, shardId);
       }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/f31bd309/contrib/blur-hive/src/test/java/test.hive
----------------------------------------------------------------------
diff --git a/contrib/blur-hive/src/test/java/test.hive b/contrib/blur-hive/src/test/java/test.hive
index 1b92dae..64baa82 100644
--- a/contrib/blur-hive/src/test/java/test.hive
+++ b/contrib/blur-hive/src/test/java/test.hive
@@ -16,41 +16,8 @@
 set mapred.job.tracker=localhost:9001;
 set hive.metastore.warehouse.dir=hdfs://localhost:9000/user/hive/warehouse;
 
-add jar file:///Users/amccurry/Development/incubator-blur/contrib/blur-hive/target/blur-hive-0.2.4-incubating-SNAPSHOT-hadoop1.jar;
-
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/blur-console-0.2.4-incubating-SNAPSHOT-hadoop1-webapp.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/blur-core-0.2.4-incubating-SNAPSHOT-hadoop1.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/blur-gui-0.2.4-incubating-SNAPSHOT-hadoop1.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/blur-gui-0.2.4-incubating-SNAPSHOT-hadoop1.war;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/blur-mapred-hadoop1-0.2.4-incubating-SNAPSHOT-hadoop1.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/blur-query-0.2.4-incubating-SNAPSHOT-hadoop1.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/blur-shell-0.2.4-incubating-SNAPSHOT-hadoop1.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/blur-store-0.2.4-incubating-SNAPSHOT-hadoop1.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/blur-thrift-0.2.4-incubating-SNAPSHOT-hadoop1.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/blur-util-0.2.4-incubating-SNAPSHOT-hadoop1.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/commons-cli-1.2.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/concurrentlinkedhashmap-lru-1.3.2.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/guava-14.0.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/jackson-annotations-2.1.1.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/jackson-core-2.1.1.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/jackson-databind-2.1.1.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/jline-2.10.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/json-20090211.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/lucene-analyzers-common-4.3.0.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/lucene-codecs-4.3.0.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/lucene-core-4.3.0.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/lucene-highlighter-4.3.0.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/lucene-memory-4.3.0.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/lucene-queries-4.3.0.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/lucene-queryparser-4.3.0.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/lucene-sandbox-4.3.0.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/lucene-spatial-4.3.0.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/metrics-core-2.2.0.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/metrics-ganglia-2.2.0.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/metrics-graphite-2.2.0.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/metrics-servlet-2.2.0.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/spatial4j-0.3.jar;
-add jar /Users/amccurry/Development/incubator-blur/distribution/target/apache-blur-0.2.4-incubating-SNAPSHOT-hadoop1-bin/lib/zookeeper-3.4.5.jar;
+-- add jar file:///Users/amccurry/Development/incubator-blur/contrib/blur-hive/target/blur-hive-0.2.4-incubating-SNAPSHOT-hadoop1-all.jar;
+add jar file:///Users/amccurry/Development/incubator-blur/contrib/blur-hive/target/all-mod.zip;
 
 create database if not exists test;
 use test;


Mime
View raw message