incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: Fixed BLUR-106.
Date Sat, 25 May 2013 17:07:21 GMT
Updated Branches:
  refs/heads/0.1.5 fbee0e711 -> 8eaca7c10


Fixed BLUR-106.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/8eaca7c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/8eaca7c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/8eaca7c1

Branch: refs/heads/0.1.5
Commit: 8eaca7c10d1d0abfa97a88abd0659103d85f1d1e
Parents: fbee0e7
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Sat May 25 13:06:42 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Sat May 25 13:06:42 2013 -0400

----------------------------------------------------------------------
 .../org/apache/blur/manager/BlurPartitioner.java   |   49 ++++++++++++++-
 .../java/org/apache/blur/manager/IndexManager.java |    3 +-
 .../apache/blur/thrift/BlurControllerServer.java   |    3 +-
 .../org/apache/blur/thrift/MutationHelper.java     |   11 +---
 4 files changed, 51 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8eaca7c1/src/blur-core/src/main/java/org/apache/blur/manager/BlurPartitioner.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/BlurPartitioner.java b/src/blur-core/src/main/java/org/apache/blur/manager/BlurPartitioner.java
index 9ca3a9a..899cd17 100644
--- a/src/blur-core/src/main/java/org/apache/blur/manager/BlurPartitioner.java
+++ b/src/blur-core/src/main/java/org/apache/blur/manager/BlurPartitioner.java
@@ -16,11 +16,56 @@ package org.apache.blur.manager;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.mapreduce.Partitioner;
 
-public class BlurPartitioner<BytesWritable, V> extends Partitioner<BytesWritable,
V> {
+/**
+ * Simple hashing class used to guide the rows to the correct shards during
+ * MapReduce jobs as well as during normal runtime operations.
+ */
+public class BlurPartitioner extends Partitioner<Text, Writable> {
+
+  /**
+   * Gets the shard from the the rowId, based on the number of shards in the
+   * table.
+   * 
+   * @param rowId
+   *          the rowId
+   * @param numberOfShardsInTable
+   *          the number of shards in the table.
+   * @return the shard where this rowId should be stored.
+   */
+  public int getShard(String rowId, int numberOfShardsInTable) {
+    return getShard(new Text(rowId), numberOfShardsInTable);
+  }
+
+  /**
+   * Gets the shard from the the rowId, based on the number of shards in the
+   * table.
+   * 
+   * @param rowId
+   *          the rowId
+   * @param numberOfShardsInTable
+   *          the number of shards in the table.
+   * @return the shard where this rowId should be stored.
+   */
+  public int getShard(Text rowId, int numberOfShardsInTable) {
+    return getPartition(rowId, null, numberOfShardsInTable);
+  }
 
-  public int getPartition(BytesWritable key, V value, int numReduceTasks) {
+  /**
+   * Gets the partition or reducer from the the rowId, based on the number of
+   * shards in the table.
+   * 
+   * @param rowId
+   *          the rowId
+   * @param numberOfShardsInTable
+   *          the number of shards in the table.
+   * @return the partition where this rowId should be processed.
+   */
+  @Override
+  public int getPartition(Text key, Writable value, int numReduceTasks) {
     return (key.hashCode() & Integer.MAX_VALUE) % numReduceTasks;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8eaca7c1/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
index 9938219..457e261 100644
--- a/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
+++ b/src/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java
@@ -86,7 +86,6 @@ import org.apache.blur.utils.ForkJoin;
 import org.apache.blur.utils.ForkJoin.Merger;
 import org.apache.blur.utils.ForkJoin.ParallelCall;
 import org.apache.blur.utils.ResetableDocumentStoredFieldVisitor;
-import org.apache.hadoop.io.BytesWritable;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.CorruptIndexException;
@@ -127,7 +126,7 @@ public class IndexManager {
   private int _threadCount;
   private QueryStatusManager _statusManager = new QueryStatusManager();
   private boolean _closed;
-  private BlurPartitioner<BytesWritable, Void> _blurPartitioner = new BlurPartitioner<BytesWritable,
Void>();
+  private BlurPartitioner _blurPartitioner = new BlurPartitioner();
   private BlurFilterCache _filterCache = new DefaultBlurFilterCache();
   private long _defaultParallelCallTimeout = TimeUnit.MINUTES.toMillis(1);
   private Meter _recordsMeter;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8eaca7c1/src/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
b/src/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
index f7ff594..79c9b61 100644
--- a/src/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
+++ b/src/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java
@@ -74,7 +74,6 @@ import org.apache.blur.utils.ForkJoin.ParallelCall;
 import org.apache.blur.zookeeper.WatchChildren;
 import org.apache.blur.zookeeper.WatchChildren.OnChange;
 import org.apache.blur.zookeeper.WatchNodeExistance;
-import org.apache.hadoop.io.BytesWritable;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.ZooDefs.Ids;
@@ -105,7 +104,7 @@ public class BlurControllerServer extends TableAdmin implements Iface
{
   private int _threadCount = 64;
   private AtomicBoolean _closed = new AtomicBoolean();
   private Map<String, Integer> _tableShardCountMap = new ConcurrentHashMap<String,
Integer>();
-  private BlurPartitioner<BytesWritable, Void> _blurPartitioner = new BlurPartitioner<BytesWritable,
Void>();
+  private BlurPartitioner _blurPartitioner = new BlurPartitioner();
   private String _nodeName;
   private int _remoteFetchCount = 100;
   private long _maxTimeToLive = TimeUnit.MINUTES.toMillis(1);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8eaca7c1/src/blur-core/src/main/java/org/apache/blur/thrift/MutationHelper.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/MutationHelper.java b/src/blur-core/src/main/java/org/apache/blur/thrift/MutationHelper.java
index b6a26a0..3ca394e 100644
--- a/src/blur-core/src/main/java/org/apache/blur/thrift/MutationHelper.java
+++ b/src/blur-core/src/main/java/org/apache/blur/thrift/MutationHelper.java
@@ -25,14 +25,11 @@ import org.apache.blur.thrift.generated.Row;
 import org.apache.blur.thrift.generated.RowMutation;
 import org.apache.blur.utils.BlurConstants;
 import org.apache.blur.utils.BlurUtil;
-import org.apache.hadoop.io.BytesWritable;
-
 
 public class MutationHelper {
 
-  public static String getShardName(String table, String rowId, int numberOfShards, BlurPartitioner<BytesWritable,
?> blurPartitioner) {
-    BytesWritable key = getKey(rowId);
-    int partition = blurPartitioner.getPartition(key, null, numberOfShards);
+  public static String getShardName(String table, String rowId, int numberOfShards, BlurPartitioner
blurPartitioner) {
+    int partition = blurPartitioner.getShard(rowId, numberOfShards);
     return BlurUtil.getShardName(BlurConstants.SHARD_PREFIX, partition);
   }
 
@@ -48,10 +45,6 @@ public class MutationHelper {
     }
   }
 
-  public static BytesWritable getKey(String rowId) {
-    return new BytesWritable(rowId.getBytes());
-  }
-
   public static Row getRowFromMutations(String id, List<RecordMutation> recordMutations)
{
     Row row = new Row().setId(id);
     for (RecordMutation mutation : recordMutations) {


Mime
View raw message