incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [1/2] git commit: The document fetch is now working.
Date Tue, 27 Nov 2012 03:06:13 GMT
Updated Branches:
  refs/heads/0.2-dev-removing-old-thrift ce72e5444 -> 46d9a00c3


The document fetch is now working.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/46d9a00c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/46d9a00c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/46d9a00c

Branch: refs/heads/0.2-dev-removing-old-thrift
Commit: 46d9a00c36402ef8b86742753e354c00bd9cfebd
Parents: 68cf6ed
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Mon Nov 26 21:46:21 2012 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Mon Nov 26 21:46:21 2012 -0500

----------------------------------------------------------------------
 .../org/apache/blur/thrift/BlurShardServer.java    |   36 +++++++++++----
 .../main/java/org/apache/blur/utils/BlurUtil.java  |   20 +++++---
 .../apache/blur/utils/ThriftLuceneConversion.java  |   24 +++++++++-
 .../org/apache/blur/thrift/ITBlurClusterTest.java  |   11 +++-
 .../blur/testsuite/AddDocumentsLuceneApiTable.java |    2 +-
 .../org/apache/blur/testsuite/CreateTable.java     |   19 +++-----
 6 files changed, 78 insertions(+), 34 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/46d9a00c/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java b/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
index d362653..2fe9aca 100644
--- a/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
+++ b/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
@@ -19,6 +19,8 @@ package org.apache.blur.thrift;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_CACHE_MAX_QUERYCACHE_ELEMENTS;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_CACHE_MAX_TIMETOLIVE;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_DATA_FETCH_THREAD_COUNT;
+import static org.apache.blur.utils.ThriftLuceneConversion.setShardIndexTopDocs;
+import static org.apache.blur.utils.ThriftLuceneConversion.setShardIndexTopFieldDocs;
 import static org.apache.blur.utils.ThriftLuceneConversion.toThrift;
 
 import java.io.IOException;
@@ -294,26 +296,22 @@ public class BlurShardServer extends TableAdmin implements Iface {
 
     private TopFieldDocs addShardIndex(TopFieldDocs topFieldDocs) {
       topFieldDocs.setShardIndex(shardIndex);
-      List<org.apache.blur.thrift.generated.ScoreDoc> scoreDocs = topFieldDocs.getScoreDocs();
-      for (org.apache.blur.thrift.generated.ScoreDoc sd : scoreDocs) {
-        long docLocation = sd.getDocLocation();
-        // docLocation.setShardIndex(shardIndex);
-      }
       return topFieldDocs;
     }
 
     private TopFieldDocs doSearch() throws IOException {
       if (after == null) {
         if (sort == null) {
-          return toThrift(searcher.search(query, filter, count));
+          return toThrift(setShardIndexTopDocs(shardIndex, searcher.search(query, filter,
count)));
         } else {
-          return toThrift(searcher.search(query, filter, count, sort, doDocScores, doMaxScore));
+          return toThrift(setShardIndexTopFieldDocs(shardIndex, searcher.search(query, filter,
count, sort, doDocScores, doMaxScore)));
         }
       } else {
         if (sort == null) {
-          return toThrift(searcher.searchAfter(after, query, filter, count));
+          return toThrift(setShardIndexTopDocs(shardIndex, searcher.searchAfter(after, query,
filter, count)));
         } else {
-          return toThrift((org.apache.lucene.search.TopFieldDocs) searcher.searchAfter(after,
query, filter, count, sort, doDocScores, doMaxScore));
+          return toThrift(setShardIndexTopFieldDocs(shardIndex,
+              (org.apache.lucene.search.TopFieldDocs) searcher.searchAfter(after, query,
filter, count, sort, doDocScores, doMaxScore)));
         }
       }
     }
@@ -382,7 +380,11 @@ public class BlurShardServer extends TableAdmin implements Iface {
         int shardIndex = BlurUtil.getShardIndex(docLocation);
         int docId = BlurUtil.getDocumentId(docLocation);
         IndexSearcher searcher = searchers.get(shardIndex);
-        result.add(toThrift(searcher.document(docId, fieldsToLoad)));
+        if (searcher == null) {
+          result.addAll(forwardDoc(session, shardIndex, docLocation, fieldsToLoad));
+        } else {
+          result.add(toThrift(searcher.document(docId, fieldsToLoad)));
+        }
       }
       return result;
     } catch (Throwable t) {
@@ -391,6 +393,20 @@ public class BlurShardServer extends TableAdmin implements Iface {
     }
   }
 
+  private List<Document> forwardDoc(final Session session, int shardIndex, final Long
docLocation, final Set<String> fieldsToLoad) throws BlurException, TException, IOException
{
+    // TODO Make more efficient by making a single call to a server for many
+    // docLocations
+    String table = session.getTableName();
+    System.out.println(table + " " + shardIndex);
+    Connection connection = getConnection(table, shardIndex);
+    return BlurClientManager.execute(connection, new BlurCommand<List<Document>>()
{
+      @Override
+      public List<Document> call(Client client) throws BlurException, TException {
+        return client.doc(session, Arrays.asList(docLocation), fieldsToLoad);
+      }
+    });
+  }
+
   @Override
   public void closeReadSession(Session session) throws BlurException, TException {
     SessionInfo sessionInfo = getSessionInfo(session);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/46d9a00c/src/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java b/src/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
index 7ba2eed..b9cb4cc 100644
--- a/src/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
+++ b/src/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
@@ -42,6 +42,7 @@ import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants;
 import org.apache.blur.metrics.BlurMetrics;
 import org.apache.blur.metrics.BlurMetrics.MethodCall;
 import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.blur.thrift.generated.ScoreDoc;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
@@ -119,7 +120,6 @@ public class BlurUtil {
     }
   }
 
-
   public static void createIfMissing(ZooKeeper zookeeper, String path) throws KeeperException,
InterruptedException {
     if (zookeeper.exists(path, false) == null) {
       try {
@@ -497,18 +497,24 @@ public class BlurUtil {
   }
 
   public static int getShardIndex(long docLocation) {
-    // TODO Auto-generated method stub
-    return 0;
+    return (int) (docLocation >> 32);
   }
 
   public static int getDocumentId(long docLocation) {
-    // TODO Auto-generated method stub
-    return 0;
+    return (int) (docLocation & 0xFFFFFFFF);
   }
 
   public static long getDocLocation(int shardIndex, int doc) {
-    // TODO Auto-generated method stub
-    return 0;
+    long docLocation = (long) shardIndex << 32L;
+    return docLocation | ((long) doc & 0xFFFFFFFFL);
+  }
+
+  public static List<Long> getDocLocations(List<ScoreDoc> scoreDocs) {
+    List<Long> docLocations = new ArrayList<Long>(scoreDocs.size());
+    for (ScoreDoc scoreDoc : scoreDocs) {
+      docLocations.add(scoreDoc.getDocLocation());
+    }
+    return docLocations;
   }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/46d9a00c/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java
b/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java
index 543c82c..affa0a6 100644
--- a/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java
+++ b/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java
@@ -44,6 +44,28 @@ public class ThriftLuceneConversion {
     return toThrift(new org.apache.lucene.search.TopFieldDocs(topDocs.totalHits, topDocs.scoreDocs,
null, Float.NaN));
   }
 
+  public static TopDocs setShardIndexTopDocs(int shardIndex, TopDocs topDocs) {
+    org.apache.lucene.search.ScoreDoc[] scoreDocs = topDocs.scoreDocs;
+    for (int i = 0; i < scoreDocs.length; i++) {
+      org.apache.lucene.search.ScoreDoc scoreDoc = scoreDocs[i];
+      if (scoreDoc != null) {
+        scoreDoc.shardIndex = shardIndex;
+      }
+    }
+    return topDocs;
+  }
+  
+  public static org.apache.lucene.search.TopFieldDocs setShardIndexTopFieldDocs(int shardIndex,
org.apache.lucene.search.TopFieldDocs topDocs) {
+    org.apache.lucene.search.ScoreDoc[] scoreDocs = topDocs.scoreDocs;
+    for (int i = 0; i < scoreDocs.length; i++) {
+      org.apache.lucene.search.ScoreDoc scoreDoc = scoreDocs[i];
+      if (scoreDoc != null) {
+        scoreDoc.shardIndex = shardIndex;
+      }
+    }
+    return topDocs;
+  }
+
   public static org.apache.lucene.search.ScoreDoc toLucene(org.apache.blur.thrift.generated.ScoreDoc
scoreDoc) {
     if (scoreDoc == null) {
       return null;
@@ -289,7 +311,7 @@ public class ThriftLuceneConversion {
 
   public static Query toLuceneQuery(ByteBuffer byteBuffer) throws IOException {
     DataInputBuffer in = new DataInputBuffer();
-    in.reset(byteBuffer.array(),byteBuffer.arrayOffset() + byteBuffer.position() , byteBuffer.remaining());
+    in.reset(byteBuffer.array(), byteBuffer.arrayOffset() + byteBuffer.position(), byteBuffer.remaining());
     QueryWritable qw = new QueryWritable();
     qw.readFields(in);
     return qw.getQuery();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/46d9a00c/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java b/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java
index 28d1144..3a8971e 100644
--- a/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java
+++ b/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java
@@ -41,6 +41,7 @@ import org.apache.blur.thrift.generated.TYPE;
 import org.apache.blur.thrift.generated.TableDescriptor;
 import org.apache.blur.thrift.generated.TopFieldDocs;
 import org.apache.blur.thrift.generated.UpdatePackage;
+import org.apache.blur.utils.BlurUtil;
 import org.apache.blur.utils.ThriftLuceneConversion;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -143,6 +144,10 @@ public class ITBlurClusterTest {
     long totalHits = 0;
     for (TopFieldDocs fieldDocs : results) {
       totalHits += fieldDocs.getTotalHits();
+      List<Document> docs = client.doc(session, BlurUtil.getDocLocations(fieldDocs.getScoreDocs()),
null);
+      for (Document document : docs) {
+        System.out.println(document);
+      }
     }
     assertEquals(SHARD_COUNT * 100, totalHits);
   }
@@ -190,7 +195,7 @@ public class ITBlurClusterTest {
     }
     assertEquals(SHARD_COUNT * 100, totalHits);
   }
-  
+
   @Test
   public void testAddDocumentsPerformance() throws BlurException, TException, InterruptedException,
IOException {
     Iface client = getClient();
@@ -227,9 +232,9 @@ public class ITBlurClusterTest {
     long t2 = System.nanoTime();
 
     client.blockUntilGenerationIsVisible(generations, true);
-    
+
     long t3 = System.nanoTime();
-    
+
     System.out.println("Add time [" + (t2 - t1) / 1000000000.0 + "]");
     System.out.println("Block time [" + (t3 - t2) / 1000000000.0 + "]");
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/46d9a00c/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/AddDocumentsLuceneApiTable.java
----------------------------------------------------------------------
diff --git a/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/AddDocumentsLuceneApiTable.java
b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/AddDocumentsLuceneApiTable.java
index 2983b01..2f6be97 100644
--- a/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/AddDocumentsLuceneApiTable.java
+++ b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/AddDocumentsLuceneApiTable.java
@@ -26,7 +26,7 @@ import java.util.UUID;
 import org.apache.blur.thrift.generated.BlurException;
 import org.apache.blur.thrift.generated.Document;
 import org.apache.blur.thrift.generated.Field;
-import org.apache.blur.thrift.generated.ShardServer.Client;
+import org.apache.blur.thrift.generated.Blur.Client;
 import org.apache.blur.thrift.generated.MutateOptions;
 import org.apache.blur.thrift.generated.TYPE;
 import org.apache.lucene.queryparser.classic.ParseException;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/46d9a00c/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/CreateTable.java
----------------------------------------------------------------------
diff --git a/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/CreateTable.java b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/CreateTable.java
index 2b4e697..b7d4bda 100644
--- a/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/CreateTable.java
+++ b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/CreateTable.java
@@ -19,10 +19,9 @@ package org.apache.blur.testsuite;
 import java.io.IOException;
 
 import org.apache.blur.thrift.BlurClient;
-import org.apache.blur.thrift.generated.AnalyzerDefinition;
+import org.apache.blur.thrift.generated.Blur.Iface;
 import org.apache.blur.thrift.generated.BlurException;
 import org.apache.blur.thrift.generated.TableDescriptor;
-import org.apache.blur.thrift.generated.Blur.Iface;
 import org.apache.thrift.TException;
 
 
@@ -30,19 +29,15 @@ public class CreateTable {
 
   public static void main(String[] args) throws BlurException, TException, IOException {
     String connectionStr = args[0];
-    final String cluster = args[1];
-    final String tableName = args[2];
-    int shardCount = Integer.parseInt(args[3]);
-    String uri = args[4];
+    final String tableName = args[1];
+    int shardCount = Integer.parseInt(args[2]);
+    String uri = args[3];
 
     final TableDescriptor tableDescriptor = new TableDescriptor();
-    tableDescriptor.analyzerDefinition = new AnalyzerDefinition();
-    tableDescriptor.cluster = cluster;
-    tableDescriptor.name = tableName;
-    tableDescriptor.readOnly = false;
+    tableDescriptor.setName ( tableName);
 
-    tableDescriptor.shardCount = shardCount;
-    tableDescriptor.tableUri = uri;
+    tableDescriptor.setShardCount(shardCount);
+    tableDescriptor.setStoragePath( uri);
 
     Iface client = BlurClient.getClient(connectionStr);
     client.createTable(tableDescriptor);


Mime
View raw message