incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [2/5] git commit: Added document fetching.
Date Wed, 31 Oct 2012 01:14:49 GMT
Added document fetching.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/0e8a136f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/0e8a136f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/0e8a136f

Branch: refs/heads/0.2-dev
Commit: 0e8a136f339348c7514af976aa28aae91827bce2
Parents: 7d6b160
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Tue Oct 30 21:13:11 2012 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Tue Oct 30 21:13:11 2012 -0400

----------------------------------------------------------------------
 .../org/apache/blur/thrift/BlurShardServer.java    |   21 ++-
 .../java/org/apache/blur/thrift/SessionInfo.java   |   17 ++
 .../main/java/org/apache/blur/utils/BlurUtil.java  |  157 ++++++++++++++-
 3 files changed, 189 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e8a136f/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java b/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
index 1085579..0b6cc89 100644
--- a/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
+++ b/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
@@ -524,15 +524,26 @@ public class BlurShardServer extends TableAdmin implements Iface {
   }
 
   @Override
-  public List<Document> doc(Session session, List<DocLocation> docLocations,
Set<String> fields) throws BlurException, TException {
-    // TODO Auto-generated method stub
-    return null;
+  public List<Document> doc(Session session, List<DocLocation> docLocations,
Set<String> fieldsToLoad) throws BlurException, TException {
+    try {
+      SessionInfo sessionInfo = getSessionInfo(session);
+      Map<Integer, IndexSearcher> searchers = sessionInfo.getSearchers();
+      List<Document> result = new ArrayList<Document>();
+      for (DocLocation docLocation : docLocations) {
+        IndexSearcher searcher = searchers.get(docLocation.getShardIndex());
+        result.add(BlurUtil.convert(searcher.document(docLocation.getDoc(), fieldsToLoad)));
+      }
+      return result;
+    } catch (Throwable t) {
+      LOG.error("Unknown error", t);
+      throw new BException(t.getMessage(), t);
+    }
   }
 
   @Override
   public void closeReadSession(Session session) throws BlurException, TException {
-    // TODO Auto-generated method stub
-
+    SessionInfo sessionInfo = getSessionInfo(session);
+    sessionInfo.releaseReaders();
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e8a136f/src/blur-core/src/main/java/org/apache/blur/thrift/SessionInfo.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/SessionInfo.java b/src/blur-core/src/main/java/org/apache/blur/thrift/SessionInfo.java
index e147423..435d1c7 100644
--- a/src/blur-core/src/main/java/org/apache/blur/thrift/SessionInfo.java
+++ b/src/blur-core/src/main/java/org/apache/blur/thrift/SessionInfo.java
@@ -1,13 +1,19 @@
 package org.apache.blur.thrift;
 
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Map.Entry;
 
 import org.apache.blur.analysis.BlurAnalyzer;
+import org.apache.blur.log.Log;
+import org.apache.blur.log.LogFactory;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.IndexSearcher;
 
 public class SessionInfo {
+  
+  private static final Log LOG = LogFactory.getLog(SessionInfo.class);
 
   private String uuid;
   private BlurAnalyzer analyzer;
@@ -53,6 +59,17 @@ public class SessionInfo {
   public void setSearchers(Map<Integer, IndexSearcher> searchers) {
     this.searchers = searchers;
   }
+
+  public void releaseReaders() {
+    for (Entry<Integer, IndexReader> entry : readers.entrySet()) {
+      IndexReader reader = entry.getValue();
+      try {
+        reader.decRef();
+      } catch (IOException e) {
+        LOG.error("Unknown exception while trying to decRef on reader [{0}]",e,reader);
+      }
+    }
+  }
   
   
   

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e8a136f/src/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java b/src/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
index df6ceec..3468507 100644
--- a/src/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
+++ b/src/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
@@ -57,6 +57,7 @@ import org.apache.blur.thrift.generated.BlurResults;
 import org.apache.blur.thrift.generated.Column;
 import org.apache.blur.thrift.generated.DocLocation;
 import org.apache.blur.thrift.generated.FetchResult;
+import org.apache.blur.thrift.generated.Field;
 import org.apache.blur.thrift.generated.QueryArgs;
 import org.apache.blur.thrift.generated.Record;
 import org.apache.blur.thrift.generated.RecordMutation;
@@ -69,6 +70,7 @@ import org.apache.blur.thrift.generated.Selector;
 import org.apache.blur.thrift.generated.SimpleQuery;
 import org.apache.blur.thrift.generated.SortField;
 import org.apache.blur.thrift.generated.SortType;
+import org.apache.blur.thrift.generated.TYPE;
 import org.apache.blur.thrift.generated.TopFieldDocs;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
@@ -79,6 +81,7 @@ import org.apache.hadoop.io.DataInputBuffer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.AtomicReader;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.SlowCompositeReaderWrapper;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Filter;
@@ -88,6 +91,7 @@ import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField.Type;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.thrift.TBase;
 import org.apache.thrift.TException;
@@ -788,7 +792,7 @@ public class BlurUtil {
   }
 
   public static List<ByteBuffer> convert(Object[] fields) {
-//    throw new RuntimeException("Not supported YET!");
+    // throw new RuntimeException("Not supported YET!");
     LOG.error("Not supported YET!");
     return null;
   }
@@ -821,4 +825,155 @@ public class BlurUtil {
     }
     return new org.apache.lucene.search.ScoreDoc(scoreDoc.docLocation.doc, (float) scoreDoc.score,
scoreDoc.docLocation.shardIndex);
   }
+
+  public static org.apache.blur.thrift.generated.Document convert(org.apache.lucene.document.Document
document) {
+    org.apache.blur.thrift.generated.Document result = new org.apache.blur.thrift.generated.Document();
+    List<IndexableField> fields = document.getFields();
+    for (IndexableField field : fields) {
+      result.addToFields(convert(field));
+    }
+    return result;
+  }
+
+  public static Field convert(IndexableField field) {
+    Field result = new Field();
+    result.setBoost(field.boost()); 
+    result.setName(field.name());
+    Number number = field.numericValue();
+    if (number != null) {
+      if (number instanceof Byte || number instanceof Short || number instanceof Integer)
{
+        result.setType(TYPE.INT);
+        result.setValue(convert((Integer) number));
+      } else if (number instanceof Long) {
+        result.setType(TYPE.LONG);
+        result.setValue(convert((Long) number));
+      } else if (number instanceof Float) {
+        result.setType(TYPE.FLOAT);
+        result.setValue(convert((Float) number));
+      } else if (number instanceof Double) {
+        result.setType(TYPE.DOUBLE);
+        result.setValue(convert((Double) number));
+      } else {
+        throw new IllegalArgumentException("cannot store numeric type " + number.getClass());
+      }
+    } else {
+      BytesRef bytes = field.binaryValue();
+      if (bytes != null) {
+        result.setType(TYPE.BINARY);
+        result.setValue(convert(bytes));
+      } else {
+        result.setType(TYPE.STRING);
+        result.setValue(convert(field.stringValue()));
+      }
+    }
+    return result;
+  }
+
+  public static ByteBuffer convert(String val) {
+    if (val == null) {
+      return null;
+    }
+    return convert(new BytesRef(val));
+  }
+
+  public static ByteBuffer convert(BytesRef val) {
+    if (val == null) {
+      return null;
+    }
+    return flip(ByteBuffer.allocate(val.length).put(val.bytes, val.offset, val.length));
+  }
+
+  private static ByteBuffer flip(ByteBuffer buffer) {
+    buffer.flip();
+    return buffer;
+  }
+
+  public static ByteBuffer convert(Double val) {
+    if (val == null) {
+      return null;
+    }
+    return flip(ByteBuffer.allocate(8).putDouble(val));
+  }
+
+  public static ByteBuffer convert(Long val) {
+    if (val == null) {
+      return null;
+    }
+    return flip(ByteBuffer.allocate(8).putLong(val));
+  }
+
+  public static ByteBuffer convert(Float val) {
+    if (val == null) {
+      return null;
+    }
+    return flip(ByteBuffer.allocate(4).putFloat(val));
+  }
+
+  public static ByteBuffer convert(Integer val) {
+    if (val == null) {
+      return null;
+    }
+    return flip(ByteBuffer.allocate(4).putInt(val));
+  }
+
+  public static List<DocLocation> getDocLocations(List<ScoreDoc> scoreDocs) {
+    List<DocLocation> result = new ArrayList<DocLocation>(scoreDocs.size());
+    for (ScoreDoc scoreDoc : scoreDocs) {
+      result.add(scoreDoc.getDocLocation());
+    }
+    return result;
+  }
+
+  public static String toString(org.apache.blur.thrift.generated.Document document) {
+    StringBuilder builder = new StringBuilder();
+    builder.append('{');
+    for (Field f : document.getFields()) {
+      if (builder.length() != 0) {
+        builder.append(',');
+      }
+      builder.append(toString(f));
+    }
+    builder.append('}');
+    return builder.toString();
+  }
+
+  public static String toString(Field f) {
+    StringBuilder builder = new StringBuilder();
+    builder.append('{');
+    String name = f.getName();
+    builder.append(name);
+    builder.append(',');
+    
+    TYPE type = f.getType();
+    builder.append(type.name());
+    builder.append(',');
+    
+    byte[] value = f.getValue();
+    builder.append(toString(type, value));
+    builder.append(',');
+    
+    double boost = f.getBoost();
+    builder.append(boost);
+    builder.append('}');
+    return builder.toString();
+  }
+
+  public static String toString(TYPE type, byte[] value) {
+    switch (type) {
+    case STRING:
+      return new BytesRef(value).utf8ToString();
+    case INT:
+      return Integer.toString(ByteBuffer.wrap(value).getInt());
+    case LONG:
+      return Long.toString(ByteBuffer.wrap(value).getLong());
+    case FLOAT:
+      return Float.toString(ByteBuffer.wrap(value).getFloat());
+    case DOUBLE:
+      return Double.toString(ByteBuffer.wrap(value).getDouble());
+    case BINARY:
+      return ByteBuffer.wrap(value).toString();
+    default:
+      throw new RuntimeException("Type unknown.");
+    }
+  }
 }


Mime
View raw message